1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/aec3/suppression_gain.h"
12
13 #include <math.h>
14 #include <stddef.h>
15
16 #include <algorithm>
17 #include <numeric>
18
19 #include "modules/audio_processing/aec3/dominant_nearend_detector.h"
20 #include "modules/audio_processing/aec3/moving_average.h"
21 #include "modules/audio_processing/aec3/subband_nearend_detector.h"
22 #include "modules/audio_processing/aec3/vector_math.h"
23 #include "modules/audio_processing/logging/apm_data_dumper.h"
24 #include "rtc_base/atomic_ops.h"
25 #include "rtc_base/checks.h"
26
27 namespace webrtc {
28 namespace {
29
PostprocessGains(std::array<float,kFftLengthBy2Plus1> * gain)30 void PostprocessGains(std::array<float, kFftLengthBy2Plus1>* gain) {
31 // TODO(gustaf): Investigate if this can be relaxed to achieve higher
32 // transparency above 2 kHz.
33
34 // Limit the low frequency gains to avoid the impact of the high-pass filter
35 // on the lower-frequency gain influencing the overall achieved gain.
36 (*gain)[0] = (*gain)[1] = std::min((*gain)[1], (*gain)[2]);
37
38 // Limit the high frequency gains to avoid the impact of the anti-aliasing
39 // filter on the upper-frequency gains influencing the overall achieved
40 // gain. TODO(peah): Update this when new anti-aliasing filters are
41 // implemented.
42 constexpr size_t kAntiAliasingImpactLimit = (64 * 2000) / 8000;
43 const float min_upper_gain = (*gain)[kAntiAliasingImpactLimit];
44 std::for_each(
45 gain->begin() + kAntiAliasingImpactLimit, gain->end() - 1,
46 [min_upper_gain](float& a) { a = std::min(a, min_upper_gain); });
47 (*gain)[kFftLengthBy2] = (*gain)[kFftLengthBy2Minus1];
48
49 // Limits the gain in the frequencies for which the adaptive filter has not
50 // converged.
51 // TODO(peah): Make adaptive to take the actual filter error into account.
52 constexpr size_t kUpperAccurateBandPlus1 = 29;
53
54 constexpr float oneByBandsInSum =
55 1 / static_cast<float>(kUpperAccurateBandPlus1 - 20);
56 const float hf_gain_bound =
57 std::accumulate(gain->begin() + 20,
58 gain->begin() + kUpperAccurateBandPlus1, 0.f) *
59 oneByBandsInSum;
60
61 std::for_each(gain->begin() + kUpperAccurateBandPlus1, gain->end(),
62 [hf_gain_bound](float& a) { a = std::min(a, hf_gain_bound); });
63 }
64
65 // Scales the echo according to assessed audibility at the other end.
WeightEchoForAudibility(const EchoCanceller3Config & config,rtc::ArrayView<const float> echo,rtc::ArrayView<float> weighted_echo)66 void WeightEchoForAudibility(const EchoCanceller3Config& config,
67 rtc::ArrayView<const float> echo,
68 rtc::ArrayView<float> weighted_echo) {
69 RTC_DCHECK_EQ(kFftLengthBy2Plus1, echo.size());
70 RTC_DCHECK_EQ(kFftLengthBy2Plus1, weighted_echo.size());
71
72 auto weigh = [](float threshold, float normalizer, size_t begin, size_t end,
73 rtc::ArrayView<const float> echo,
74 rtc::ArrayView<float> weighted_echo) {
75 for (size_t k = begin; k < end; ++k) {
76 if (echo[k] < threshold) {
77 float tmp = (threshold - echo[k]) * normalizer;
78 weighted_echo[k] = echo[k] * std::max(0.f, 1.f - tmp * tmp);
79 } else {
80 weighted_echo[k] = echo[k];
81 }
82 }
83 };
84
85 float threshold = config.echo_audibility.floor_power *
86 config.echo_audibility.audibility_threshold_lf;
87 float normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
88 weigh(threshold, normalizer, 0, 3, echo, weighted_echo);
89
90 threshold = config.echo_audibility.floor_power *
91 config.echo_audibility.audibility_threshold_mf;
92 normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
93 weigh(threshold, normalizer, 3, 7, echo, weighted_echo);
94
95 threshold = config.echo_audibility.floor_power *
96 config.echo_audibility.audibility_threshold_hf;
97 normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
98 weigh(threshold, normalizer, 7, kFftLengthBy2Plus1, echo, weighted_echo);
99 }
100
101 } // namespace
102
103 int SuppressionGain::instance_count_ = 0;
104
UpperBandsGain(rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> echo_spectrum,rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> comfort_noise_spectrum,const absl::optional<int> & narrow_peak_band,bool saturated_echo,const std::vector<std::vector<std::vector<float>>> & render,const std::array<float,kFftLengthBy2Plus1> & low_band_gain) const105 float SuppressionGain::UpperBandsGain(
106 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> echo_spectrum,
107 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
108 comfort_noise_spectrum,
109 const absl::optional<int>& narrow_peak_band,
110 bool saturated_echo,
111 const std::vector<std::vector<std::vector<float>>>& render,
112 const std::array<float, kFftLengthBy2Plus1>& low_band_gain) const {
113 RTC_DCHECK_LT(0, render.size());
114 if (render.size() == 1) {
115 return 1.f;
116 }
117 const size_t num_render_channels = render[0].size();
118
119 if (narrow_peak_band &&
120 (*narrow_peak_band > static_cast<int>(kFftLengthBy2Plus1 - 10))) {
121 return 0.001f;
122 }
123
124 constexpr size_t kLowBandGainLimit = kFftLengthBy2 / 2;
125 const float gain_below_8_khz = *std::min_element(
126 low_band_gain.begin() + kLowBandGainLimit, low_band_gain.end());
127
128 // Always attenuate the upper bands when there is saturated echo.
129 if (saturated_echo) {
130 return std::min(0.001f, gain_below_8_khz);
131 }
132
133 // Compute the upper and lower band energies.
134 const auto sum_of_squares = [](float a, float b) { return a + b * b; };
135 float low_band_energy = 0.f;
136 for (size_t ch = 0; ch < num_render_channels; ++ch) {
137 const float channel_energy = std::accumulate(
138 render[0][0].begin(), render[0][0].end(), 0.f, sum_of_squares);
139 low_band_energy = std::max(low_band_energy, channel_energy);
140 }
141 float high_band_energy = 0.f;
142 for (size_t k = 1; k < render.size(); ++k) {
143 for (size_t ch = 0; ch < num_render_channels; ++ch) {
144 const float energy = std::accumulate(
145 render[k][ch].begin(), render[k][ch].end(), 0.f, sum_of_squares);
146 high_band_energy = std::max(high_band_energy, energy);
147 }
148 }
149
150 // If there is more power in the lower frequencies than the upper frequencies,
151 // or if the power in upper frequencies is low, do not bound the gain in the
152 // upper bands.
153 float anti_howling_gain;
154 const float activation_threshold =
155 kBlockSize * config_.suppressor.high_bands_suppression
156 .anti_howling_activation_threshold;
157 if (high_band_energy < std::max(low_band_energy, activation_threshold)) {
158 anti_howling_gain = 1.f;
159 } else {
160 // In all other cases, bound the gain for upper frequencies.
161 RTC_DCHECK_LE(low_band_energy, high_band_energy);
162 RTC_DCHECK_NE(0.f, high_band_energy);
163 anti_howling_gain =
164 config_.suppressor.high_bands_suppression.anti_howling_gain *
165 sqrtf(low_band_energy / high_band_energy);
166 }
167
168 float gain_bound = 1.f;
169 if (!dominant_nearend_detector_->IsNearendState()) {
170 // Bound the upper gain during significant echo activity.
171 const auto& cfg = config_.suppressor.high_bands_suppression;
172 auto low_frequency_energy = [](rtc::ArrayView<const float> spectrum) {
173 RTC_DCHECK_LE(16, spectrum.size());
174 return std::accumulate(spectrum.begin() + 1, spectrum.begin() + 16, 0.f);
175 };
176 for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
177 const float echo_sum = low_frequency_energy(echo_spectrum[ch]);
178 const float noise_sum = low_frequency_energy(comfort_noise_spectrum[ch]);
179 if (echo_sum > cfg.enr_threshold * noise_sum) {
180 gain_bound = cfg.max_gain_during_echo;
181 break;
182 }
183 }
184 }
185
186 // Choose the gain as the minimum of the lower and upper gains.
187 return std::min(std::min(gain_below_8_khz, anti_howling_gain), gain_bound);
188 }
189
190 // Computes the gain to reduce the echo to a non audible level.
GainToNoAudibleEcho(const std::array<float,kFftLengthBy2Plus1> & nearend,const std::array<float,kFftLengthBy2Plus1> & echo,const std::array<float,kFftLengthBy2Plus1> & masker,std::array<float,kFftLengthBy2Plus1> * gain) const191 void SuppressionGain::GainToNoAudibleEcho(
192 const std::array<float, kFftLengthBy2Plus1>& nearend,
193 const std::array<float, kFftLengthBy2Plus1>& echo,
194 const std::array<float, kFftLengthBy2Plus1>& masker,
195 std::array<float, kFftLengthBy2Plus1>* gain) const {
196 const auto& p = dominant_nearend_detector_->IsNearendState() ? nearend_params_
197 : normal_params_;
198 for (size_t k = 0; k < gain->size(); ++k) {
199 float enr = echo[k] / (nearend[k] + 1.f); // Echo-to-nearend ratio.
200 float emr = echo[k] / (masker[k] + 1.f); // Echo-to-masker (noise) ratio.
201 float g = 1.0f;
202 if (enr > p.enr_transparent_[k] && emr > p.emr_transparent_[k]) {
203 g = (p.enr_suppress_[k] - enr) /
204 (p.enr_suppress_[k] - p.enr_transparent_[k]);
205 g = std::max(g, p.emr_transparent_[k] / emr);
206 }
207 (*gain)[k] = g;
208 }
209 }
210
211 // Compute the minimum gain as the attenuating gain to put the signal just
212 // above the zero sample values.
GetMinGain(rtc::ArrayView<const float> weighted_residual_echo,rtc::ArrayView<const float> last_nearend,rtc::ArrayView<const float> last_echo,bool low_noise_render,bool saturated_echo,rtc::ArrayView<float> min_gain) const213 void SuppressionGain::GetMinGain(
214 rtc::ArrayView<const float> weighted_residual_echo,
215 rtc::ArrayView<const float> last_nearend,
216 rtc::ArrayView<const float> last_echo,
217 bool low_noise_render,
218 bool saturated_echo,
219 rtc::ArrayView<float> min_gain) const {
220 if (!saturated_echo) {
221 const float min_echo_power =
222 low_noise_render ? config_.echo_audibility.low_render_limit
223 : config_.echo_audibility.normal_render_limit;
224
225 for (size_t k = 0; k < min_gain.size(); ++k) {
226 min_gain[k] = weighted_residual_echo[k] > 0.f
227 ? min_echo_power / weighted_residual_echo[k]
228 : 1.f;
229 min_gain[k] = std::min(min_gain[k], 1.f);
230 }
231
232 const bool is_nearend_state = dominant_nearend_detector_->IsNearendState();
233 for (size_t k = 0; k < 6; ++k) {
234 const auto& dec = is_nearend_state ? nearend_params_.max_dec_factor_lf
235 : normal_params_.max_dec_factor_lf;
236
237 // Make sure the gains of the low frequencies do not decrease too
238 // quickly after strong nearend.
239 if (last_nearend[k] > last_echo[k]) {
240 min_gain[k] = std::max(min_gain[k], last_gain_[k] * dec);
241 min_gain[k] = std::min(min_gain[k], 1.f);
242 }
243 }
244 } else {
245 std::fill(min_gain.begin(), min_gain.end(), 0.f);
246 }
247 }
248
249 // Compute the maximum gain by limiting the gain increase from the previous
250 // gain.
GetMaxGain(rtc::ArrayView<float> max_gain) const251 void SuppressionGain::GetMaxGain(rtc::ArrayView<float> max_gain) const {
252 const auto& inc = dominant_nearend_detector_->IsNearendState()
253 ? nearend_params_.max_inc_factor
254 : normal_params_.max_inc_factor;
255 const auto& floor = config_.suppressor.floor_first_increase;
256 for (size_t k = 0; k < max_gain.size(); ++k) {
257 max_gain[k] = std::min(std::max(last_gain_[k] * inc, floor), 1.f);
258 }
259 }
260
LowerBandGain(bool low_noise_render,const AecState & aec_state,rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> suppressor_input,rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> residual_echo,rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> comfort_noise,std::array<float,kFftLengthBy2Plus1> * gain)261 void SuppressionGain::LowerBandGain(
262 bool low_noise_render,
263 const AecState& aec_state,
264 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
265 suppressor_input,
266 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> residual_echo,
267 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> comfort_noise,
268 std::array<float, kFftLengthBy2Plus1>* gain) {
269 gain->fill(1.f);
270 const bool saturated_echo = aec_state.SaturatedEcho();
271 std::array<float, kFftLengthBy2Plus1> max_gain;
272 GetMaxGain(max_gain);
273
274 for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
275 std::array<float, kFftLengthBy2Plus1> G;
276 std::array<float, kFftLengthBy2Plus1> nearend;
277 nearend_smoothers_[ch].Average(suppressor_input[ch], nearend);
278
279 // Weight echo power in terms of audibility.
280 std::array<float, kFftLengthBy2Plus1> weighted_residual_echo;
281 WeightEchoForAudibility(config_, residual_echo[ch], weighted_residual_echo);
282
283 std::array<float, kFftLengthBy2Plus1> min_gain;
284 GetMinGain(weighted_residual_echo, last_nearend_[ch], last_echo_[ch],
285 low_noise_render, saturated_echo, min_gain);
286
287 GainToNoAudibleEcho(nearend, weighted_residual_echo, comfort_noise[0], &G);
288
289 // Clamp gains.
290 for (size_t k = 0; k < gain->size(); ++k) {
291 G[k] = std::max(std::min(G[k], max_gain[k]), min_gain[k]);
292 (*gain)[k] = std::min((*gain)[k], G[k]);
293 }
294
295 // Store data required for the gain computation of the next block.
296 std::copy(nearend.begin(), nearend.end(), last_nearend_[ch].begin());
297 std::copy(weighted_residual_echo.begin(), weighted_residual_echo.end(),
298 last_echo_[ch].begin());
299 }
300
301 // Limit high-frequency gains.
302 PostprocessGains(gain);
303
304 // Store computed gains.
305 std::copy(gain->begin(), gain->end(), last_gain_.begin());
306
307 // Transform gains to amplitude domain.
308 aec3::VectorMath(optimization_).Sqrt(*gain);
309 }
310
SuppressionGain(const EchoCanceller3Config & config,Aec3Optimization optimization,int sample_rate_hz,size_t num_capture_channels)311 SuppressionGain::SuppressionGain(const EchoCanceller3Config& config,
312 Aec3Optimization optimization,
313 int sample_rate_hz,
314 size_t num_capture_channels)
315 : data_dumper_(
316 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
317 optimization_(optimization),
318 config_(config),
319 num_capture_channels_(num_capture_channels),
320 state_change_duration_blocks_(
321 static_cast<int>(config_.filter.config_change_duration_blocks)),
322 last_nearend_(num_capture_channels_, {0}),
323 last_echo_(num_capture_channels_, {0}),
324 nearend_smoothers_(
325 num_capture_channels_,
326 aec3::MovingAverage(kFftLengthBy2Plus1,
327 config.suppressor.nearend_average_blocks)),
328 nearend_params_(config_.suppressor.nearend_tuning),
329 normal_params_(config_.suppressor.normal_tuning) {
330 RTC_DCHECK_LT(0, state_change_duration_blocks_);
331 last_gain_.fill(1.f);
332 if (config_.suppressor.use_subband_nearend_detection) {
333 dominant_nearend_detector_ = std::make_unique<SubbandNearendDetector>(
334 config_.suppressor.subband_nearend_detection, num_capture_channels_);
335 } else {
336 dominant_nearend_detector_ = std::make_unique<DominantNearendDetector>(
337 config_.suppressor.dominant_nearend_detection, num_capture_channels_);
338 }
339 RTC_DCHECK(dominant_nearend_detector_);
340 }
341
342 SuppressionGain::~SuppressionGain() = default;
343
GetGain(rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> nearend_spectrum,rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> echo_spectrum,rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> residual_echo_spectrum,rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> comfort_noise_spectrum,const RenderSignalAnalyzer & render_signal_analyzer,const AecState & aec_state,const std::vector<std::vector<std::vector<float>>> & render,float * high_bands_gain,std::array<float,kFftLengthBy2Plus1> * low_band_gain)344 void SuppressionGain::GetGain(
345 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
346 nearend_spectrum,
347 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> echo_spectrum,
348 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
349 residual_echo_spectrum,
350 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
351 comfort_noise_spectrum,
352 const RenderSignalAnalyzer& render_signal_analyzer,
353 const AecState& aec_state,
354 const std::vector<std::vector<std::vector<float>>>& render,
355 float* high_bands_gain,
356 std::array<float, kFftLengthBy2Plus1>* low_band_gain) {
357 RTC_DCHECK(high_bands_gain);
358 RTC_DCHECK(low_band_gain);
359
360 // Update the nearend state selection.
361 dominant_nearend_detector_->Update(nearend_spectrum, residual_echo_spectrum,
362 comfort_noise_spectrum, initial_state_);
363
364 // Compute gain for the lower band.
365 bool low_noise_render = low_render_detector_.Detect(render);
366 LowerBandGain(low_noise_render, aec_state, nearend_spectrum,
367 residual_echo_spectrum, comfort_noise_spectrum, low_band_gain);
368
369 // Compute the gain for the upper bands.
370 const absl::optional<int> narrow_peak_band =
371 render_signal_analyzer.NarrowPeakBand();
372
373 *high_bands_gain =
374 UpperBandsGain(echo_spectrum, comfort_noise_spectrum, narrow_peak_band,
375 aec_state.SaturatedEcho(), render, *low_band_gain);
376 }
377
SetInitialState(bool state)378 void SuppressionGain::SetInitialState(bool state) {
379 initial_state_ = state;
380 if (state) {
381 initial_state_change_counter_ = state_change_duration_blocks_;
382 } else {
383 initial_state_change_counter_ = 0;
384 }
385 }
386
387 // Detects when the render signal can be considered to have low power and
388 // consist of stationary noise.
Detect(const std::vector<std::vector<std::vector<float>>> & render)389 bool SuppressionGain::LowNoiseRenderDetector::Detect(
390 const std::vector<std::vector<std::vector<float>>>& render) {
391 float x2_sum = 0.f;
392 float x2_max = 0.f;
393 for (const auto& x_ch : render[0]) {
394 for (const auto& x_k : x_ch) {
395 const float x2 = x_k * x_k;
396 x2_sum += x2;
397 x2_max = std::max(x2_max, x2);
398 }
399 }
400 const size_t num_render_channels = render[0].size();
401 x2_sum = x2_sum / num_render_channels;
402 ;
403
404 constexpr float kThreshold = 50.f * 50.f * 64.f;
405 const bool low_noise_render =
406 average_power_ < kThreshold && x2_max < 3 * average_power_;
407 average_power_ = average_power_ * 0.9f + x2_sum * 0.1f;
408 return low_noise_render;
409 }
410
GainParameters(const EchoCanceller3Config::Suppressor::Tuning & tuning)411 SuppressionGain::GainParameters::GainParameters(
412 const EchoCanceller3Config::Suppressor::Tuning& tuning)
413 : max_inc_factor(tuning.max_inc_factor),
414 max_dec_factor_lf(tuning.max_dec_factor_lf) {
415 // Compute per-band masking thresholds.
416 constexpr size_t kLastLfBand = 5;
417 constexpr size_t kFirstHfBand = 8;
418 RTC_DCHECK_LT(kLastLfBand, kFirstHfBand);
419 auto& lf = tuning.mask_lf;
420 auto& hf = tuning.mask_hf;
421 RTC_DCHECK_LT(lf.enr_transparent, lf.enr_suppress);
422 RTC_DCHECK_LT(hf.enr_transparent, hf.enr_suppress);
423 for (size_t k = 0; k < kFftLengthBy2Plus1; k++) {
424 float a;
425 if (k <= kLastLfBand) {
426 a = 0.f;
427 } else if (k < kFirstHfBand) {
428 a = (k - kLastLfBand) / static_cast<float>(kFirstHfBand - kLastLfBand);
429 } else {
430 a = 1.f;
431 }
432 enr_transparent_[k] = (1 - a) * lf.enr_transparent + a * hf.enr_transparent;
433 enr_suppress_[k] = (1 - a) * lf.enr_suppress + a * hf.enr_suppress;
434 emr_transparent_[k] = (1 - a) * lf.emr_transparent + a * hf.emr_transparent;
435 }
436 }
437
438 } // namespace webrtc
439