1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/aec3/aec_state.h"
12
13 #include <math.h>
14
15 #include <numeric>
16 #include <vector>
17
18 #include "api/array_view.h"
19 #include "modules/audio_processing/logging/apm_data_dumper.h"
20 #include "rtc_base/atomicops.h"
21 #include "rtc_base/checks.h"
22
23 namespace webrtc {
24 namespace {
25
26 // Computes delay of the adaptive filter.
EstimateFilterDelay(const std::vector<std::array<float,kFftLengthBy2Plus1>> & adaptive_filter_frequency_response)27 int EstimateFilterDelay(
28 const std::vector<std::array<float, kFftLengthBy2Plus1>>&
29 adaptive_filter_frequency_response) {
30 const auto& H2 = adaptive_filter_frequency_response;
31 constexpr size_t kUpperBin = kFftLengthBy2 - 5;
32 RTC_DCHECK_GE(kAdaptiveFilterLength, H2.size());
33 std::array<int, kAdaptiveFilterLength> delays;
34 delays.fill(0);
35 for (size_t k = 1; k < kUpperBin; ++k) {
36 // Find the maximum of H2[j].
37 size_t peak = 0;
38 for (size_t j = 0; j < H2.size(); ++j) {
39 if (H2[j][k] > H2[peak][k]) {
40 peak = j;
41 }
42 }
43 ++delays[peak];
44 }
45
46 return std::distance(delays.begin(),
47 std::max_element(delays.begin(), delays.end()));
48 }
49
50 } // namespace
51
52 int AecState::instance_count_ = 0;
53
AecState(const EchoCanceller3Config & config)54 AecState::AecState(const EchoCanceller3Config& config)
55 : data_dumper_(
56 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
57 erle_estimator_(config.erle.min, config.erle.max_l, config.erle.max_h),
58 config_(config),
59 reverb_decay_(config_.ep_strength.default_len) {
60 max_render_.fill(0.f);
61 }
62
63 AecState::~AecState() = default;
64
HandleEchoPathChange(const EchoPathVariability & echo_path_variability)65 void AecState::HandleEchoPathChange(
66 const EchoPathVariability& echo_path_variability) {
67 if (echo_path_variability.AudioPathChanged()) {
68 blocks_since_last_saturation_ = 0;
69 usable_linear_estimate_ = false;
70 echo_leakage_detected_ = false;
71 capture_signal_saturation_ = false;
72 echo_saturation_ = false;
73 previous_max_sample_ = 0.f;
74 max_render_.fill(0.f);
75
76 if (echo_path_variability.delay_change) {
77 force_zero_gain_counter_ = 0;
78 blocks_with_filter_adaptation_ = 0;
79 blocks_with_strong_render_ = 0;
80 initial_state_ = true;
81 linear_echo_estimate_ = false;
82 sufficient_filter_updates_ = false;
83 render_received_ = false;
84 force_zero_gain_ = true;
85 capture_block_counter_ = 0;
86 }
87 if (echo_path_variability.gain_change) {
88 capture_block_counter_ = kNumBlocksPerSecond;
89 }
90 }
91 }
92
Update(const std::vector<std::array<float,kFftLengthBy2Plus1>> & adaptive_filter_frequency_response,const std::array<float,kAdaptiveFilterTimeDomainLength> & adaptive_filter_impulse_response,bool converged_filter,const rtc::Optional<size_t> & external_delay_samples,const RenderBuffer & render_buffer,const std::array<float,kFftLengthBy2Plus1> & E2_main,const std::array<float,kFftLengthBy2Plus1> & Y2,rtc::ArrayView<const float> x,const std::array<float,kBlockSize> & s,bool echo_leakage_detected)93 void AecState::Update(const std::vector<std::array<float, kFftLengthBy2Plus1>>&
94 adaptive_filter_frequency_response,
95 const std::array<float, kAdaptiveFilterTimeDomainLength>&
96 adaptive_filter_impulse_response,
97 bool converged_filter,
98 const rtc::Optional<size_t>& external_delay_samples,
99 const RenderBuffer& render_buffer,
100 const std::array<float, kFftLengthBy2Plus1>& E2_main,
101 const std::array<float, kFftLengthBy2Plus1>& Y2,
102 rtc::ArrayView<const float> x,
103 const std::array<float, kBlockSize>& s,
104 bool echo_leakage_detected) {
105 // Store input parameters.
106 echo_leakage_detected_ = echo_leakage_detected;
107
108 // Update counters.
109 ++capture_block_counter_;
110
111 // Force zero echo suppression gain after an echo path change to allow at
112 // least some render data to be collected in order to avoid an initial echo
113 // burst.
114 force_zero_gain_ = (++force_zero_gain_counter_) < kNumBlocksPerSecond / 5;
115
116 // Estimate delays.
117 filter_delay_ = EstimateFilterDelay(adaptive_filter_frequency_response);
118 external_delay_ =
119 external_delay_samples
120 ? rtc::Optional<size_t>(*external_delay_samples / kBlockSize)
121 : rtc::nullopt;
122
123 // Update the ERL and ERLE measures.
124 if (converged_filter && capture_block_counter_ >= 2 * kNumBlocksPerSecond) {
125 const auto& X2 = render_buffer.Spectrum(*filter_delay_);
126 erle_estimator_.Update(X2, Y2, E2_main);
127 erl_estimator_.Update(X2, Y2);
128 }
129
130 // Update the echo audibility evaluator.
131 echo_audibility_.Update(x, s, converged_filter);
132
133 // Detect and flag echo saturation.
134 // TODO(peah): Add the delay in this computation to ensure that the render and
135 // capture signals are properly aligned.
136 RTC_DCHECK_LT(0, x.size());
137 const float max_sample = fabs(*std::max_element(
138 x.begin(), x.end(), [](float a, float b) { return a * a < b * b; }));
139
140 if (config_.ep_strength.echo_can_saturate) {
141 const bool saturated_echo =
142 (previous_max_sample_ > 200.f) && SaturatedCapture();
143
144 // Counts the blocks since saturation.
145 constexpr size_t kSaturationLeakageBlocks = 20;
146
147 // Set flag for potential presence of saturated echo
148 blocks_since_last_saturation_ =
149 saturated_echo ? 0 : blocks_since_last_saturation_ + 1;
150
151 echo_saturation_ = blocks_since_last_saturation_ < kSaturationLeakageBlocks;
152 } else {
153 echo_saturation_ = false;
154 }
155 previous_max_sample_ = max_sample;
156
157 // TODO(peah): Move?
158 sufficient_filter_updates_ =
159 blocks_with_filter_adaptation_ >= kEchoPathChangeConvergenceBlocks;
160 initial_state_ = capture_block_counter_ < 3 * kNumBlocksPerSecond;
161
162 // Flag whether the linear filter estimate is usable.
163 usable_linear_estimate_ =
164 (!echo_saturation_) && (converged_filter || SufficientFilterUpdates()) &&
165 capture_block_counter_ >= 2 * kNumBlocksPerSecond && external_delay_;
166
167 linear_echo_estimate_ = UsableLinearEstimate() && !TransparentMode();
168
169 // After an amount of active render samples for which an echo should have been
170 // detected in the capture signal if the ERL was not infinite, flag that a
171 // transparent mode should be entered.
172 const float x_energy = std::inner_product(x.begin(), x.end(), x.begin(), 0.f);
173 const bool active_render_block =
174 x_energy > (config_.render_levels.active_render_limit *
175 config_.render_levels.active_render_limit) *
176 kFftLengthBy2;
177
178 if (active_render_block) {
179 render_received_ = true;
180 }
181
182 // Update counters.
183 blocks_with_filter_adaptation_ +=
184 (active_render_block && (!SaturatedCapture()) ? 1 : 0);
185
186 transparent_mode_ = !converged_filter &&
187 (!render_received_ || blocks_with_filter_adaptation_ >=
188 5 * kNumBlocksPerSecond);
189
190 // Update the room reverb estimate.
191 UpdateReverb(adaptive_filter_impulse_response);
192 }
193
UpdateReverb(const std::array<float,kAdaptiveFilterTimeDomainLength> & impulse_response)194 void AecState::UpdateReverb(
195 const std::array<float, kAdaptiveFilterTimeDomainLength>&
196 impulse_response) {
197 if ((!(filter_delay_ && usable_linear_estimate_)) ||
198 (*filter_delay_ > kAdaptiveFilterLength - 4)) {
199 return;
200 }
201
202 // Form the data to match against by squaring the impulse response
203 // coefficients.
204 std::array<float, kAdaptiveFilterTimeDomainLength> matching_data;
205 std::transform(impulse_response.begin(), impulse_response.end(),
206 matching_data.begin(), [](float a) { return a * a; });
207
208 // Avoid matching against noise in the model by subtracting an estimate of the
209 // model noise power.
210 constexpr size_t kTailLength = 64;
211 constexpr size_t tail_index = kAdaptiveFilterTimeDomainLength - kTailLength;
212 const float tail_power = *std::max_element(matching_data.begin() + tail_index,
213 matching_data.end());
214 std::for_each(matching_data.begin(), matching_data.begin() + tail_index,
215 [tail_power](float& a) { a = std::max(0.f, a - tail_power); });
216
217 // Identify the peak index of the impulse response.
218 const size_t peak_index = *std::max_element(
219 matching_data.begin(), matching_data.begin() + tail_index);
220
221 if (peak_index + 128 < tail_index) {
222 size_t start_index = peak_index + 64;
223 // Compute the matching residual error for the current candidate to match.
224 float residual_sqr_sum = 0.f;
225 float d_k = reverb_decay_to_test_;
226 for (size_t k = start_index; k < tail_index; ++k) {
227 if (matching_data[start_index + 1] == 0.f) {
228 break;
229 }
230
231 float residual = matching_data[k] - matching_data[peak_index] * d_k;
232 residual_sqr_sum += residual * residual;
233 d_k *= reverb_decay_to_test_;
234 }
235
236 // If needed, update the best candidate for the reverb decay.
237 if (reverb_decay_candidate_residual_ < 0.f ||
238 residual_sqr_sum < reverb_decay_candidate_residual_) {
239 reverb_decay_candidate_residual_ = residual_sqr_sum;
240 reverb_decay_candidate_ = reverb_decay_to_test_;
241 }
242 }
243
244 // Compute the next reverb candidate to evaluate such that all candidates will
245 // be evaluated within one second.
246 reverb_decay_to_test_ += (0.9965f - 0.9f) / (5 * kNumBlocksPerSecond);
247
248 // If all reverb candidates have been evaluated, choose the best one as the
249 // reverb decay.
250 if (reverb_decay_to_test_ >= 0.9965f) {
251 if (reverb_decay_candidate_residual_ < 0.f) {
252 // Transform the decay to be in the unit of blocks.
253 reverb_decay_ = powf(reverb_decay_candidate_, kFftLengthBy2);
254
255 // Limit the estimated reverb_decay_ to the maximum one needed in practice
256 // to minimize the impact of incorrect estimates.
257 reverb_decay_ = std::min(config_.ep_strength.default_len, reverb_decay_);
258 }
259 reverb_decay_to_test_ = 0.9f;
260 reverb_decay_candidate_residual_ = -1.f;
261 }
262
263 // For noisy impulse responses, assume a fixed tail length.
264 if (tail_power > 0.0005f) {
265 reverb_decay_ = config_.ep_strength.default_len;
266 }
267 data_dumper_->DumpRaw("aec3_reverb_decay", reverb_decay_);
268 data_dumper_->DumpRaw("aec3_tail_power", tail_power);
269 }
270
Update(rtc::ArrayView<const float> x,const std::array<float,kBlockSize> & s,bool converged_filter)271 void AecState::EchoAudibility::Update(rtc::ArrayView<const float> x,
272 const std::array<float, kBlockSize>& s,
273 bool converged_filter) {
274 auto result_x = std::minmax_element(x.begin(), x.end());
275 auto result_s = std::minmax_element(s.begin(), s.end());
276 const float x_abs =
277 std::max(fabsf(*result_x.first), fabsf(*result_x.second));
278 const float s_abs =
279 std::max(fabsf(*result_s.first), fabsf(*result_s.second));
280
281 if (converged_filter) {
282 if (x_abs < 20.f) {
283 ++low_farend_counter_;
284 } else {
285 low_farend_counter_ = 0;
286 }
287 } else {
288 if (x_abs < 100.f) {
289 ++low_farend_counter_;
290 } else {
291 low_farend_counter_ = 0;
292 }
293 }
294
295 // The echo is deemed as not audible if the echo estimate is on the level of
296 // the quantization noise in the FFTs and the nearend level is sufficiently
297 // strong to mask that by ensuring that the playout and AGC gains do not boost
298 // any residual echo that is below the quantization noise level. Furthermore,
299 // cases where the render signal is very close to zero are also identified as
300 // not producing audible echo.
301 inaudible_echo_ = (max_nearend_ > 500 && s_abs < 30.f) ||
302 (!converged_filter && x_abs < 500);
303 inaudible_echo_ = inaudible_echo_ || low_farend_counter_ > 20;
304 }
305
UpdateWithOutput(rtc::ArrayView<const float> e)306 void AecState::EchoAudibility::UpdateWithOutput(rtc::ArrayView<const float> e) {
307 const float e_max = *std::max_element(e.begin(), e.end());
308 const float e_min = *std::min_element(e.begin(), e.end());
309 const float e_abs = std::max(fabsf(e_max), fabsf(e_min));
310
311 if (max_nearend_ < e_abs) {
312 max_nearend_ = e_abs;
313 max_nearend_counter_ = 0;
314 } else {
315 if (++max_nearend_counter_ > 5 * kNumBlocksPerSecond) {
316 max_nearend_ *= 0.995f;
317 }
318 }
319 }
320
321 } // namespace webrtc
322