1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/audio_processing/aec3/echo_remover.h"
11
12 #include <math.h>
13 #include <algorithm>
14 #include <memory>
15 #include <numeric>
16 #include <string>
17
18 #include "api/array_view.h"
19 #include "modules/audio_processing/aec3/aec3_common.h"
20 #include "modules/audio_processing/aec3/aec_state.h"
21 #include "modules/audio_processing/aec3/comfort_noise_generator.h"
22 #include "modules/audio_processing/aec3/echo_path_variability.h"
23 #include "modules/audio_processing/aec3/echo_remover_metrics.h"
24 #include "modules/audio_processing/aec3/fft_data.h"
25 #include "modules/audio_processing/aec3/output_selector.h"
26 #include "modules/audio_processing/aec3/render_buffer.h"
27 #include "modules/audio_processing/aec3/render_delay_buffer.h"
28 #include "modules/audio_processing/aec3/residual_echo_estimator.h"
29 #include "modules/audio_processing/aec3/subtractor.h"
30 #include "modules/audio_processing/aec3/suppression_filter.h"
31 #include "modules/audio_processing/aec3/suppression_gain.h"
32 #include "modules/audio_processing/logging/apm_data_dumper.h"
33 #include "rtc_base/atomicops.h"
34 #include "rtc_base/constructormagic.h"
35
36 namespace webrtc {
37
38 namespace {
39
LinearEchoPower(const FftData & E,const FftData & Y,std::array<float,kFftLengthBy2Plus1> * S2)40 void LinearEchoPower(const FftData& E,
41 const FftData& Y,
42 std::array<float, kFftLengthBy2Plus1>* S2) {
43 for (size_t k = 0; k < E.re.size(); ++k) {
44 (*S2)[k] = (Y.re[k] - E.re[k]) * (Y.re[k] - E.re[k]) +
45 (Y.im[k] - E.im[k]) * (Y.im[k] - E.im[k]);
46 }
47 }
48
49 // Class for removing the echo from the capture signal.
50 class EchoRemoverImpl final : public EchoRemover {
51 public:
52 explicit EchoRemoverImpl(const EchoCanceller3Config& config,
53 int sample_rate_hz);
54 ~EchoRemoverImpl() override;
55
56 void GetMetrics(EchoControl::Metrics* metrics) const override;
57
58 // Removes the echo from a block of samples from the capture signal. The
59 // supplied render signal is assumed to be pre-aligned with the capture
60 // signal.
61 void ProcessCapture(const rtc::Optional<size_t>& echo_path_delay_samples,
62 const EchoPathVariability& echo_path_variability,
63 bool capture_signal_saturation,
64 const RenderBuffer& render_buffer,
65 std::vector<std::vector<float>>* capture) override;
66
67 // Updates the status on whether echo leakage is detected in the output of the
68 // echo remover.
UpdateEchoLeakageStatus(bool leakage_detected)69 void UpdateEchoLeakageStatus(bool leakage_detected) override {
70 echo_leakage_detected_ = leakage_detected;
71 }
72
73 private:
74 static int instance_count_;
75 const EchoCanceller3Config config_;
76 const Aec3Fft fft_;
77 std::unique_ptr<ApmDataDumper> data_dumper_;
78 const Aec3Optimization optimization_;
79 const int sample_rate_hz_;
80 Subtractor subtractor_;
81 SuppressionGain suppression_gain_;
82 ComfortNoiseGenerator cng_;
83 SuppressionFilter suppression_filter_;
84 RenderSignalAnalyzer render_signal_analyzer_;
85 OutputSelector output_selector_;
86 ResidualEchoEstimator residual_echo_estimator_;
87 bool echo_leakage_detected_ = false;
88 AecState aec_state_;
89 EchoRemoverMetrics metrics_;
90
91 RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl);
92 };
93
94 int EchoRemoverImpl::instance_count_ = 0;
95
EchoRemoverImpl(const EchoCanceller3Config & config,int sample_rate_hz)96 EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config,
97 int sample_rate_hz)
98 : config_(config),
99 fft_(),
100 data_dumper_(
101 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
102 optimization_(DetectOptimization()),
103 sample_rate_hz_(sample_rate_hz),
104 subtractor_(data_dumper_.get(), optimization_),
105 suppression_gain_(config_, optimization_),
106 cng_(optimization_),
107 suppression_filter_(sample_rate_hz_),
108 residual_echo_estimator_(config_),
109 aec_state_(config_) {
110 RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
111 }
112
113 EchoRemoverImpl::~EchoRemoverImpl() = default;
114
GetMetrics(EchoControl::Metrics * metrics) const115 void EchoRemoverImpl::GetMetrics(EchoControl::Metrics* metrics) const {
116 // Echo return loss (ERL) is inverted to go from gain to attenuation.
117 metrics->echo_return_loss = -10.0 * log10(aec_state_.ErlTimeDomain());
118 metrics->echo_return_loss_enhancement =
119 10.0 * log10(aec_state_.ErleTimeDomain());
120 }
121
ProcessCapture(const rtc::Optional<size_t> & echo_path_delay_samples,const EchoPathVariability & echo_path_variability,bool capture_signal_saturation,const RenderBuffer & render_buffer,std::vector<std::vector<float>> * capture)122 void EchoRemoverImpl::ProcessCapture(
123 const rtc::Optional<size_t>& echo_path_delay_samples,
124 const EchoPathVariability& echo_path_variability,
125 bool capture_signal_saturation,
126 const RenderBuffer& render_buffer,
127 std::vector<std::vector<float>>* capture) {
128 const std::vector<std::vector<float>>& x = render_buffer.MostRecentBlock();
129 std::vector<std::vector<float>>* y = capture;
130
131 RTC_DCHECK(y);
132 RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_));
133 RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_));
134 RTC_DCHECK_EQ(x[0].size(), kBlockSize);
135 RTC_DCHECK_EQ((*y)[0].size(), kBlockSize);
136 const std::vector<float>& x0 = x[0];
137 std::vector<float>& y0 = (*y)[0];
138
139 data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, &y0[0],
140 LowestBandRate(sample_rate_hz_), 1);
141 data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, &x0[0],
142 LowestBandRate(sample_rate_hz_), 1);
143 data_dumper_->DumpRaw("aec3_echo_remover_capture_input", y0);
144 data_dumper_->DumpRaw("aec3_echo_remover_render_input", x0);
145
146 aec_state_.UpdateCaptureSaturation(capture_signal_saturation);
147
148 if (echo_path_variability.AudioPathChanged()) {
149 subtractor_.HandleEchoPathChange(echo_path_variability);
150 aec_state_.HandleEchoPathChange(echo_path_variability);
151 }
152
153 std::array<float, kFftLengthBy2Plus1> Y2;
154 std::array<float, kFftLengthBy2Plus1> R2;
155 std::array<float, kFftLengthBy2Plus1> S2_linear;
156 std::array<float, kFftLengthBy2Plus1> G;
157 float high_bands_gain;
158 FftData Y;
159 FftData comfort_noise;
160 FftData high_band_comfort_noise;
161 SubtractorOutput subtractor_output;
162 FftData& E_main = subtractor_output.E_main;
163 auto& E2_main = subtractor_output.E2_main;
164 auto& E2_shadow = subtractor_output.E2_shadow;
165 auto& e_main = subtractor_output.e_main;
166
167 // Analyze the render signal.
168 render_signal_analyzer_.Update(render_buffer, aec_state_.FilterDelay());
169
170 // Perform linear echo cancellation.
171 subtractor_.Process(render_buffer, y0, render_signal_analyzer_, aec_state_,
172 &subtractor_output);
173
174 // Compute spectra.
175 fft_.ZeroPaddedFft(y0, &Y);
176 LinearEchoPower(E_main, Y, &S2_linear);
177 Y.Spectrum(optimization_, &Y2);
178
179 // Update the AEC state information.
180 aec_state_.Update(subtractor_.FilterFrequencyResponse(),
181 subtractor_.FilterImpulseResponse(),
182 subtractor_.ConvergedFilter(), echo_path_delay_samples,
183 render_buffer, E2_main, Y2, x0, subtractor_output.s_main,
184 echo_leakage_detected_);
185
186 // Choose the linear output.
187 output_selector_.FormLinearOutput(!aec_state_.TransparentMode(), e_main, y0);
188 data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0],
189 LowestBandRate(sample_rate_hz_), 1);
190 data_dumper_->DumpRaw("aec3_output_linear", y0);
191 const auto& E2 = output_selector_.UseSubtractorOutput() ? E2_main : Y2;
192
193 // Estimate the residual echo power.
194 residual_echo_estimator_.Estimate(aec_state_, render_buffer, S2_linear, Y2,
195 &R2);
196
197 // Estimate the comfort noise.
198 cng_.Compute(aec_state_, Y2, &comfort_noise, &high_band_comfort_noise);
199
200 // A choose and apply echo suppression gain.
201 suppression_gain_.GetGain(E2, R2, cng_.NoiseSpectrum(),
202 render_signal_analyzer_, aec_state_, x,
203 &high_bands_gain, &G);
204 suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G,
205 high_bands_gain, y);
206
207 // Update the metrics.
208 metrics_.Update(aec_state_, cng_.NoiseSpectrum(), G);
209
210 // Update the aec state with the aec output characteristics.
211 aec_state_.UpdateWithOutput(y0);
212
213 // Debug outputs for the purpose of development and analysis.
214 data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize,
215 &subtractor_output.s_main[0],
216 LowestBandRate(sample_rate_hz_), 1);
217 data_dumper_->DumpRaw("aec3_output", y0);
218 data_dumper_->DumpRaw("aec3_narrow_render",
219 render_signal_analyzer_.NarrowPeakBand() ? 1 : 0);
220 data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum());
221 data_dumper_->DumpRaw("aec3_suppressor_gain", G);
222 data_dumper_->DumpWav("aec3_output",
223 rtc::ArrayView<const float>(&y0[0], kBlockSize),
224 LowestBandRate(sample_rate_hz_), 1);
225 data_dumper_->DumpRaw("aec3_using_subtractor_output",
226 output_selector_.UseSubtractorOutput() ? 1 : 0);
227 data_dumper_->DumpRaw("aec3_E2", E2);
228 data_dumper_->DumpRaw("aec3_E2_main", E2_main);
229 data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow);
230 data_dumper_->DumpRaw("aec3_S2_linear", S2_linear);
231 data_dumper_->DumpRaw("aec3_Y2", Y2);
232 data_dumper_->DumpRaw("aec3_X2", render_buffer.Spectrum(0));
233 data_dumper_->DumpRaw("aec3_R2", R2);
234 data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle());
235 data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl());
236 data_dumper_->DumpRaw("aec3_active_render", aec_state_.ActiveRender());
237 data_dumper_->DumpRaw("aec3_usable_linear_estimate",
238 aec_state_.UsableLinearEstimate());
239 data_dumper_->DumpRaw(
240 "aec3_filter_delay",
241 aec_state_.FilterDelay() ? *aec_state_.FilterDelay() : -1);
242 data_dumper_->DumpRaw(
243 "aec3_external_delay",
244 aec_state_.ExternalDelay() ? *aec_state_.ExternalDelay() : -1);
245 data_dumper_->DumpRaw("aec3_capture_saturation",
246 aec_state_.SaturatedCapture() ? 1 : 0);
247 }
248
249 } // namespace
250
Create(const EchoCanceller3Config & config,int sample_rate_hz)251 EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config,
252 int sample_rate_hz) {
253 return new EchoRemoverImpl(config, sample_rate_hz);
254 }
255
256 } // namespace webrtc
257