1 /*
2  *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #include "modules/audio_processing/aec3/echo_remover.h"
11 
12 #include <math.h>
13 #include <algorithm>
14 #include <memory>
15 #include <numeric>
16 #include <string>
17 
18 #include "api/array_view.h"
19 #include "modules/audio_processing/aec3/aec3_common.h"
20 #include "modules/audio_processing/aec3/aec_state.h"
21 #include "modules/audio_processing/aec3/comfort_noise_generator.h"
22 #include "modules/audio_processing/aec3/echo_path_variability.h"
23 #include "modules/audio_processing/aec3/echo_remover_metrics.h"
24 #include "modules/audio_processing/aec3/fft_data.h"
25 #include "modules/audio_processing/aec3/output_selector.h"
26 #include "modules/audio_processing/aec3/render_buffer.h"
27 #include "modules/audio_processing/aec3/render_delay_buffer.h"
28 #include "modules/audio_processing/aec3/residual_echo_estimator.h"
29 #include "modules/audio_processing/aec3/subtractor.h"
30 #include "modules/audio_processing/aec3/suppression_filter.h"
31 #include "modules/audio_processing/aec3/suppression_gain.h"
32 #include "modules/audio_processing/logging/apm_data_dumper.h"
33 #include "rtc_base/atomicops.h"
34 #include "rtc_base/constructormagic.h"
35 
36 namespace webrtc {
37 
38 namespace {
39 
LinearEchoPower(const FftData & E,const FftData & Y,std::array<float,kFftLengthBy2Plus1> * S2)40 void LinearEchoPower(const FftData& E,
41                      const FftData& Y,
42                      std::array<float, kFftLengthBy2Plus1>* S2) {
43   for (size_t k = 0; k < E.re.size(); ++k) {
44     (*S2)[k] = (Y.re[k] - E.re[k]) * (Y.re[k] - E.re[k]) +
45                (Y.im[k] - E.im[k]) * (Y.im[k] - E.im[k]);
46   }
47 }
48 
49 // Class for removing the echo from the capture signal.
50 class EchoRemoverImpl final : public EchoRemover {
51  public:
52   explicit EchoRemoverImpl(const EchoCanceller3Config& config,
53                            int sample_rate_hz);
54   ~EchoRemoverImpl() override;
55 
56   void GetMetrics(EchoControl::Metrics* metrics) const override;
57 
58   // Removes the echo from a block of samples from the capture signal. The
59   // supplied render signal is assumed to be pre-aligned with the capture
60   // signal.
61   void ProcessCapture(const rtc::Optional<size_t>& echo_path_delay_samples,
62                       const EchoPathVariability& echo_path_variability,
63                       bool capture_signal_saturation,
64                       const RenderBuffer& render_buffer,
65                       std::vector<std::vector<float>>* capture) override;
66 
67   // Updates the status on whether echo leakage is detected in the output of the
68   // echo remover.
UpdateEchoLeakageStatus(bool leakage_detected)69   void UpdateEchoLeakageStatus(bool leakage_detected) override {
70     echo_leakage_detected_ = leakage_detected;
71   }
72 
73  private:
74   static int instance_count_;
75   const EchoCanceller3Config config_;
76   const Aec3Fft fft_;
77   std::unique_ptr<ApmDataDumper> data_dumper_;
78   const Aec3Optimization optimization_;
79   const int sample_rate_hz_;
80   Subtractor subtractor_;
81   SuppressionGain suppression_gain_;
82   ComfortNoiseGenerator cng_;
83   SuppressionFilter suppression_filter_;
84   RenderSignalAnalyzer render_signal_analyzer_;
85   OutputSelector output_selector_;
86   ResidualEchoEstimator residual_echo_estimator_;
87   bool echo_leakage_detected_ = false;
88   AecState aec_state_;
89   EchoRemoverMetrics metrics_;
90 
91   RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl);
92 };
93 
94 int EchoRemoverImpl::instance_count_ = 0;
95 
EchoRemoverImpl(const EchoCanceller3Config & config,int sample_rate_hz)96 EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config,
97                                  int sample_rate_hz)
98     : config_(config),
99       fft_(),
100       data_dumper_(
101           new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
102       optimization_(DetectOptimization()),
103       sample_rate_hz_(sample_rate_hz),
104       subtractor_(data_dumper_.get(), optimization_),
105       suppression_gain_(config_, optimization_),
106       cng_(optimization_),
107       suppression_filter_(sample_rate_hz_),
108       residual_echo_estimator_(config_),
109       aec_state_(config_) {
110   RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
111 }
112 
113 EchoRemoverImpl::~EchoRemoverImpl() = default;
114 
GetMetrics(EchoControl::Metrics * metrics) const115 void EchoRemoverImpl::GetMetrics(EchoControl::Metrics* metrics) const {
116   // Echo return loss (ERL) is inverted to go from gain to attenuation.
117   metrics->echo_return_loss = -10.0 * log10(aec_state_.ErlTimeDomain());
118   metrics->echo_return_loss_enhancement =
119       10.0 * log10(aec_state_.ErleTimeDomain());
120 }
121 
ProcessCapture(const rtc::Optional<size_t> & echo_path_delay_samples,const EchoPathVariability & echo_path_variability,bool capture_signal_saturation,const RenderBuffer & render_buffer,std::vector<std::vector<float>> * capture)122 void EchoRemoverImpl::ProcessCapture(
123     const rtc::Optional<size_t>& echo_path_delay_samples,
124     const EchoPathVariability& echo_path_variability,
125     bool capture_signal_saturation,
126     const RenderBuffer& render_buffer,
127     std::vector<std::vector<float>>* capture) {
128   const std::vector<std::vector<float>>& x = render_buffer.MostRecentBlock();
129   std::vector<std::vector<float>>* y = capture;
130 
131   RTC_DCHECK(y);
132   RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_));
133   RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_));
134   RTC_DCHECK_EQ(x[0].size(), kBlockSize);
135   RTC_DCHECK_EQ((*y)[0].size(), kBlockSize);
136   const std::vector<float>& x0 = x[0];
137   std::vector<float>& y0 = (*y)[0];
138 
139   data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, &y0[0],
140                         LowestBandRate(sample_rate_hz_), 1);
141   data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, &x0[0],
142                         LowestBandRate(sample_rate_hz_), 1);
143   data_dumper_->DumpRaw("aec3_echo_remover_capture_input", y0);
144   data_dumper_->DumpRaw("aec3_echo_remover_render_input", x0);
145 
146   aec_state_.UpdateCaptureSaturation(capture_signal_saturation);
147 
148   if (echo_path_variability.AudioPathChanged()) {
149     subtractor_.HandleEchoPathChange(echo_path_variability);
150     aec_state_.HandleEchoPathChange(echo_path_variability);
151   }
152 
153   std::array<float, kFftLengthBy2Plus1> Y2;
154   std::array<float, kFftLengthBy2Plus1> R2;
155   std::array<float, kFftLengthBy2Plus1> S2_linear;
156   std::array<float, kFftLengthBy2Plus1> G;
157   float high_bands_gain;
158   FftData Y;
159   FftData comfort_noise;
160   FftData high_band_comfort_noise;
161   SubtractorOutput subtractor_output;
162   FftData& E_main = subtractor_output.E_main;
163   auto& E2_main = subtractor_output.E2_main;
164   auto& E2_shadow = subtractor_output.E2_shadow;
165   auto& e_main = subtractor_output.e_main;
166 
167   // Analyze the render signal.
168   render_signal_analyzer_.Update(render_buffer, aec_state_.FilterDelay());
169 
170   // Perform linear echo cancellation.
171   subtractor_.Process(render_buffer, y0, render_signal_analyzer_, aec_state_,
172                       &subtractor_output);
173 
174   // Compute spectra.
175   fft_.ZeroPaddedFft(y0, &Y);
176   LinearEchoPower(E_main, Y, &S2_linear);
177   Y.Spectrum(optimization_, &Y2);
178 
179   // Update the AEC state information.
180   aec_state_.Update(subtractor_.FilterFrequencyResponse(),
181                     subtractor_.FilterImpulseResponse(),
182                     subtractor_.ConvergedFilter(), echo_path_delay_samples,
183                     render_buffer, E2_main, Y2, x0, subtractor_output.s_main,
184                     echo_leakage_detected_);
185 
186   // Choose the linear output.
187   output_selector_.FormLinearOutput(!aec_state_.TransparentMode(), e_main, y0);
188   data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0],
189                         LowestBandRate(sample_rate_hz_), 1);
190   data_dumper_->DumpRaw("aec3_output_linear", y0);
191   const auto& E2 = output_selector_.UseSubtractorOutput() ? E2_main : Y2;
192 
193   // Estimate the residual echo power.
194   residual_echo_estimator_.Estimate(aec_state_, render_buffer, S2_linear, Y2,
195                                     &R2);
196 
197   // Estimate the comfort noise.
198   cng_.Compute(aec_state_, Y2, &comfort_noise, &high_band_comfort_noise);
199 
200   // A choose and apply echo suppression gain.
201   suppression_gain_.GetGain(E2, R2, cng_.NoiseSpectrum(),
202                             render_signal_analyzer_, aec_state_, x,
203                             &high_bands_gain, &G);
204   suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G,
205                                 high_bands_gain, y);
206 
207   // Update the metrics.
208   metrics_.Update(aec_state_, cng_.NoiseSpectrum(), G);
209 
210   // Update the aec state with the aec output characteristics.
211   aec_state_.UpdateWithOutput(y0);
212 
213   // Debug outputs for the purpose of development and analysis.
214   data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize,
215                         &subtractor_output.s_main[0],
216                         LowestBandRate(sample_rate_hz_), 1);
217   data_dumper_->DumpRaw("aec3_output", y0);
218   data_dumper_->DumpRaw("aec3_narrow_render",
219                         render_signal_analyzer_.NarrowPeakBand() ? 1 : 0);
220   data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum());
221   data_dumper_->DumpRaw("aec3_suppressor_gain", G);
222   data_dumper_->DumpWav("aec3_output",
223                         rtc::ArrayView<const float>(&y0[0], kBlockSize),
224                         LowestBandRate(sample_rate_hz_), 1);
225   data_dumper_->DumpRaw("aec3_using_subtractor_output",
226                         output_selector_.UseSubtractorOutput() ? 1 : 0);
227   data_dumper_->DumpRaw("aec3_E2", E2);
228   data_dumper_->DumpRaw("aec3_E2_main", E2_main);
229   data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow);
230   data_dumper_->DumpRaw("aec3_S2_linear", S2_linear);
231   data_dumper_->DumpRaw("aec3_Y2", Y2);
232   data_dumper_->DumpRaw("aec3_X2", render_buffer.Spectrum(0));
233   data_dumper_->DumpRaw("aec3_R2", R2);
234   data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle());
235   data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl());
236   data_dumper_->DumpRaw("aec3_active_render", aec_state_.ActiveRender());
237   data_dumper_->DumpRaw("aec3_usable_linear_estimate",
238                         aec_state_.UsableLinearEstimate());
239   data_dumper_->DumpRaw(
240       "aec3_filter_delay",
241       aec_state_.FilterDelay() ? *aec_state_.FilterDelay() : -1);
242   data_dumper_->DumpRaw(
243       "aec3_external_delay",
244       aec_state_.ExternalDelay() ? *aec_state_.ExternalDelay() : -1);
245   data_dumper_->DumpRaw("aec3_capture_saturation",
246                         aec_state_.SaturatedCapture() ? 1 : 0);
247 }
248 
249 }  // namespace
250 
Create(const EchoCanceller3Config & config,int sample_rate_hz)251 EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config,
252                                  int sample_rate_hz) {
253   return new EchoRemoverImpl(config, sample_rate_hz);
254 }
255 
256 }  // namespace webrtc
257