1 /*
2  *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_
12 #define MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_
13 
14 #include <algorithm>
15 #include <memory>
16 #include <vector>
17 
18 #include "api/array_view.h"
19 #include "api/optional.h"
20 #include "modules/audio_processing/aec3/aec3_common.h"
21 #include "modules/audio_processing/aec3/echo_path_variability.h"
22 #include "modules/audio_processing/aec3/erl_estimator.h"
23 #include "modules/audio_processing/aec3/erle_estimator.h"
24 #include "modules/audio_processing/aec3/render_buffer.h"
25 #include "modules/audio_processing/include/audio_processing.h"
26 #include "rtc_base/constructormagic.h"
27 
28 namespace webrtc {
29 
30 class ApmDataDumper;
31 
32 // Handles the state and the conditions for the echo removal functionality.
33 class AecState {
34  public:
35   explicit AecState(const EchoCanceller3Config& config);
36   ~AecState();
37 
38   // Returns whether the linear filter estimate is usable.
UsableLinearEstimate()39   bool UsableLinearEstimate() const { return usable_linear_estimate_; }
40 
41   // Returns whether there has been echo leakage detected.
EchoLeakageDetected()42   bool EchoLeakageDetected() const { return echo_leakage_detected_; }
43 
44   // Returns whether the render signal is currently active.
45   // TODO(peah): Deprecate this in an upcoming CL.
ActiveRender()46   bool ActiveRender() const { return blocks_with_filter_adaptation_ > 200; }
47 
48   // Returns the ERLE.
Erle()49   const std::array<float, kFftLengthBy2Plus1>& Erle() const {
50     return erle_estimator_.Erle();
51   }
52 
53   // Returns the time-domain ERLE.
ErleTimeDomain()54   float ErleTimeDomain() const { return erle_estimator_.ErleTimeDomain(); }
55 
56   // Returns the ERL.
Erl()57   const std::array<float, kFftLengthBy2Plus1>& Erl() const {
58     return erl_estimator_.Erl();
59   }
60 
61   // Returns the time-domain ERL.
ErlTimeDomain()62   float ErlTimeDomain() const { return erl_estimator_.ErlTimeDomain(); }
63 
64   // Returns the delay estimate based on the linear filter.
FilterDelay()65   rtc::Optional<size_t> FilterDelay() const { return filter_delay_; }
66 
67   // Returns the externally provided delay.
ExternalDelay()68   rtc::Optional<size_t> ExternalDelay() const { return external_delay_; }
69 
70   // Returns whether the capture signal is saturated.
SaturatedCapture()71   bool SaturatedCapture() const { return capture_signal_saturation_; }
72 
73   // Returns whether the echo signal is saturated.
SaturatedEcho()74   bool SaturatedEcho() const { return echo_saturation_; }
75 
76   // Returns whether the echo path can saturate.
SaturatingEchoPath()77   bool SaturatingEchoPath() const { return saturating_echo_path_; }
78 
79   // Updates the capture signal saturation.
UpdateCaptureSaturation(bool capture_signal_saturation)80   void UpdateCaptureSaturation(bool capture_signal_saturation) {
81     capture_signal_saturation_ = capture_signal_saturation;
82   }
83 
84   // Returns whether the transparent mode is active
TransparentMode()85   bool TransparentMode() const { return transparent_mode_; }
86 
87   // Takes appropriate action at an echo path change.
88   void HandleEchoPathChange(const EchoPathVariability& echo_path_variability);
89 
90   // Returns the decay factor for the echo reverberation.
ReverbDecay()91   float ReverbDecay() const { return reverb_decay_; }
92 
93   // Returns whether the echo suppression gain should be forced to zero.
ForcedZeroGain()94   bool ForcedZeroGain() const { return force_zero_gain_; }
95 
96   // Returns whether the echo in the capture signal is audible.
InaudibleEcho()97   bool InaudibleEcho() const { return echo_audibility_.InaudibleEcho(); }
98 
99   // Updates the aec state with the AEC output signal.
UpdateWithOutput(rtc::ArrayView<const float> e)100   void UpdateWithOutput(rtc::ArrayView<const float> e) {
101     echo_audibility_.UpdateWithOutput(e);
102   }
103 
104   // Returns whether the linear filter should have been able to adapt properly.
SufficientFilterUpdates()105   bool SufficientFilterUpdates() const { return sufficient_filter_updates_; }
106 
107   // Returns whether the echo subtractor can be used to determine the residual
108   // echo.
LinearEchoEstimate()109   bool LinearEchoEstimate() const { return linear_echo_estimate_; }
110 
111   // Returns whether the AEC is in an initial state.
InitialState()112   bool InitialState() const { return initial_state_; }
113 
114   // Updates the aec state.
115   void Update(const std::vector<std::array<float, kFftLengthBy2Plus1>>&
116                   adaptive_filter_frequency_response,
117               const std::array<float, kAdaptiveFilterTimeDomainLength>&
118                   adaptive_filter_impulse_response,
119               bool converged_filter,
120               const rtc::Optional<size_t>& external_delay_samples,
121               const RenderBuffer& render_buffer,
122               const std::array<float, kFftLengthBy2Plus1>& E2_main,
123               const std::array<float, kFftLengthBy2Plus1>& Y2,
124               rtc::ArrayView<const float> x,
125               const std::array<float, kBlockSize>& s_main,
126               bool echo_leakage_detected);
127 
128  private:
129   class EchoAudibility {
130    public:
131     void Update(rtc::ArrayView<const float> x,
132                 const std::array<float, kBlockSize>& s,
133                 bool converged_filter);
134     void UpdateWithOutput(rtc::ArrayView<const float> e);
InaudibleEcho()135     bool InaudibleEcho() const { return inaudible_echo_; }
136 
137    private:
138     float max_nearend_ = 0.f;
139     size_t max_nearend_counter_ = 0;
140     size_t low_farend_counter_ = 0;
141     bool inaudible_echo_ = false;
142   };
143 
144   void UpdateReverb(const std::array<float, kAdaptiveFilterTimeDomainLength>&
145                         impulse_response);
146 
147   static int instance_count_;
148   std::unique_ptr<ApmDataDumper> data_dumper_;
149   ErlEstimator erl_estimator_;
150   ErleEstimator erle_estimator_;
151   size_t capture_block_counter_ = 0;
152   size_t blocks_with_filter_adaptation_ = 0;
153   size_t blocks_with_strong_render_ = 0;
154   bool usable_linear_estimate_ = false;
155   bool echo_leakage_detected_ = false;
156   bool capture_signal_saturation_ = false;
157   bool echo_saturation_ = false;
158   bool transparent_mode_ = false;
159   float previous_max_sample_ = 0.f;
160   std::array<float, kAdaptiveFilterLength> max_render_;
161   bool force_zero_gain_ = false;
162   bool render_received_ = false;
163   size_t force_zero_gain_counter_ = 0;
164   rtc::Optional<size_t> filter_delay_;
165   rtc::Optional<size_t> external_delay_;
166   size_t blocks_since_last_saturation_ = 1000;
167   float reverb_decay_to_test_ = 0.9f;
168   float reverb_decay_candidate_ = 0.f;
169   float reverb_decay_candidate_residual_ = -1.f;
170   EchoAudibility echo_audibility_;
171   const EchoCanceller3Config config_;
172   float reverb_decay_;
173   bool saturating_echo_path_ = false;
174   bool initial_state_ = true;
175   bool linear_echo_estimate_ = false;
176   bool sufficient_filter_updates_ = false;
177 
178   RTC_DISALLOW_COPY_AND_ASSIGN(AecState);
179 };
180 
181 }  // namespace webrtc
182 
183 #endif  // MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_
184