1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/residual_echo_detector.h"
12
13 #include <algorithm>
14 #include <numeric>
15
16 #include "modules/audio_processing/audio_buffer.h"
17 #include "modules/audio_processing/logging/apm_data_dumper.h"
18 #include "rtc_base/atomicops.h"
19 #include "rtc_base/logging.h"
20 #include "system_wrappers/include/metrics.h"
21
22 namespace {
23
Power(rtc::ArrayView<const float> input)24 float Power(rtc::ArrayView<const float> input) {
25 if (input.empty()) {
26 return 0.f;
27 }
28 return std::inner_product(input.begin(), input.end(), input.begin(), 0.f) /
29 input.size();
30 }
31
32 constexpr size_t kLookbackFrames = 650;
33 // TODO(ivoc): Verify the size of this buffer.
34 constexpr size_t kRenderBufferSize = 30;
35 constexpr float kAlpha = 0.001f;
36 // 10 seconds of data, updated every 10 ms.
37 constexpr size_t kAggregationBufferSize = 10 * 100;
38
39 } // namespace
40
41 namespace webrtc {
42
43 int ResidualEchoDetector::instance_count_ = 0;
44
ResidualEchoDetector()45 ResidualEchoDetector::ResidualEchoDetector()
46 : data_dumper_(
47 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
48 render_buffer_(kRenderBufferSize),
49 render_power_(kLookbackFrames),
50 render_power_mean_(kLookbackFrames),
51 render_power_std_dev_(kLookbackFrames),
52 covariances_(kLookbackFrames),
53 recent_likelihood_max_(kAggregationBufferSize) {}
54
55 ResidualEchoDetector::~ResidualEchoDetector() = default;
56
AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio)57 void ResidualEchoDetector::AnalyzeRenderAudio(
58 rtc::ArrayView<const float> render_audio) {
59 // Dump debug data assuming 48 kHz sample rate (if this assumption is not
60 // valid the dumped audio will need to be converted offline accordingly).
61 data_dumper_->DumpWav("ed_render", render_audio.size(), render_audio.data(),
62 48000, 1);
63
64 if (render_buffer_.Size() == 0) {
65 frames_since_zero_buffer_size_ = 0;
66 } else if (frames_since_zero_buffer_size_ >= kRenderBufferSize) {
67 // This can happen in a few cases: at the start of a call, due to a glitch
68 // or due to clock drift. The excess capture value will be ignored.
69 // TODO(ivoc): Include how often this happens in APM stats.
70 render_buffer_.Pop();
71 frames_since_zero_buffer_size_ = 0;
72 }
73 ++frames_since_zero_buffer_size_;
74 float power = Power(render_audio);
75 render_buffer_.Push(power);
76 }
77
AnalyzeCaptureAudio(rtc::ArrayView<const float> capture_audio)78 void ResidualEchoDetector::AnalyzeCaptureAudio(
79 rtc::ArrayView<const float> capture_audio) {
80 // Dump debug data assuming 48 kHz sample rate (if this assumption is not
81 // valid the dumped audio will need to be converted offline accordingly).
82 data_dumper_->DumpWav("ed_capture", capture_audio.size(),
83 capture_audio.data(), 48000, 1);
84
85 if (first_process_call_) {
86 // On the first process call (so the start of a call), we must flush the
87 // render buffer, otherwise the render data will be delayed.
88 render_buffer_.Clear();
89 first_process_call_ = false;
90 }
91
92 // Get the next render value.
93 const rtc::Optional<float> buffered_render_power = render_buffer_.Pop();
94 if (!buffered_render_power) {
95 // This can happen in a few cases: at the start of a call, due to a glitch
96 // or due to clock drift. The excess capture value will be ignored.
97 // TODO(ivoc): Include how often this happens in APM stats.
98 return;
99 }
100 // Update the render statistics, and store the statistics in circular buffers.
101 render_statistics_.Update(*buffered_render_power);
102 RTC_DCHECK_LT(next_insertion_index_, kLookbackFrames);
103 render_power_[next_insertion_index_] = *buffered_render_power;
104 render_power_mean_[next_insertion_index_] = render_statistics_.mean();
105 render_power_std_dev_[next_insertion_index_] =
106 render_statistics_.std_deviation();
107
108 // Get the next capture value, update capture statistics and add the relevant
109 // values to the buffers.
110 const float capture_power = Power(capture_audio);
111 capture_statistics_.Update(capture_power);
112 const float capture_mean = capture_statistics_.mean();
113 const float capture_std_deviation = capture_statistics_.std_deviation();
114
115 // Update the covariance values and determine the new echo likelihood.
116 echo_likelihood_ = 0.f;
117 size_t read_index = next_insertion_index_;
118
119 int best_delay = -1;
120 for (size_t delay = 0; delay < covariances_.size(); ++delay) {
121 RTC_DCHECK_LT(read_index, render_power_.size());
122 covariances_[delay].Update(capture_power, capture_mean,
123 capture_std_deviation, render_power_[read_index],
124 render_power_mean_[read_index],
125 render_power_std_dev_[read_index]);
126 read_index = read_index > 0 ? read_index - 1 : kLookbackFrames - 1;
127
128 if (covariances_[delay].normalized_cross_correlation() > echo_likelihood_) {
129 echo_likelihood_ = covariances_[delay].normalized_cross_correlation();
130 best_delay = static_cast<int>(delay);
131 }
132 }
133 // This is a temporary log message to help find the underlying cause for echo
134 // likelihoods > 1.0.
135 // TODO(ivoc): Remove once the issue is resolved.
136 if (echo_likelihood_ > 1.1f) {
137 // Make sure we don't spam the log.
138 if (log_counter_ < 5 && best_delay != -1) {
139 size_t read_index = kLookbackFrames + next_insertion_index_ - best_delay;
140 if (read_index >= kLookbackFrames) {
141 read_index -= kLookbackFrames;
142 }
143 RTC_DCHECK_LT(read_index, render_power_.size());
144 RTC_LOG_F(LS_ERROR) << "Echo detector internal state: {"
145 << "Echo likelihood: " << echo_likelihood_
146 << ", Best Delay: " << best_delay << ", Covariance: "
147 << covariances_[best_delay].covariance()
148 << ", Last capture power: " << capture_power
149 << ", Capture mean: " << capture_mean
150 << ", Capture_standard deviation: "
151 << capture_std_deviation << ", Last render power: "
152 << render_power_[read_index]
153 << ", Render mean: " << render_power_mean_[read_index]
154 << ", Render standard deviation: "
155 << render_power_std_dev_[read_index]
156 << ", Reliability: " << reliability_ << "}";
157 log_counter_++;
158 }
159 }
160 RTC_DCHECK_LT(echo_likelihood_, 1.1f);
161
162 reliability_ = (1.0f - kAlpha) * reliability_ + kAlpha * 1.0f;
163 echo_likelihood_ *= reliability_;
164 // This is a temporary fix to prevent echo likelihood values > 1.0.
165 // TODO(ivoc): Find the root cause of this issue and fix it.
166 echo_likelihood_ = std::min(echo_likelihood_, 1.0f);
167 int echo_percentage = static_cast<int>(echo_likelihood_ * 100);
168 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.ResidualEchoDetector.EchoLikelihood",
169 echo_percentage, 0, 100, 100 /* number of bins */);
170
171 // Update the buffer of recent likelihood values.
172 recent_likelihood_max_.Update(echo_likelihood_);
173
174 // Update the next insertion index.
175 next_insertion_index_ = next_insertion_index_ < (kLookbackFrames - 1)
176 ? next_insertion_index_ + 1
177 : 0;
178 }
179
Initialize()180 void ResidualEchoDetector::Initialize() {
181 render_buffer_.Clear();
182 std::fill(render_power_.begin(), render_power_.end(), 0.f);
183 std::fill(render_power_mean_.begin(), render_power_mean_.end(), 0.f);
184 std::fill(render_power_std_dev_.begin(), render_power_std_dev_.end(), 0.f);
185 render_statistics_.Clear();
186 capture_statistics_.Clear();
187 recent_likelihood_max_.Clear();
188 for (auto& cov : covariances_) {
189 cov.Clear();
190 }
191 echo_likelihood_ = 0.f;
192 next_insertion_index_ = 0;
193 reliability_ = 0.f;
194 }
195
PackRenderAudioBuffer(AudioBuffer * audio,std::vector<float> * packed_buffer)196 void ResidualEchoDetector::PackRenderAudioBuffer(
197 AudioBuffer* audio,
198 std::vector<float>* packed_buffer) {
199 packed_buffer->clear();
200 packed_buffer->insert(packed_buffer->end(), audio->channels_f()[0],
201 audio->channels_f()[0] + audio->num_frames());
202 }
203
204 } // namespace webrtc
205