1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/audio_processing/aec3/echo_canceller3.h"
11
12 #include <sstream>
13
14 #include "modules/audio_processing/logging/apm_data_dumper.h"
15 #include "rtc_base/atomicops.h"
16
17 namespace webrtc {
18
19 namespace {
20
21 enum class EchoCanceller3ApiCall { kCapture, kRender };
22
DetectSaturation(rtc::ArrayView<const float> y)23 bool DetectSaturation(rtc::ArrayView<const float> y) {
24 for (auto y_k : y) {
25 if (y_k >= 32700.0f || y_k <= -32700.0f) {
26 return true;
27 }
28 }
29 return false;
30 }
31
FillSubFrameView(AudioBuffer * frame,size_t sub_frame_index,std::vector<rtc::ArrayView<float>> * sub_frame_view)32 void FillSubFrameView(AudioBuffer* frame,
33 size_t sub_frame_index,
34 std::vector<rtc::ArrayView<float>>* sub_frame_view) {
35 RTC_DCHECK_GE(1, sub_frame_index);
36 RTC_DCHECK_LE(0, sub_frame_index);
37 RTC_DCHECK_EQ(frame->num_bands(), sub_frame_view->size());
38 for (size_t k = 0; k < sub_frame_view->size(); ++k) {
39 (*sub_frame_view)[k] = rtc::ArrayView<float>(
40 &frame->split_bands_f(0)[k][sub_frame_index * kSubFrameLength],
41 kSubFrameLength);
42 }
43 }
44
FillSubFrameView(std::vector<std::vector<float>> * frame,size_t sub_frame_index,std::vector<rtc::ArrayView<float>> * sub_frame_view)45 void FillSubFrameView(std::vector<std::vector<float>>* frame,
46 size_t sub_frame_index,
47 std::vector<rtc::ArrayView<float>>* sub_frame_view) {
48 RTC_DCHECK_GE(1, sub_frame_index);
49 RTC_DCHECK_EQ(frame->size(), sub_frame_view->size());
50 for (size_t k = 0; k < frame->size(); ++k) {
51 (*sub_frame_view)[k] = rtc::ArrayView<float>(
52 &(*frame)[k][sub_frame_index * kSubFrameLength], kSubFrameLength);
53 }
54 }
55
ProcessCaptureFrameContent(AudioBuffer * capture,bool level_change,bool saturated_microphone_signal,size_t sub_frame_index,FrameBlocker * capture_blocker,BlockFramer * output_framer,BlockProcessor * block_processor,std::vector<std::vector<float>> * block,std::vector<rtc::ArrayView<float>> * sub_frame_view)56 void ProcessCaptureFrameContent(
57 AudioBuffer* capture,
58 bool level_change,
59 bool saturated_microphone_signal,
60 size_t sub_frame_index,
61 FrameBlocker* capture_blocker,
62 BlockFramer* output_framer,
63 BlockProcessor* block_processor,
64 std::vector<std::vector<float>>* block,
65 std::vector<rtc::ArrayView<float>>* sub_frame_view) {
66 FillSubFrameView(capture, sub_frame_index, sub_frame_view);
67 capture_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block);
68 block_processor->ProcessCapture(level_change, saturated_microphone_signal,
69 block);
70 output_framer->InsertBlockAndExtractSubFrame(*block, sub_frame_view);
71 }
72
ProcessRemainingCaptureFrameContent(bool level_change,bool saturated_microphone_signal,FrameBlocker * capture_blocker,BlockFramer * output_framer,BlockProcessor * block_processor,std::vector<std::vector<float>> * block)73 void ProcessRemainingCaptureFrameContent(
74 bool level_change,
75 bool saturated_microphone_signal,
76 FrameBlocker* capture_blocker,
77 BlockFramer* output_framer,
78 BlockProcessor* block_processor,
79 std::vector<std::vector<float>>* block) {
80 if (!capture_blocker->IsBlockAvailable()) {
81 return;
82 }
83
84 capture_blocker->ExtractBlock(block);
85 block_processor->ProcessCapture(level_change, saturated_microphone_signal,
86 block);
87 output_framer->InsertBlock(*block);
88 }
89
BufferRenderFrameContent(std::vector<std::vector<float>> * render_frame,size_t sub_frame_index,FrameBlocker * render_blocker,BlockProcessor * block_processor,std::vector<std::vector<float>> * block,std::vector<rtc::ArrayView<float>> * sub_frame_view)90 void BufferRenderFrameContent(
91 std::vector<std::vector<float>>* render_frame,
92 size_t sub_frame_index,
93 FrameBlocker* render_blocker,
94 BlockProcessor* block_processor,
95 std::vector<std::vector<float>>* block,
96 std::vector<rtc::ArrayView<float>>* sub_frame_view) {
97 FillSubFrameView(render_frame, sub_frame_index, sub_frame_view);
98 render_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block);
99 block_processor->BufferRender(*block);
100 }
101
BufferRemainingRenderFrameContent(FrameBlocker * render_blocker,BlockProcessor * block_processor,std::vector<std::vector<float>> * block)102 void BufferRemainingRenderFrameContent(FrameBlocker* render_blocker,
103 BlockProcessor* block_processor,
104 std::vector<std::vector<float>>* block) {
105 if (!render_blocker->IsBlockAvailable()) {
106 return;
107 }
108 render_blocker->ExtractBlock(block);
109 block_processor->BufferRender(*block);
110 }
111
CopyBufferIntoFrame(AudioBuffer * buffer,size_t num_bands,size_t frame_length,std::vector<std::vector<float>> * frame)112 void CopyBufferIntoFrame(AudioBuffer* buffer,
113 size_t num_bands,
114 size_t frame_length,
115 std::vector<std::vector<float>>* frame) {
116 RTC_DCHECK_EQ(num_bands, frame->size());
117 RTC_DCHECK_EQ(frame_length, (*frame)[0].size());
118 for (size_t k = 0; k < num_bands; ++k) {
119 rtc::ArrayView<float> buffer_view(&buffer->split_bands_f(0)[k][0],
120 frame_length);
121 std::copy(buffer_view.begin(), buffer_view.end(), (*frame)[k].begin());
122 }
123 }
124
125 // [B,A] = butter(2,100/4000,'high')
126 const CascadedBiQuadFilter::BiQuadCoefficients
127 kHighPassFilterCoefficients_8kHz = {{0.94598f, -1.89195f, 0.94598f},
128 {-1.88903f, 0.89487f}};
129 const int kNumberOfHighPassBiQuads_8kHz = 1;
130
131 // [B,A] = butter(2,100/8000,'high')
132 const CascadedBiQuadFilter::BiQuadCoefficients
133 kHighPassFilterCoefficients_16kHz = {{0.97261f, -1.94523f, 0.97261f},
134 {-1.94448f, 0.94598f}};
135 const int kNumberOfHighPassBiQuads_16kHz = 1;
136
137 } // namespace
138
139 class EchoCanceller3::RenderWriter {
140 public:
141 RenderWriter(ApmDataDumper* data_dumper,
142 SwapQueue<std::vector<std::vector<float>>,
143 Aec3RenderQueueItemVerifier>* render_transfer_queue,
144 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter,
145 int sample_rate_hz,
146 int frame_length,
147 int num_bands);
148 ~RenderWriter();
149 void Insert(AudioBuffer* input);
150
151 private:
152 ApmDataDumper* data_dumper_;
153 const int sample_rate_hz_;
154 const size_t frame_length_;
155 const int num_bands_;
156 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter_;
157 std::vector<std::vector<float>> render_queue_input_frame_;
158 SwapQueue<std::vector<std::vector<float>>, Aec3RenderQueueItemVerifier>*
159 render_transfer_queue_;
160 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderWriter);
161 };
162
RenderWriter(ApmDataDumper * data_dumper,SwapQueue<std::vector<std::vector<float>>,Aec3RenderQueueItemVerifier> * render_transfer_queue,std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter,int sample_rate_hz,int frame_length,int num_bands)163 EchoCanceller3::RenderWriter::RenderWriter(
164 ApmDataDumper* data_dumper,
165 SwapQueue<std::vector<std::vector<float>>, Aec3RenderQueueItemVerifier>*
166 render_transfer_queue,
167 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter,
168 int sample_rate_hz,
169 int frame_length,
170 int num_bands)
171 : data_dumper_(data_dumper),
172 sample_rate_hz_(sample_rate_hz),
173 frame_length_(frame_length),
174 num_bands_(num_bands),
175 render_highpass_filter_(std::move(render_highpass_filter)),
176 render_queue_input_frame_(num_bands_,
177 std::vector<float>(frame_length_, 0.f)),
178 render_transfer_queue_(render_transfer_queue) {
179 RTC_DCHECK(data_dumper);
180 }
181
182 EchoCanceller3::RenderWriter::~RenderWriter() = default;
183
Insert(AudioBuffer * input)184 void EchoCanceller3::RenderWriter::Insert(AudioBuffer* input) {
185 RTC_DCHECK_EQ(1, input->num_channels());
186 RTC_DCHECK_EQ(frame_length_, input->num_frames_per_band());
187 data_dumper_->DumpWav("aec3_render_input", frame_length_,
188 &input->split_bands_f(0)[0][0],
189 LowestBandRate(sample_rate_hz_), 1);
190
191 CopyBufferIntoFrame(input, num_bands_, frame_length_,
192 &render_queue_input_frame_);
193
194 if (render_highpass_filter_) {
195 render_highpass_filter_->Process(render_queue_input_frame_[0]);
196 }
197
198 static_cast<void>(render_transfer_queue_->Insert(&render_queue_input_frame_));
199 }
200
201 int EchoCanceller3::instance_count_ = 0;
202
EchoCanceller3(const EchoCanceller3Config & config,int sample_rate_hz,bool use_highpass_filter)203 EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config,
204 int sample_rate_hz,
205 bool use_highpass_filter)
206 : EchoCanceller3(sample_rate_hz,
207 use_highpass_filter,
208 std::unique_ptr<BlockProcessor>(
209 BlockProcessor::Create(config, sample_rate_hz))) {}
EchoCanceller3(int sample_rate_hz,bool use_highpass_filter,std::unique_ptr<BlockProcessor> block_processor)210 EchoCanceller3::EchoCanceller3(int sample_rate_hz,
211 bool use_highpass_filter,
212 std::unique_ptr<BlockProcessor> block_processor)
213 : data_dumper_(
214 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
215 sample_rate_hz_(sample_rate_hz),
216 num_bands_(NumBandsForRate(sample_rate_hz_)),
217 frame_length_(rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)),
218 output_framer_(num_bands_),
219 capture_blocker_(num_bands_),
220 render_blocker_(num_bands_),
221 render_transfer_queue_(
222 kRenderTransferQueueSize,
223 std::vector<std::vector<float>>(
224 num_bands_,
225 std::vector<float>(frame_length_, 0.f)),
226 Aec3RenderQueueItemVerifier(num_bands_, frame_length_)),
227 block_processor_(std::move(block_processor)),
228 render_queue_output_frame_(num_bands_,
229 std::vector<float>(frame_length_, 0.f)),
230 block_(num_bands_, std::vector<float>(kBlockSize, 0.f)),
231 sub_frame_view_(num_bands_) {
232 RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
233
234 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter;
235 if (use_highpass_filter) {
236 render_highpass_filter.reset(new CascadedBiQuadFilter(
237 sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz
238 : kHighPassFilterCoefficients_16kHz,
239 sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz
240 : kNumberOfHighPassBiQuads_16kHz));
241 capture_highpass_filter_.reset(new CascadedBiQuadFilter(
242 sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz
243 : kHighPassFilterCoefficients_16kHz,
244 sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz
245 : kNumberOfHighPassBiQuads_16kHz));
246 }
247
248 render_writer_.reset(
249 new RenderWriter(data_dumper_.get(), &render_transfer_queue_,
250 std::move(render_highpass_filter), sample_rate_hz_,
251 frame_length_, num_bands_));
252
253 RTC_DCHECK_EQ(num_bands_, std::max(sample_rate_hz_, 16000) / 16000);
254 RTC_DCHECK_GE(kMaxNumBands, num_bands_);
255 }
256
257 EchoCanceller3::~EchoCanceller3() = default;
258
AnalyzeRender(AudioBuffer * render)259 void EchoCanceller3::AnalyzeRender(AudioBuffer* render) {
260 RTC_DCHECK_RUNS_SERIALIZED(&render_race_checker_);
261 RTC_DCHECK(render);
262 data_dumper_->DumpRaw("aec3_call_order",
263 static_cast<int>(EchoCanceller3ApiCall::kRender));
264
265 return render_writer_->Insert(render);
266 }
267
AnalyzeCapture(AudioBuffer * capture)268 void EchoCanceller3::AnalyzeCapture(AudioBuffer* capture) {
269 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
270 RTC_DCHECK(capture);
271 data_dumper_->DumpWav("aec3_capture_analyze_input", capture->num_frames(),
272 capture->channels_f()[0], sample_rate_hz_, 1);
273
274 saturated_microphone_signal_ = false;
275 for (size_t k = 0; k < capture->num_channels(); ++k) {
276 saturated_microphone_signal_ |=
277 DetectSaturation(rtc::ArrayView<const float>(capture->channels_f()[k],
278 capture->num_frames()));
279 if (saturated_microphone_signal_) {
280 break;
281 }
282 }
283 }
284
ProcessCapture(AudioBuffer * capture,bool level_change)285 void EchoCanceller3::ProcessCapture(AudioBuffer* capture, bool level_change) {
286 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
287 RTC_DCHECK(capture);
288 RTC_DCHECK_EQ(1u, capture->num_channels());
289 RTC_DCHECK_EQ(num_bands_, capture->num_bands());
290 RTC_DCHECK_EQ(frame_length_, capture->num_frames_per_band());
291 data_dumper_->DumpRaw("aec3_call_order",
292 static_cast<int>(EchoCanceller3ApiCall::kCapture));
293
294 rtc::ArrayView<float> capture_lower_band =
295 rtc::ArrayView<float>(&capture->split_bands_f(0)[0][0], frame_length_);
296
297 data_dumper_->DumpWav("aec3_capture_input", capture_lower_band,
298 LowestBandRate(sample_rate_hz_), 1);
299
300 EmptyRenderQueue();
301
302 if (capture_highpass_filter_) {
303 capture_highpass_filter_->Process(capture_lower_band);
304 }
305
306 ProcessCaptureFrameContent(
307 capture, level_change, saturated_microphone_signal_, 0, &capture_blocker_,
308 &output_framer_, block_processor_.get(), &block_, &sub_frame_view_);
309
310 if (sample_rate_hz_ != 8000) {
311 ProcessCaptureFrameContent(
312 capture, level_change, saturated_microphone_signal_, 1,
313 &capture_blocker_, &output_framer_, block_processor_.get(), &block_,
314 &sub_frame_view_);
315 }
316
317 ProcessRemainingCaptureFrameContent(
318 level_change, saturated_microphone_signal_, &capture_blocker_,
319 &output_framer_, block_processor_.get(), &block_);
320
321 data_dumper_->DumpWav("aec3_capture_output", frame_length_,
322 &capture->split_bands_f(0)[0][0],
323 LowestBandRate(sample_rate_hz_), 1);
324 }
325
GetMetrics() const326 EchoControl::Metrics EchoCanceller3::GetMetrics() const {
327 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
328 Metrics metrics;
329 block_processor_->GetMetrics(&metrics);
330 return metrics;
331 }
332
Validate(const EchoCanceller3Config & config)333 bool EchoCanceller3::Validate(const EchoCanceller3Config& config) {
334 return true;
335 }
336
EmptyRenderQueue()337 void EchoCanceller3::EmptyRenderQueue() {
338 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
339 bool frame_to_buffer =
340 render_transfer_queue_.Remove(&render_queue_output_frame_);
341 while (frame_to_buffer) {
342 BufferRenderFrameContent(&render_queue_output_frame_, 0, &render_blocker_,
343 block_processor_.get(), &block_, &sub_frame_view_);
344
345 if (sample_rate_hz_ != 8000) {
346 BufferRenderFrameContent(&render_queue_output_frame_, 1, &render_blocker_,
347 block_processor_.get(), &block_,
348 &sub_frame_view_);
349 }
350
351 BufferRemainingRenderFrameContent(&render_blocker_, block_processor_.get(),
352 &block_);
353
354 frame_to_buffer =
355 render_transfer_queue_.Remove(&render_queue_output_frame_);
356 }
357 }
358
EchoCanceller3Factory()359 EchoCanceller3Factory::EchoCanceller3Factory() {}
360
EchoCanceller3Factory(const EchoCanceller3Config & config)361 EchoCanceller3Factory::EchoCanceller3Factory(const EchoCanceller3Config& config)
362 : config_(config) {
363 // Revert to default configuration if needed.
364 if (!EchoCanceller3::Validate(config_)) {
365 config_ = EchoCanceller3Config();
366 }
367 }
368
Create(int sample_rate_hz)369 std::unique_ptr<EchoControl> EchoCanceller3Factory::Create(int sample_rate_hz) {
370 return std::unique_ptr<EchoControl>(
371 new EchoCanceller3(config_, sample_rate_hz, true));
372 }
373
374 } // namespace webrtc
375