1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/audio_processing/aec3/block_processor.h"
11
12 #include "api/optional.h"
13 #include "modules/audio_processing/aec3/aec3_common.h"
14 #include "modules/audio_processing/aec3/block_processor_metrics.h"
15 #include "modules/audio_processing/aec3/echo_path_variability.h"
16 #include "modules/audio_processing/logging/apm_data_dumper.h"
17 #include "rtc_base/atomicops.h"
18 #include "rtc_base/constructormagic.h"
19 #include "rtc_base/logging.h"
20
21 namespace webrtc {
22 namespace {
23
24 enum class BlockProcessorApiCall { kCapture, kRender };
25
26 class BlockProcessorImpl final : public BlockProcessor {
27 public:
28 BlockProcessorImpl(int sample_rate_hz,
29 std::unique_ptr<RenderDelayBuffer> render_buffer,
30 std::unique_ptr<RenderDelayController> delay_controller,
31 std::unique_ptr<EchoRemover> echo_remover);
32
33 ~BlockProcessorImpl() override;
34
35 void ProcessCapture(bool echo_path_gain_change,
36 bool capture_signal_saturation,
37 std::vector<std::vector<float>>* capture_block) override;
38
39 void BufferRender(const std::vector<std::vector<float>>& block) override;
40
41 void UpdateEchoLeakageStatus(bool leakage_detected) override;
42
43 void GetMetrics(EchoControl::Metrics* metrics) const override;
44
45 private:
46 static int instance_count_;
47 bool no_capture_data_received_ = true;
48 bool no_render_data_received_ = true;
49 std::unique_ptr<ApmDataDumper> data_dumper_;
50 const size_t sample_rate_hz_;
51 std::unique_ptr<RenderDelayBuffer> render_buffer_;
52 std::unique_ptr<RenderDelayController> delay_controller_;
53 std::unique_ptr<EchoRemover> echo_remover_;
54 BlockProcessorMetrics metrics_;
55 bool render_buffer_overrun_occurred_ = false;
56 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl);
57 };
58
59 int BlockProcessorImpl::instance_count_ = 0;
60
BlockProcessorImpl(int sample_rate_hz,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)61 BlockProcessorImpl::BlockProcessorImpl(
62 int sample_rate_hz,
63 std::unique_ptr<RenderDelayBuffer> render_buffer,
64 std::unique_ptr<RenderDelayController> delay_controller,
65 std::unique_ptr<EchoRemover> echo_remover)
66 : data_dumper_(
67 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
68 sample_rate_hz_(sample_rate_hz),
69 render_buffer_(std::move(render_buffer)),
70 delay_controller_(std::move(delay_controller)),
71 echo_remover_(std::move(echo_remover)) {
72 RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
73 }
74
75 BlockProcessorImpl::~BlockProcessorImpl() = default;
76
ProcessCapture(bool echo_path_gain_change,bool capture_signal_saturation,std::vector<std::vector<float>> * capture_block)77 void BlockProcessorImpl::ProcessCapture(
78 bool echo_path_gain_change,
79 bool capture_signal_saturation,
80 std::vector<std::vector<float>>* capture_block) {
81 RTC_DCHECK(capture_block);
82 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size());
83 RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size());
84 data_dumper_->DumpRaw("aec3_processblock_call_order",
85 static_cast<int>(BlockProcessorApiCall::kCapture));
86 data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize,
87 &(*capture_block)[0][0],
88 LowestBandRate(sample_rate_hz_), 1);
89
90 // Do not start processing until render data has been buffered as that will
91 // cause the buffers to be wrongly aligned.
92 no_capture_data_received_ = false;
93 if (no_render_data_received_) {
94 return;
95 }
96
97 data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,
98 &(*capture_block)[0][0],
99 LowestBandRate(sample_rate_hz_), 1);
100
101 bool render_buffer_underrun = false;
102 if (render_buffer_overrun_occurred_) {
103 // Reset the render buffers and the alignment functionality when there has
104 // been a render buffer overrun as the buffer alignment may be noncausal.
105 delay_controller_->Reset();
106 render_buffer_->Reset();
107 RTC_LOG(LS_WARNING) << "Reset due to detected render buffer overrun.";
108 }
109
110 // Update the render buffers with new render data, filling the buffers with
111 // empty blocks when there is no render data available.
112 render_buffer_underrun = !render_buffer_->UpdateBuffers();
113 if (render_buffer_underrun) {
114 RTC_LOG(LS_WARNING) << "Render API jitter buffer underrun.";
115 }
116
117 // Compute and and apply the render delay required to achieve proper signal
118 // alignment.
119 const size_t old_delay = render_buffer_->Delay();
120 const size_t new_delay = delay_controller_->GetDelay(
121 render_buffer_->GetDownsampledRenderBuffer(), (*capture_block)[0]);
122
123 bool delay_change;
124 if (new_delay >= kMinEchoPathDelayBlocks) {
125 render_buffer_->SetDelay(new_delay);
126 const size_t achieved_delay = render_buffer_->Delay();
127 delay_change = old_delay != achieved_delay || old_delay != new_delay ||
128 render_buffer_overrun_occurred_;
129
130 // Inform the delay controller of the actually set delay to allow it to
131 // properly react to a non-feasible delay.
132 delay_controller_->SetDelay(achieved_delay);
133 } else {
134 delay_controller_->Reset();
135 render_buffer_->Reset();
136 delay_change = true;
137 RTC_LOG(LS_WARNING) << "Reset due to noncausal delay.";
138 }
139
140 // Remove the echo from the capture signal.
141 echo_remover_->ProcessCapture(
142 delay_controller_->AlignmentHeadroomSamples(),
143 EchoPathVariability(echo_path_gain_change, delay_change),
144 capture_signal_saturation, render_buffer_->GetRenderBuffer(),
145 capture_block);
146
147 // Update the metrics.
148 metrics_.UpdateCapture(render_buffer_underrun);
149
150 render_buffer_overrun_occurred_ = false;
151 }
152
BufferRender(const std::vector<std::vector<float>> & block)153 void BlockProcessorImpl::BufferRender(
154 const std::vector<std::vector<float>>& block) {
155 RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size());
156 RTC_DCHECK_EQ(kBlockSize, block[0].size());
157 data_dumper_->DumpRaw("aec3_processblock_call_order",
158 static_cast<int>(BlockProcessorApiCall::kRender));
159 data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize,
160 &block[0][0], LowestBandRate(sample_rate_hz_), 1);
161
162 no_render_data_received_ = false;
163
164 // Do not start buffer render data until capture data has been received as
165 // that data may give a false alignment.
166 if (no_capture_data_received_) {
167 return;
168 }
169
170 data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize,
171 &block[0][0], LowestBandRate(sample_rate_hz_), 1);
172
173 // Buffer the render data.
174 render_buffer_overrun_occurred_ = !render_buffer_->Insert(block);
175
176 // Update the metrics.
177 metrics_.UpdateRender(render_buffer_overrun_occurred_);
178 }
179
UpdateEchoLeakageStatus(bool leakage_detected)180 void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {
181 echo_remover_->UpdateEchoLeakageStatus(leakage_detected);
182 }
183
GetMetrics(EchoControl::Metrics * metrics) const184 void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const {
185 echo_remover_->GetMetrics(metrics);
186 const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4;
187 metrics->delay_ms = static_cast<int>(render_buffer_->Delay()) * block_size_ms;
188 }
189
190 } // namespace
191
Create(const EchoCanceller3Config & config,int sample_rate_hz)192 BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,
193 int sample_rate_hz) {
194 std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create(
195 NumBandsForRate(sample_rate_hz), config.delay.down_sampling_factor,
196 GetDownSampledBufferSize(config.delay.down_sampling_factor,
197 config.delay.num_filters),
198 GetRenderDelayBufferSize(config.delay.down_sampling_factor,
199 config.delay.num_filters)));
200 std::unique_ptr<RenderDelayController> delay_controller(
201 RenderDelayController::Create(config, sample_rate_hz));
202 std::unique_ptr<EchoRemover> echo_remover(
203 EchoRemover::Create(config, sample_rate_hz));
204 return Create(config, sample_rate_hz, std::move(render_buffer),
205 std::move(delay_controller), std::move(echo_remover));
206 }
207
Create(const EchoCanceller3Config & config,int sample_rate_hz,std::unique_ptr<RenderDelayBuffer> render_buffer)208 BlockProcessor* BlockProcessor::Create(
209 const EchoCanceller3Config& config,
210 int sample_rate_hz,
211 std::unique_ptr<RenderDelayBuffer> render_buffer) {
212 std::unique_ptr<RenderDelayController> delay_controller(
213 RenderDelayController::Create(config, sample_rate_hz));
214 std::unique_ptr<EchoRemover> echo_remover(
215 EchoRemover::Create(config, sample_rate_hz));
216 return Create(config, sample_rate_hz, std::move(render_buffer),
217 std::move(delay_controller), std::move(echo_remover));
218 }
219
Create(const EchoCanceller3Config & config,int sample_rate_hz,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)220 BlockProcessor* BlockProcessor::Create(
221 const EchoCanceller3Config& config,
222 int sample_rate_hz,
223 std::unique_ptr<RenderDelayBuffer> render_buffer,
224 std::unique_ptr<RenderDelayController> delay_controller,
225 std::unique_ptr<EchoRemover> echo_remover) {
226 return new BlockProcessorImpl(sample_rate_hz, std::move(render_buffer),
227 std::move(delay_controller),
228 std::move(echo_remover));
229 }
230
231 } // namespace webrtc
232