1 /*
2  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #include "modules/audio_processing/aec3/block_processor.h"
11 
12 #include "api/optional.h"
13 #include "modules/audio_processing/aec3/aec3_common.h"
14 #include "modules/audio_processing/aec3/block_processor_metrics.h"
15 #include "modules/audio_processing/aec3/echo_path_variability.h"
16 #include "modules/audio_processing/logging/apm_data_dumper.h"
17 #include "rtc_base/atomicops.h"
18 #include "rtc_base/constructormagic.h"
19 #include "rtc_base/logging.h"
20 
21 namespace webrtc {
22 namespace {
23 
24 enum class BlockProcessorApiCall { kCapture, kRender };
25 
26 class BlockProcessorImpl final : public BlockProcessor {
27  public:
28   BlockProcessorImpl(int sample_rate_hz,
29                      std::unique_ptr<RenderDelayBuffer> render_buffer,
30                      std::unique_ptr<RenderDelayController> delay_controller,
31                      std::unique_ptr<EchoRemover> echo_remover);
32 
33   ~BlockProcessorImpl() override;
34 
35   void ProcessCapture(bool echo_path_gain_change,
36                       bool capture_signal_saturation,
37                       std::vector<std::vector<float>>* capture_block) override;
38 
39   void BufferRender(const std::vector<std::vector<float>>& block) override;
40 
41   void UpdateEchoLeakageStatus(bool leakage_detected) override;
42 
43   void GetMetrics(EchoControl::Metrics* metrics) const override;
44 
45  private:
46   static int instance_count_;
47   bool no_capture_data_received_ = true;
48   bool no_render_data_received_ = true;
49   std::unique_ptr<ApmDataDumper> data_dumper_;
50   const size_t sample_rate_hz_;
51   std::unique_ptr<RenderDelayBuffer> render_buffer_;
52   std::unique_ptr<RenderDelayController> delay_controller_;
53   std::unique_ptr<EchoRemover> echo_remover_;
54   BlockProcessorMetrics metrics_;
55   bool render_buffer_overrun_occurred_ = false;
56   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl);
57 };
58 
59 int BlockProcessorImpl::instance_count_ = 0;
60 
BlockProcessorImpl(int sample_rate_hz,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)61 BlockProcessorImpl::BlockProcessorImpl(
62     int sample_rate_hz,
63     std::unique_ptr<RenderDelayBuffer> render_buffer,
64     std::unique_ptr<RenderDelayController> delay_controller,
65     std::unique_ptr<EchoRemover> echo_remover)
66     : data_dumper_(
67           new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
68       sample_rate_hz_(sample_rate_hz),
69       render_buffer_(std::move(render_buffer)),
70       delay_controller_(std::move(delay_controller)),
71       echo_remover_(std::move(echo_remover)) {
72   RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
73 }
74 
75 BlockProcessorImpl::~BlockProcessorImpl() = default;
76 
ProcessCapture(bool echo_path_gain_change,bool capture_signal_saturation,std::vector<std::vector<float>> * capture_block)77 void BlockProcessorImpl::ProcessCapture(
78     bool echo_path_gain_change,
79     bool capture_signal_saturation,
80     std::vector<std::vector<float>>* capture_block) {
81   RTC_DCHECK(capture_block);
82   RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size());
83   RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size());
84   data_dumper_->DumpRaw("aec3_processblock_call_order",
85                         static_cast<int>(BlockProcessorApiCall::kCapture));
86   data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize,
87                         &(*capture_block)[0][0],
88                         LowestBandRate(sample_rate_hz_), 1);
89 
90   // Do not start processing until render data has been buffered as that will
91   // cause the buffers to be wrongly aligned.
92   no_capture_data_received_ = false;
93   if (no_render_data_received_) {
94     return;
95   }
96 
97   data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,
98                         &(*capture_block)[0][0],
99                         LowestBandRate(sample_rate_hz_), 1);
100 
101   bool render_buffer_underrun = false;
102   if (render_buffer_overrun_occurred_) {
103     // Reset the render buffers and the alignment functionality when there has
104     // been a render buffer overrun as the buffer alignment may be noncausal.
105     delay_controller_->Reset();
106     render_buffer_->Reset();
107     RTC_LOG(LS_WARNING) << "Reset due to detected render buffer overrun.";
108   }
109 
110   // Update the render buffers with new render data, filling the buffers with
111   // empty blocks when there is no render data available.
112   render_buffer_underrun = !render_buffer_->UpdateBuffers();
113   if (render_buffer_underrun) {
114     RTC_LOG(LS_WARNING) << "Render API jitter buffer underrun.";
115   }
116 
117   // Compute and and apply the render delay required to achieve proper signal
118   // alignment.
119   const size_t old_delay = render_buffer_->Delay();
120   const size_t new_delay = delay_controller_->GetDelay(
121       render_buffer_->GetDownsampledRenderBuffer(), (*capture_block)[0]);
122 
123   bool delay_change;
124   if (new_delay >= kMinEchoPathDelayBlocks) {
125     render_buffer_->SetDelay(new_delay);
126     const size_t achieved_delay = render_buffer_->Delay();
127     delay_change = old_delay != achieved_delay || old_delay != new_delay ||
128                    render_buffer_overrun_occurred_;
129 
130     // Inform the delay controller of the actually set delay to allow it to
131     // properly react to a non-feasible delay.
132     delay_controller_->SetDelay(achieved_delay);
133   } else {
134     delay_controller_->Reset();
135     render_buffer_->Reset();
136     delay_change = true;
137     RTC_LOG(LS_WARNING) << "Reset due to noncausal delay.";
138   }
139 
140   // Remove the echo from the capture signal.
141   echo_remover_->ProcessCapture(
142       delay_controller_->AlignmentHeadroomSamples(),
143       EchoPathVariability(echo_path_gain_change, delay_change),
144       capture_signal_saturation, render_buffer_->GetRenderBuffer(),
145       capture_block);
146 
147   // Update the metrics.
148   metrics_.UpdateCapture(render_buffer_underrun);
149 
150   render_buffer_overrun_occurred_ = false;
151 }
152 
BufferRender(const std::vector<std::vector<float>> & block)153 void BlockProcessorImpl::BufferRender(
154     const std::vector<std::vector<float>>& block) {
155   RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size());
156   RTC_DCHECK_EQ(kBlockSize, block[0].size());
157   data_dumper_->DumpRaw("aec3_processblock_call_order",
158                         static_cast<int>(BlockProcessorApiCall::kRender));
159   data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize,
160                         &block[0][0], LowestBandRate(sample_rate_hz_), 1);
161 
162   no_render_data_received_ = false;
163 
164   // Do not start buffer render data until capture data has been received as
165   // that data may give a false alignment.
166   if (no_capture_data_received_) {
167     return;
168   }
169 
170   data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize,
171                         &block[0][0], LowestBandRate(sample_rate_hz_), 1);
172 
173   // Buffer the render data.
174   render_buffer_overrun_occurred_ = !render_buffer_->Insert(block);
175 
176   // Update the metrics.
177   metrics_.UpdateRender(render_buffer_overrun_occurred_);
178 }
179 
UpdateEchoLeakageStatus(bool leakage_detected)180 void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {
181   echo_remover_->UpdateEchoLeakageStatus(leakage_detected);
182 }
183 
GetMetrics(EchoControl::Metrics * metrics) const184 void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const {
185   echo_remover_->GetMetrics(metrics);
186   const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4;
187   metrics->delay_ms = static_cast<int>(render_buffer_->Delay()) * block_size_ms;
188 }
189 
190 }  // namespace
191 
Create(const EchoCanceller3Config & config,int sample_rate_hz)192 BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,
193                                        int sample_rate_hz) {
194   std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create(
195       NumBandsForRate(sample_rate_hz), config.delay.down_sampling_factor,
196       GetDownSampledBufferSize(config.delay.down_sampling_factor,
197                                config.delay.num_filters),
198       GetRenderDelayBufferSize(config.delay.down_sampling_factor,
199                                config.delay.num_filters)));
200   std::unique_ptr<RenderDelayController> delay_controller(
201       RenderDelayController::Create(config, sample_rate_hz));
202   std::unique_ptr<EchoRemover> echo_remover(
203       EchoRemover::Create(config, sample_rate_hz));
204   return Create(config, sample_rate_hz, std::move(render_buffer),
205                 std::move(delay_controller), std::move(echo_remover));
206 }
207 
Create(const EchoCanceller3Config & config,int sample_rate_hz,std::unique_ptr<RenderDelayBuffer> render_buffer)208 BlockProcessor* BlockProcessor::Create(
209     const EchoCanceller3Config& config,
210     int sample_rate_hz,
211     std::unique_ptr<RenderDelayBuffer> render_buffer) {
212   std::unique_ptr<RenderDelayController> delay_controller(
213       RenderDelayController::Create(config, sample_rate_hz));
214   std::unique_ptr<EchoRemover> echo_remover(
215       EchoRemover::Create(config, sample_rate_hz));
216   return Create(config, sample_rate_hz, std::move(render_buffer),
217                 std::move(delay_controller), std::move(echo_remover));
218 }
219 
Create(const EchoCanceller3Config & config,int sample_rate_hz,std::unique_ptr<RenderDelayBuffer> render_buffer,std::unique_ptr<RenderDelayController> delay_controller,std::unique_ptr<EchoRemover> echo_remover)220 BlockProcessor* BlockProcessor::Create(
221     const EchoCanceller3Config& config,
222     int sample_rate_hz,
223     std::unique_ptr<RenderDelayBuffer> render_buffer,
224     std::unique_ptr<RenderDelayController> delay_controller,
225     std::unique_ptr<EchoRemover> echo_remover) {
226   return new BlockProcessorImpl(sample_rate_hz, std::move(render_buffer),
227                                 std::move(delay_controller),
228                                 std::move(echo_remover));
229 }
230 
231 }  // namespace webrtc
232