1 /*
2  *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/audio_processing/aec3/render_delay_buffer.h"
12 
13 #include <string.h>
14 #include <algorithm>
15 
16 #include "modules/audio_processing/aec3/aec3_common.h"
17 #include "modules/audio_processing/aec3/block_processor.h"
18 #include "modules/audio_processing/aec3/decimator.h"
19 #include "modules/audio_processing/aec3/fft_data.h"
20 #include "rtc_base/atomicops.h"
21 #include "rtc_base/checks.h"
22 #include "rtc_base/constructormagic.h"
23 #include "rtc_base/logging.h"
24 
25 namespace webrtc {
26 namespace {
27 
28 class ApiCallJitterBuffer {
29  public:
ApiCallJitterBuffer(size_t num_bands)30   explicit ApiCallJitterBuffer(size_t num_bands) {
31     buffer_.fill(std::vector<std::vector<float>>(
32         num_bands, std::vector<float>(kBlockSize, 0.f)));
33   }
34 
35   ~ApiCallJitterBuffer() = default;
36 
Reset()37   void Reset() {
38     size_ = 0;
39     last_insert_index_ = 0;
40   }
41 
Insert(const std::vector<std::vector<float>> & block)42   void Insert(const std::vector<std::vector<float>>& block) {
43     RTC_DCHECK_LT(size_, buffer_.size());
44     last_insert_index_ = (last_insert_index_ + 1) % buffer_.size();
45     RTC_DCHECK_EQ(buffer_[last_insert_index_].size(), block.size());
46     RTC_DCHECK_EQ(buffer_[last_insert_index_][0].size(), block[0].size());
47     for (size_t k = 0; k < block.size(); ++k) {
48       std::copy(block[k].begin(), block[k].end(),
49                 buffer_[last_insert_index_][k].begin());
50     }
51     ++size_;
52   }
53 
Remove(std::vector<std::vector<float>> * block)54   void Remove(std::vector<std::vector<float>>* block) {
55     RTC_DCHECK_LT(0, size_);
56     --size_;
57     const size_t extract_index =
58         (last_insert_index_ - size_ + buffer_.size()) % buffer_.size();
59     for (size_t k = 0; k < block->size(); ++k) {
60       std::copy(buffer_[extract_index][k].begin(),
61                 buffer_[extract_index][k].end(), (*block)[k].begin());
62     }
63   }
64 
Size() const65   size_t Size() const { return size_; }
Full() const66   bool Full() const { return size_ >= (buffer_.size()); }
Empty() const67   bool Empty() const { return size_ == 0; }
68 
69  private:
70   std::array<std::vector<std::vector<float>>, kMaxApiCallsJitterBlocks> buffer_;
71   size_t size_ = 0;
72   int last_insert_index_ = 0;
73 };
74 
75 class RenderDelayBufferImpl final : public RenderDelayBuffer {
76  public:
77   RenderDelayBufferImpl(size_t num_bands,
78                         size_t down_sampling_factor,
79                         size_t downsampled_render_buffer_size,
80                         size_t render_delay_buffer_size);
81   ~RenderDelayBufferImpl() override;
82 
83   void Reset() override;
84   bool Insert(const std::vector<std::vector<float>>& block) override;
85   bool UpdateBuffers() override;
86   void SetDelay(size_t delay) override;
Delay() const87   size_t Delay() const override { return delay_; }
88 
GetRenderBuffer() const89   const RenderBuffer& GetRenderBuffer() const override { return fft_buffer_; }
90 
GetDownsampledRenderBuffer() const91   const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override {
92     return downsampled_render_buffer_;
93   }
94 
95  private:
96   static int instance_count_;
97   std::unique_ptr<ApmDataDumper> data_dumper_;
98   const Aec3Optimization optimization_;
99   const size_t down_sampling_factor_;
100   const size_t sub_block_size_;
101   std::vector<std::vector<std::vector<float>>> buffer_;
102   size_t delay_ = 0;
103   size_t last_insert_index_ = 0;
104   RenderBuffer fft_buffer_;
105   DownsampledRenderBuffer downsampled_render_buffer_;
106   Decimator render_decimator_;
107   ApiCallJitterBuffer api_call_jitter_buffer_;
108   const std::vector<std::vector<float>> zero_block_;
109   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayBufferImpl);
110 };
111 
112 int RenderDelayBufferImpl::instance_count_ = 0;
113 
RenderDelayBufferImpl(size_t num_bands,size_t down_sampling_factor,size_t downsampled_render_buffer_size,size_t render_delay_buffer_size)114 RenderDelayBufferImpl::RenderDelayBufferImpl(
115     size_t num_bands,
116     size_t down_sampling_factor,
117     size_t downsampled_render_buffer_size,
118     size_t render_delay_buffer_size)
119     : data_dumper_(
120           new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
121       optimization_(DetectOptimization()),
122       down_sampling_factor_(down_sampling_factor),
123       sub_block_size_(down_sampling_factor_ > 0
124                           ? kBlockSize / down_sampling_factor
125                           : kBlockSize),
126       buffer_(
127           render_delay_buffer_size,
128           std::vector<std::vector<float>>(num_bands,
129                                           std::vector<float>(kBlockSize, 0.f))),
130       fft_buffer_(
131           optimization_,
132           num_bands,
133           std::max(kUnknownDelayRenderWindowSize, kAdaptiveFilterLength),
134           std::vector<size_t>(1, kAdaptiveFilterLength)),
135       downsampled_render_buffer_(downsampled_render_buffer_size),
136       render_decimator_(down_sampling_factor_),
137       api_call_jitter_buffer_(num_bands),
138       zero_block_(num_bands, std::vector<float>(kBlockSize, 0.f)) {
139   RTC_DCHECK_LT(buffer_.size(), downsampled_render_buffer_.buffer.size());
140 }
141 
142 RenderDelayBufferImpl::~RenderDelayBufferImpl() = default;
143 
Reset()144 void RenderDelayBufferImpl::Reset() {
145   // Empty all data in the buffers.
146   delay_ = 0;
147   last_insert_index_ = 0;
148   downsampled_render_buffer_.position = 0;
149   std::fill(downsampled_render_buffer_.buffer.begin(),
150             downsampled_render_buffer_.buffer.end(), 0.f);
151   fft_buffer_.Clear();
152   api_call_jitter_buffer_.Reset();
153   for (auto& c : buffer_) {
154     for (auto& b : c) {
155       std::fill(b.begin(), b.end(), 0.f);
156     }
157   }
158 }
159 
Insert(const std::vector<std::vector<float>> & block)160 bool RenderDelayBufferImpl::Insert(
161     const std::vector<std::vector<float>>& block) {
162   RTC_DCHECK_EQ(block.size(), buffer_[0].size());
163   RTC_DCHECK_EQ(block[0].size(), buffer_[0][0].size());
164 
165   if (api_call_jitter_buffer_.Full()) {
166     // Report buffer overrun and let the caller handle the overrun.
167     return false;
168   }
169   api_call_jitter_buffer_.Insert(block);
170 
171   return true;
172 }
173 
UpdateBuffers()174 bool RenderDelayBufferImpl::UpdateBuffers() {
175   bool underrun = true;
176   // Update the buffers with a new block if such is available, otherwise insert
177   // a block of silence.
178   if (api_call_jitter_buffer_.Size() > 0) {
179     last_insert_index_ = (last_insert_index_ + 1) % buffer_.size();
180     api_call_jitter_buffer_.Remove(&buffer_[last_insert_index_]);
181     underrun = false;
182   }
183 
184   downsampled_render_buffer_.position =
185       (downsampled_render_buffer_.position - sub_block_size_ +
186        downsampled_render_buffer_.buffer.size()) %
187       downsampled_render_buffer_.buffer.size();
188 
189   rtc::ArrayView<const float> input(
190       underrun ? zero_block_[0].data() : buffer_[last_insert_index_][0].data(),
191       kBlockSize);
192   rtc::ArrayView<float> output(downsampled_render_buffer_.buffer.data() +
193                                    downsampled_render_buffer_.position,
194                                sub_block_size_);
195   data_dumper_->DumpWav("aec3_render_decimator_input", input.size(),
196                         input.data(), 16000, 1);
197   render_decimator_.Decimate(input, output);
198   data_dumper_->DumpWav("aec3_render_decimator_output", output.size(),
199                         output.data(), 16000 / down_sampling_factor_, 1);
200   for (size_t k = 0; k < output.size() / 2; ++k) {
201     float tmp = output[k];
202     output[k] = output[output.size() - 1 - k];
203     output[output.size() - 1 - k] = tmp;
204   }
205 
206   if (underrun) {
207     fft_buffer_.Insert(zero_block_);
208   } else {
209     fft_buffer_.Insert(buffer_[(last_insert_index_ - delay_ + buffer_.size()) %
210                                buffer_.size()]);
211   }
212   return !underrun;
213 }
214 
SetDelay(size_t delay)215 void RenderDelayBufferImpl::SetDelay(size_t delay) {
216   if (delay_ == delay) {
217     return;
218   }
219 
220   // If there is a new delay set, clear the fft buffer.
221   fft_buffer_.Clear();
222 
223   if ((buffer_.size() - 1) < delay) {
224     // If the desired delay is larger than the delay buffer, shorten the delay
225     // buffer size to achieve the desired alignment with the available buffer
226     // size.
227     downsampled_render_buffer_.position =
228         (downsampled_render_buffer_.position +
229          sub_block_size_ * (delay - (buffer_.size() - 1))) %
230         downsampled_render_buffer_.buffer.size();
231 
232     last_insert_index_ =
233         (last_insert_index_ - (delay - (buffer_.size() - 1)) + buffer_.size()) %
234         buffer_.size();
235     delay_ = buffer_.size() - 1;
236   } else {
237     delay_ = delay;
238   }
239 }
240 
241 }  // namespace
242 
Create(size_t num_bands,size_t down_sampling_factor,size_t downsampled_render_buffer_size,size_t render_delay_buffer_size)243 RenderDelayBuffer* RenderDelayBuffer::Create(
244     size_t num_bands,
245     size_t down_sampling_factor,
246     size_t downsampled_render_buffer_size,
247     size_t render_delay_buffer_size) {
248   return new RenderDelayBufferImpl(num_bands, down_sampling_factor,
249                                    downsampled_render_buffer_size,
250                                    render_delay_buffer_size);
251 }
252 
253 }  // namespace webrtc
254