1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/audio_processing/aec3/render_delay_controller.h"
11
12 #include <stddef.h>
13
14 #include <algorithm>
15 #include <memory>
16
17 #include "absl/types/optional.h"
18 #include "api/array_view.h"
19 #include "api/audio/echo_canceller3_config.h"
20 #include "modules/audio_processing/aec3/aec3_common.h"
21 #include "modules/audio_processing/aec3/delay_estimate.h"
22 #include "modules/audio_processing/aec3/downsampled_render_buffer.h"
23 #include "modules/audio_processing/aec3/echo_path_delay_estimator.h"
24 #include "modules/audio_processing/aec3/render_delay_controller_metrics.h"
25 #include "modules/audio_processing/logging/apm_data_dumper.h"
26 #include "rtc_base/atomic_ops.h"
27 #include "rtc_base/checks.h"
28 #include "rtc_base/constructor_magic.h"
29
30 namespace webrtc {
31
32 namespace {
33
34 class RenderDelayControllerImpl final : public RenderDelayController {
35 public:
36 RenderDelayControllerImpl(const EchoCanceller3Config& config,
37 int sample_rate_hz,
38 size_t num_capture_channels);
39 ~RenderDelayControllerImpl() override;
40 void Reset(bool reset_delay_confidence) override;
41 void LogRenderCall() override;
42 absl::optional<DelayEstimate> GetDelay(
43 const DownsampledRenderBuffer& render_buffer,
44 size_t render_delay_buffer_delay,
45 const std::vector<std::vector<float>>& capture) override;
46 bool HasClockdrift() const override;
47
48 private:
49 static int instance_count_;
50 std::unique_ptr<ApmDataDumper> data_dumper_;
51 const int hysteresis_limit_blocks_;
52 const int delay_headroom_samples_;
53 absl::optional<DelayEstimate> delay_;
54 EchoPathDelayEstimator delay_estimator_;
55 RenderDelayControllerMetrics metrics_;
56 absl::optional<DelayEstimate> delay_samples_;
57 size_t capture_call_counter_ = 0;
58 int delay_change_counter_ = 0;
59 DelayEstimate::Quality last_delay_estimate_quality_;
60 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl);
61 };
62
ComputeBufferDelay(const absl::optional<DelayEstimate> & current_delay,int hysteresis_limit_blocks,int delay_headroom_samples,DelayEstimate estimated_delay)63 DelayEstimate ComputeBufferDelay(
64 const absl::optional<DelayEstimate>& current_delay,
65 int hysteresis_limit_blocks,
66 int delay_headroom_samples,
67 DelayEstimate estimated_delay) {
68 // Subtract delay headroom.
69 const int delay_with_headroom_samples = std::max(
70 static_cast<int>(estimated_delay.delay) - delay_headroom_samples, 0);
71
72 // Compute the buffer delay increase required to achieve the desired latency.
73 size_t new_delay_blocks = delay_with_headroom_samples >> kBlockSizeLog2;
74
75 // Add hysteresis.
76 if (current_delay) {
77 size_t current_delay_blocks = current_delay->delay;
78 if (new_delay_blocks > current_delay_blocks &&
79 new_delay_blocks <= current_delay_blocks + hysteresis_limit_blocks) {
80 new_delay_blocks = current_delay_blocks;
81 }
82 }
83
84 DelayEstimate new_delay = estimated_delay;
85 new_delay.delay = new_delay_blocks;
86 return new_delay;
87 }
88
89 int RenderDelayControllerImpl::instance_count_ = 0;
90
RenderDelayControllerImpl(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_capture_channels)91 RenderDelayControllerImpl::RenderDelayControllerImpl(
92 const EchoCanceller3Config& config,
93 int sample_rate_hz,
94 size_t num_capture_channels)
95 : data_dumper_(
96 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
97 hysteresis_limit_blocks_(
98 static_cast<int>(config.delay.hysteresis_limit_blocks)),
99 delay_headroom_samples_(config.delay.delay_headroom_samples),
100 delay_estimator_(data_dumper_.get(), config, num_capture_channels),
101 last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) {
102 RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
103 delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, 0);
104 }
105
106 RenderDelayControllerImpl::~RenderDelayControllerImpl() = default;
107
Reset(bool reset_delay_confidence)108 void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) {
109 delay_ = absl::nullopt;
110 delay_samples_ = absl::nullopt;
111 delay_estimator_.Reset(reset_delay_confidence);
112 delay_change_counter_ = 0;
113 if (reset_delay_confidence) {
114 last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse;
115 }
116 }
117
LogRenderCall()118 void RenderDelayControllerImpl::LogRenderCall() {}
119
GetDelay(const DownsampledRenderBuffer & render_buffer,size_t render_delay_buffer_delay,const std::vector<std::vector<float>> & capture)120 absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
121 const DownsampledRenderBuffer& render_buffer,
122 size_t render_delay_buffer_delay,
123 const std::vector<std::vector<float>>& capture) {
124 RTC_DCHECK_EQ(kBlockSize, capture[0].size());
125 ++capture_call_counter_;
126
127 auto delay_samples = delay_estimator_.EstimateDelay(render_buffer, capture);
128
129 if (delay_samples) {
130 if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
131 delay_change_counter_ = 0;
132 }
133 if (delay_samples_) {
134 delay_samples_->blocks_since_last_change =
135 delay_samples_->delay == delay_samples->delay
136 ? delay_samples_->blocks_since_last_change + 1
137 : 0;
138 delay_samples_->blocks_since_last_update = 0;
139 delay_samples_->delay = delay_samples->delay;
140 delay_samples_->quality = delay_samples->quality;
141 } else {
142 delay_samples_ = delay_samples;
143 }
144 } else {
145 if (delay_samples_) {
146 ++delay_samples_->blocks_since_last_change;
147 ++delay_samples_->blocks_since_last_update;
148 }
149 }
150
151 if (delay_change_counter_ < 2 * kNumBlocksPerSecond) {
152 ++delay_change_counter_;
153 }
154
155 if (delay_samples_) {
156 // Compute the render delay buffer delay.
157 const bool use_hysteresis =
158 last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined &&
159 delay_samples_->quality == DelayEstimate::Quality::kRefined;
160 delay_ = ComputeBufferDelay(delay_,
161 use_hysteresis ? hysteresis_limit_blocks_ : 0,
162 delay_headroom_samples_, *delay_samples_);
163 last_delay_estimate_quality_ = delay_samples_->quality;
164 }
165
166 metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)
167 : absl::nullopt,
168 delay_ ? delay_->delay : 0, 0, delay_estimator_.Clockdrift());
169
170 data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
171 delay_samples ? delay_samples->delay : 0);
172 data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay",
173 delay_ ? delay_->delay : 0);
174
175 return delay_;
176 }
177
HasClockdrift() const178 bool RenderDelayControllerImpl::HasClockdrift() const {
179 return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone;
180 }
181
182 } // namespace
183
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_capture_channels)184 RenderDelayController* RenderDelayController::Create(
185 const EchoCanceller3Config& config,
186 int sample_rate_hz,
187 size_t num_capture_channels) {
188 return new RenderDelayControllerImpl(config, sample_rate_hz,
189 num_capture_channels);
190 }
191
192 } // namespace webrtc
193