1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
12
13 #include <string.h>
14
15 #include "webrtc/base/checks.h"
16 #include "webrtc/modules/audio_processing/aec/aec_core.h"
17 #include "webrtc/modules/audio_processing/aec/echo_cancellation.h"
18 #include "webrtc/modules/audio_processing/audio_buffer.h"
19
20 namespace webrtc {
21
22 namespace {
MapSetting(EchoCancellation::SuppressionLevel level)23 int16_t MapSetting(EchoCancellation::SuppressionLevel level) {
24 switch (level) {
25 case EchoCancellation::kLowSuppression:
26 return kAecNlpConservative;
27 case EchoCancellation::kModerateSuppression:
28 return kAecNlpModerate;
29 case EchoCancellation::kHighSuppression:
30 return kAecNlpAggressive;
31 }
32 RTC_NOTREACHED();
33 return -1;
34 }
35
MapError(int err)36 AudioProcessing::Error MapError(int err) {
37 switch (err) {
38 case AEC_UNSUPPORTED_FUNCTION_ERROR:
39 return AudioProcessing::kUnsupportedFunctionError;
40 case AEC_BAD_PARAMETER_ERROR:
41 return AudioProcessing::kBadParameterError;
42 case AEC_BAD_PARAMETER_WARNING:
43 return AudioProcessing::kBadStreamParameterWarning;
44 default:
45 // AEC_UNSPECIFIED_ERROR
46 // AEC_UNINITIALIZED_ERROR
47 // AEC_NULL_POINTER_ERROR
48 return AudioProcessing::kUnspecifiedError;
49 }
50 }
51
52 } // namespace
53
54 struct EchoCancellationImpl::StreamProperties {
55 StreamProperties() = delete;
StreamPropertieswebrtc::EchoCancellationImpl::StreamProperties56 StreamProperties(int sample_rate_hz,
57 size_t num_reverse_channels,
58 size_t num_output_channels,
59 size_t num_proc_channels)
60 : sample_rate_hz(sample_rate_hz),
61 num_reverse_channels(num_reverse_channels),
62 num_output_channels(num_output_channels),
63 num_proc_channels(num_proc_channels) {}
64
65 const int sample_rate_hz;
66 const size_t num_reverse_channels;
67 const size_t num_output_channels;
68 const size_t num_proc_channels;
69 };
70
71 class EchoCancellationImpl::Canceller {
72 public:
Canceller()73 Canceller() {
74 state_ = WebRtcAec_Create();
75 RTC_DCHECK(state_);
76 }
77
~Canceller()78 ~Canceller() {
79 RTC_CHECK(state_);
80 WebRtcAec_Free(state_);
81 }
82
state()83 void* state() { return state_; }
84
Initialize(int sample_rate_hz)85 void Initialize(int sample_rate_hz) {
86 // TODO(ajm): Drift compensation is disabled in practice. If restored, it
87 // should be managed internally and not depend on the hardware sample rate.
88 // For now, just hardcode a 48 kHz value.
89 const int error = WebRtcAec_Init(state_, sample_rate_hz, 48000);
90 RTC_DCHECK_EQ(0, error);
91 }
92
93 private:
94 void* state_;
95 };
96
EchoCancellationImpl(rtc::CriticalSection * crit_render,rtc::CriticalSection * crit_capture)97 EchoCancellationImpl::EchoCancellationImpl(rtc::CriticalSection* crit_render,
98 rtc::CriticalSection* crit_capture)
99 : crit_render_(crit_render),
100 crit_capture_(crit_capture),
101 drift_compensation_enabled_(false),
102 metrics_enabled_(false),
103 suppression_level_(kModerateSuppression),
104 stream_drift_samples_(0),
105 was_stream_drift_set_(false),
106 stream_has_echo_(false),
107 delay_logging_enabled_(false),
108 extended_filter_enabled_(false),
109 delay_agnostic_enabled_(false) {
110 RTC_DCHECK(crit_render);
111 RTC_DCHECK(crit_capture);
112 }
113
114 EchoCancellationImpl::~EchoCancellationImpl() = default;
115
ProcessRenderAudio(rtc::ArrayView<const float> packed_render_audio)116 void EchoCancellationImpl::ProcessRenderAudio(
117 rtc::ArrayView<const float> packed_render_audio) {
118 rtc::CritScope cs_capture(crit_capture_);
119 if (!enabled_) {
120 return;
121 }
122
123 RTC_DCHECK(stream_properties_);
124 size_t handle_index = 0;
125 size_t buffer_index = 0;
126 const size_t num_frames_per_band =
127 packed_render_audio.size() / (stream_properties_->num_output_channels *
128 stream_properties_->num_reverse_channels);
129 for (size_t i = 0; i < stream_properties_->num_output_channels; i++) {
130 for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) {
131 WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(),
132 &packed_render_audio[buffer_index],
133 num_frames_per_band);
134
135 buffer_index += num_frames_per_band;
136 }
137 }
138 }
139
140
ProcessCaptureAudio(AudioBuffer * audio,int stream_delay_ms)141 int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio,
142 int stream_delay_ms) {
143 rtc::CritScope cs_capture(crit_capture_);
144 if (!enabled_) {
145 return AudioProcessing::kNoError;
146 }
147
148 if (drift_compensation_enabled_ && !was_stream_drift_set_) {
149 return AudioProcessing::kStreamParameterNotSetError;
150 }
151
152 RTC_DCHECK(stream_properties_);
153 RTC_DCHECK_GE(160, audio->num_frames_per_band());
154 RTC_DCHECK_EQ(audio->num_channels(), stream_properties_->num_proc_channels);
155
156 int err = AudioProcessing::kNoError;
157
158 // The ordering convention must be followed to pass to the correct AEC.
159 size_t handle_index = 0;
160 stream_has_echo_ = false;
161 for (size_t i = 0; i < audio->num_channels(); i++) {
162 for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) {
163 err = WebRtcAec_Process(
164 cancellers_[handle_index]->state(), audio->split_bands_const_f(i),
165 audio->num_bands(), audio->split_bands_f(i),
166 audio->num_frames_per_band(), stream_delay_ms, stream_drift_samples_);
167
168 if (err != AudioProcessing::kNoError) {
169 err = MapError(err);
170 // TODO(ajm): Figure out how to return warnings properly.
171 if (err != AudioProcessing::kBadStreamParameterWarning) {
172 return err;
173 }
174 }
175
176 int status = 0;
177 err = WebRtcAec_get_echo_status(cancellers_[handle_index]->state(),
178 &status);
179 if (err != AudioProcessing::kNoError) {
180 return MapError(err);
181 }
182
183 if (status == 1) {
184 stream_has_echo_ = true;
185 }
186
187 handle_index++;
188 }
189 }
190
191 was_stream_drift_set_ = false;
192 return AudioProcessing::kNoError;
193 }
194
Enable(bool enable)195 int EchoCancellationImpl::Enable(bool enable) {
196 // Run in a single-threaded manner.
197 rtc::CritScope cs_render(crit_render_);
198 rtc::CritScope cs_capture(crit_capture_);
199
200 if (enable && !enabled_) {
201 enabled_ = enable; // Must be set before Initialize() is called.
202
203 // TODO(peah): Simplify once the Enable function has been removed from
204 // the public APM API.
205 RTC_DCHECK(stream_properties_);
206 Initialize(stream_properties_->sample_rate_hz,
207 stream_properties_->num_reverse_channels,
208 stream_properties_->num_output_channels,
209 stream_properties_->num_proc_channels);
210 } else {
211 enabled_ = enable;
212 }
213 return AudioProcessing::kNoError;
214 }
215
is_enabled() const216 bool EchoCancellationImpl::is_enabled() const {
217 rtc::CritScope cs(crit_capture_);
218 return enabled_;
219 }
220
set_suppression_level(SuppressionLevel level)221 int EchoCancellationImpl::set_suppression_level(SuppressionLevel level) {
222 {
223 if (MapSetting(level) == -1) {
224 return AudioProcessing::kBadParameterError;
225 }
226 rtc::CritScope cs(crit_capture_);
227 suppression_level_ = level;
228 }
229 return Configure();
230 }
231
suppression_level() const232 EchoCancellation::SuppressionLevel EchoCancellationImpl::suppression_level()
233 const {
234 rtc::CritScope cs(crit_capture_);
235 return suppression_level_;
236 }
237
enable_drift_compensation(bool enable)238 int EchoCancellationImpl::enable_drift_compensation(bool enable) {
239 {
240 rtc::CritScope cs(crit_capture_);
241 drift_compensation_enabled_ = enable;
242 }
243 return Configure();
244 }
245
is_drift_compensation_enabled() const246 bool EchoCancellationImpl::is_drift_compensation_enabled() const {
247 rtc::CritScope cs(crit_capture_);
248 return drift_compensation_enabled_;
249 }
250
set_stream_drift_samples(int drift)251 void EchoCancellationImpl::set_stream_drift_samples(int drift) {
252 rtc::CritScope cs(crit_capture_);
253 was_stream_drift_set_ = true;
254 stream_drift_samples_ = drift;
255 }
256
stream_drift_samples() const257 int EchoCancellationImpl::stream_drift_samples() const {
258 rtc::CritScope cs(crit_capture_);
259 return stream_drift_samples_;
260 }
261
enable_metrics(bool enable)262 int EchoCancellationImpl::enable_metrics(bool enable) {
263 {
264 rtc::CritScope cs(crit_capture_);
265 metrics_enabled_ = enable;
266 }
267 return Configure();
268 }
269
are_metrics_enabled() const270 bool EchoCancellationImpl::are_metrics_enabled() const {
271 rtc::CritScope cs(crit_capture_);
272 return enabled_ && metrics_enabled_;
273 }
274
275 // TODO(ajm): we currently just use the metrics from the first AEC. Think more
276 // aboue the best way to extend this to multi-channel.
GetMetrics(Metrics * metrics)277 int EchoCancellationImpl::GetMetrics(Metrics* metrics) {
278 rtc::CritScope cs(crit_capture_);
279 if (metrics == NULL) {
280 return AudioProcessing::kNullPointerError;
281 }
282
283 if (!enabled_ || !metrics_enabled_) {
284 return AudioProcessing::kNotEnabledError;
285 }
286
287 AecMetrics my_metrics;
288 memset(&my_metrics, 0, sizeof(my_metrics));
289 memset(metrics, 0, sizeof(Metrics));
290
291 const int err = WebRtcAec_GetMetrics(cancellers_[0]->state(), &my_metrics);
292 if (err != AudioProcessing::kNoError) {
293 return MapError(err);
294 }
295
296 metrics->residual_echo_return_loss.instant = my_metrics.rerl.instant;
297 metrics->residual_echo_return_loss.average = my_metrics.rerl.average;
298 metrics->residual_echo_return_loss.maximum = my_metrics.rerl.max;
299 metrics->residual_echo_return_loss.minimum = my_metrics.rerl.min;
300
301 metrics->echo_return_loss.instant = my_metrics.erl.instant;
302 metrics->echo_return_loss.average = my_metrics.erl.average;
303 metrics->echo_return_loss.maximum = my_metrics.erl.max;
304 metrics->echo_return_loss.minimum = my_metrics.erl.min;
305
306 metrics->echo_return_loss_enhancement.instant = my_metrics.erle.instant;
307 metrics->echo_return_loss_enhancement.average = my_metrics.erle.average;
308 metrics->echo_return_loss_enhancement.maximum = my_metrics.erle.max;
309 metrics->echo_return_loss_enhancement.minimum = my_metrics.erle.min;
310
311 metrics->a_nlp.instant = my_metrics.aNlp.instant;
312 metrics->a_nlp.average = my_metrics.aNlp.average;
313 metrics->a_nlp.maximum = my_metrics.aNlp.max;
314 metrics->a_nlp.minimum = my_metrics.aNlp.min;
315
316 metrics->divergent_filter_fraction = my_metrics.divergent_filter_fraction;
317 return AudioProcessing::kNoError;
318 }
319
stream_has_echo() const320 bool EchoCancellationImpl::stream_has_echo() const {
321 rtc::CritScope cs(crit_capture_);
322 return stream_has_echo_;
323 }
324
enable_delay_logging(bool enable)325 int EchoCancellationImpl::enable_delay_logging(bool enable) {
326 {
327 rtc::CritScope cs(crit_capture_);
328 delay_logging_enabled_ = enable;
329 }
330 return Configure();
331 }
332
is_delay_logging_enabled() const333 bool EchoCancellationImpl::is_delay_logging_enabled() const {
334 rtc::CritScope cs(crit_capture_);
335 return enabled_ && delay_logging_enabled_;
336 }
337
is_delay_agnostic_enabled() const338 bool EchoCancellationImpl::is_delay_agnostic_enabled() const {
339 rtc::CritScope cs(crit_capture_);
340 return delay_agnostic_enabled_;
341 }
342
GetExperimentsDescription()343 std::string EchoCancellationImpl::GetExperimentsDescription() {
344 rtc::CritScope cs(crit_capture_);
345 return refined_adaptive_filter_enabled_ ? "RefinedAdaptiveFilter;" : "";
346 }
347
is_refined_adaptive_filter_enabled() const348 bool EchoCancellationImpl::is_refined_adaptive_filter_enabled() const {
349 rtc::CritScope cs(crit_capture_);
350 return refined_adaptive_filter_enabled_;
351 }
352
is_extended_filter_enabled() const353 bool EchoCancellationImpl::is_extended_filter_enabled() const {
354 rtc::CritScope cs(crit_capture_);
355 return extended_filter_enabled_;
356 }
357
358 // TODO(bjornv): How should we handle the multi-channel case?
GetDelayMetrics(int * median,int * std)359 int EchoCancellationImpl::GetDelayMetrics(int* median, int* std) {
360 rtc::CritScope cs(crit_capture_);
361 float fraction_poor_delays = 0;
362 return GetDelayMetrics(median, std, &fraction_poor_delays);
363 }
364
GetDelayMetrics(int * median,int * std,float * fraction_poor_delays)365 int EchoCancellationImpl::GetDelayMetrics(int* median, int* std,
366 float* fraction_poor_delays) {
367 rtc::CritScope cs(crit_capture_);
368 if (median == NULL) {
369 return AudioProcessing::kNullPointerError;
370 }
371 if (std == NULL) {
372 return AudioProcessing::kNullPointerError;
373 }
374
375 if (!enabled_ || !delay_logging_enabled_) {
376 return AudioProcessing::kNotEnabledError;
377 }
378
379 const int err = WebRtcAec_GetDelayMetrics(cancellers_[0]->state(), median,
380 std, fraction_poor_delays);
381 if (err != AudioProcessing::kNoError) {
382 return MapError(err);
383 }
384
385 return AudioProcessing::kNoError;
386 }
387
aec_core() const388 struct AecCore* EchoCancellationImpl::aec_core() const {
389 rtc::CritScope cs(crit_capture_);
390 if (!enabled_) {
391 return NULL;
392 }
393 return WebRtcAec_aec_core(cancellers_[0]->state());
394 }
395
Initialize(int sample_rate_hz,size_t num_reverse_channels,size_t num_output_channels,size_t num_proc_channels)396 void EchoCancellationImpl::Initialize(int sample_rate_hz,
397 size_t num_reverse_channels,
398 size_t num_output_channels,
399 size_t num_proc_channels) {
400 rtc::CritScope cs_render(crit_render_);
401 rtc::CritScope cs_capture(crit_capture_);
402
403 stream_properties_.reset(
404 new StreamProperties(sample_rate_hz, num_reverse_channels,
405 num_output_channels, num_proc_channels));
406
407 if (!enabled_) {
408 return;
409 }
410
411 const size_t num_cancellers_required =
412 NumCancellersRequired(stream_properties_->num_output_channels,
413 stream_properties_->num_reverse_channels);
414 if (num_cancellers_required > cancellers_.size()) {
415 const size_t cancellers_old_size = cancellers_.size();
416 cancellers_.resize(num_cancellers_required);
417
418 for (size_t i = cancellers_old_size; i < cancellers_.size(); ++i) {
419 cancellers_[i].reset(new Canceller());
420 }
421 }
422
423 for (auto& canceller : cancellers_) {
424 canceller->Initialize(sample_rate_hz);
425 }
426
427 Configure();
428 }
429
GetSystemDelayInSamples() const430 int EchoCancellationImpl::GetSystemDelayInSamples() const {
431 rtc::CritScope cs(crit_capture_);
432 RTC_DCHECK(enabled_);
433 // Report the delay for the first AEC component.
434 return WebRtcAec_system_delay(
435 WebRtcAec_aec_core(cancellers_[0]->state()));
436 }
437
PackRenderAudioBuffer(const AudioBuffer * audio,size_t num_output_channels,size_t num_channels,std::vector<float> * packed_buffer)438 void EchoCancellationImpl::PackRenderAudioBuffer(
439 const AudioBuffer* audio,
440 size_t num_output_channels,
441 size_t num_channels,
442 std::vector<float>* packed_buffer) {
443 RTC_DCHECK_GE(160, audio->num_frames_per_band());
444 RTC_DCHECK_EQ(num_channels, audio->num_channels());
445
446 packed_buffer->clear();
447 // The ordering convention must be followed to pass the correct data.
448 for (size_t i = 0; i < num_output_channels; i++) {
449 for (size_t j = 0; j < audio->num_channels(); j++) {
450 // Buffer the samples in the render queue.
451 packed_buffer->insert(packed_buffer->end(),
452 audio->split_bands_const_f(j)[kBand0To8kHz],
453 (audio->split_bands_const_f(j)[kBand0To8kHz] +
454 audio->num_frames_per_band()));
455 }
456 }
457 }
458
SetExtraOptions(const webrtc::Config & config)459 void EchoCancellationImpl::SetExtraOptions(const webrtc::Config& config) {
460 {
461 rtc::CritScope cs(crit_capture_);
462 extended_filter_enabled_ = config.Get<ExtendedFilter>().enabled;
463 delay_agnostic_enabled_ = config.Get<DelayAgnostic>().enabled;
464 refined_adaptive_filter_enabled_ =
465 config.Get<RefinedAdaptiveFilter>().enabled;
466 }
467 Configure();
468 }
469
Configure()470 int EchoCancellationImpl::Configure() {
471 rtc::CritScope cs_render(crit_render_);
472 rtc::CritScope cs_capture(crit_capture_);
473 AecConfig config;
474 config.metricsMode = metrics_enabled_;
475 config.nlpMode = MapSetting(suppression_level_);
476 config.skewMode = drift_compensation_enabled_;
477 config.delay_logging = delay_logging_enabled_;
478
479 int error = AudioProcessing::kNoError;
480 for (auto& canceller : cancellers_) {
481 WebRtcAec_enable_extended_filter(WebRtcAec_aec_core(canceller->state()),
482 extended_filter_enabled_ ? 1 : 0);
483 WebRtcAec_enable_delay_agnostic(WebRtcAec_aec_core(canceller->state()),
484 delay_agnostic_enabled_ ? 1 : 0);
485 WebRtcAec_enable_refined_adaptive_filter(
486 WebRtcAec_aec_core(canceller->state()),
487 refined_adaptive_filter_enabled_);
488 const int handle_error = WebRtcAec_set_config(canceller->state(), config);
489 if (handle_error != AudioProcessing::kNoError) {
490 error = AudioProcessing::kNoError;
491 }
492 }
493 return error;
494 }
495
NumCancellersRequired(size_t num_output_channels,size_t num_reverse_channels)496 size_t EchoCancellationImpl::NumCancellersRequired(
497 size_t num_output_channels,
498 size_t num_reverse_channels) {
499 return num_output_channels * num_reverse_channels;
500 }
501
502 } // namespace webrtc
503