1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ 12 #define WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ 13 14 // MSVC++ requires this to be set before any other includes to get M_PI. 15 #define _USE_MATH_DEFINES 16 17 #include <math.h> 18 #include <stddef.h> // size_t 19 #include <stdio.h> // FILE 20 #include <vector> 21 22 #include "webrtc/base/arraysize.h" 23 #include "webrtc/base/platform_file.h" 24 #include "webrtc/common.h" 25 #include "webrtc/modules/audio_processing/beamformer/array_util.h" 26 #include "webrtc/typedefs.h" 27 28 struct AecCore; 29 30 namespace webrtc { 31 32 class AudioFrame; 33 34 template<typename T> 35 class Beamformer; 36 37 class StreamConfig; 38 class ProcessingConfig; 39 40 class EchoCancellation; 41 class EchoControlMobile; 42 class GainControl; 43 class HighPassFilter; 44 class LevelEstimator; 45 class NoiseSuppression; 46 class VoiceDetection; 47 48 // Use to enable the extended filter mode in the AEC, along with robustness 49 // measures around the reported system delays. It comes with a significant 50 // increase in AEC complexity, but is much more robust to unreliable reported 51 // delays. 52 // 53 // Detailed changes to the algorithm: 54 // - The filter length is changed from 48 to 128 ms. This comes with tuning of 55 // several parameters: i) filter adaptation stepsize and error threshold; 56 // ii) non-linear processing smoothing and overdrive. 57 // - Option to ignore the reported delays on platforms which we deem 58 // sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c. 59 // - Faster startup times by removing the excessive "startup phase" processing 60 // of reported delays. 61 // - Much more conservative adjustments to the far-end read pointer. We smooth 62 // the delay difference more heavily, and back off from the difference more. 63 // Adjustments force a readaptation of the filter, so they should be avoided 64 // except when really necessary. 65 struct ExtendedFilter { ExtendedFilterExtendedFilter66 ExtendedFilter() : enabled(false) {} ExtendedFilterExtendedFilter67 explicit ExtendedFilter(bool enabled) : enabled(enabled) {} 68 bool enabled; 69 }; 70 71 // Enables delay-agnostic echo cancellation. This feature relies on internally 72 // estimated delays between the process and reverse streams, thus not relying 73 // on reported system delays. This configuration only applies to 74 // EchoCancellation and not EchoControlMobile. It can be set in the constructor 75 // or using AudioProcessing::SetExtraOptions(). 76 struct DelayAgnostic { DelayAgnosticDelayAgnostic77 DelayAgnostic() : enabled(false) {} DelayAgnosticDelayAgnostic78 explicit DelayAgnostic(bool enabled) : enabled(enabled) {} 79 bool enabled; 80 }; 81 82 // Use to enable experimental gain control (AGC). At startup the experimental 83 // AGC moves the microphone volume up to |startup_min_volume| if the current 84 // microphone volume is set too low. The value is clamped to its operating range 85 // [12, 255]. Here, 255 maps to 100%. 86 // 87 // Must be provided through AudioProcessing::Create(Confg&). 88 #if defined(WEBRTC_CHROMIUM_BUILD) 89 static const int kAgcStartupMinVolume = 85; 90 #else 91 static const int kAgcStartupMinVolume = 0; 92 #endif // defined(WEBRTC_CHROMIUM_BUILD) 93 struct ExperimentalAgc { ExperimentalAgcExperimentalAgc94 ExperimentalAgc() : enabled(true), startup_min_volume(kAgcStartupMinVolume) {} ExperimentalAgcExperimentalAgc95 explicit ExperimentalAgc(bool enabled) 96 : enabled(enabled), startup_min_volume(kAgcStartupMinVolume) {} ExperimentalAgcExperimentalAgc97 ExperimentalAgc(bool enabled, int startup_min_volume) 98 : enabled(enabled), startup_min_volume(startup_min_volume) {} 99 bool enabled; 100 int startup_min_volume; 101 }; 102 103 // Use to enable experimental noise suppression. It can be set in the 104 // constructor or using AudioProcessing::SetExtraOptions(). 105 struct ExperimentalNs { ExperimentalNsExperimentalNs106 ExperimentalNs() : enabled(false) {} ExperimentalNsExperimentalNs107 explicit ExperimentalNs(bool enabled) : enabled(enabled) {} 108 bool enabled; 109 }; 110 111 // Use to enable beamforming. Must be provided through the constructor. It will 112 // have no impact if used with AudioProcessing::SetExtraOptions(). 113 struct Beamforming { BeamformingBeamforming114 Beamforming() 115 : enabled(false), 116 array_geometry(), 117 target_direction( 118 SphericalPointf(static_cast<float>(M_PI) / 2.f, 0.f, 1.f)) {} BeamformingBeamforming119 Beamforming(bool enabled, const std::vector<Point>& array_geometry) 120 : Beamforming(enabled, 121 array_geometry, 122 SphericalPointf(static_cast<float>(M_PI) / 2.f, 0.f, 1.f)) { 123 } BeamformingBeamforming124 Beamforming(bool enabled, 125 const std::vector<Point>& array_geometry, 126 SphericalPointf target_direction) 127 : enabled(enabled), 128 array_geometry(array_geometry), 129 target_direction(target_direction) {} 130 const bool enabled; 131 const std::vector<Point> array_geometry; 132 const SphericalPointf target_direction; 133 }; 134 135 // Use to enable intelligibility enhancer in audio processing. Must be provided 136 // though the constructor. It will have no impact if used with 137 // AudioProcessing::SetExtraOptions(). 138 // 139 // Note: If enabled and the reverse stream has more than one output channel, 140 // the reverse stream will become an upmixed mono signal. 141 struct Intelligibility { IntelligibilityIntelligibility142 Intelligibility() : enabled(false) {} IntelligibilityIntelligibility143 explicit Intelligibility(bool enabled) : enabled(enabled) {} 144 bool enabled; 145 }; 146 147 // The Audio Processing Module (APM) provides a collection of voice processing 148 // components designed for real-time communications software. 149 // 150 // APM operates on two audio streams on a frame-by-frame basis. Frames of the 151 // primary stream, on which all processing is applied, are passed to 152 // |ProcessStream()|. Frames of the reverse direction stream, which are used for 153 // analysis by some components, are passed to |AnalyzeReverseStream()|. On the 154 // client-side, this will typically be the near-end (capture) and far-end 155 // (render) streams, respectively. APM should be placed in the signal chain as 156 // close to the audio hardware abstraction layer (HAL) as possible. 157 // 158 // On the server-side, the reverse stream will normally not be used, with 159 // processing occurring on each incoming stream. 160 // 161 // Component interfaces follow a similar pattern and are accessed through 162 // corresponding getters in APM. All components are disabled at create-time, 163 // with default settings that are recommended for most situations. New settings 164 // can be applied without enabling a component. Enabling a component triggers 165 // memory allocation and initialization to allow it to start processing the 166 // streams. 167 // 168 // Thread safety is provided with the following assumptions to reduce locking 169 // overhead: 170 // 1. The stream getters and setters are called from the same thread as 171 // ProcessStream(). More precisely, stream functions are never called 172 // concurrently with ProcessStream(). 173 // 2. Parameter getters are never called concurrently with the corresponding 174 // setter. 175 // 176 // APM accepts only linear PCM audio data in chunks of 10 ms. The int16 177 // interfaces use interleaved data, while the float interfaces use deinterleaved 178 // data. 179 // 180 // Usage example, omitting error checking: 181 // AudioProcessing* apm = AudioProcessing::Create(0); 182 // 183 // apm->high_pass_filter()->Enable(true); 184 // 185 // apm->echo_cancellation()->enable_drift_compensation(false); 186 // apm->echo_cancellation()->Enable(true); 187 // 188 // apm->noise_reduction()->set_level(kHighSuppression); 189 // apm->noise_reduction()->Enable(true); 190 // 191 // apm->gain_control()->set_analog_level_limits(0, 255); 192 // apm->gain_control()->set_mode(kAdaptiveAnalog); 193 // apm->gain_control()->Enable(true); 194 // 195 // apm->voice_detection()->Enable(true); 196 // 197 // // Start a voice call... 198 // 199 // // ... Render frame arrives bound for the audio HAL ... 200 // apm->AnalyzeReverseStream(render_frame); 201 // 202 // // ... Capture frame arrives from the audio HAL ... 203 // // Call required set_stream_ functions. 204 // apm->set_stream_delay_ms(delay_ms); 205 // apm->gain_control()->set_stream_analog_level(analog_level); 206 // 207 // apm->ProcessStream(capture_frame); 208 // 209 // // Call required stream_ functions. 210 // analog_level = apm->gain_control()->stream_analog_level(); 211 // has_voice = apm->stream_has_voice(); 212 // 213 // // Repeate render and capture processing for the duration of the call... 214 // // Start a new call... 215 // apm->Initialize(); 216 // 217 // // Close the application... 218 // delete apm; 219 // 220 class AudioProcessing { 221 public: 222 // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone. 223 enum ChannelLayout { 224 kMono, 225 // Left, right. 226 kStereo, 227 // Mono, keyboard mic. 228 kMonoAndKeyboard, 229 // Left, right, keyboard mic. 230 kStereoAndKeyboard 231 }; 232 233 // Creates an APM instance. Use one instance for every primary audio stream 234 // requiring processing. On the client-side, this would typically be one 235 // instance for the near-end stream, and additional instances for each far-end 236 // stream which requires processing. On the server-side, this would typically 237 // be one instance for every incoming stream. 238 static AudioProcessing* Create(); 239 // Allows passing in an optional configuration at create-time. 240 static AudioProcessing* Create(const Config& config); 241 // Only for testing. 242 static AudioProcessing* Create(const Config& config, 243 Beamformer<float>* beamformer); ~AudioProcessing()244 virtual ~AudioProcessing() {} 245 246 // Initializes internal states, while retaining all user settings. This 247 // should be called before beginning to process a new audio stream. However, 248 // it is not necessary to call before processing the first stream after 249 // creation. 250 // 251 // It is also not necessary to call if the audio parameters (sample 252 // rate and number of channels) have changed. Passing updated parameters 253 // directly to |ProcessStream()| and |AnalyzeReverseStream()| is permissible. 254 // If the parameters are known at init-time though, they may be provided. 255 virtual int Initialize() = 0; 256 257 // The int16 interfaces require: 258 // - only |NativeRate|s be used 259 // - that the input, output and reverse rates must match 260 // - that |processing_config.output_stream()| matches 261 // |processing_config.input_stream()|. 262 // 263 // The float interfaces accept arbitrary rates and support differing input and 264 // output layouts, but the output must have either one channel or the same 265 // number of channels as the input. 266 virtual int Initialize(const ProcessingConfig& processing_config) = 0; 267 268 // Initialize with unpacked parameters. See Initialize() above for details. 269 // 270 // TODO(mgraczyk): Remove once clients are updated to use the new interface. 271 virtual int Initialize(int input_sample_rate_hz, 272 int output_sample_rate_hz, 273 int reverse_sample_rate_hz, 274 ChannelLayout input_layout, 275 ChannelLayout output_layout, 276 ChannelLayout reverse_layout) = 0; 277 278 // Pass down additional options which don't have explicit setters. This 279 // ensures the options are applied immediately. 280 virtual void SetExtraOptions(const Config& config) = 0; 281 282 // TODO(ajm): Only intended for internal use. Make private and friend the 283 // necessary classes? 284 virtual int proc_sample_rate_hz() const = 0; 285 virtual int proc_split_sample_rate_hz() const = 0; 286 virtual int num_input_channels() const = 0; 287 virtual int num_output_channels() const = 0; 288 virtual int num_reverse_channels() const = 0; 289 290 // Set to true when the output of AudioProcessing will be muted or in some 291 // other way not used. Ideally, the captured audio would still be processed, 292 // but some components may change behavior based on this information. 293 // Default false. 294 virtual void set_output_will_be_muted(bool muted) = 0; 295 296 // Processes a 10 ms |frame| of the primary audio stream. On the client-side, 297 // this is the near-end (or captured) audio. 298 // 299 // If needed for enabled functionality, any function with the set_stream_ tag 300 // must be called prior to processing the current frame. Any getter function 301 // with the stream_ tag which is needed should be called after processing. 302 // 303 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| 304 // members of |frame| must be valid. If changed from the previous call to this 305 // method, it will trigger an initialization. 306 virtual int ProcessStream(AudioFrame* frame) = 0; 307 308 // Accepts deinterleaved float audio with the range [-1, 1]. Each element 309 // of |src| points to a channel buffer, arranged according to 310 // |input_layout|. At output, the channels will be arranged according to 311 // |output_layout| at |output_sample_rate_hz| in |dest|. 312 // 313 // The output layout must have one channel or as many channels as the input. 314 // |src| and |dest| may use the same memory, if desired. 315 // 316 // TODO(mgraczyk): Remove once clients are updated to use the new interface. 317 virtual int ProcessStream(const float* const* src, 318 size_t samples_per_channel, 319 int input_sample_rate_hz, 320 ChannelLayout input_layout, 321 int output_sample_rate_hz, 322 ChannelLayout output_layout, 323 float* const* dest) = 0; 324 325 // Accepts deinterleaved float audio with the range [-1, 1]. Each element of 326 // |src| points to a channel buffer, arranged according to |input_stream|. At 327 // output, the channels will be arranged according to |output_stream| in 328 // |dest|. 329 // 330 // The output must have one channel or as many channels as the input. |src| 331 // and |dest| may use the same memory, if desired. 332 virtual int ProcessStream(const float* const* src, 333 const StreamConfig& input_config, 334 const StreamConfig& output_config, 335 float* const* dest) = 0; 336 337 // Analyzes a 10 ms |frame| of the reverse direction audio stream. The frame 338 // will not be modified. On the client-side, this is the far-end (or to be 339 // rendered) audio. 340 // 341 // It is only necessary to provide this if echo processing is enabled, as the 342 // reverse stream forms the echo reference signal. It is recommended, but not 343 // necessary, to provide if gain control is enabled. On the server-side this 344 // typically will not be used. If you're not sure what to pass in here, 345 // chances are you don't need to use it. 346 // 347 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| 348 // members of |frame| must be valid. |sample_rate_hz_| must correspond to 349 // |input_sample_rate_hz()| 350 // 351 // TODO(ajm): add const to input; requires an implementation fix. 352 // DEPRECATED: Use |ProcessReverseStream| instead. 353 // TODO(ekm): Remove once all users have updated to |ProcessReverseStream|. 354 virtual int AnalyzeReverseStream(AudioFrame* frame) = 0; 355 356 // Same as |AnalyzeReverseStream|, but may modify |frame| if intelligibility 357 // is enabled. 358 virtual int ProcessReverseStream(AudioFrame* frame) = 0; 359 360 // Accepts deinterleaved float audio with the range [-1, 1]. Each element 361 // of |data| points to a channel buffer, arranged according to |layout|. 362 // TODO(mgraczyk): Remove once clients are updated to use the new interface. 363 virtual int AnalyzeReverseStream(const float* const* data, 364 size_t samples_per_channel, 365 int rev_sample_rate_hz, 366 ChannelLayout layout) = 0; 367 368 // Accepts deinterleaved float audio with the range [-1, 1]. Each element of 369 // |data| points to a channel buffer, arranged according to |reverse_config|. 370 virtual int ProcessReverseStream(const float* const* src, 371 const StreamConfig& reverse_input_config, 372 const StreamConfig& reverse_output_config, 373 float* const* dest) = 0; 374 375 // This must be called if and only if echo processing is enabled. 376 // 377 // Sets the |delay| in ms between AnalyzeReverseStream() receiving a far-end 378 // frame and ProcessStream() receiving a near-end frame containing the 379 // corresponding echo. On the client-side this can be expressed as 380 // delay = (t_render - t_analyze) + (t_process - t_capture) 381 // where, 382 // - t_analyze is the time a frame is passed to AnalyzeReverseStream() and 383 // t_render is the time the first sample of the same frame is rendered by 384 // the audio hardware. 385 // - t_capture is the time the first sample of a frame is captured by the 386 // audio hardware and t_pull is the time the same frame is passed to 387 // ProcessStream(). 388 virtual int set_stream_delay_ms(int delay) = 0; 389 virtual int stream_delay_ms() const = 0; 390 virtual bool was_stream_delay_set() const = 0; 391 392 // Call to signal that a key press occurred (true) or did not occur (false) 393 // with this chunk of audio. 394 virtual void set_stream_key_pressed(bool key_pressed) = 0; 395 396 // Sets a delay |offset| in ms to add to the values passed in through 397 // set_stream_delay_ms(). May be positive or negative. 398 // 399 // Note that this could cause an otherwise valid value passed to 400 // set_stream_delay_ms() to return an error. 401 virtual void set_delay_offset_ms(int offset) = 0; 402 virtual int delay_offset_ms() const = 0; 403 404 // Starts recording debugging information to a file specified by |filename|, 405 // a NULL-terminated string. If there is an ongoing recording, the old file 406 // will be closed, and recording will continue in the newly specified file. 407 // An already existing file will be overwritten without warning. 408 static const size_t kMaxFilenameSize = 1024; 409 virtual int StartDebugRecording(const char filename[kMaxFilenameSize]) = 0; 410 411 // Same as above but uses an existing file handle. Takes ownership 412 // of |handle| and closes it at StopDebugRecording(). 413 virtual int StartDebugRecording(FILE* handle) = 0; 414 415 // Same as above but uses an existing PlatformFile handle. Takes ownership 416 // of |handle| and closes it at StopDebugRecording(). 417 // TODO(xians): Make this interface pure virtual. StartDebugRecordingForPlatformFile(rtc::PlatformFile handle)418 virtual int StartDebugRecordingForPlatformFile(rtc::PlatformFile handle) { 419 return -1; 420 } 421 422 // Stops recording debugging information, and closes the file. Recording 423 // cannot be resumed in the same file (without overwriting it). 424 virtual int StopDebugRecording() = 0; 425 426 // Use to send UMA histograms at end of a call. Note that all histogram 427 // specific member variables are reset. 428 virtual void UpdateHistogramsOnCallEnd() = 0; 429 430 // These provide access to the component interfaces and should never return 431 // NULL. The pointers will be valid for the lifetime of the APM instance. 432 // The memory for these objects is entirely managed internally. 433 virtual EchoCancellation* echo_cancellation() const = 0; 434 virtual EchoControlMobile* echo_control_mobile() const = 0; 435 virtual GainControl* gain_control() const = 0; 436 virtual HighPassFilter* high_pass_filter() const = 0; 437 virtual LevelEstimator* level_estimator() const = 0; 438 virtual NoiseSuppression* noise_suppression() const = 0; 439 virtual VoiceDetection* voice_detection() const = 0; 440 441 struct Statistic { 442 int instant; // Instantaneous value. 443 int average; // Long-term average. 444 int maximum; // Long-term maximum. 445 int minimum; // Long-term minimum. 446 }; 447 448 enum Error { 449 // Fatal errors. 450 kNoError = 0, 451 kUnspecifiedError = -1, 452 kCreationFailedError = -2, 453 kUnsupportedComponentError = -3, 454 kUnsupportedFunctionError = -4, 455 kNullPointerError = -5, 456 kBadParameterError = -6, 457 kBadSampleRateError = -7, 458 kBadDataLengthError = -8, 459 kBadNumberChannelsError = -9, 460 kFileError = -10, 461 kStreamParameterNotSetError = -11, 462 kNotEnabledError = -12, 463 464 // Warnings are non-fatal. 465 // This results when a set_stream_ parameter is out of range. Processing 466 // will continue, but the parameter may have been truncated. 467 kBadStreamParameterWarning = -13 468 }; 469 470 enum NativeRate { 471 kSampleRate8kHz = 8000, 472 kSampleRate16kHz = 16000, 473 kSampleRate32kHz = 32000, 474 kSampleRate48kHz = 48000 475 }; 476 477 static const int kNativeSampleRatesHz[]; 478 static const size_t kNumNativeSampleRates; 479 static const int kMaxNativeSampleRateHz; 480 static const int kMaxAECMSampleRateHz; 481 482 static const int kChunkSizeMs = 10; 483 }; 484 485 class StreamConfig { 486 public: 487 // sample_rate_hz: The sampling rate of the stream. 488 // 489 // num_channels: The number of audio channels in the stream, excluding the 490 // keyboard channel if it is present. When passing a 491 // StreamConfig with an array of arrays T*[N], 492 // 493 // N == {num_channels + 1 if has_keyboard 494 // {num_channels if !has_keyboard 495 // 496 // has_keyboard: True if the stream has a keyboard channel. When has_keyboard 497 // is true, the last channel in any corresponding list of 498 // channels is the keyboard channel. 499 StreamConfig(int sample_rate_hz = 0, 500 int num_channels = 0, 501 bool has_keyboard = false) sample_rate_hz_(sample_rate_hz)502 : sample_rate_hz_(sample_rate_hz), 503 num_channels_(num_channels), 504 has_keyboard_(has_keyboard), 505 num_frames_(calculate_frames(sample_rate_hz)) {} 506 set_sample_rate_hz(int value)507 void set_sample_rate_hz(int value) { 508 sample_rate_hz_ = value; 509 num_frames_ = calculate_frames(value); 510 } set_num_channels(int value)511 void set_num_channels(int value) { num_channels_ = value; } set_has_keyboard(bool value)512 void set_has_keyboard(bool value) { has_keyboard_ = value; } 513 sample_rate_hz()514 int sample_rate_hz() const { return sample_rate_hz_; } 515 516 // The number of channels in the stream, not including the keyboard channel if 517 // present. num_channels()518 int num_channels() const { return num_channels_; } 519 has_keyboard()520 bool has_keyboard() const { return has_keyboard_; } num_frames()521 size_t num_frames() const { return num_frames_; } num_samples()522 size_t num_samples() const { return num_channels_ * num_frames_; } 523 524 bool operator==(const StreamConfig& other) const { 525 return sample_rate_hz_ == other.sample_rate_hz_ && 526 num_channels_ == other.num_channels_ && 527 has_keyboard_ == other.has_keyboard_; 528 } 529 530 bool operator!=(const StreamConfig& other) const { return !(*this == other); } 531 532 private: calculate_frames(int sample_rate_hz)533 static size_t calculate_frames(int sample_rate_hz) { 534 return static_cast<size_t>( 535 AudioProcessing::kChunkSizeMs * sample_rate_hz / 1000); 536 } 537 538 int sample_rate_hz_; 539 int num_channels_; 540 bool has_keyboard_; 541 size_t num_frames_; 542 }; 543 544 class ProcessingConfig { 545 public: 546 enum StreamName { 547 kInputStream, 548 kOutputStream, 549 kReverseInputStream, 550 kReverseOutputStream, 551 kNumStreamNames, 552 }; 553 input_stream()554 const StreamConfig& input_stream() const { 555 return streams[StreamName::kInputStream]; 556 } output_stream()557 const StreamConfig& output_stream() const { 558 return streams[StreamName::kOutputStream]; 559 } reverse_input_stream()560 const StreamConfig& reverse_input_stream() const { 561 return streams[StreamName::kReverseInputStream]; 562 } reverse_output_stream()563 const StreamConfig& reverse_output_stream() const { 564 return streams[StreamName::kReverseOutputStream]; 565 } 566 input_stream()567 StreamConfig& input_stream() { return streams[StreamName::kInputStream]; } output_stream()568 StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; } reverse_input_stream()569 StreamConfig& reverse_input_stream() { 570 return streams[StreamName::kReverseInputStream]; 571 } reverse_output_stream()572 StreamConfig& reverse_output_stream() { 573 return streams[StreamName::kReverseOutputStream]; 574 } 575 576 bool operator==(const ProcessingConfig& other) const { 577 for (int i = 0; i < StreamName::kNumStreamNames; ++i) { 578 if (this->streams[i] != other.streams[i]) { 579 return false; 580 } 581 } 582 return true; 583 } 584 585 bool operator!=(const ProcessingConfig& other) const { 586 return !(*this == other); 587 } 588 589 StreamConfig streams[StreamName::kNumStreamNames]; 590 }; 591 592 // The acoustic echo cancellation (AEC) component provides better performance 593 // than AECM but also requires more processing power and is dependent on delay 594 // stability and reporting accuracy. As such it is well-suited and recommended 595 // for PC and IP phone applications. 596 // 597 // Not recommended to be enabled on the server-side. 598 class EchoCancellation { 599 public: 600 // EchoCancellation and EchoControlMobile may not be enabled simultaneously. 601 // Enabling one will disable the other. 602 virtual int Enable(bool enable) = 0; 603 virtual bool is_enabled() const = 0; 604 605 // Differences in clock speed on the primary and reverse streams can impact 606 // the AEC performance. On the client-side, this could be seen when different 607 // render and capture devices are used, particularly with webcams. 608 // 609 // This enables a compensation mechanism, and requires that 610 // set_stream_drift_samples() be called. 611 virtual int enable_drift_compensation(bool enable) = 0; 612 virtual bool is_drift_compensation_enabled() const = 0; 613 614 // Sets the difference between the number of samples rendered and captured by 615 // the audio devices since the last call to |ProcessStream()|. Must be called 616 // if drift compensation is enabled, prior to |ProcessStream()|. 617 virtual void set_stream_drift_samples(int drift) = 0; 618 virtual int stream_drift_samples() const = 0; 619 620 enum SuppressionLevel { 621 kLowSuppression, 622 kModerateSuppression, 623 kHighSuppression 624 }; 625 626 // Sets the aggressiveness of the suppressor. A higher level trades off 627 // double-talk performance for increased echo suppression. 628 virtual int set_suppression_level(SuppressionLevel level) = 0; 629 virtual SuppressionLevel suppression_level() const = 0; 630 631 // Returns false if the current frame almost certainly contains no echo 632 // and true if it _might_ contain echo. 633 virtual bool stream_has_echo() const = 0; 634 635 // Enables the computation of various echo metrics. These are obtained 636 // through |GetMetrics()|. 637 virtual int enable_metrics(bool enable) = 0; 638 virtual bool are_metrics_enabled() const = 0; 639 640 // Each statistic is reported in dB. 641 // P_far: Far-end (render) signal power. 642 // P_echo: Near-end (capture) echo signal power. 643 // P_out: Signal power at the output of the AEC. 644 // P_a: Internal signal power at the point before the AEC's non-linear 645 // processor. 646 struct Metrics { 647 // RERL = ERL + ERLE 648 AudioProcessing::Statistic residual_echo_return_loss; 649 650 // ERL = 10log_10(P_far / P_echo) 651 AudioProcessing::Statistic echo_return_loss; 652 653 // ERLE = 10log_10(P_echo / P_out) 654 AudioProcessing::Statistic echo_return_loss_enhancement; 655 656 // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a) 657 AudioProcessing::Statistic a_nlp; 658 }; 659 660 // TODO(ajm): discuss the metrics update period. 661 virtual int GetMetrics(Metrics* metrics) = 0; 662 663 // Enables computation and logging of delay values. Statistics are obtained 664 // through |GetDelayMetrics()|. 665 virtual int enable_delay_logging(bool enable) = 0; 666 virtual bool is_delay_logging_enabled() const = 0; 667 668 // The delay metrics consists of the delay |median| and the delay standard 669 // deviation |std|. It also consists of the fraction of delay estimates 670 // |fraction_poor_delays| that can make the echo cancellation perform poorly. 671 // The values are aggregated until the first call to |GetDelayMetrics()| and 672 // afterwards aggregated and updated every second. 673 // Note that if there are several clients pulling metrics from 674 // |GetDelayMetrics()| during a session the first call from any of them will 675 // change to one second aggregation window for all. 676 // TODO(bjornv): Deprecated, remove. 677 virtual int GetDelayMetrics(int* median, int* std) = 0; 678 virtual int GetDelayMetrics(int* median, int* std, 679 float* fraction_poor_delays) = 0; 680 681 // Returns a pointer to the low level AEC component. In case of multiple 682 // channels, the pointer to the first one is returned. A NULL pointer is 683 // returned when the AEC component is disabled or has not been initialized 684 // successfully. 685 virtual struct AecCore* aec_core() const = 0; 686 687 protected: ~EchoCancellation()688 virtual ~EchoCancellation() {} 689 }; 690 691 // The acoustic echo control for mobile (AECM) component is a low complexity 692 // robust option intended for use on mobile devices. 693 // 694 // Not recommended to be enabled on the server-side. 695 class EchoControlMobile { 696 public: 697 // EchoCancellation and EchoControlMobile may not be enabled simultaneously. 698 // Enabling one will disable the other. 699 virtual int Enable(bool enable) = 0; 700 virtual bool is_enabled() const = 0; 701 702 // Recommended settings for particular audio routes. In general, the louder 703 // the echo is expected to be, the higher this value should be set. The 704 // preferred setting may vary from device to device. 705 enum RoutingMode { 706 kQuietEarpieceOrHeadset, 707 kEarpiece, 708 kLoudEarpiece, 709 kSpeakerphone, 710 kLoudSpeakerphone 711 }; 712 713 // Sets echo control appropriate for the audio routing |mode| on the device. 714 // It can and should be updated during a call if the audio routing changes. 715 virtual int set_routing_mode(RoutingMode mode) = 0; 716 virtual RoutingMode routing_mode() const = 0; 717 718 // Comfort noise replaces suppressed background noise to maintain a 719 // consistent signal level. 720 virtual int enable_comfort_noise(bool enable) = 0; 721 virtual bool is_comfort_noise_enabled() const = 0; 722 723 // A typical use case is to initialize the component with an echo path from a 724 // previous call. The echo path is retrieved using |GetEchoPath()|, typically 725 // at the end of a call. The data can then be stored for later use as an 726 // initializer before the next call, using |SetEchoPath()|. 727 // 728 // Controlling the echo path this way requires the data |size_bytes| to match 729 // the internal echo path size. This size can be acquired using 730 // |echo_path_size_bytes()|. |SetEchoPath()| causes an entire reset, worth 731 // noting if it is to be called during an ongoing call. 732 // 733 // It is possible that version incompatibilities may result in a stored echo 734 // path of the incorrect size. In this case, the stored path should be 735 // discarded. 736 virtual int SetEchoPath(const void* echo_path, size_t size_bytes) = 0; 737 virtual int GetEchoPath(void* echo_path, size_t size_bytes) const = 0; 738 739 // The returned path size is guaranteed not to change for the lifetime of 740 // the application. 741 static size_t echo_path_size_bytes(); 742 743 protected: ~EchoControlMobile()744 virtual ~EchoControlMobile() {} 745 }; 746 747 // The automatic gain control (AGC) component brings the signal to an 748 // appropriate range. This is done by applying a digital gain directly and, in 749 // the analog mode, prescribing an analog gain to be applied at the audio HAL. 750 // 751 // Recommended to be enabled on the client-side. 752 class GainControl { 753 public: 754 virtual int Enable(bool enable) = 0; 755 virtual bool is_enabled() const = 0; 756 757 // When an analog mode is set, this must be called prior to |ProcessStream()| 758 // to pass the current analog level from the audio HAL. Must be within the 759 // range provided to |set_analog_level_limits()|. 760 virtual int set_stream_analog_level(int level) = 0; 761 762 // When an analog mode is set, this should be called after |ProcessStream()| 763 // to obtain the recommended new analog level for the audio HAL. It is the 764 // users responsibility to apply this level. 765 virtual int stream_analog_level() = 0; 766 767 enum Mode { 768 // Adaptive mode intended for use if an analog volume control is available 769 // on the capture device. It will require the user to provide coupling 770 // between the OS mixer controls and AGC through the |stream_analog_level()| 771 // functions. 772 // 773 // It consists of an analog gain prescription for the audio device and a 774 // digital compression stage. 775 kAdaptiveAnalog, 776 777 // Adaptive mode intended for situations in which an analog volume control 778 // is unavailable. It operates in a similar fashion to the adaptive analog 779 // mode, but with scaling instead applied in the digital domain. As with 780 // the analog mode, it additionally uses a digital compression stage. 781 kAdaptiveDigital, 782 783 // Fixed mode which enables only the digital compression stage also used by 784 // the two adaptive modes. 785 // 786 // It is distinguished from the adaptive modes by considering only a 787 // short time-window of the input signal. It applies a fixed gain through 788 // most of the input level range, and compresses (gradually reduces gain 789 // with increasing level) the input signal at higher levels. This mode is 790 // preferred on embedded devices where the capture signal level is 791 // predictable, so that a known gain can be applied. 792 kFixedDigital 793 }; 794 795 virtual int set_mode(Mode mode) = 0; 796 virtual Mode mode() const = 0; 797 798 // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels 799 // from digital full-scale). The convention is to use positive values. For 800 // instance, passing in a value of 3 corresponds to -3 dBFs, or a target 801 // level 3 dB below full-scale. Limited to [0, 31]. 802 // 803 // TODO(ajm): use a negative value here instead, if/when VoE will similarly 804 // update its interface. 805 virtual int set_target_level_dbfs(int level) = 0; 806 virtual int target_level_dbfs() const = 0; 807 808 // Sets the maximum |gain| the digital compression stage may apply, in dB. A 809 // higher number corresponds to greater compression, while a value of 0 will 810 // leave the signal uncompressed. Limited to [0, 90]. 811 virtual int set_compression_gain_db(int gain) = 0; 812 virtual int compression_gain_db() const = 0; 813 814 // When enabled, the compression stage will hard limit the signal to the 815 // target level. Otherwise, the signal will be compressed but not limited 816 // above the target level. 817 virtual int enable_limiter(bool enable) = 0; 818 virtual bool is_limiter_enabled() const = 0; 819 820 // Sets the |minimum| and |maximum| analog levels of the audio capture device. 821 // Must be set if and only if an analog mode is used. Limited to [0, 65535]. 822 virtual int set_analog_level_limits(int minimum, 823 int maximum) = 0; 824 virtual int analog_level_minimum() const = 0; 825 virtual int analog_level_maximum() const = 0; 826 827 // Returns true if the AGC has detected a saturation event (period where the 828 // signal reaches digital full-scale) in the current frame and the analog 829 // level cannot be reduced. 830 // 831 // This could be used as an indicator to reduce or disable analog mic gain at 832 // the audio HAL. 833 virtual bool stream_is_saturated() const = 0; 834 835 protected: ~GainControl()836 virtual ~GainControl() {} 837 }; 838 839 // A filtering component which removes DC offset and low-frequency noise. 840 // Recommended to be enabled on the client-side. 841 class HighPassFilter { 842 public: 843 virtual int Enable(bool enable) = 0; 844 virtual bool is_enabled() const = 0; 845 846 protected: ~HighPassFilter()847 virtual ~HighPassFilter() {} 848 }; 849 850 // An estimation component used to retrieve level metrics. 851 class LevelEstimator { 852 public: 853 virtual int Enable(bool enable) = 0; 854 virtual bool is_enabled() const = 0; 855 856 // Returns the root mean square (RMS) level in dBFs (decibels from digital 857 // full-scale), or alternately dBov. It is computed over all primary stream 858 // frames since the last call to RMS(). The returned value is positive but 859 // should be interpreted as negative. It is constrained to [0, 127]. 860 // 861 // The computation follows: https://tools.ietf.org/html/rfc6465 862 // with the intent that it can provide the RTP audio level indication. 863 // 864 // Frames passed to ProcessStream() with an |_energy| of zero are considered 865 // to have been muted. The RMS of the frame will be interpreted as -127. 866 virtual int RMS() = 0; 867 868 protected: ~LevelEstimator()869 virtual ~LevelEstimator() {} 870 }; 871 872 // The noise suppression (NS) component attempts to remove noise while 873 // retaining speech. Recommended to be enabled on the client-side. 874 // 875 // Recommended to be enabled on the client-side. 876 class NoiseSuppression { 877 public: 878 virtual int Enable(bool enable) = 0; 879 virtual bool is_enabled() const = 0; 880 881 // Determines the aggressiveness of the suppression. Increasing the level 882 // will reduce the noise level at the expense of a higher speech distortion. 883 enum Level { 884 kLow, 885 kModerate, 886 kHigh, 887 kVeryHigh 888 }; 889 890 virtual int set_level(Level level) = 0; 891 virtual Level level() const = 0; 892 893 // Returns the internally computed prior speech probability of current frame 894 // averaged over output channels. This is not supported in fixed point, for 895 // which |kUnsupportedFunctionError| is returned. 896 virtual float speech_probability() const = 0; 897 898 protected: ~NoiseSuppression()899 virtual ~NoiseSuppression() {} 900 }; 901 902 // The voice activity detection (VAD) component analyzes the stream to 903 // determine if voice is present. A facility is also provided to pass in an 904 // external VAD decision. 905 // 906 // In addition to |stream_has_voice()| the VAD decision is provided through the 907 // |AudioFrame| passed to |ProcessStream()|. The |vad_activity_| member will be 908 // modified to reflect the current decision. 909 class VoiceDetection { 910 public: 911 virtual int Enable(bool enable) = 0; 912 virtual bool is_enabled() const = 0; 913 914 // Returns true if voice is detected in the current frame. Should be called 915 // after |ProcessStream()|. 916 virtual bool stream_has_voice() const = 0; 917 918 // Some of the APM functionality requires a VAD decision. In the case that 919 // a decision is externally available for the current frame, it can be passed 920 // in here, before |ProcessStream()| is called. 921 // 922 // VoiceDetection does _not_ need to be enabled to use this. If it happens to 923 // be enabled, detection will be skipped for any frame in which an external 924 // VAD decision is provided. 925 virtual int set_stream_has_voice(bool has_voice) = 0; 926 927 // Specifies the likelihood that a frame will be declared to contain voice. 928 // A higher value makes it more likely that speech will not be clipped, at 929 // the expense of more noise being detected as voice. 930 enum Likelihood { 931 kVeryLowLikelihood, 932 kLowLikelihood, 933 kModerateLikelihood, 934 kHighLikelihood 935 }; 936 937 virtual int set_likelihood(Likelihood likelihood) = 0; 938 virtual Likelihood likelihood() const = 0; 939 940 // Sets the |size| of the frames in ms on which the VAD will operate. Larger 941 // frames will improve detection accuracy, but reduce the frequency of 942 // updates. 943 // 944 // This does not impact the size of frames passed to |ProcessStream()|. 945 virtual int set_frame_size_ms(int size) = 0; 946 virtual int frame_size_ms() const = 0; 947 948 protected: ~VoiceDetection()949 virtual ~VoiceDetection() {} 950 }; 951 } // namespace webrtc 952 953 #endif // WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ 954