1/* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#import <AVFoundation/AVFoundation.h> 12#import <Foundation/Foundation.h> 13 14#include "audio_device_ios.h" 15 16#include <cmath> 17 18#include "api/array_view.h" 19#include "helpers.h" 20#include "modules/audio_device/fine_audio_buffer.h" 21#include "rtc_base/atomic_ops.h" 22#include "rtc_base/checks.h" 23#include "rtc_base/logging.h" 24#include "rtc_base/thread.h" 25#include "rtc_base/thread_annotations.h" 26#include "rtc_base/time_utils.h" 27#include "system_wrappers/include/field_trial.h" 28#include "system_wrappers/include/metrics.h" 29 30#import "base/RTCLogging.h" 31#import "components/audio/RTCAudioSession+Private.h" 32#import "components/audio/RTCAudioSession.h" 33#import "components/audio/RTCAudioSessionConfiguration.h" 34#import "components/audio/RTCNativeAudioSessionDelegateAdapter.h" 35 36namespace webrtc { 37namespace ios_adm { 38 39#define LOGI() RTC_LOG(LS_INFO) << "AudioDeviceIOS::" 40 41#define LOG_AND_RETURN_IF_ERROR(error, message) \ 42 do { \ 43 OSStatus err = error; \ 44 if (err) { \ 45 RTC_LOG(LS_ERROR) << message << ": " << err; \ 46 return false; \ 47 } \ 48 } while (0) 49 50#define LOG_IF_ERROR(error, message) \ 51 do { \ 52 OSStatus err = error; \ 53 if (err) { \ 54 RTC_LOG(LS_ERROR) << message << ": " << err; \ 55 } \ 56 } while (0) 57 58// Hardcoded delay estimates based on real measurements. 59// TODO(henrika): these value is not used in combination with built-in AEC. 60// Can most likely be removed. 61const UInt16 kFixedPlayoutDelayEstimate = 30; 62const UInt16 kFixedRecordDelayEstimate = 30; 63 64enum AudioDeviceMessageType : uint32_t { 65 kMessageTypeInterruptionBegin, 66 kMessageTypeInterruptionEnd, 67 kMessageTypeValidRouteChange, 68 kMessageTypeCanPlayOrRecordChange, 69 kMessageTypePlayoutGlitchDetected, 70 kMessageOutputVolumeChange, 71}; 72 73using ios::CheckAndLogError; 74 75#if !defined(NDEBUG) 76// Returns true when the code runs on a device simulator. 77static bool DeviceIsSimulator() { 78 return ios::GetDeviceName() == "x86_64"; 79} 80 81// Helper method that logs essential device information strings. 82static void LogDeviceInfo() { 83 RTC_LOG(LS_INFO) << "LogDeviceInfo"; 84 @autoreleasepool { 85 RTC_LOG(LS_INFO) << " system name: " << ios::GetSystemName(); 86 RTC_LOG(LS_INFO) << " system version: " << ios::GetSystemVersionAsString(); 87 RTC_LOG(LS_INFO) << " device type: " << ios::GetDeviceType(); 88 RTC_LOG(LS_INFO) << " device name: " << ios::GetDeviceName(); 89 RTC_LOG(LS_INFO) << " process name: " << ios::GetProcessName(); 90 RTC_LOG(LS_INFO) << " process ID: " << ios::GetProcessID(); 91 RTC_LOG(LS_INFO) << " OS version: " << ios::GetOSVersionString(); 92 RTC_LOG(LS_INFO) << " processing cores: " << ios::GetProcessorCount(); 93 RTC_LOG(LS_INFO) << " low power mode: " << ios::GetLowPowerModeEnabled(); 94#if TARGET_IPHONE_SIMULATOR 95 RTC_LOG(LS_INFO) << " TARGET_IPHONE_SIMULATOR is defined"; 96#endif 97 RTC_LOG(LS_INFO) << " DeviceIsSimulator: " << DeviceIsSimulator(); 98 } 99} 100#endif // !defined(NDEBUG) 101 102AudioDeviceIOS::AudioDeviceIOS(bool bypass_voice_processing) 103 : bypass_voice_processing_(bypass_voice_processing), 104 audio_device_buffer_(nullptr), 105 audio_unit_(nullptr), 106 recording_(0), 107 playing_(0), 108 initialized_(false), 109 audio_is_initialized_(false), 110 is_interrupted_(false), 111 has_configured_session_(false), 112 num_detected_playout_glitches_(0), 113 last_playout_time_(0), 114 num_playout_callbacks_(0), 115 last_output_volume_change_time_(0) { 116 LOGI() << "ctor" << ios::GetCurrentThreadDescription() 117 << ",bypass_voice_processing=" << bypass_voice_processing_; 118 io_thread_checker_.Detach(); 119 thread_checker_.Detach(); 120 thread_ = rtc::Thread::Current(); 121 122 audio_session_observer_ = [[RTCNativeAudioSessionDelegateAdapter alloc] initWithObserver:this]; 123} 124 125AudioDeviceIOS::~AudioDeviceIOS() { 126 RTC_DCHECK(thread_checker_.IsCurrent()); 127 LOGI() << "~dtor" << ios::GetCurrentThreadDescription(); 128 thread_->Clear(this); 129 Terminate(); 130 audio_session_observer_ = nil; 131} 132 133void AudioDeviceIOS::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { 134 LOGI() << "AttachAudioBuffer"; 135 RTC_DCHECK(audioBuffer); 136 RTC_DCHECK(thread_checker_.IsCurrent()); 137 audio_device_buffer_ = audioBuffer; 138} 139 140AudioDeviceGeneric::InitStatus AudioDeviceIOS::Init() { 141 LOGI() << "Init"; 142 io_thread_checker_.Detach(); 143 thread_checker_.Detach(); 144 145 RTC_DCHECK_RUN_ON(&thread_checker_); 146 if (initialized_) { 147 return InitStatus::OK; 148 } 149#if !defined(NDEBUG) 150 LogDeviceInfo(); 151#endif 152 // Store the preferred sample rate and preferred number of channels already 153 // here. They have not been set and confirmed yet since configureForWebRTC 154 // is not called until audio is about to start. However, it makes sense to 155 // store the parameters now and then verify at a later stage. 156 RTC_OBJC_TYPE(RTCAudioSessionConfiguration)* config = 157 [RTC_OBJC_TYPE(RTCAudioSessionConfiguration) webRTCConfiguration]; 158 playout_parameters_.reset(config.sampleRate, config.outputNumberOfChannels); 159 record_parameters_.reset(config.sampleRate, config.inputNumberOfChannels); 160 // Ensure that the audio device buffer (ADB) knows about the internal audio 161 // parameters. Note that, even if we are unable to get a mono audio session, 162 // we will always tell the I/O audio unit to do a channel format conversion 163 // to guarantee mono on the "input side" of the audio unit. 164 UpdateAudioDeviceBuffer(); 165 initialized_ = true; 166 return InitStatus::OK; 167} 168 169int32_t AudioDeviceIOS::Terminate() { 170 LOGI() << "Terminate"; 171 RTC_DCHECK_RUN_ON(&thread_checker_); 172 if (!initialized_) { 173 return 0; 174 } 175 StopPlayout(); 176 StopRecording(); 177 initialized_ = false; 178 return 0; 179} 180 181bool AudioDeviceIOS::Initialized() const { 182 RTC_DCHECK_RUN_ON(&thread_checker_); 183 return initialized_; 184} 185 186int32_t AudioDeviceIOS::InitPlayout() { 187 LOGI() << "InitPlayout"; 188 RTC_DCHECK_RUN_ON(&thread_checker_); 189 RTC_DCHECK(initialized_); 190 RTC_DCHECK(!audio_is_initialized_); 191 RTC_DCHECK(!playing_); 192 if (!audio_is_initialized_) { 193 if (!InitPlayOrRecord()) { 194 RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitPlayout!"; 195 return -1; 196 } 197 } 198 audio_is_initialized_ = true; 199 return 0; 200} 201 202bool AudioDeviceIOS::PlayoutIsInitialized() const { 203 RTC_DCHECK_RUN_ON(&thread_checker_); 204 return audio_is_initialized_; 205} 206 207bool AudioDeviceIOS::RecordingIsInitialized() const { 208 RTC_DCHECK_RUN_ON(&thread_checker_); 209 return audio_is_initialized_; 210} 211 212int32_t AudioDeviceIOS::InitRecording() { 213 LOGI() << "InitRecording"; 214 RTC_DCHECK_RUN_ON(&thread_checker_); 215 RTC_DCHECK(initialized_); 216 RTC_DCHECK(!audio_is_initialized_); 217 RTC_DCHECK(!recording_); 218 if (!audio_is_initialized_) { 219 if (!InitPlayOrRecord()) { 220 RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitRecording!"; 221 return -1; 222 } 223 } 224 audio_is_initialized_ = true; 225 return 0; 226} 227 228int32_t AudioDeviceIOS::StartPlayout() { 229 LOGI() << "StartPlayout"; 230 RTC_DCHECK_RUN_ON(&thread_checker_); 231 RTC_DCHECK(audio_is_initialized_); 232 RTC_DCHECK(!playing_); 233 RTC_DCHECK(audio_unit_); 234 if (fine_audio_buffer_) { 235 fine_audio_buffer_->ResetPlayout(); 236 } 237 if (!recording_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 238 if (!audio_unit_->Start()) { 239 RTCLogError(@"StartPlayout failed to start audio unit."); 240 return -1; 241 } 242 RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; 243 } 244 rtc::AtomicOps::ReleaseStore(&playing_, 1); 245 num_playout_callbacks_ = 0; 246 num_detected_playout_glitches_ = 0; 247 return 0; 248} 249 250int32_t AudioDeviceIOS::StopPlayout() { 251 LOGI() << "StopPlayout"; 252 RTC_DCHECK_RUN_ON(&thread_checker_); 253 if (!audio_is_initialized_ || !playing_) { 254 return 0; 255 } 256 if (!recording_) { 257 ShutdownPlayOrRecord(); 258 audio_is_initialized_ = false; 259 } 260 rtc::AtomicOps::ReleaseStore(&playing_, 0); 261 262 // Derive average number of calls to OnGetPlayoutData() between detected 263 // audio glitches and add the result to a histogram. 264 int average_number_of_playout_callbacks_between_glitches = 100000; 265 RTC_DCHECK_GE(num_playout_callbacks_, num_detected_playout_glitches_); 266 if (num_detected_playout_glitches_ > 0) { 267 average_number_of_playout_callbacks_between_glitches = 268 num_playout_callbacks_ / num_detected_playout_glitches_; 269 } 270 RTC_HISTOGRAM_COUNTS_100000("WebRTC.Audio.AveragePlayoutCallbacksBetweenGlitches", 271 average_number_of_playout_callbacks_between_glitches); 272 RTCLog(@"Average number of playout callbacks between glitches: %d", 273 average_number_of_playout_callbacks_between_glitches); 274 return 0; 275} 276 277bool AudioDeviceIOS::Playing() const { 278 return playing_; 279} 280 281int32_t AudioDeviceIOS::StartRecording() { 282 LOGI() << "StartRecording"; 283 RTC_DCHECK_RUN_ON(&thread_checker_); 284 RTC_DCHECK(audio_is_initialized_); 285 RTC_DCHECK(!recording_); 286 RTC_DCHECK(audio_unit_); 287 if (fine_audio_buffer_) { 288 fine_audio_buffer_->ResetRecord(); 289 } 290 if (!playing_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 291 if (!audio_unit_->Start()) { 292 RTCLogError(@"StartRecording failed to start audio unit."); 293 return -1; 294 } 295 RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; 296 } 297 rtc::AtomicOps::ReleaseStore(&recording_, 1); 298 return 0; 299} 300 301int32_t AudioDeviceIOS::StopRecording() { 302 LOGI() << "StopRecording"; 303 RTC_DCHECK_RUN_ON(&thread_checker_); 304 if (!audio_is_initialized_ || !recording_) { 305 return 0; 306 } 307 if (!playing_) { 308 ShutdownPlayOrRecord(); 309 audio_is_initialized_ = false; 310 } 311 rtc::AtomicOps::ReleaseStore(&recording_, 0); 312 return 0; 313} 314 315bool AudioDeviceIOS::Recording() const { 316 return recording_; 317} 318 319int32_t AudioDeviceIOS::PlayoutDelay(uint16_t& delayMS) const { 320 delayMS = kFixedPlayoutDelayEstimate; 321 return 0; 322} 323 324int AudioDeviceIOS::GetPlayoutAudioParameters(AudioParameters* params) const { 325 LOGI() << "GetPlayoutAudioParameters"; 326 RTC_DCHECK(playout_parameters_.is_valid()); 327 RTC_DCHECK(thread_checker_.IsCurrent()); 328 *params = playout_parameters_; 329 return 0; 330} 331 332int AudioDeviceIOS::GetRecordAudioParameters(AudioParameters* params) const { 333 LOGI() << "GetRecordAudioParameters"; 334 RTC_DCHECK(record_parameters_.is_valid()); 335 RTC_DCHECK(thread_checker_.IsCurrent()); 336 *params = record_parameters_; 337 return 0; 338} 339 340void AudioDeviceIOS::OnInterruptionBegin() { 341 RTC_DCHECK(thread_); 342 LOGI() << "OnInterruptionBegin"; 343 thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionBegin); 344} 345 346void AudioDeviceIOS::OnInterruptionEnd() { 347 RTC_DCHECK(thread_); 348 LOGI() << "OnInterruptionEnd"; 349 thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionEnd); 350} 351 352void AudioDeviceIOS::OnValidRouteChange() { 353 RTC_DCHECK(thread_); 354 thread_->Post(RTC_FROM_HERE, this, kMessageTypeValidRouteChange); 355} 356 357void AudioDeviceIOS::OnCanPlayOrRecordChange(bool can_play_or_record) { 358 RTC_DCHECK(thread_); 359 thread_->Post(RTC_FROM_HERE, 360 this, 361 kMessageTypeCanPlayOrRecordChange, 362 new rtc::TypedMessageData<bool>(can_play_or_record)); 363} 364 365void AudioDeviceIOS::OnChangedOutputVolume() { 366 RTC_DCHECK(thread_); 367 thread_->Post(RTC_FROM_HERE, this, kMessageOutputVolumeChange); 368} 369 370OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, 371 const AudioTimeStamp* time_stamp, 372 UInt32 bus_number, 373 UInt32 num_frames, 374 AudioBufferList* /* io_data */) { 375 RTC_DCHECK_RUN_ON(&io_thread_checker_); 376 OSStatus result = noErr; 377 // Simply return if recording is not enabled. 378 if (!rtc::AtomicOps::AcquireLoad(&recording_)) return result; 379 380 // Set the size of our own audio buffer and clear it first to avoid copying 381 // in combination with potential reallocations. 382 // On real iOS devices, the size will only be set once (at first callback). 383 record_audio_buffer_.Clear(); 384 record_audio_buffer_.SetSize(num_frames); 385 386 // Allocate AudioBuffers to be used as storage for the received audio. 387 // The AudioBufferList structure works as a placeholder for the 388 // AudioBuffer structure, which holds a pointer to the actual data buffer 389 // in |record_audio_buffer_|. Recorded audio will be rendered into this memory 390 // at each input callback when calling AudioUnitRender(). 391 AudioBufferList audio_buffer_list; 392 audio_buffer_list.mNumberBuffers = 1; 393 AudioBuffer* audio_buffer = &audio_buffer_list.mBuffers[0]; 394 audio_buffer->mNumberChannels = record_parameters_.channels(); 395 audio_buffer->mDataByteSize = 396 record_audio_buffer_.size() * VoiceProcessingAudioUnit::kBytesPerSample; 397 audio_buffer->mData = reinterpret_cast<int8_t*>(record_audio_buffer_.data()); 398 399 // Obtain the recorded audio samples by initiating a rendering cycle. 400 // Since it happens on the input bus, the |io_data| parameter is a reference 401 // to the preallocated audio buffer list that the audio unit renders into. 402 // We can make the audio unit provide a buffer instead in io_data, but we 403 // currently just use our own. 404 // TODO(henrika): should error handling be improved? 405 result = audio_unit_->Render(flags, time_stamp, bus_number, num_frames, &audio_buffer_list); 406 if (result != noErr) { 407 RTCLogError(@"Failed to render audio."); 408 return result; 409 } 410 411 // Get a pointer to the recorded audio and send it to the WebRTC ADB. 412 // Use the FineAudioBuffer instance to convert between native buffer size 413 // and the 10ms buffer size used by WebRTC. 414 fine_audio_buffer_->DeliverRecordedData(record_audio_buffer_, kFixedRecordDelayEstimate); 415 return noErr; 416} 417 418OSStatus AudioDeviceIOS::OnGetPlayoutData(AudioUnitRenderActionFlags* flags, 419 const AudioTimeStamp* time_stamp, 420 UInt32 bus_number, 421 UInt32 num_frames, 422 AudioBufferList* io_data) { 423 RTC_DCHECK_RUN_ON(&io_thread_checker_); 424 // Verify 16-bit, noninterleaved mono PCM signal format. 425 RTC_DCHECK_EQ(1, io_data->mNumberBuffers); 426 AudioBuffer* audio_buffer = &io_data->mBuffers[0]; 427 RTC_DCHECK_EQ(1, audio_buffer->mNumberChannels); 428 429 // Produce silence and give audio unit a hint about it if playout is not 430 // activated. 431 if (!rtc::AtomicOps::AcquireLoad(&playing_)) { 432 const size_t size_in_bytes = audio_buffer->mDataByteSize; 433 RTC_CHECK_EQ(size_in_bytes / VoiceProcessingAudioUnit::kBytesPerSample, num_frames); 434 *flags |= kAudioUnitRenderAction_OutputIsSilence; 435 memset(static_cast<int8_t*>(audio_buffer->mData), 0, size_in_bytes); 436 return noErr; 437 } 438 439 // Measure time since last call to OnGetPlayoutData() and see if it is larger 440 // than a well defined threshold which depends on the current IO buffer size. 441 // If so, we have an indication of a glitch in the output audio since the 442 // core audio layer will most likely run dry in this state. 443 ++num_playout_callbacks_; 444 const int64_t now_time = rtc::TimeMillis(); 445 if (time_stamp->mSampleTime != num_frames) { 446 const int64_t delta_time = now_time - last_playout_time_; 447 const int glitch_threshold = 1.6 * playout_parameters_.GetBufferSizeInMilliseconds(); 448 if (delta_time > glitch_threshold) { 449 RTCLogWarning(@"Possible playout audio glitch detected.\n" 450 " Time since last OnGetPlayoutData was %lld ms.\n", 451 delta_time); 452 // Exclude extreme delta values since they do most likely not correspond 453 // to a real glitch. Instead, the most probable cause is that a headset 454 // has been plugged in or out. There are more direct ways to detect 455 // audio device changes (see HandleValidRouteChange()) but experiments 456 // show that using it leads to more complex implementations. 457 // TODO(henrika): more tests might be needed to come up with an even 458 // better upper limit. 459 if (glitch_threshold < 120 && delta_time > 120) { 460 RTCLog(@"Glitch warning is ignored. Probably caused by device switch."); 461 } else { 462 thread_->Post(RTC_FROM_HERE, this, kMessageTypePlayoutGlitchDetected); 463 } 464 } 465 } 466 last_playout_time_ = now_time; 467 468 // Read decoded 16-bit PCM samples from WebRTC (using a size that matches 469 // the native I/O audio unit) and copy the result to the audio buffer in the 470 // |io_data| destination. 471 fine_audio_buffer_->GetPlayoutData( 472 rtc::ArrayView<int16_t>(static_cast<int16_t*>(audio_buffer->mData), num_frames), 473 kFixedPlayoutDelayEstimate); 474 return noErr; 475} 476 477void AudioDeviceIOS::OnMessage(rtc::Message* msg) { 478 switch (msg->message_id) { 479 case kMessageTypeInterruptionBegin: 480 HandleInterruptionBegin(); 481 break; 482 case kMessageTypeInterruptionEnd: 483 HandleInterruptionEnd(); 484 break; 485 case kMessageTypeValidRouteChange: 486 HandleValidRouteChange(); 487 break; 488 case kMessageTypeCanPlayOrRecordChange: { 489 rtc::TypedMessageData<bool>* data = static_cast<rtc::TypedMessageData<bool>*>(msg->pdata); 490 HandleCanPlayOrRecordChange(data->data()); 491 delete data; 492 break; 493 } 494 case kMessageTypePlayoutGlitchDetected: 495 HandlePlayoutGlitchDetected(); 496 break; 497 case kMessageOutputVolumeChange: 498 HandleOutputVolumeChange(); 499 break; 500 } 501} 502 503void AudioDeviceIOS::HandleInterruptionBegin() { 504 RTC_DCHECK_RUN_ON(&thread_checker_); 505 RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_); 506 if (audio_unit_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { 507 RTCLog(@"Stopping the audio unit due to interruption begin."); 508 if (!audio_unit_->Stop()) { 509 RTCLogError(@"Failed to stop the audio unit for interruption begin."); 510 } 511 PrepareForNewStart(); 512 } 513 is_interrupted_ = true; 514} 515 516void AudioDeviceIOS::HandleInterruptionEnd() { 517 RTC_DCHECK_RUN_ON(&thread_checker_); 518 RTCLog(@"Interruption ended. IsInterrupted changed from %d to 0. " 519 "Updating audio unit state.", 520 is_interrupted_); 521 is_interrupted_ = false; 522 if (!audio_unit_) return; 523 if (webrtc::field_trial::IsEnabled("WebRTC-Audio-iOS-Holding")) { 524 // Work around an issue where audio does not restart properly after an interruption 525 // by restarting the audio unit when the interruption ends. 526 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { 527 audio_unit_->Stop(); 528 PrepareForNewStart(); 529 } 530 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 531 audio_unit_->Uninitialize(); 532 } 533 // Allocate new buffers given the potentially new stream format. 534 SetupAudioBuffersForActiveAudioSession(); 535 } 536 UpdateAudioUnit([RTC_OBJC_TYPE(RTCAudioSession) sharedInstance].canPlayOrRecord); 537} 538 539void AudioDeviceIOS::HandleValidRouteChange() { 540 RTC_DCHECK_RUN_ON(&thread_checker_); 541 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 542 RTCLog(@"%@", session); 543 HandleSampleRateChange(session.sampleRate); 544} 545 546void AudioDeviceIOS::HandleCanPlayOrRecordChange(bool can_play_or_record) { 547 RTCLog(@"Handling CanPlayOrRecord change to: %d", can_play_or_record); 548 UpdateAudioUnit(can_play_or_record); 549} 550 551void AudioDeviceIOS::HandleSampleRateChange(float sample_rate) { 552 RTC_DCHECK_RUN_ON(&thread_checker_); 553 RTCLog(@"Handling sample rate change to %f.", sample_rate); 554 555 // Don't do anything if we're interrupted. 556 if (is_interrupted_) { 557 RTCLog(@"Ignoring sample rate change to %f due to interruption.", sample_rate); 558 return; 559 } 560 561 // If we don't have an audio unit yet, or the audio unit is uninitialized, 562 // there is no work to do. 563 if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { 564 return; 565 } 566 567 // The audio unit is already initialized or started. 568 // Check to see if the sample rate or buffer size has changed. 569 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 570 const double session_sample_rate = session.sampleRate; 571 const NSTimeInterval session_buffer_duration = session.IOBufferDuration; 572 const size_t session_frames_per_buffer = 573 static_cast<size_t>(session_sample_rate * session_buffer_duration + .5); 574 const double current_sample_rate = playout_parameters_.sample_rate(); 575 const size_t current_frames_per_buffer = playout_parameters_.frames_per_buffer(); 576 RTCLog(@"Handling playout sample rate change to: %f\n" 577 " Session sample rate: %f frames_per_buffer: %lu\n" 578 " ADM sample rate: %f frames_per_buffer: %lu", 579 sample_rate, 580 session_sample_rate, 581 (unsigned long)session_frames_per_buffer, 582 current_sample_rate, 583 (unsigned long)current_frames_per_buffer); 584 585 // Sample rate and buffer size are the same, no work to do. 586 if (std::abs(current_sample_rate - session_sample_rate) <= DBL_EPSILON && 587 current_frames_per_buffer == session_frames_per_buffer) { 588 RTCLog(@"Ignoring sample rate change since audio parameters are intact."); 589 return; 590 } 591 592 // Extra sanity check to ensure that the new sample rate is valid. 593 if (session_sample_rate <= 0.0) { 594 RTCLogError(@"Sample rate is invalid: %f", session_sample_rate); 595 return; 596 } 597 598 // We need to adjust our format and buffer sizes. 599 // The stream format is about to be changed and it requires that we first 600 // stop and uninitialize the audio unit to deallocate its resources. 601 RTCLog(@"Stopping and uninitializing audio unit to adjust buffers."); 602 bool restart_audio_unit = false; 603 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { 604 audio_unit_->Stop(); 605 restart_audio_unit = true; 606 PrepareForNewStart(); 607 } 608 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 609 audio_unit_->Uninitialize(); 610 } 611 612 // Allocate new buffers given the new stream format. 613 SetupAudioBuffersForActiveAudioSession(); 614 615 // Initialize the audio unit again with the new sample rate. 616 RTC_DCHECK_EQ(playout_parameters_.sample_rate(), session_sample_rate); 617 if (!audio_unit_->Initialize(session_sample_rate)) { 618 RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", session_sample_rate); 619 return; 620 } 621 622 // Restart the audio unit if it was already running. 623 if (restart_audio_unit && !audio_unit_->Start()) { 624 RTCLogError(@"Failed to start audio unit with sample rate: %f", session_sample_rate); 625 return; 626 } 627 RTCLog(@"Successfully handled sample rate change."); 628} 629 630void AudioDeviceIOS::HandlePlayoutGlitchDetected() { 631 RTC_DCHECK_RUN_ON(&thread_checker_); 632 // Don't update metrics if we're interrupted since a "glitch" is expected 633 // in this state. 634 if (is_interrupted_) { 635 RTCLog(@"Ignoring audio glitch due to interruption."); 636 return; 637 } 638 // Avoid doing glitch detection for two seconds after a volume change 639 // has been detected to reduce the risk of false alarm. 640 if (last_output_volume_change_time_ > 0 && 641 rtc::TimeSince(last_output_volume_change_time_) < 2000) { 642 RTCLog(@"Ignoring audio glitch due to recent output volume change."); 643 return; 644 } 645 num_detected_playout_glitches_++; 646 RTCLog(@"Number of detected playout glitches: %lld", num_detected_playout_glitches_); 647 648 int64_t glitch_count = num_detected_playout_glitches_; 649 dispatch_async(dispatch_get_main_queue(), ^{ 650 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 651 [session notifyDidDetectPlayoutGlitch:glitch_count]; 652 }); 653} 654 655void AudioDeviceIOS::HandleOutputVolumeChange() { 656 RTC_DCHECK_RUN_ON(&thread_checker_); 657 RTCLog(@"Output volume change detected."); 658 // Store time of this detection so it can be used to defer detection of 659 // glitches too close in time to this event. 660 last_output_volume_change_time_ = rtc::TimeMillis(); 661} 662 663void AudioDeviceIOS::UpdateAudioDeviceBuffer() { 664 LOGI() << "UpdateAudioDevicebuffer"; 665 // AttachAudioBuffer() is called at construction by the main class but check 666 // just in case. 667 RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; 668 RTC_DCHECK_GT(playout_parameters_.sample_rate(), 0); 669 RTC_DCHECK_GT(record_parameters_.sample_rate(), 0); 670 RTC_DCHECK_EQ(playout_parameters_.channels(), 1); 671 RTC_DCHECK_EQ(record_parameters_.channels(), 1); 672 // Inform the audio device buffer (ADB) about the new audio format. 673 audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate()); 674 audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels()); 675 audio_device_buffer_->SetRecordingSampleRate(record_parameters_.sample_rate()); 676 audio_device_buffer_->SetRecordingChannels(record_parameters_.channels()); 677} 678 679void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() { 680 LOGI() << "SetupAudioBuffersForActiveAudioSession"; 681 // Verify the current values once the audio session has been activated. 682 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 683 double sample_rate = session.sampleRate; 684 NSTimeInterval io_buffer_duration = session.IOBufferDuration; 685 RTCLog(@"%@", session); 686 687 // Log a warning message for the case when we are unable to set the preferred 688 // hardware sample rate but continue and use the non-ideal sample rate after 689 // reinitializing the audio parameters. Most BT headsets only support 8kHz or 690 // 16kHz. 691 RTC_OBJC_TYPE(RTCAudioSessionConfiguration)* webRTCConfig = 692 [RTC_OBJC_TYPE(RTCAudioSessionConfiguration) webRTCConfiguration]; 693 if (sample_rate != webRTCConfig.sampleRate) { 694 RTC_LOG(LS_WARNING) << "Unable to set the preferred sample rate"; 695 } 696 697 // Crash reports indicates that it can happen in rare cases that the reported 698 // sample rate is less than or equal to zero. If that happens and if a valid 699 // sample rate has already been set during initialization, the best guess we 700 // can do is to reuse the current sample rate. 701 if (sample_rate <= DBL_EPSILON && playout_parameters_.sample_rate() > 0) { 702 RTCLogError(@"Reported rate is invalid: %f. " 703 "Using %d as sample rate instead.", 704 sample_rate, playout_parameters_.sample_rate()); 705 sample_rate = playout_parameters_.sample_rate(); 706 } 707 708 // At this stage, we also know the exact IO buffer duration and can add 709 // that info to the existing audio parameters where it is converted into 710 // number of audio frames. 711 // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz. 712 // Hence, 128 is the size we expect to see in upcoming render callbacks. 713 playout_parameters_.reset(sample_rate, playout_parameters_.channels(), io_buffer_duration); 714 RTC_DCHECK(playout_parameters_.is_complete()); 715 record_parameters_.reset(sample_rate, record_parameters_.channels(), io_buffer_duration); 716 RTC_DCHECK(record_parameters_.is_complete()); 717 RTC_LOG(LS_INFO) << " frames per I/O buffer: " << playout_parameters_.frames_per_buffer(); 718 RTC_LOG(LS_INFO) << " bytes per I/O buffer: " << playout_parameters_.GetBytesPerBuffer(); 719 RTC_DCHECK_EQ(playout_parameters_.GetBytesPerBuffer(), record_parameters_.GetBytesPerBuffer()); 720 721 // Update the ADB parameters since the sample rate might have changed. 722 UpdateAudioDeviceBuffer(); 723 724 // Create a modified audio buffer class which allows us to ask for, 725 // or deliver, any number of samples (and not only multiple of 10ms) to match 726 // the native audio unit buffer size. 727 RTC_DCHECK(audio_device_buffer_); 728 fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_)); 729} 730 731bool AudioDeviceIOS::CreateAudioUnit() { 732 RTC_DCHECK(!audio_unit_); 733 734 audio_unit_.reset(new VoiceProcessingAudioUnit(bypass_voice_processing_, this)); 735 if (!audio_unit_->Init()) { 736 audio_unit_.reset(); 737 return false; 738 } 739 740 return true; 741} 742 743void AudioDeviceIOS::UpdateAudioUnit(bool can_play_or_record) { 744 RTC_DCHECK_RUN_ON(&thread_checker_); 745 RTCLog(@"Updating audio unit state. CanPlayOrRecord=%d IsInterrupted=%d", 746 can_play_or_record, 747 is_interrupted_); 748 749 if (is_interrupted_) { 750 RTCLog(@"Ignoring audio unit update due to interruption."); 751 return; 752 } 753 754 // If we're not initialized we don't need to do anything. Audio unit will 755 // be initialized on initialization. 756 if (!audio_is_initialized_) return; 757 758 // If we're initialized, we must have an audio unit. 759 RTC_DCHECK(audio_unit_); 760 761 bool should_initialize_audio_unit = false; 762 bool should_uninitialize_audio_unit = false; 763 bool should_start_audio_unit = false; 764 bool should_stop_audio_unit = false; 765 766 switch (audio_unit_->GetState()) { 767 case VoiceProcessingAudioUnit::kInitRequired: 768 RTCLog(@"VPAU state: InitRequired"); 769 RTC_NOTREACHED(); 770 break; 771 case VoiceProcessingAudioUnit::kUninitialized: 772 RTCLog(@"VPAU state: Uninitialized"); 773 should_initialize_audio_unit = can_play_or_record; 774 should_start_audio_unit = should_initialize_audio_unit && (playing_ || recording_); 775 break; 776 case VoiceProcessingAudioUnit::kInitialized: 777 RTCLog(@"VPAU state: Initialized"); 778 should_start_audio_unit = can_play_or_record && (playing_ || recording_); 779 should_uninitialize_audio_unit = !can_play_or_record; 780 break; 781 case VoiceProcessingAudioUnit::kStarted: 782 RTCLog(@"VPAU state: Started"); 783 RTC_DCHECK(playing_ || recording_); 784 should_stop_audio_unit = !can_play_or_record; 785 should_uninitialize_audio_unit = should_stop_audio_unit; 786 break; 787 } 788 789 if (should_initialize_audio_unit) { 790 RTCLog(@"Initializing audio unit for UpdateAudioUnit"); 791 ConfigureAudioSession(); 792 SetupAudioBuffersForActiveAudioSession(); 793 if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { 794 RTCLogError(@"Failed to initialize audio unit."); 795 return; 796 } 797 } 798 799 if (should_start_audio_unit) { 800 RTCLog(@"Starting audio unit for UpdateAudioUnit"); 801 // Log session settings before trying to start audio streaming. 802 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 803 RTCLog(@"%@", session); 804 if (!audio_unit_->Start()) { 805 RTCLogError(@"Failed to start audio unit."); 806 return; 807 } 808 } 809 810 if (should_stop_audio_unit) { 811 RTCLog(@"Stopping audio unit for UpdateAudioUnit"); 812 if (!audio_unit_->Stop()) { 813 RTCLogError(@"Failed to stop audio unit."); 814 PrepareForNewStart(); 815 return; 816 } 817 PrepareForNewStart(); 818 } 819 820 if (should_uninitialize_audio_unit) { 821 RTCLog(@"Uninitializing audio unit for UpdateAudioUnit"); 822 audio_unit_->Uninitialize(); 823 UnconfigureAudioSession(); 824 } 825} 826 827bool AudioDeviceIOS::ConfigureAudioSession() { 828 RTC_DCHECK_RUN_ON(&thread_checker_); 829 RTCLog(@"Configuring audio session."); 830 if (has_configured_session_) { 831 RTCLogWarning(@"Audio session already configured."); 832 return false; 833 } 834 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 835 [session lockForConfiguration]; 836 bool success = [session configureWebRTCSession:nil]; 837 [session unlockForConfiguration]; 838 if (success) { 839 has_configured_session_ = true; 840 RTCLog(@"Configured audio session."); 841 } else { 842 RTCLog(@"Failed to configure audio session."); 843 } 844 return success; 845} 846 847bool AudioDeviceIOS::ConfigureAudioSessionLocked() { 848 RTC_DCHECK_RUN_ON(&thread_checker_); 849 RTCLog(@"Configuring audio session."); 850 if (has_configured_session_) { 851 RTCLogWarning(@"Audio session already configured."); 852 return false; 853 } 854 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 855 bool success = [session configureWebRTCSession:nil]; 856 if (success) { 857 has_configured_session_ = true; 858 RTCLog(@"Configured audio session."); 859 } else { 860 RTCLog(@"Failed to configure audio session."); 861 } 862 return success; 863} 864 865void AudioDeviceIOS::UnconfigureAudioSession() { 866 RTC_DCHECK_RUN_ON(&thread_checker_); 867 RTCLog(@"Unconfiguring audio session."); 868 if (!has_configured_session_) { 869 RTCLogWarning(@"Audio session already unconfigured."); 870 return; 871 } 872 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 873 [session lockForConfiguration]; 874 [session unconfigureWebRTCSession:nil]; 875 [session endWebRTCSession:nil]; 876 [session unlockForConfiguration]; 877 has_configured_session_ = false; 878 RTCLog(@"Unconfigured audio session."); 879} 880 881bool AudioDeviceIOS::InitPlayOrRecord() { 882 LOGI() << "InitPlayOrRecord"; 883 RTC_DCHECK_RUN_ON(&thread_checker_); 884 885 // There should be no audio unit at this point. 886 if (!CreateAudioUnit()) { 887 return false; 888 } 889 890 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 891 // Subscribe to audio session events. 892 [session pushDelegate:audio_session_observer_]; 893 is_interrupted_ = session.isInterrupted ? true : false; 894 895 // Lock the session to make configuration changes. 896 [session lockForConfiguration]; 897 NSError* error = nil; 898 if (![session beginWebRTCSession:&error]) { 899 [session unlockForConfiguration]; 900 RTCLogError(@"Failed to begin WebRTC session: %@", error.localizedDescription); 901 audio_unit_.reset(); 902 return false; 903 } 904 905 // If we are ready to play or record, and if the audio session can be 906 // configured, then initialize the audio unit. 907 if (session.canPlayOrRecord) { 908 if (!ConfigureAudioSessionLocked()) { 909 // One possible reason for failure is if an attempt was made to use the 910 // audio session during or after a Media Services failure. 911 // See AVAudioSessionErrorCodeMediaServicesFailed for details. 912 [session unlockForConfiguration]; 913 audio_unit_.reset(); 914 return false; 915 } 916 SetupAudioBuffersForActiveAudioSession(); 917 audio_unit_->Initialize(playout_parameters_.sample_rate()); 918 } 919 920 // Release the lock. 921 [session unlockForConfiguration]; 922 return true; 923} 924 925void AudioDeviceIOS::ShutdownPlayOrRecord() { 926 LOGI() << "ShutdownPlayOrRecord"; 927 RTC_DCHECK_RUN_ON(&thread_checker_); 928 929 // Stop the audio unit to prevent any additional audio callbacks. 930 audio_unit_->Stop(); 931 932 // Close and delete the voice-processing I/O unit. 933 audio_unit_.reset(); 934 935 // Detach thread checker for the AURemoteIO::IOThread to ensure that the 936 // next session uses a fresh thread id. 937 io_thread_checker_.Detach(); 938 939 // Remove audio session notification observers. 940 RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; 941 [session removeDelegate:audio_session_observer_]; 942 943 // All I/O should be stopped or paused prior to deactivating the audio 944 // session, hence we deactivate as last action. 945 UnconfigureAudioSession(); 946} 947 948void AudioDeviceIOS::PrepareForNewStart() { 949 LOGI() << "PrepareForNewStart"; 950 // The audio unit has been stopped and preparations are needed for an upcoming 951 // restart. It will result in audio callbacks from a new native I/O thread 952 // which means that we must detach thread checkers here to be prepared for an 953 // upcoming new audio stream. 954 io_thread_checker_.Detach(); 955} 956 957bool AudioDeviceIOS::IsInterrupted() { 958 return is_interrupted_; 959} 960 961#pragma mark - Not Implemented 962 963int32_t AudioDeviceIOS::ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const { 964 audioLayer = AudioDeviceModule::kPlatformDefaultAudio; 965 return 0; 966} 967 968int16_t AudioDeviceIOS::PlayoutDevices() { 969 // TODO(henrika): improve. 970 RTC_LOG_F(LS_WARNING) << "Not implemented"; 971 return (int16_t)1; 972} 973 974int16_t AudioDeviceIOS::RecordingDevices() { 975 // TODO(henrika): improve. 976 RTC_LOG_F(LS_WARNING) << "Not implemented"; 977 return (int16_t)1; 978} 979 980int32_t AudioDeviceIOS::InitSpeaker() { 981 return 0; 982} 983 984bool AudioDeviceIOS::SpeakerIsInitialized() const { 985 return true; 986} 987 988int32_t AudioDeviceIOS::SpeakerVolumeIsAvailable(bool& available) { 989 available = false; 990 return 0; 991} 992 993int32_t AudioDeviceIOS::SetSpeakerVolume(uint32_t volume) { 994 RTC_NOTREACHED() << "Not implemented"; 995 return -1; 996} 997 998int32_t AudioDeviceIOS::SpeakerVolume(uint32_t& volume) const { 999 RTC_NOTREACHED() << "Not implemented"; 1000 return -1; 1001} 1002 1003int32_t AudioDeviceIOS::MaxSpeakerVolume(uint32_t& maxVolume) const { 1004 RTC_NOTREACHED() << "Not implemented"; 1005 return -1; 1006} 1007 1008int32_t AudioDeviceIOS::MinSpeakerVolume(uint32_t& minVolume) const { 1009 RTC_NOTREACHED() << "Not implemented"; 1010 return -1; 1011} 1012 1013int32_t AudioDeviceIOS::SpeakerMuteIsAvailable(bool& available) { 1014 available = false; 1015 return 0; 1016} 1017 1018int32_t AudioDeviceIOS::SetSpeakerMute(bool enable) { 1019 RTC_NOTREACHED() << "Not implemented"; 1020 return -1; 1021} 1022 1023int32_t AudioDeviceIOS::SpeakerMute(bool& enabled) const { 1024 RTC_NOTREACHED() << "Not implemented"; 1025 return -1; 1026} 1027 1028int32_t AudioDeviceIOS::SetPlayoutDevice(uint16_t index) { 1029 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1030 return 0; 1031} 1032 1033int32_t AudioDeviceIOS::SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType) { 1034 RTC_NOTREACHED() << "Not implemented"; 1035 return -1; 1036} 1037 1038int32_t AudioDeviceIOS::InitMicrophone() { 1039 return 0; 1040} 1041 1042bool AudioDeviceIOS::MicrophoneIsInitialized() const { 1043 return true; 1044} 1045 1046int32_t AudioDeviceIOS::MicrophoneMuteIsAvailable(bool& available) { 1047 available = false; 1048 return 0; 1049} 1050 1051int32_t AudioDeviceIOS::SetMicrophoneMute(bool enable) { 1052 RTC_NOTREACHED() << "Not implemented"; 1053 return -1; 1054} 1055 1056int32_t AudioDeviceIOS::MicrophoneMute(bool& enabled) const { 1057 RTC_NOTREACHED() << "Not implemented"; 1058 return -1; 1059} 1060 1061int32_t AudioDeviceIOS::StereoRecordingIsAvailable(bool& available) { 1062 available = false; 1063 return 0; 1064} 1065 1066int32_t AudioDeviceIOS::SetStereoRecording(bool enable) { 1067 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1068 return -1; 1069} 1070 1071int32_t AudioDeviceIOS::StereoRecording(bool& enabled) const { 1072 enabled = false; 1073 return 0; 1074} 1075 1076int32_t AudioDeviceIOS::StereoPlayoutIsAvailable(bool& available) { 1077 available = false; 1078 return 0; 1079} 1080 1081int32_t AudioDeviceIOS::SetStereoPlayout(bool enable) { 1082 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1083 return -1; 1084} 1085 1086int32_t AudioDeviceIOS::StereoPlayout(bool& enabled) const { 1087 enabled = false; 1088 return 0; 1089} 1090 1091int32_t AudioDeviceIOS::MicrophoneVolumeIsAvailable(bool& available) { 1092 available = false; 1093 return 0; 1094} 1095 1096int32_t AudioDeviceIOS::SetMicrophoneVolume(uint32_t volume) { 1097 RTC_NOTREACHED() << "Not implemented"; 1098 return -1; 1099} 1100 1101int32_t AudioDeviceIOS::MicrophoneVolume(uint32_t& volume) const { 1102 RTC_NOTREACHED() << "Not implemented"; 1103 return -1; 1104} 1105 1106int32_t AudioDeviceIOS::MaxMicrophoneVolume(uint32_t& maxVolume) const { 1107 RTC_NOTREACHED() << "Not implemented"; 1108 return -1; 1109} 1110 1111int32_t AudioDeviceIOS::MinMicrophoneVolume(uint32_t& minVolume) const { 1112 RTC_NOTREACHED() << "Not implemented"; 1113 return -1; 1114} 1115 1116int32_t AudioDeviceIOS::PlayoutDeviceName(uint16_t index, 1117 char name[kAdmMaxDeviceNameSize], 1118 char guid[kAdmMaxGuidSize]) { 1119 RTC_NOTREACHED() << "Not implemented"; 1120 return -1; 1121} 1122 1123int32_t AudioDeviceIOS::RecordingDeviceName(uint16_t index, 1124 char name[kAdmMaxDeviceNameSize], 1125 char guid[kAdmMaxGuidSize]) { 1126 RTC_NOTREACHED() << "Not implemented"; 1127 return -1; 1128} 1129 1130int32_t AudioDeviceIOS::SetRecordingDevice(uint16_t index) { 1131 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1132 return 0; 1133} 1134 1135int32_t AudioDeviceIOS::SetRecordingDevice(AudioDeviceModule::WindowsDeviceType) { 1136 RTC_NOTREACHED() << "Not implemented"; 1137 return -1; 1138} 1139 1140int32_t AudioDeviceIOS::PlayoutIsAvailable(bool& available) { 1141 available = true; 1142 return 0; 1143} 1144 1145int32_t AudioDeviceIOS::RecordingIsAvailable(bool& available) { 1146 available = true; 1147 return 0; 1148} 1149 1150} // namespace ios_adm 1151} // namespace webrtc 1152