1/* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#import <AVFoundation/AVFoundation.h> 12#import <Foundation/Foundation.h> 13 14#include "audio_device_ios.h" 15 16#include <cmath> 17 18#include "api/array_view.h" 19#include "helpers.h" 20#include "modules/audio_device/fine_audio_buffer.h" 21#include "rtc_base/atomic_ops.h" 22#include "rtc_base/bind.h" 23#include "rtc_base/checks.h" 24#include "rtc_base/critical_section.h" 25#include "rtc_base/logging.h" 26#include "rtc_base/thread.h" 27#include "rtc_base/thread_annotations.h" 28#include "rtc_base/time_utils.h" 29#include "system_wrappers/include/field_trial.h" 30#include "system_wrappers/include/metrics.h" 31 32#import "base/RTCLogging.h" 33#import "components/audio/RTCAudioSession+Private.h" 34#import "components/audio/RTCAudioSession.h" 35#import "components/audio/RTCAudioSessionConfiguration.h" 36#import "components/audio/RTCNativeAudioSessionDelegateAdapter.h" 37 38namespace webrtc { 39namespace ios_adm { 40 41#define LOGI() RTC_LOG(LS_INFO) << "AudioDeviceIOS::" 42 43#define LOG_AND_RETURN_IF_ERROR(error, message) \ 44 do { \ 45 OSStatus err = error; \ 46 if (err) { \ 47 RTC_LOG(LS_ERROR) << message << ": " << err; \ 48 return false; \ 49 } \ 50 } while (0) 51 52#define LOG_IF_ERROR(error, message) \ 53 do { \ 54 OSStatus err = error; \ 55 if (err) { \ 56 RTC_LOG(LS_ERROR) << message << ": " << err; \ 57 } \ 58 } while (0) 59 60// Hardcoded delay estimates based on real measurements. 61// TODO(henrika): these value is not used in combination with built-in AEC. 62// Can most likely be removed. 63const UInt16 kFixedPlayoutDelayEstimate = 30; 64const UInt16 kFixedRecordDelayEstimate = 30; 65 66enum AudioDeviceMessageType : uint32_t { 67 kMessageTypeInterruptionBegin, 68 kMessageTypeInterruptionEnd, 69 kMessageTypeValidRouteChange, 70 kMessageTypeCanPlayOrRecordChange, 71 kMessageTypePlayoutGlitchDetected, 72 kMessageOutputVolumeChange, 73}; 74 75using ios::CheckAndLogError; 76 77#if !defined(NDEBUG) 78// Returns true when the code runs on a device simulator. 79static bool DeviceIsSimulator() { 80 return ios::GetDeviceName() == "x86_64"; 81} 82 83// Helper method that logs essential device information strings. 84static void LogDeviceInfo() { 85 RTC_LOG(LS_INFO) << "LogDeviceInfo"; 86 @autoreleasepool { 87 RTC_LOG(LS_INFO) << " system name: " << ios::GetSystemName(); 88 RTC_LOG(LS_INFO) << " system version: " << ios::GetSystemVersionAsString(); 89 RTC_LOG(LS_INFO) << " device type: " << ios::GetDeviceType(); 90 RTC_LOG(LS_INFO) << " device name: " << ios::GetDeviceName(); 91 RTC_LOG(LS_INFO) << " process name: " << ios::GetProcessName(); 92 RTC_LOG(LS_INFO) << " process ID: " << ios::GetProcessID(); 93 RTC_LOG(LS_INFO) << " OS version: " << ios::GetOSVersionString(); 94 RTC_LOG(LS_INFO) << " processing cores: " << ios::GetProcessorCount(); 95 RTC_LOG(LS_INFO) << " low power mode: " << ios::GetLowPowerModeEnabled(); 96#if TARGET_IPHONE_SIMULATOR 97 RTC_LOG(LS_INFO) << " TARGET_IPHONE_SIMULATOR is defined"; 98#endif 99 RTC_LOG(LS_INFO) << " DeviceIsSimulator: " << DeviceIsSimulator(); 100 } 101} 102#endif // !defined(NDEBUG) 103 104AudioDeviceIOS::AudioDeviceIOS() 105 : audio_device_buffer_(nullptr), 106 audio_unit_(nullptr), 107 recording_(0), 108 playing_(0), 109 initialized_(false), 110 audio_is_initialized_(false), 111 is_interrupted_(false), 112 has_configured_session_(false), 113 num_detected_playout_glitches_(0), 114 last_playout_time_(0), 115 num_playout_callbacks_(0), 116 last_output_volume_change_time_(0) { 117 LOGI() << "ctor" << ios::GetCurrentThreadDescription(); 118 io_thread_checker_.Detach(); 119 thread_checker_.Detach(); 120 thread_ = rtc::Thread::Current(); 121 122 audio_session_observer_ = [[RTCNativeAudioSessionDelegateAdapter alloc] initWithObserver:this]; 123} 124 125AudioDeviceIOS::~AudioDeviceIOS() { 126 RTC_DCHECK(thread_checker_.IsCurrent()); 127 LOGI() << "~dtor" << ios::GetCurrentThreadDescription(); 128 Terminate(); 129 audio_session_observer_ = nil; 130} 131 132void AudioDeviceIOS::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { 133 LOGI() << "AttachAudioBuffer"; 134 RTC_DCHECK(audioBuffer); 135 RTC_DCHECK(thread_checker_.IsCurrent()); 136 audio_device_buffer_ = audioBuffer; 137} 138 139AudioDeviceGeneric::InitStatus AudioDeviceIOS::Init() { 140 LOGI() << "Init"; 141 io_thread_checker_.Detach(); 142 thread_checker_.Detach(); 143 144 RTC_DCHECK_RUN_ON(&thread_checker_); 145 if (initialized_) { 146 return InitStatus::OK; 147 } 148#if !defined(NDEBUG) 149 LogDeviceInfo(); 150#endif 151 // Store the preferred sample rate and preferred number of channels already 152 // here. They have not been set and confirmed yet since configureForWebRTC 153 // is not called until audio is about to start. However, it makes sense to 154 // store the parameters now and then verify at a later stage. 155 RTCAudioSessionConfiguration* config = [RTCAudioSessionConfiguration webRTCConfiguration]; 156 playout_parameters_.reset(config.sampleRate, config.outputNumberOfChannels); 157 record_parameters_.reset(config.sampleRate, config.inputNumberOfChannels); 158 // Ensure that the audio device buffer (ADB) knows about the internal audio 159 // parameters. Note that, even if we are unable to get a mono audio session, 160 // we will always tell the I/O audio unit to do a channel format conversion 161 // to guarantee mono on the "input side" of the audio unit. 162 UpdateAudioDeviceBuffer(); 163 initialized_ = true; 164 return InitStatus::OK; 165} 166 167int32_t AudioDeviceIOS::Terminate() { 168 LOGI() << "Terminate"; 169 RTC_DCHECK_RUN_ON(&thread_checker_); 170 if (!initialized_) { 171 return 0; 172 } 173 StopPlayout(); 174 StopRecording(); 175 initialized_ = false; 176 return 0; 177} 178 179bool AudioDeviceIOS::Initialized() const { 180 RTC_DCHECK_RUN_ON(&thread_checker_); 181 return initialized_; 182} 183 184int32_t AudioDeviceIOS::InitPlayout() { 185 LOGI() << "InitPlayout"; 186 RTC_DCHECK_RUN_ON(&thread_checker_); 187 RTC_DCHECK(initialized_); 188 RTC_DCHECK(!audio_is_initialized_); 189 RTC_DCHECK(!playing_); 190 if (!audio_is_initialized_) { 191 if (!InitPlayOrRecord()) { 192 RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitPlayout!"; 193 return -1; 194 } 195 } 196 audio_is_initialized_ = true; 197 return 0; 198} 199 200bool AudioDeviceIOS::PlayoutIsInitialized() const { 201 RTC_DCHECK_RUN_ON(&thread_checker_); 202 return audio_is_initialized_; 203} 204 205bool AudioDeviceIOS::RecordingIsInitialized() const { 206 RTC_DCHECK_RUN_ON(&thread_checker_); 207 return audio_is_initialized_; 208} 209 210int32_t AudioDeviceIOS::InitRecording() { 211 LOGI() << "InitRecording"; 212 RTC_DCHECK_RUN_ON(&thread_checker_); 213 RTC_DCHECK(initialized_); 214 RTC_DCHECK(!audio_is_initialized_); 215 RTC_DCHECK(!recording_); 216 if (!audio_is_initialized_) { 217 if (!InitPlayOrRecord()) { 218 RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitRecording!"; 219 return -1; 220 } 221 } 222 audio_is_initialized_ = true; 223 return 0; 224} 225 226int32_t AudioDeviceIOS::StartPlayout() { 227 LOGI() << "StartPlayout"; 228 RTC_DCHECK_RUN_ON(&thread_checker_); 229 RTC_DCHECK(audio_is_initialized_); 230 RTC_DCHECK(!playing_); 231 RTC_DCHECK(audio_unit_); 232 if (fine_audio_buffer_) { 233 fine_audio_buffer_->ResetPlayout(); 234 } 235 if (!recording_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 236 if (!audio_unit_->Start()) { 237 RTCLogError(@"StartPlayout failed to start audio unit."); 238 return -1; 239 } 240 RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; 241 } 242 rtc::AtomicOps::ReleaseStore(&playing_, 1); 243 num_playout_callbacks_ = 0; 244 num_detected_playout_glitches_ = 0; 245 return 0; 246} 247 248int32_t AudioDeviceIOS::StopPlayout() { 249 LOGI() << "StopPlayout"; 250 RTC_DCHECK_RUN_ON(&thread_checker_); 251 if (!audio_is_initialized_ || !playing_) { 252 return 0; 253 } 254 if (!recording_) { 255 ShutdownPlayOrRecord(); 256 audio_is_initialized_ = false; 257 } 258 rtc::AtomicOps::ReleaseStore(&playing_, 0); 259 260 // Derive average number of calls to OnGetPlayoutData() between detected 261 // audio glitches and add the result to a histogram. 262 int average_number_of_playout_callbacks_between_glitches = 100000; 263 RTC_DCHECK_GE(num_playout_callbacks_, num_detected_playout_glitches_); 264 if (num_detected_playout_glitches_ > 0) { 265 average_number_of_playout_callbacks_between_glitches = 266 num_playout_callbacks_ / num_detected_playout_glitches_; 267 } 268 RTC_HISTOGRAM_COUNTS_100000("WebRTC.Audio.AveragePlayoutCallbacksBetweenGlitches", 269 average_number_of_playout_callbacks_between_glitches); 270 RTCLog(@"Average number of playout callbacks between glitches: %d", 271 average_number_of_playout_callbacks_between_glitches); 272 return 0; 273} 274 275bool AudioDeviceIOS::Playing() const { 276 return playing_; 277} 278 279int32_t AudioDeviceIOS::StartRecording() { 280 LOGI() << "StartRecording"; 281 RTC_DCHECK_RUN_ON(&thread_checker_); 282 RTC_DCHECK(audio_is_initialized_); 283 RTC_DCHECK(!recording_); 284 RTC_DCHECK(audio_unit_); 285 if (fine_audio_buffer_) { 286 fine_audio_buffer_->ResetRecord(); 287 } 288 if (!playing_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 289 if (!audio_unit_->Start()) { 290 RTCLogError(@"StartRecording failed to start audio unit."); 291 return -1; 292 } 293 RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; 294 } 295 rtc::AtomicOps::ReleaseStore(&recording_, 1); 296 return 0; 297} 298 299int32_t AudioDeviceIOS::StopRecording() { 300 LOGI() << "StopRecording"; 301 RTC_DCHECK_RUN_ON(&thread_checker_); 302 if (!audio_is_initialized_ || !recording_) { 303 return 0; 304 } 305 if (!playing_) { 306 ShutdownPlayOrRecord(); 307 audio_is_initialized_ = false; 308 } 309 rtc::AtomicOps::ReleaseStore(&recording_, 0); 310 return 0; 311} 312 313bool AudioDeviceIOS::Recording() const { 314 return recording_; 315} 316 317int32_t AudioDeviceIOS::PlayoutDelay(uint16_t& delayMS) const { 318 delayMS = kFixedPlayoutDelayEstimate; 319 return 0; 320} 321 322int AudioDeviceIOS::GetPlayoutAudioParameters(AudioParameters* params) const { 323 LOGI() << "GetPlayoutAudioParameters"; 324 RTC_DCHECK(playout_parameters_.is_valid()); 325 RTC_DCHECK(thread_checker_.IsCurrent()); 326 *params = playout_parameters_; 327 return 0; 328} 329 330int AudioDeviceIOS::GetRecordAudioParameters(AudioParameters* params) const { 331 LOGI() << "GetRecordAudioParameters"; 332 RTC_DCHECK(record_parameters_.is_valid()); 333 RTC_DCHECK(thread_checker_.IsCurrent()); 334 *params = record_parameters_; 335 return 0; 336} 337 338void AudioDeviceIOS::OnInterruptionBegin() { 339 RTC_DCHECK(thread_); 340 LOGI() << "OnInterruptionBegin"; 341 thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionBegin); 342} 343 344void AudioDeviceIOS::OnInterruptionEnd() { 345 RTC_DCHECK(thread_); 346 LOGI() << "OnInterruptionEnd"; 347 thread_->Post(RTC_FROM_HERE, this, kMessageTypeInterruptionEnd); 348} 349 350void AudioDeviceIOS::OnValidRouteChange() { 351 RTC_DCHECK(thread_); 352 thread_->Post(RTC_FROM_HERE, this, kMessageTypeValidRouteChange); 353} 354 355void AudioDeviceIOS::OnCanPlayOrRecordChange(bool can_play_or_record) { 356 RTC_DCHECK(thread_); 357 thread_->Post(RTC_FROM_HERE, 358 this, 359 kMessageTypeCanPlayOrRecordChange, 360 new rtc::TypedMessageData<bool>(can_play_or_record)); 361} 362 363void AudioDeviceIOS::OnChangedOutputVolume() { 364 RTC_DCHECK(thread_); 365 thread_->Post(RTC_FROM_HERE, this, kMessageOutputVolumeChange); 366} 367 368OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, 369 const AudioTimeStamp* time_stamp, 370 UInt32 bus_number, 371 UInt32 num_frames, 372 AudioBufferList* /* io_data */) { 373 RTC_DCHECK_RUN_ON(&io_thread_checker_); 374 OSStatus result = noErr; 375 // Simply return if recording is not enabled. 376 if (!rtc::AtomicOps::AcquireLoad(&recording_)) return result; 377 378 // Set the size of our own audio buffer and clear it first to avoid copying 379 // in combination with potential reallocations. 380 // On real iOS devices, the size will only be set once (at first callback). 381 record_audio_buffer_.Clear(); 382 record_audio_buffer_.SetSize(num_frames); 383 384 // Allocate AudioBuffers to be used as storage for the received audio. 385 // The AudioBufferList structure works as a placeholder for the 386 // AudioBuffer structure, which holds a pointer to the actual data buffer 387 // in |record_audio_buffer_|. Recorded audio will be rendered into this memory 388 // at each input callback when calling AudioUnitRender(). 389 AudioBufferList audio_buffer_list; 390 audio_buffer_list.mNumberBuffers = 1; 391 AudioBuffer* audio_buffer = &audio_buffer_list.mBuffers[0]; 392 audio_buffer->mNumberChannels = record_parameters_.channels(); 393 audio_buffer->mDataByteSize = 394 record_audio_buffer_.size() * VoiceProcessingAudioUnit::kBytesPerSample; 395 audio_buffer->mData = reinterpret_cast<int8_t*>(record_audio_buffer_.data()); 396 397 // Obtain the recorded audio samples by initiating a rendering cycle. 398 // Since it happens on the input bus, the |io_data| parameter is a reference 399 // to the preallocated audio buffer list that the audio unit renders into. 400 // We can make the audio unit provide a buffer instead in io_data, but we 401 // currently just use our own. 402 // TODO(henrika): should error handling be improved? 403 result = audio_unit_->Render(flags, time_stamp, bus_number, num_frames, &audio_buffer_list); 404 if (result != noErr) { 405 RTCLogError(@"Failed to render audio."); 406 return result; 407 } 408 409 // Get a pointer to the recorded audio and send it to the WebRTC ADB. 410 // Use the FineAudioBuffer instance to convert between native buffer size 411 // and the 10ms buffer size used by WebRTC. 412 fine_audio_buffer_->DeliverRecordedData(record_audio_buffer_, kFixedRecordDelayEstimate); 413 return noErr; 414} 415 416OSStatus AudioDeviceIOS::OnGetPlayoutData(AudioUnitRenderActionFlags* flags, 417 const AudioTimeStamp* time_stamp, 418 UInt32 bus_number, 419 UInt32 num_frames, 420 AudioBufferList* io_data) { 421 RTC_DCHECK_RUN_ON(&io_thread_checker_); 422 // Verify 16-bit, noninterleaved mono PCM signal format. 423 RTC_DCHECK_EQ(1, io_data->mNumberBuffers); 424 AudioBuffer* audio_buffer = &io_data->mBuffers[0]; 425 RTC_DCHECK_EQ(1, audio_buffer->mNumberChannels); 426 427 // Produce silence and give audio unit a hint about it if playout is not 428 // activated. 429 if (!rtc::AtomicOps::AcquireLoad(&playing_)) { 430 const size_t size_in_bytes = audio_buffer->mDataByteSize; 431 RTC_CHECK_EQ(size_in_bytes / VoiceProcessingAudioUnit::kBytesPerSample, num_frames); 432 *flags |= kAudioUnitRenderAction_OutputIsSilence; 433 memset(static_cast<int8_t*>(audio_buffer->mData), 0, size_in_bytes); 434 return noErr; 435 } 436 437 // Measure time since last call to OnGetPlayoutData() and see if it is larger 438 // than a well defined threshold which depends on the current IO buffer size. 439 // If so, we have an indication of a glitch in the output audio since the 440 // core audio layer will most likely run dry in this state. 441 ++num_playout_callbacks_; 442 const int64_t now_time = rtc::TimeMillis(); 443 if (time_stamp->mSampleTime != num_frames) { 444 const int64_t delta_time = now_time - last_playout_time_; 445 const int glitch_threshold = 1.6 * playout_parameters_.GetBufferSizeInMilliseconds(); 446 if (delta_time > glitch_threshold) { 447 RTCLogWarning(@"Possible playout audio glitch detected.\n" 448 " Time since last OnGetPlayoutData was %lld ms.\n", 449 delta_time); 450 // Exclude extreme delta values since they do most likely not correspond 451 // to a real glitch. Instead, the most probable cause is that a headset 452 // has been plugged in or out. There are more direct ways to detect 453 // audio device changes (see HandleValidRouteChange()) but experiments 454 // show that using it leads to more complex implementations. 455 // TODO(henrika): more tests might be needed to come up with an even 456 // better upper limit. 457 if (glitch_threshold < 120 && delta_time > 120) { 458 RTCLog(@"Glitch warning is ignored. Probably caused by device switch."); 459 } else { 460 thread_->Post(RTC_FROM_HERE, this, kMessageTypePlayoutGlitchDetected); 461 } 462 } 463 } 464 last_playout_time_ = now_time; 465 466 // Read decoded 16-bit PCM samples from WebRTC (using a size that matches 467 // the native I/O audio unit) and copy the result to the audio buffer in the 468 // |io_data| destination. 469 fine_audio_buffer_->GetPlayoutData( 470 rtc::ArrayView<int16_t>(static_cast<int16_t*>(audio_buffer->mData), num_frames), 471 kFixedPlayoutDelayEstimate); 472 return noErr; 473} 474 475void AudioDeviceIOS::OnMessage(rtc::Message* msg) { 476 switch (msg->message_id) { 477 case kMessageTypeInterruptionBegin: 478 HandleInterruptionBegin(); 479 break; 480 case kMessageTypeInterruptionEnd: 481 HandleInterruptionEnd(); 482 break; 483 case kMessageTypeValidRouteChange: 484 HandleValidRouteChange(); 485 break; 486 case kMessageTypeCanPlayOrRecordChange: { 487 rtc::TypedMessageData<bool>* data = static_cast<rtc::TypedMessageData<bool>*>(msg->pdata); 488 HandleCanPlayOrRecordChange(data->data()); 489 delete data; 490 break; 491 } 492 case kMessageTypePlayoutGlitchDetected: 493 HandlePlayoutGlitchDetected(); 494 break; 495 case kMessageOutputVolumeChange: 496 HandleOutputVolumeChange(); 497 break; 498 } 499} 500 501void AudioDeviceIOS::HandleInterruptionBegin() { 502 RTC_DCHECK_RUN_ON(&thread_checker_); 503 RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_); 504 if (audio_unit_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { 505 RTCLog(@"Stopping the audio unit due to interruption begin."); 506 if (!audio_unit_->Stop()) { 507 RTCLogError(@"Failed to stop the audio unit for interruption begin."); 508 } else { 509 PrepareForNewStart(); 510 } 511 } 512 is_interrupted_ = true; 513} 514 515void AudioDeviceIOS::HandleInterruptionEnd() { 516 RTC_DCHECK_RUN_ON(&thread_checker_); 517 RTCLog(@"Interruption ended. IsInterrupted changed from %d to 0. " 518 "Updating audio unit state.", 519 is_interrupted_); 520 is_interrupted_ = false; 521 if (!audio_unit_) return; 522 if (webrtc::field_trial::IsEnabled("WebRTC-Audio-iOS-Holding")) { 523 // Work around an issue where audio does not restart properly after an interruption 524 // by restarting the audio unit when the interruption ends. 525 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { 526 audio_unit_->Stop(); 527 PrepareForNewStart(); 528 } 529 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 530 audio_unit_->Uninitialize(); 531 } 532 // Allocate new buffers given the potentially new stream format. 533 SetupAudioBuffersForActiveAudioSession(); 534 } 535 UpdateAudioUnit([RTCAudioSession sharedInstance].canPlayOrRecord); 536} 537 538void AudioDeviceIOS::HandleValidRouteChange() { 539 RTC_DCHECK_RUN_ON(&thread_checker_); 540 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 541 RTCLog(@"%@", session); 542 HandleSampleRateChange(session.sampleRate); 543} 544 545void AudioDeviceIOS::HandleCanPlayOrRecordChange(bool can_play_or_record) { 546 RTCLog(@"Handling CanPlayOrRecord change to: %d", can_play_or_record); 547 UpdateAudioUnit(can_play_or_record); 548} 549 550void AudioDeviceIOS::HandleSampleRateChange(float sample_rate) { 551 RTC_DCHECK_RUN_ON(&thread_checker_); 552 RTCLog(@"Handling sample rate change to %f.", sample_rate); 553 554 // Don't do anything if we're interrupted. 555 if (is_interrupted_) { 556 RTCLog(@"Ignoring sample rate change to %f due to interruption.", sample_rate); 557 return; 558 } 559 560 // If we don't have an audio unit yet, or the audio unit is uninitialized, 561 // there is no work to do. 562 if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { 563 return; 564 } 565 566 // The audio unit is already initialized or started. 567 // Check to see if the sample rate or buffer size has changed. 568 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 569 const double session_sample_rate = session.sampleRate; 570 const NSTimeInterval session_buffer_duration = session.IOBufferDuration; 571 const size_t session_frames_per_buffer = 572 static_cast<size_t>(session_sample_rate * session_buffer_duration + .5); 573 const double current_sample_rate = playout_parameters_.sample_rate(); 574 const size_t current_frames_per_buffer = playout_parameters_.frames_per_buffer(); 575 RTCLog(@"Handling playout sample rate change to: %f\n" 576 " Session sample rate: %f frames_per_buffer: %lu\n" 577 " ADM sample rate: %f frames_per_buffer: %lu", 578 sample_rate, 579 session_sample_rate, 580 (unsigned long)session_frames_per_buffer, 581 current_sample_rate, 582 (unsigned long)current_frames_per_buffer); 583 584 // Sample rate and buffer size are the same, no work to do. 585 if (std::abs(current_sample_rate - session_sample_rate) <= DBL_EPSILON && 586 current_frames_per_buffer == session_frames_per_buffer) { 587 RTCLog(@"Ignoring sample rate change since audio parameters are intact."); 588 return; 589 } 590 591 // Extra sanity check to ensure that the new sample rate is valid. 592 if (session_sample_rate <= 0.0) { 593 RTCLogError(@"Sample rate is invalid: %f", session_sample_rate); 594 return; 595 } 596 597 // We need to adjust our format and buffer sizes. 598 // The stream format is about to be changed and it requires that we first 599 // stop and uninitialize the audio unit to deallocate its resources. 600 RTCLog(@"Stopping and uninitializing audio unit to adjust buffers."); 601 bool restart_audio_unit = false; 602 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { 603 audio_unit_->Stop(); 604 restart_audio_unit = true; 605 PrepareForNewStart(); 606 } 607 if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { 608 audio_unit_->Uninitialize(); 609 } 610 611 // Allocate new buffers given the new stream format. 612 SetupAudioBuffersForActiveAudioSession(); 613 614 // Initialize the audio unit again with the new sample rate. 615 RTC_DCHECK_EQ(playout_parameters_.sample_rate(), session_sample_rate); 616 if (!audio_unit_->Initialize(session_sample_rate)) { 617 RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", session_sample_rate); 618 return; 619 } 620 621 // Restart the audio unit if it was already running. 622 if (restart_audio_unit && !audio_unit_->Start()) { 623 RTCLogError(@"Failed to start audio unit with sample rate: %f", session_sample_rate); 624 return; 625 } 626 RTCLog(@"Successfully handled sample rate change."); 627} 628 629void AudioDeviceIOS::HandlePlayoutGlitchDetected() { 630 RTC_DCHECK_RUN_ON(&thread_checker_); 631 // Don't update metrics if we're interrupted since a "glitch" is expected 632 // in this state. 633 if (is_interrupted_) { 634 RTCLog(@"Ignoring audio glitch due to interruption."); 635 return; 636 } 637 // Avoid doing glitch detection for two seconds after a volume change 638 // has been detected to reduce the risk of false alarm. 639 if (last_output_volume_change_time_ > 0 && 640 rtc::TimeSince(last_output_volume_change_time_) < 2000) { 641 RTCLog(@"Ignoring audio glitch due to recent output volume change."); 642 return; 643 } 644 num_detected_playout_glitches_++; 645 RTCLog(@"Number of detected playout glitches: %lld", num_detected_playout_glitches_); 646 647 int64_t glitch_count = num_detected_playout_glitches_; 648 dispatch_async(dispatch_get_main_queue(), ^{ 649 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 650 [session notifyDidDetectPlayoutGlitch:glitch_count]; 651 }); 652} 653 654void AudioDeviceIOS::HandleOutputVolumeChange() { 655 RTC_DCHECK_RUN_ON(&thread_checker_); 656 RTCLog(@"Output volume change detected."); 657 // Store time of this detection so it can be used to defer detection of 658 // glitches too close in time to this event. 659 last_output_volume_change_time_ = rtc::TimeMillis(); 660} 661 662void AudioDeviceIOS::UpdateAudioDeviceBuffer() { 663 LOGI() << "UpdateAudioDevicebuffer"; 664 // AttachAudioBuffer() is called at construction by the main class but check 665 // just in case. 666 RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; 667 RTC_DCHECK_GT(playout_parameters_.sample_rate(), 0); 668 RTC_DCHECK_GT(record_parameters_.sample_rate(), 0); 669 RTC_DCHECK_EQ(playout_parameters_.channels(), 1); 670 RTC_DCHECK_EQ(record_parameters_.channels(), 1); 671 // Inform the audio device buffer (ADB) about the new audio format. 672 audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate()); 673 audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels()); 674 audio_device_buffer_->SetRecordingSampleRate(record_parameters_.sample_rate()); 675 audio_device_buffer_->SetRecordingChannels(record_parameters_.channels()); 676} 677 678void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() { 679 LOGI() << "SetupAudioBuffersForActiveAudioSession"; 680 // Verify the current values once the audio session has been activated. 681 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 682 double sample_rate = session.sampleRate; 683 NSTimeInterval io_buffer_duration = session.IOBufferDuration; 684 RTCLog(@"%@", session); 685 686 // Log a warning message for the case when we are unable to set the preferred 687 // hardware sample rate but continue and use the non-ideal sample rate after 688 // reinitializing the audio parameters. Most BT headsets only support 8kHz or 689 // 16kHz. 690 RTCAudioSessionConfiguration* webRTCConfig = [RTCAudioSessionConfiguration webRTCConfiguration]; 691 if (sample_rate != webRTCConfig.sampleRate) { 692 RTC_LOG(LS_WARNING) << "Unable to set the preferred sample rate"; 693 } 694 695 // Crash reports indicates that it can happen in rare cases that the reported 696 // sample rate is less than or equal to zero. If that happens and if a valid 697 // sample rate has already been set during initialization, the best guess we 698 // can do is to reuse the current sample rate. 699 if (sample_rate <= DBL_EPSILON && playout_parameters_.sample_rate() > 0) { 700 RTCLogError(@"Reported rate is invalid: %f. " 701 "Using %d as sample rate instead.", 702 sample_rate, playout_parameters_.sample_rate()); 703 sample_rate = playout_parameters_.sample_rate(); 704 } 705 706 // At this stage, we also know the exact IO buffer duration and can add 707 // that info to the existing audio parameters where it is converted into 708 // number of audio frames. 709 // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz. 710 // Hence, 128 is the size we expect to see in upcoming render callbacks. 711 playout_parameters_.reset(sample_rate, playout_parameters_.channels(), io_buffer_duration); 712 RTC_DCHECK(playout_parameters_.is_complete()); 713 record_parameters_.reset(sample_rate, record_parameters_.channels(), io_buffer_duration); 714 RTC_DCHECK(record_parameters_.is_complete()); 715 RTC_LOG(LS_INFO) << " frames per I/O buffer: " << playout_parameters_.frames_per_buffer(); 716 RTC_LOG(LS_INFO) << " bytes per I/O buffer: " << playout_parameters_.GetBytesPerBuffer(); 717 RTC_DCHECK_EQ(playout_parameters_.GetBytesPerBuffer(), record_parameters_.GetBytesPerBuffer()); 718 719 // Update the ADB parameters since the sample rate might have changed. 720 UpdateAudioDeviceBuffer(); 721 722 // Create a modified audio buffer class which allows us to ask for, 723 // or deliver, any number of samples (and not only multiple of 10ms) to match 724 // the native audio unit buffer size. 725 RTC_DCHECK(audio_device_buffer_); 726 fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_)); 727} 728 729bool AudioDeviceIOS::CreateAudioUnit() { 730 RTC_DCHECK(!audio_unit_); 731 732 audio_unit_.reset(new VoiceProcessingAudioUnit(this)); 733 if (!audio_unit_->Init()) { 734 audio_unit_.reset(); 735 return false; 736 } 737 738 return true; 739} 740 741void AudioDeviceIOS::UpdateAudioUnit(bool can_play_or_record) { 742 RTC_DCHECK_RUN_ON(&thread_checker_); 743 RTCLog(@"Updating audio unit state. CanPlayOrRecord=%d IsInterrupted=%d", 744 can_play_or_record, 745 is_interrupted_); 746 747 if (is_interrupted_) { 748 RTCLog(@"Ignoring audio unit update due to interruption."); 749 return; 750 } 751 752 // If we're not initialized we don't need to do anything. Audio unit will 753 // be initialized on initialization. 754 if (!audio_is_initialized_) return; 755 756 // If we're initialized, we must have an audio unit. 757 RTC_DCHECK(audio_unit_); 758 759 bool should_initialize_audio_unit = false; 760 bool should_uninitialize_audio_unit = false; 761 bool should_start_audio_unit = false; 762 bool should_stop_audio_unit = false; 763 764 switch (audio_unit_->GetState()) { 765 case VoiceProcessingAudioUnit::kInitRequired: 766 RTCLog(@"VPAU state: InitRequired"); 767 RTC_NOTREACHED(); 768 break; 769 case VoiceProcessingAudioUnit::kUninitialized: 770 RTCLog(@"VPAU state: Uninitialized"); 771 should_initialize_audio_unit = can_play_or_record; 772 should_start_audio_unit = should_initialize_audio_unit && (playing_ || recording_); 773 break; 774 case VoiceProcessingAudioUnit::kInitialized: 775 RTCLog(@"VPAU state: Initialized"); 776 should_start_audio_unit = can_play_or_record && (playing_ || recording_); 777 should_uninitialize_audio_unit = !can_play_or_record; 778 break; 779 case VoiceProcessingAudioUnit::kStarted: 780 RTCLog(@"VPAU state: Started"); 781 RTC_DCHECK(playing_ || recording_); 782 should_stop_audio_unit = !can_play_or_record; 783 should_uninitialize_audio_unit = should_stop_audio_unit; 784 break; 785 } 786 787 if (should_initialize_audio_unit) { 788 RTCLog(@"Initializing audio unit for UpdateAudioUnit"); 789 ConfigureAudioSession(); 790 SetupAudioBuffersForActiveAudioSession(); 791 if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { 792 RTCLogError(@"Failed to initialize audio unit."); 793 return; 794 } 795 } 796 797 if (should_start_audio_unit) { 798 RTCLog(@"Starting audio unit for UpdateAudioUnit"); 799 // Log session settings before trying to start audio streaming. 800 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 801 RTCLog(@"%@", session); 802 if (!audio_unit_->Start()) { 803 RTCLogError(@"Failed to start audio unit."); 804 return; 805 } 806 } 807 808 if (should_stop_audio_unit) { 809 RTCLog(@"Stopping audio unit for UpdateAudioUnit"); 810 if (!audio_unit_->Stop()) { 811 RTCLogError(@"Failed to stop audio unit."); 812 return; 813 } 814 } 815 816 if (should_uninitialize_audio_unit) { 817 RTCLog(@"Uninitializing audio unit for UpdateAudioUnit"); 818 audio_unit_->Uninitialize(); 819 UnconfigureAudioSession(); 820 } 821} 822 823bool AudioDeviceIOS::ConfigureAudioSession() { 824 RTC_DCHECK_RUN_ON(&thread_checker_); 825 RTCLog(@"Configuring audio session."); 826 if (has_configured_session_) { 827 RTCLogWarning(@"Audio session already configured."); 828 return false; 829 } 830 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 831 [session lockForConfiguration]; 832 bool success = [session configureWebRTCSession:nil]; 833 [session unlockForConfiguration]; 834 if (success) { 835 has_configured_session_ = true; 836 RTCLog(@"Configured audio session."); 837 } else { 838 RTCLog(@"Failed to configure audio session."); 839 } 840 return success; 841} 842 843void AudioDeviceIOS::UnconfigureAudioSession() { 844 RTC_DCHECK_RUN_ON(&thread_checker_); 845 RTCLog(@"Unconfiguring audio session."); 846 if (!has_configured_session_) { 847 RTCLogWarning(@"Audio session already unconfigured."); 848 return; 849 } 850 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 851 [session lockForConfiguration]; 852 [session unconfigureWebRTCSession:nil]; 853 [session endWebRTCSession:nil]; 854 [session unlockForConfiguration]; 855 has_configured_session_ = false; 856 RTCLog(@"Unconfigured audio session."); 857} 858 859bool AudioDeviceIOS::InitPlayOrRecord() { 860 LOGI() << "InitPlayOrRecord"; 861 RTC_DCHECK_RUN_ON(&thread_checker_); 862 863 // There should be no audio unit at this point. 864 if (!CreateAudioUnit()) { 865 return false; 866 } 867 868 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 869 // Subscribe to audio session events. 870 [session pushDelegate:audio_session_observer_]; 871 is_interrupted_ = session.isInterrupted ? true : false; 872 873 // Lock the session to make configuration changes. 874 [session lockForConfiguration]; 875 NSError* error = nil; 876 if (![session beginWebRTCSession:&error]) { 877 [session unlockForConfiguration]; 878 RTCLogError(@"Failed to begin WebRTC session: %@", error.localizedDescription); 879 audio_unit_.reset(); 880 return false; 881 } 882 883 // If we are ready to play or record, and if the audio session can be 884 // configured, then initialize the audio unit. 885 if (session.canPlayOrRecord) { 886 if (!ConfigureAudioSession()) { 887 // One possible reason for failure is if an attempt was made to use the 888 // audio session during or after a Media Services failure. 889 // See AVAudioSessionErrorCodeMediaServicesFailed for details. 890 [session unlockForConfiguration]; 891 audio_unit_.reset(); 892 return false; 893 } 894 SetupAudioBuffersForActiveAudioSession(); 895 audio_unit_->Initialize(playout_parameters_.sample_rate()); 896 } 897 898 // Release the lock. 899 [session unlockForConfiguration]; 900 return true; 901} 902 903void AudioDeviceIOS::ShutdownPlayOrRecord() { 904 LOGI() << "ShutdownPlayOrRecord"; 905 RTC_DCHECK_RUN_ON(&thread_checker_); 906 907 // Stop the audio unit to prevent any additional audio callbacks. 908 audio_unit_->Stop(); 909 910 // Close and delete the voice-processing I/O unit. 911 audio_unit_.reset(); 912 913 // Detach thread checker for the AURemoteIO::IOThread to ensure that the 914 // next session uses a fresh thread id. 915 io_thread_checker_.Detach(); 916 917 // Remove audio session notification observers. 918 RTCAudioSession* session = [RTCAudioSession sharedInstance]; 919 [session removeDelegate:audio_session_observer_]; 920 921 // All I/O should be stopped or paused prior to deactivating the audio 922 // session, hence we deactivate as last action. 923 UnconfigureAudioSession(); 924} 925 926void AudioDeviceIOS::PrepareForNewStart() { 927 LOGI() << "PrepareForNewStart"; 928 // The audio unit has been stopped and preparations are needed for an upcoming 929 // restart. It will result in audio callbacks from a new native I/O thread 930 // which means that we must detach thread checkers here to be prepared for an 931 // upcoming new audio stream. 932 io_thread_checker_.Detach(); 933} 934 935bool AudioDeviceIOS::IsInterrupted() { 936 return is_interrupted_; 937} 938 939#pragma mark - Not Implemented 940 941int32_t AudioDeviceIOS::ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const { 942 audioLayer = AudioDeviceModule::kPlatformDefaultAudio; 943 return 0; 944} 945 946int16_t AudioDeviceIOS::PlayoutDevices() { 947 // TODO(henrika): improve. 948 RTC_LOG_F(LS_WARNING) << "Not implemented"; 949 return (int16_t)1; 950} 951 952int16_t AudioDeviceIOS::RecordingDevices() { 953 // TODO(henrika): improve. 954 RTC_LOG_F(LS_WARNING) << "Not implemented"; 955 return (int16_t)1; 956} 957 958int32_t AudioDeviceIOS::InitSpeaker() { 959 return 0; 960} 961 962bool AudioDeviceIOS::SpeakerIsInitialized() const { 963 return true; 964} 965 966int32_t AudioDeviceIOS::SpeakerVolumeIsAvailable(bool& available) { 967 available = false; 968 return 0; 969} 970 971int32_t AudioDeviceIOS::SetSpeakerVolume(uint32_t volume) { 972 RTC_NOTREACHED() << "Not implemented"; 973 return -1; 974} 975 976int32_t AudioDeviceIOS::SpeakerVolume(uint32_t& volume) const { 977 RTC_NOTREACHED() << "Not implemented"; 978 return -1; 979} 980 981int32_t AudioDeviceIOS::MaxSpeakerVolume(uint32_t& maxVolume) const { 982 RTC_NOTREACHED() << "Not implemented"; 983 return -1; 984} 985 986int32_t AudioDeviceIOS::MinSpeakerVolume(uint32_t& minVolume) const { 987 RTC_NOTREACHED() << "Not implemented"; 988 return -1; 989} 990 991int32_t AudioDeviceIOS::SpeakerMuteIsAvailable(bool& available) { 992 available = false; 993 return 0; 994} 995 996int32_t AudioDeviceIOS::SetSpeakerMute(bool enable) { 997 RTC_NOTREACHED() << "Not implemented"; 998 return -1; 999} 1000 1001int32_t AudioDeviceIOS::SpeakerMute(bool& enabled) const { 1002 RTC_NOTREACHED() << "Not implemented"; 1003 return -1; 1004} 1005 1006int32_t AudioDeviceIOS::SetPlayoutDevice(uint16_t index) { 1007 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1008 return 0; 1009} 1010 1011int32_t AudioDeviceIOS::SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType) { 1012 RTC_NOTREACHED() << "Not implemented"; 1013 return -1; 1014} 1015 1016int32_t AudioDeviceIOS::InitMicrophone() { 1017 return 0; 1018} 1019 1020bool AudioDeviceIOS::MicrophoneIsInitialized() const { 1021 return true; 1022} 1023 1024int32_t AudioDeviceIOS::MicrophoneMuteIsAvailable(bool& available) { 1025 available = false; 1026 return 0; 1027} 1028 1029int32_t AudioDeviceIOS::SetMicrophoneMute(bool enable) { 1030 RTC_NOTREACHED() << "Not implemented"; 1031 return -1; 1032} 1033 1034int32_t AudioDeviceIOS::MicrophoneMute(bool& enabled) const { 1035 RTC_NOTREACHED() << "Not implemented"; 1036 return -1; 1037} 1038 1039int32_t AudioDeviceIOS::StereoRecordingIsAvailable(bool& available) { 1040 available = false; 1041 return 0; 1042} 1043 1044int32_t AudioDeviceIOS::SetStereoRecording(bool enable) { 1045 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1046 return -1; 1047} 1048 1049int32_t AudioDeviceIOS::StereoRecording(bool& enabled) const { 1050 enabled = false; 1051 return 0; 1052} 1053 1054int32_t AudioDeviceIOS::StereoPlayoutIsAvailable(bool& available) { 1055 available = false; 1056 return 0; 1057} 1058 1059int32_t AudioDeviceIOS::SetStereoPlayout(bool enable) { 1060 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1061 return -1; 1062} 1063 1064int32_t AudioDeviceIOS::StereoPlayout(bool& enabled) const { 1065 enabled = false; 1066 return 0; 1067} 1068 1069int32_t AudioDeviceIOS::MicrophoneVolumeIsAvailable(bool& available) { 1070 available = false; 1071 return 0; 1072} 1073 1074int32_t AudioDeviceIOS::SetMicrophoneVolume(uint32_t volume) { 1075 RTC_NOTREACHED() << "Not implemented"; 1076 return -1; 1077} 1078 1079int32_t AudioDeviceIOS::MicrophoneVolume(uint32_t& volume) const { 1080 RTC_NOTREACHED() << "Not implemented"; 1081 return -1; 1082} 1083 1084int32_t AudioDeviceIOS::MaxMicrophoneVolume(uint32_t& maxVolume) const { 1085 RTC_NOTREACHED() << "Not implemented"; 1086 return -1; 1087} 1088 1089int32_t AudioDeviceIOS::MinMicrophoneVolume(uint32_t& minVolume) const { 1090 RTC_NOTREACHED() << "Not implemented"; 1091 return -1; 1092} 1093 1094int32_t AudioDeviceIOS::PlayoutDeviceName(uint16_t index, 1095 char name[kAdmMaxDeviceNameSize], 1096 char guid[kAdmMaxGuidSize]) { 1097 RTC_NOTREACHED() << "Not implemented"; 1098 return -1; 1099} 1100 1101int32_t AudioDeviceIOS::RecordingDeviceName(uint16_t index, 1102 char name[kAdmMaxDeviceNameSize], 1103 char guid[kAdmMaxGuidSize]) { 1104 RTC_NOTREACHED() << "Not implemented"; 1105 return -1; 1106} 1107 1108int32_t AudioDeviceIOS::SetRecordingDevice(uint16_t index) { 1109 RTC_LOG_F(LS_WARNING) << "Not implemented"; 1110 return 0; 1111} 1112 1113int32_t AudioDeviceIOS::SetRecordingDevice(AudioDeviceModule::WindowsDeviceType) { 1114 RTC_NOTREACHED() << "Not implemented"; 1115 return -1; 1116} 1117 1118int32_t AudioDeviceIOS::PlayoutIsAvailable(bool& available) { 1119 available = true; 1120 return 0; 1121} 1122 1123int32_t AudioDeviceIOS::RecordingIsAvailable(bool& available) { 1124 available = true; 1125 return 0; 1126} 1127 1128} // namespace ios_adm 1129} // namespace webrtc 1130