1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "GraphDriver.h"
8
9 #include "AudioNodeEngine.h"
10 #include "mozilla/dom/AudioContext.h"
11 #include "mozilla/dom/AudioDeviceInfo.h"
12 #include "mozilla/dom/BaseAudioContextBinding.h"
13 #include "mozilla/SchedulerGroup.h"
14 #include "mozilla/SharedThreadPool.h"
15 #include "mozilla/ClearOnShutdown.h"
16 #include "mozilla/Unused.h"
17 #include "mozilla/MathAlgorithms.h"
18 #include "CubebDeviceEnumerator.h"
19 #include "MediaTrackGraphImpl.h"
20 #include "AudioThreadRegistry.h"
21 #include "Tracing.h"
22
23 #ifdef MOZ_WEBRTC
24 # include "webrtc/MediaEngineWebRTC.h"
25 #endif
26
27 #ifdef XP_MACOSX
28 # include <sys/sysctl.h>
29 #endif
30
31 extern mozilla::LazyLogModule gMediaTrackGraphLog;
32 #ifdef LOG
33 # undef LOG
34 #endif // LOG
35 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
36
37 namespace mozilla {
38
GraphDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)39 GraphDriver::GraphDriver(GraphInterface* aGraphInterface,
40 GraphDriver* aPreviousDriver, uint32_t aSampleRate)
41 : mGraphInterface(aGraphInterface),
42 mSampleRate(aSampleRate),
43 mPreviousDriver(aPreviousDriver) {}
44
SetState(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)45 void GraphDriver::SetState(GraphTime aIterationStart, GraphTime aIterationEnd,
46 GraphTime aStateComputedTime) {
47 MOZ_ASSERT(InIteration() || !ThreadRunning());
48
49 mIterationStart = aIterationStart;
50 mIterationEnd = aIterationEnd;
51 mStateComputedTime = aStateComputedTime;
52 }
53
54 #ifdef DEBUG
InIteration() const55 bool GraphDriver::InIteration() const {
56 return OnThread() || Graph()->InDriverIteration(this);
57 }
58 #endif
59
PreviousDriver()60 GraphDriver* GraphDriver::PreviousDriver() {
61 MOZ_ASSERT(InIteration() || !ThreadRunning());
62 return mPreviousDriver;
63 }
64
SetPreviousDriver(GraphDriver * aPreviousDriver)65 void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver) {
66 MOZ_ASSERT(InIteration() || !ThreadRunning());
67 mPreviousDriver = aPreviousDriver;
68 }
69
ThreadedDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)70 ThreadedDriver::ThreadedDriver(GraphInterface* aGraphInterface,
71 GraphDriver* aPreviousDriver,
72 uint32_t aSampleRate)
73 : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
74 mThreadRunning(false) {}
75
76 class MediaTrackGraphShutdownThreadRunnable : public Runnable {
77 public:
MediaTrackGraphShutdownThreadRunnable(already_AddRefed<nsIThread> aThread)78 explicit MediaTrackGraphShutdownThreadRunnable(
79 already_AddRefed<nsIThread> aThread)
80 : Runnable("MediaTrackGraphShutdownThreadRunnable"), mThread(aThread) {}
Run()81 NS_IMETHOD Run() override {
82 TRACE("MediaTrackGraphShutdownThreadRunnable");
83 MOZ_ASSERT(NS_IsMainThread());
84 MOZ_ASSERT(mThread);
85
86 mThread->Shutdown();
87 mThread = nullptr;
88 return NS_OK;
89 }
90
91 private:
92 nsCOMPtr<nsIThread> mThread;
93 };
94
~ThreadedDriver()95 ThreadedDriver::~ThreadedDriver() {
96 if (mThread) {
97 nsCOMPtr<nsIRunnable> event =
98 new MediaTrackGraphShutdownThreadRunnable(mThread.forget());
99 SchedulerGroup::Dispatch(TaskCategory::Other, event.forget());
100 }
101 }
102
103 class MediaTrackGraphInitThreadRunnable : public Runnable {
104 public:
MediaTrackGraphInitThreadRunnable(ThreadedDriver * aDriver)105 explicit MediaTrackGraphInitThreadRunnable(ThreadedDriver* aDriver)
106 : Runnable("MediaTrackGraphInitThreadRunnable"), mDriver(aDriver) {}
Run()107 NS_IMETHOD Run() override {
108 TRACE("MediaTrackGraphInitThreadRunnable");
109 MOZ_ASSERT(!mDriver->ThreadRunning());
110 LOG(LogLevel::Debug, ("Starting a new system driver for graph %p",
111 mDriver->mGraphInterface.get()));
112
113 if (GraphDriver* previousDriver = mDriver->PreviousDriver()) {
114 LOG(LogLevel::Debug,
115 ("%p releasing an AudioCallbackDriver(%p), for graph %p",
116 mDriver.get(), previousDriver, mDriver->Graph()));
117 MOZ_ASSERT(!mDriver->AsAudioCallbackDriver());
118 RefPtr<AsyncCubebTask> releaseEvent =
119 new AsyncCubebTask(previousDriver->AsAudioCallbackDriver(),
120 AsyncCubebOperation::SHUTDOWN);
121 releaseEvent->Dispatch();
122 mDriver->SetPreviousDriver(nullptr);
123 }
124
125 mDriver->RunThread();
126 return NS_OK;
127 }
128
129 private:
130 RefPtr<ThreadedDriver> mDriver;
131 };
132
Start()133 void ThreadedDriver::Start() {
134 MOZ_ASSERT(!ThreadRunning());
135 LOG(LogLevel::Debug,
136 ("Starting thread for a SystemClockDriver %p", mGraphInterface.get()));
137 Unused << NS_WARN_IF(mThread);
138 MOZ_ASSERT(!mThread); // Ensure we haven't already started it
139
140 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphInitThreadRunnable(this);
141 // Note: mThread may be null during event->Run() if we pass to NewNamedThread!
142 // See AudioInitTask
143 nsresult rv = NS_NewNamedThread("MediaTrackGrph", getter_AddRefs(mThread));
144 if (NS_SUCCEEDED(rv)) {
145 mThread->EventTarget()->Dispatch(event.forget(), NS_DISPATCH_NORMAL);
146 }
147 }
148
Shutdown()149 void ThreadedDriver::Shutdown() {
150 NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
151 // mGraph's thread is not running so it's OK to do whatever here
152 LOG(LogLevel::Debug, ("Stopping threads for MediaTrackGraph %p", this));
153
154 if (mThread) {
155 LOG(LogLevel::Debug,
156 ("%p: Stopping ThreadedDriver's %p thread", Graph(), this));
157 mThread->Shutdown();
158 mThread = nullptr;
159 }
160 }
161
SystemClockDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)162 SystemClockDriver::SystemClockDriver(GraphInterface* aGraphInterface,
163 GraphDriver* aPreviousDriver,
164 uint32_t aSampleRate)
165 : ThreadedDriver(aGraphInterface, aPreviousDriver, aSampleRate),
166 mInitialTimeStamp(TimeStamp::Now()),
167 mCurrentTimeStamp(TimeStamp::Now()),
168 mLastTimeStamp(TimeStamp::Now()) {}
169
170 SystemClockDriver::~SystemClockDriver() = default;
171
RunThread()172 void ThreadedDriver::RunThread() {
173 mThreadRunning = true;
174 while (true) {
175 mIterationStart = mIterationEnd;
176 mIterationEnd += GetIntervalForIteration();
177
178 if (mStateComputedTime < mIterationEnd) {
179 LOG(LogLevel::Warning, ("%p: Global underrun detected", Graph()));
180 mIterationEnd = mStateComputedTime;
181 }
182
183 if (mIterationStart >= mIterationEnd) {
184 NS_ASSERTION(mIterationStart == mIterationEnd,
185 "Time can't go backwards!");
186 // This could happen due to low clock resolution, maybe?
187 LOG(LogLevel::Debug, ("%p: Time did not advance", Graph()));
188 }
189
190 GraphTime nextStateComputedTime =
191 MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(
192 mIterationEnd + MillisecondsToMediaTime(AUDIO_TARGET_MS));
193 if (nextStateComputedTime < mStateComputedTime) {
194 // A previous driver may have been processing further ahead of
195 // iterationEnd.
196 LOG(LogLevel::Warning,
197 ("%p: Prevent state from going backwards. interval[%ld; %ld] "
198 "state[%ld; "
199 "%ld]",
200 Graph(), (long)mIterationStart, (long)mIterationEnd,
201 (long)mStateComputedTime, (long)nextStateComputedTime));
202 nextStateComputedTime = mStateComputedTime;
203 }
204 LOG(LogLevel::Verbose,
205 ("%p: interval[%ld; %ld] state[%ld; %ld]", Graph(),
206 (long)mIterationStart, (long)mIterationEnd, (long)mStateComputedTime,
207 (long)nextStateComputedTime));
208
209 mStateComputedTime = nextStateComputedTime;
210 IterationResult result =
211 Graph()->OneIteration(mStateComputedTime, mIterationEnd, nullptr);
212
213 if (result.IsStop()) {
214 // Signal that we're done stopping.
215 result.Stopped();
216 break;
217 }
218 WaitForNextIteration();
219 if (GraphDriver* nextDriver = result.NextDriver()) {
220 LOG(LogLevel::Debug, ("%p: Switching to AudioCallbackDriver", Graph()));
221 result.Switched();
222 nextDriver->SetState(mIterationStart, mIterationEnd, mStateComputedTime);
223 nextDriver->Start();
224 break;
225 }
226 MOZ_ASSERT(result.IsStillProcessing());
227 }
228 mThreadRunning = false;
229 }
230
GetIntervalForIteration()231 MediaTime SystemClockDriver::GetIntervalForIteration() {
232 TimeStamp now = TimeStamp::Now();
233 MediaTime interval =
234 SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds());
235 mCurrentTimeStamp = now;
236
237 MOZ_LOG(gMediaTrackGraphLog, LogLevel::Verbose,
238 ("%p: Updating current time to %f (real %f, StateComputedTime() %f)",
239 Graph(), MediaTimeToSeconds(mIterationEnd + interval),
240 (now - mInitialTimeStamp).ToSeconds(),
241 MediaTimeToSeconds(mStateComputedTime)));
242
243 return interval;
244 }
245
EnsureNextIteration()246 void ThreadedDriver::EnsureNextIteration() {
247 mWaitHelper.EnsureNextIteration();
248 }
249
WaitForNextIteration()250 void ThreadedDriver::WaitForNextIteration() {
251 MOZ_ASSERT(mThread);
252 MOZ_ASSERT(OnThread());
253 mWaitHelper.WaitForNextIterationAtLeast(WaitInterval());
254 }
255
WaitInterval()256 TimeDuration SystemClockDriver::WaitInterval() {
257 MOZ_ASSERT(mThread);
258 MOZ_ASSERT(OnThread());
259 TimeStamp now = TimeStamp::Now();
260 int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS -
261 int64_t((now - mCurrentTimeStamp).ToMilliseconds());
262 // Make sure timeoutMS doesn't overflow 32 bits by waking up at
263 // least once a minute, if we need to wake up at all
264 timeoutMS = std::max<int64_t>(0, std::min<int64_t>(timeoutMS, 60 * 1000));
265 LOG(LogLevel::Verbose,
266 ("%p: Waiting for next iteration; at %f, timeout=%f", Graph(),
267 (now - mInitialTimeStamp).ToSeconds(), timeoutMS / 1000.0));
268
269 return TimeDuration::FromMilliseconds(timeoutMS);
270 }
271
OfflineClockDriver(GraphInterface * aGraphInterface,uint32_t aSampleRate,GraphTime aSlice)272 OfflineClockDriver::OfflineClockDriver(GraphInterface* aGraphInterface,
273 uint32_t aSampleRate, GraphTime aSlice)
274 : ThreadedDriver(aGraphInterface, nullptr, aSampleRate), mSlice(aSlice) {}
275
276 OfflineClockDriver::~OfflineClockDriver() = default;
277
RunThread()278 void OfflineClockDriver::RunThread() {
279 nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(mThread);
280 nsCOMPtr<nsIThreadObserver> observer = do_QueryInterface(Graph());
281 threadInternal->SetObserver(observer);
282
283 ThreadedDriver::RunThread();
284 }
285
GetIntervalForIteration()286 MediaTime OfflineClockDriver::GetIntervalForIteration() {
287 return MillisecondsToMediaTime(mSlice);
288 }
289
AsyncCubebTask(AudioCallbackDriver * aDriver,AsyncCubebOperation aOperation)290 AsyncCubebTask::AsyncCubebTask(AudioCallbackDriver* aDriver,
291 AsyncCubebOperation aOperation)
292 : Runnable("AsyncCubebTask"),
293 mDriver(aDriver),
294 mOperation(aOperation),
295 mShutdownGrip(aDriver->Graph()) {
296 NS_WARNING_ASSERTION(
297 mDriver->mAudioStream || aOperation == AsyncCubebOperation::INIT,
298 "No audio stream!");
299 }
300
301 AsyncCubebTask::~AsyncCubebTask() = default;
302
303 NS_IMETHODIMP
Run()304 AsyncCubebTask::Run() {
305 MOZ_ASSERT(mDriver);
306
307 switch (mOperation) {
308 case AsyncCubebOperation::INIT: {
309 LOG(LogLevel::Debug, ("%p: AsyncCubebOperation::INIT driver=%p",
310 mDriver->Graph(), mDriver.get()));
311 mDriver->Init();
312 break;
313 }
314 case AsyncCubebOperation::SHUTDOWN: {
315 LOG(LogLevel::Debug, ("%p: AsyncCubebOperation::SHUTDOWN driver=%p",
316 mDriver->Graph(), mDriver.get()));
317 mDriver->Stop();
318 mDriver = nullptr;
319 mShutdownGrip = nullptr;
320 break;
321 }
322 default:
323 MOZ_CRASH("Operation not implemented.");
324 }
325
326 // The thread will kill itself after a bit
327 return NS_OK;
328 }
329
TrackAndPromiseForOperation(MediaTrack * aTrack,dom::AudioContextOperation aOperation,AbstractThread * aMainThread,MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise> && aHolder)330 TrackAndPromiseForOperation::TrackAndPromiseForOperation(
331 MediaTrack* aTrack, dom::AudioContextOperation aOperation,
332 AbstractThread* aMainThread,
333 MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise>&& aHolder)
334 : mTrack(aTrack),
335 mOperation(aOperation),
336 mMainThread(aMainThread),
337 mHolder(std::move(aHolder)) {}
338
TrackAndPromiseForOperation(TrackAndPromiseForOperation && aOther)339 TrackAndPromiseForOperation::TrackAndPromiseForOperation(
340 TrackAndPromiseForOperation&& aOther) noexcept
341 : mTrack(std::move(aOther.mTrack)),
342 mOperation(aOther.mOperation),
343 mMainThread(std::move(aOther.mMainThread)),
344 mHolder(std::move(aOther.mHolder)) {}
345
346 /* Helper to proxy the GraphInterface methods used by a running
347 * mFallbackDriver. */
348 class AudioCallbackDriver::FallbackWrapper : public GraphInterface {
349 public:
FallbackWrapper(RefPtr<GraphInterface> aGraph,RefPtr<AudioCallbackDriver> aOwner,uint32_t aSampleRate,GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)350 FallbackWrapper(RefPtr<GraphInterface> aGraph,
351 RefPtr<AudioCallbackDriver> aOwner, uint32_t aSampleRate,
352 GraphTime aIterationStart, GraphTime aIterationEnd,
353 GraphTime aStateComputedTime)
354 : mGraph(std::move(aGraph)),
355 mOwner(std::move(aOwner)),
356 mFallbackDriver(
357 MakeRefPtr<SystemClockDriver>(this, nullptr, aSampleRate)),
358 mIterationStart(aIterationStart),
359 mIterationEnd(aIterationEnd),
360 mStateComputedTime(aStateComputedTime) {
361 mFallbackDriver->SetState(mIterationStart, mIterationEnd,
362 mStateComputedTime);
363 }
364
365 NS_DECL_THREADSAFE_ISUPPORTS
366
367 /* Proxied SystemClockDriver methods */
SetState(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)368 void SetState(GraphTime aIterationStart, GraphTime aIterationEnd,
369 GraphTime aStateComputedTime) {
370 mIterationStart = aIterationStart;
371 mIterationEnd = aIterationEnd;
372 mStateComputedTime = aStateComputedTime;
373 mFallbackDriver->SetState(aIterationStart, aIterationEnd,
374 aStateComputedTime);
375 }
Start()376 void Start() { mFallbackDriver->Start(); }
Shutdown()377 MOZ_CAN_RUN_SCRIPT void Shutdown() {
378 RefPtr<SystemClockDriver> driver = mFallbackDriver;
379 driver->Shutdown();
380 }
EnsureNextIteration()381 void EnsureNextIteration() { mFallbackDriver->EnsureNextIteration(); }
382 #ifdef DEBUG
InIteration()383 bool InIteration() { return mFallbackDriver->InIteration(); }
384 #endif
OnThread()385 bool OnThread() { return mFallbackDriver->OnThread(); }
386
387 /* GraphInterface methods */
NotifyOutputData(AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)388 void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
389 TrackRate aRate, uint32_t aChannels) override {
390 MOZ_CRASH("Unexpected NotifyOutputData from fallback SystemClockDriver");
391 }
NotifyInputStopped()392 void NotifyInputStopped() override {
393 MOZ_CRASH("Unexpected NotifyInputStopped from fallback SystemClockDriver");
394 }
NotifyInputData(const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels,uint32_t aAlreadyBuffered)395 void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
396 TrackRate aRate, uint32_t aChannels,
397 uint32_t aAlreadyBuffered) override {
398 MOZ_CRASH("Unexpected NotifyInputData from fallback SystemClockDriver");
399 }
DeviceChanged()400 void DeviceChanged() override {
401 MOZ_CRASH("Unexpected DeviceChanged from fallback SystemClockDriver");
402 }
403 #ifdef DEBUG
InDriverIteration(const GraphDriver * aDriver) const404 bool InDriverIteration(const GraphDriver* aDriver) const override {
405 return !mOwner->ThreadRunning() && mOwner->InIteration();
406 }
407 #endif
OneIteration(GraphTime aStateComputedEnd,GraphTime aIterationEnd,AudioMixer * aMixer)408 IterationResult OneIteration(GraphTime aStateComputedEnd,
409 GraphTime aIterationEnd,
410 AudioMixer* aMixer) override {
411 MOZ_ASSERT(!aMixer);
412
413 #ifdef DEBUG
414 AutoInCallback aic(mOwner);
415 #endif
416
417 mIterationStart = mIterationEnd;
418 mIterationEnd = aIterationEnd;
419 mStateComputedTime = aStateComputedEnd;
420 IterationResult result =
421 mGraph->OneIteration(aStateComputedEnd, aIterationEnd, aMixer);
422
423 AudioStreamState audioState = mOwner->mAudioStreamState;
424
425 MOZ_ASSERT(audioState != AudioStreamState::Stopping,
426 "The audio driver can only enter stopping if it iterated the "
427 "graph, which it can only do if there's no fallback driver");
428 if (audioState != AudioStreamState::Running && result.IsStillProcessing()) {
429 mOwner->MaybeStartAudioStream();
430 return result;
431 }
432
433 MOZ_ASSERT(result.IsStillProcessing() || result.IsStop() ||
434 result.IsSwitchDriver());
435
436 // Proxy the release of the fallback driver to a background thread, so it
437 // doesn't perform unexpected suicide.
438 IterationResult stopFallback =
439 IterationResult::CreateStop(NS_NewRunnableFunction(
440 "AudioCallbackDriver::FallbackDriverStopped",
441 [self = RefPtr<FallbackWrapper>(this), this,
442 result = std::move(result)]() mutable {
443 FallbackDriverState fallbackState =
444 result.IsStillProcessing() ? FallbackDriverState::None
445 : FallbackDriverState::Stopped;
446 mOwner->FallbackDriverStopped(mIterationStart, mIterationEnd,
447 mStateComputedTime, fallbackState);
448
449 if (fallbackState == FallbackDriverState::Stopped) {
450 #ifdef DEBUG
451 // The AudioCallbackDriver may not iterate the graph, but we'll
452 // call into it so we need to be regarded as "in iteration".
453 AutoInCallback aic(mOwner);
454 #endif
455 if (GraphDriver* nextDriver = result.NextDriver()) {
456 LOG(LogLevel::Debug,
457 ("%p: Switching from fallback to other driver.",
458 mGraph.get()));
459 result.Switched();
460 nextDriver->SetState(mIterationStart, mIterationEnd,
461 mStateComputedTime);
462 nextDriver->Start();
463 } else if (result.IsStop()) {
464 LOG(LogLevel::Debug,
465 ("%p: Stopping fallback driver.", mGraph.get()));
466 result.Stopped();
467 }
468 }
469 mOwner = nullptr;
470 NS_DispatchBackgroundTask(NS_NewRunnableFunction(
471 "AudioCallbackDriver::FallbackDriverStopped::Release",
472 [fallback = std::move(self->mFallbackDriver)] {}));
473 }));
474
475 return stopFallback;
476 }
477
478 private:
479 virtual ~FallbackWrapper() = default;
480
481 const RefPtr<GraphInterface> mGraph;
482 // Valid until mFallbackDriver has finished its last iteration.
483 RefPtr<AudioCallbackDriver> mOwner;
484 RefPtr<SystemClockDriver> mFallbackDriver;
485
486 GraphTime mIterationStart;
487 GraphTime mIterationEnd;
488 GraphTime mStateComputedTime;
489 };
490
NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)491 NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)
492
493 AudioCallbackDriver::AudioCallbackDriver(
494 GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
495 uint32_t aSampleRate, uint32_t aOutputChannelCount,
496 uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
497 CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType)
498 : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
499 mOutputChannelCount(aOutputChannelCount),
500 mInputChannelCount(aInputChannelCount),
501 mOutputDeviceID(aOutputDeviceID),
502 mInputDeviceID(aInputDeviceID),
503 mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS),
504 mStarted(false),
505 mInitShutdownThread(SharedThreadPool::Get("CubebOperation"_ns, 1)),
506 mAudioThreadId(ProfilerThreadId{}),
507 mAudioThreadIdInCb(std::thread::id()),
508 mAudioStreamState(AudioStreamState::None),
509 mFallback("AudioCallbackDriver::mFallback"),
510 mSandboxed(CubebUtils::SandboxEnabled()) {
511 LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p ctor - input: device %p, "
512 "channel %d, output: device %p, channel %d",
513 Graph(), this, mInputDeviceID, mInputChannelCount,
514 mOutputDeviceID, mOutputChannelCount));
515
516 NS_WARNING_ASSERTION(mOutputChannelCount != 0,
517 "Invalid output channel count");
518 MOZ_ASSERT(mOutputChannelCount <= 8);
519
520 const uint32_t kIdleThreadTimeoutMs = 2000;
521 mInitShutdownThread->SetIdleThreadTimeout(
522 PR_MillisecondsToInterval(kIdleThreadTimeoutMs));
523
524 if (aAudioInputType == AudioInputType::Voice) {
525 LOG(LogLevel::Debug, ("VOICE."));
526 mInputDevicePreference = CUBEB_DEVICE_PREF_VOICE;
527 CubebUtils::SetInCommunication(true);
528 } else {
529 mInputDevicePreference = CUBEB_DEVICE_PREF_ALL;
530 }
531
532 mMixer.AddCallback(WrapNotNull(this));
533 }
534
~AudioCallbackDriver()535 AudioCallbackDriver::~AudioCallbackDriver() {
536 if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
537 CubebUtils::SetInCommunication(false);
538 }
539 }
540
IsMacbookOrMacbookAir()541 bool IsMacbookOrMacbookAir() {
542 #ifdef XP_MACOSX
543 size_t len = 0;
544 sysctlbyname("hw.model", NULL, &len, NULL, 0);
545 if (len) {
546 UniquePtr<char[]> model(new char[len]);
547 // This string can be
548 // MacBook%d,%d for a normal MacBook
549 // MacBookAir%d,%d for a Macbook Air
550 sysctlbyname("hw.model", model.get(), &len, NULL, 0);
551 char* substring = strstr(model.get(), "MacBook");
552 if (substring) {
553 const size_t offset = strlen("MacBook");
554 if (!strncmp(model.get() + offset, "Air", 3) ||
555 isdigit(model[offset + 1])) {
556 return true;
557 }
558 }
559 }
560 #endif
561 return false;
562 }
563
Init()564 void AudioCallbackDriver::Init() {
565 TRACE("AudioCallbackDriver::Init");
566 MOZ_ASSERT(OnCubebOperationThread());
567 MOZ_ASSERT(mAudioStreamState == AudioStreamState::Pending);
568 FallbackDriverState fallbackState = mFallbackDriverState;
569 if (fallbackState == FallbackDriverState::Stopped) {
570 // The graph has already stopped us.
571 return;
572 }
573 bool fromFallback = fallbackState == FallbackDriverState::Running;
574 cubeb* cubebContext = CubebUtils::GetCubebContext();
575 if (!cubebContext) {
576 NS_WARNING("Could not get cubeb context.");
577 LOG(LogLevel::Warning, ("%s: Could not get cubeb context", __func__));
578 mAudioStreamState = AudioStreamState::None;
579 if (!fromFallback) {
580 CubebUtils::ReportCubebStreamInitFailure(true);
581 FallbackToSystemClockDriver();
582 }
583 return;
584 }
585
586 cubeb_stream_params output;
587 cubeb_stream_params input;
588 bool firstStream = CubebUtils::GetFirstStream();
589
590 MOZ_ASSERT(!NS_IsMainThread(),
591 "This is blocking and should never run on the main thread.");
592
593 output.rate = mSampleRate;
594
595 #ifdef MOZ_SAMPLE_TYPE_S16
596 MOZ_ASSERT(AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_S16);
597 output.format = CUBEB_SAMPLE_S16NE;
598 #else
599 MOZ_ASSERT(AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_FLOAT32);
600 output.format = CUBEB_SAMPLE_FLOAT32NE;
601 #endif
602
603 if (!mOutputChannelCount) {
604 LOG(LogLevel::Warning, ("Output number of channels is 0."));
605 mAudioStreamState = AudioStreamState::None;
606 if (!fromFallback) {
607 CubebUtils::ReportCubebStreamInitFailure(firstStream);
608 FallbackToSystemClockDriver();
609 }
610 return;
611 }
612
613 CubebUtils::AudioDeviceID forcedOutputDeviceId = nullptr;
614
615 char* forcedOutputDeviceName = CubebUtils::GetForcedOutputDevice();
616 if (forcedOutputDeviceName) {
617 RefPtr<CubebDeviceEnumerator> enumerator = Enumerator::GetInstance();
618 RefPtr<AudioDeviceInfo> device = enumerator->DeviceInfoFromName(
619 NS_ConvertUTF8toUTF16(forcedOutputDeviceName), EnumeratorSide::OUTPUT);
620 if (device && device->DeviceID()) {
621 forcedOutputDeviceId = device->DeviceID();
622 }
623 }
624
625 mBuffer = AudioCallbackBufferWrapper<AudioDataValue>(mOutputChannelCount);
626 mScratchBuffer =
627 SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2>(mOutputChannelCount);
628
629 output.channels = mOutputChannelCount;
630 AudioConfig::ChannelLayout::ChannelMap channelMap =
631 AudioConfig::ChannelLayout(mOutputChannelCount).Map();
632
633 output.layout = static_cast<uint32_t>(channelMap);
634 output.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_OUTPUT);
635 if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE &&
636 CubebUtils::RouteOutputAsVoice()) {
637 output.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
638 }
639
640 uint32_t latencyFrames = CubebUtils::GetCubebMTGLatencyInFrames(&output);
641
642 // Macbook and MacBook air don't have enough CPU to run very low latency
643 // MediaTrackGraphs, cap the minimal latency to 512 frames int this case.
644 if (IsMacbookOrMacbookAir()) {
645 latencyFrames = std::max((uint32_t)512, latencyFrames);
646 }
647
648 // On OSX, having a latency that is lower than 10ms is very common. It's
649 // not very useful when doing voice, because all the WebRTC code deal in 10ms
650 // chunks of audio. Take the first power of two above 10ms at the current
651 // rate in this case. It's probably 512, for common rates.
652 #if defined(XP_MACOSX)
653 if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
654 if (latencyFrames < mSampleRate / 100) {
655 latencyFrames = mozilla::RoundUpPow2(mSampleRate / 100);
656 }
657 }
658 #endif
659 LOG(LogLevel::Debug, ("Effective latency in frames: %d", latencyFrames));
660
661 input = output;
662 input.channels = mInputChannelCount;
663 input.layout = CUBEB_LAYOUT_UNDEFINED;
664 input.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_INPUT);
665 if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
666 input.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
667 }
668
669 cubeb_stream* stream = nullptr;
670 bool inputWanted = mInputChannelCount > 0;
671 CubebUtils::AudioDeviceID outputId = mOutputDeviceID;
672 CubebUtils::AudioDeviceID inputId = mInputDeviceID;
673
674 // XXX Only pass input input if we have an input listener. Always
675 // set up output because it's easier, and it will just get silence.
676 if (cubeb_stream_init(cubebContext, &stream, "AudioCallbackDriver", inputId,
677 inputWanted ? &input : nullptr,
678 forcedOutputDeviceId ? forcedOutputDeviceId : outputId,
679 &output, latencyFrames, DataCallback_s, StateCallback_s,
680 this) == CUBEB_OK) {
681 mAudioStream.own(stream);
682 DebugOnly<int> rv =
683 cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale());
684 NS_WARNING_ASSERTION(
685 rv == CUBEB_OK,
686 "Could not set the audio stream volume in GraphDriver.cpp");
687 CubebUtils::ReportCubebBackendUsed();
688 } else {
689 NS_WARNING(
690 "Could not create a cubeb stream for MediaTrackGraph, falling "
691 "back to a SystemClockDriver");
692 mAudioStreamState = AudioStreamState::None;
693 // Only report failures when we're not coming from a driver that was
694 // created itself as a fallback driver because of a previous audio driver
695 // failure.
696 if (!fromFallback) {
697 CubebUtils::ReportCubebStreamInitFailure(firstStream);
698 FallbackToSystemClockDriver();
699 }
700 return;
701 }
702
703 #ifdef XP_MACOSX
704 PanOutputIfNeeded(inputWanted);
705 #endif
706
707 cubeb_stream_register_device_changed_callback(
708 mAudioStream, AudioCallbackDriver::DeviceChangedCallback_s);
709
710 // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable. This
711 // is intended for diagnosing issues, and only works if the content sandbox is
712 // disabled.
713 mInputStreamFile.Open("GraphDriverInput", input.channels, input.rate);
714 mOutputStreamFile.Open("GraphDriverOutput", output.channels, output.rate);
715
716 if (NS_WARN_IF(!StartStream())) {
717 LOG(LogLevel::Warning,
718 ("%p: AudioCallbackDriver couldn't start a cubeb stream.", Graph()));
719 return;
720 }
721
722 LOG(LogLevel::Debug, ("%p: AudioCallbackDriver started.", Graph()));
723 }
724
Start()725 void AudioCallbackDriver::Start() {
726 MOZ_ASSERT(!IsStarted());
727 MOZ_ASSERT(mAudioStreamState == AudioStreamState::None);
728 MOZ_ASSERT_IF(PreviousDriver(), PreviousDriver()->InIteration());
729 mAudioStreamState = AudioStreamState::Pending;
730
731 if (mFallbackDriverState == FallbackDriverState::None) {
732 // Starting an audio driver could take a while. We start a system driver in
733 // the meantime so that the graph is kept running.
734 FallbackToSystemClockDriver();
735 }
736
737 if (mPreviousDriver) {
738 if (mPreviousDriver->AsAudioCallbackDriver()) {
739 LOG(LogLevel::Debug, ("Releasing audio driver off main thread."));
740 RefPtr<AsyncCubebTask> releaseEvent =
741 new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(),
742 AsyncCubebOperation::SHUTDOWN);
743 releaseEvent->Dispatch();
744 } else {
745 LOG(LogLevel::Debug,
746 ("Dropping driver reference for SystemClockDriver."));
747 MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver());
748 }
749 mPreviousDriver = nullptr;
750 }
751
752 LOG(LogLevel::Debug, ("Starting new audio driver off main thread, "
753 "to ensure it runs after previous shutdown."));
754 RefPtr<AsyncCubebTask> initEvent =
755 new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT);
756 initEvent->Dispatch();
757 }
758
StartStream()759 bool AudioCallbackDriver::StartStream() {
760 TRACE("AudioCallbackDriver::StartStream");
761 MOZ_ASSERT(!IsStarted() && OnCubebOperationThread());
762 // Set mStarted before cubeb_stream_start, since starting the cubeb stream can
763 // result in a callback (that may read mStarted) before mStarted would
764 // otherwise be set to true.
765 mStarted = true;
766 if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
767 NS_WARNING("Could not start cubeb stream for MTG.");
768 mStarted = false;
769 return false;
770 }
771
772 return true;
773 }
774
Stop()775 void AudioCallbackDriver::Stop() {
776 TRACE("AudioCallbackDriver::Stop");
777 MOZ_ASSERT(OnCubebOperationThread());
778 cubeb_stream_register_device_changed_callback(mAudioStream, nullptr);
779 if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) {
780 NS_WARNING("Could not stop cubeb stream for MTG.");
781 }
782 }
783
Shutdown()784 void AudioCallbackDriver::Shutdown() {
785 MOZ_ASSERT(NS_IsMainThread());
786 RefPtr<FallbackWrapper> fallback;
787 {
788 auto fallbackLock = mFallback.Lock();
789 fallback = fallbackLock.ref();
790 fallbackLock.ref() = nullptr;
791 }
792 if (fallback) {
793 LOG(LogLevel::Debug,
794 ("%p: Releasing fallback driver %p.", Graph(), fallback.get()));
795 fallback->Shutdown();
796 }
797
798 LOG(LogLevel::Debug,
799 ("%p: Releasing audio driver off main thread (GraphDriver::Shutdown).",
800 Graph()));
801
802 RefPtr<AsyncCubebTask> releaseEvent =
803 new AsyncCubebTask(this, AsyncCubebOperation::SHUTDOWN);
804 releaseEvent->Dispatch(NS_DISPATCH_SYNC);
805 }
806
807 /* static */
DataCallback_s(cubeb_stream * aStream,void * aUser,const void * aInputBuffer,void * aOutputBuffer,long aFrames)808 long AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream, void* aUser,
809 const void* aInputBuffer,
810 void* aOutputBuffer, long aFrames) {
811 AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
812 return driver->DataCallback(static_cast<const AudioDataValue*>(aInputBuffer),
813 static_cast<AudioDataValue*>(aOutputBuffer),
814 aFrames);
815 }
816
817 /* static */
StateCallback_s(cubeb_stream * aStream,void * aUser,cubeb_state aState)818 void AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void* aUser,
819 cubeb_state aState) {
820 AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
821 driver->StateCallback(aState);
822 }
823
824 /* static */
DeviceChangedCallback_s(void * aUser)825 void AudioCallbackDriver::DeviceChangedCallback_s(void* aUser) {
826 AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
827 driver->DeviceChangedCallback();
828 }
829
AutoInCallback(AudioCallbackDriver * aDriver)830 AudioCallbackDriver::AutoInCallback::AutoInCallback(
831 AudioCallbackDriver* aDriver)
832 : mDriver(aDriver) {
833 MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::thread::id());
834 mDriver->mAudioThreadIdInCb = std::this_thread::get_id();
835 }
836
~AutoInCallback()837 AudioCallbackDriver::AutoInCallback::~AutoInCallback() {
838 MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::this_thread::get_id());
839 mDriver->mAudioThreadIdInCb = std::thread::id();
840 }
841
CheckThreadIdChanged()842 bool AudioCallbackDriver::CheckThreadIdChanged() {
843 ProfilerThreadId id = profiler_current_thread_id();
844 if (id != mAudioThreadId) {
845 mAudioThreadId = id;
846 return true;
847 }
848 return false;
849 }
850
DataCallback(const AudioDataValue * aInputBuffer,AudioDataValue * aOutputBuffer,long aFrames)851 long AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer,
852 AudioDataValue* aOutputBuffer,
853 long aFrames) {
854 if (!mSandboxed && CheckThreadIdChanged()) {
855 CubebUtils::GetAudioThreadRegistry()->Register(mAudioThreadId);
856 }
857
858 if (mAudioStreamState.compareExchange(AudioStreamState::Pending,
859 AudioStreamState::Running)) {
860 LOG(LogLevel::Verbose, ("%p: AudioCallbackDriver %p First audio callback "
861 "close the Fallback driver",
862 Graph(), this));
863 }
864
865 FallbackDriverState fallbackState = mFallbackDriverState;
866 if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Running)) {
867 // Wait for the fallback driver to stop. Wake it up so it can stop if it's
868 // sleeping.
869 LOG(LogLevel::Verbose,
870 ("%p: AudioCallbackDriver %p Waiting for the Fallback driver to stop",
871 Graph(), this));
872 EnsureNextIteration();
873 PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
874 return aFrames;
875 }
876
877 if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Stopped)) {
878 // We're supposed to stop.
879 PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
880 if (!mSandboxed) {
881 CubebUtils::GetAudioThreadRegistry()->Unregister(mAudioThreadId);
882 }
883 return aFrames - 1;
884 }
885
886 MOZ_ASSERT(ThreadRunning());
887 TRACE_AUDIO_CALLBACK_BUDGET(aFrames, mSampleRate);
888 TRACE("AudioCallbackDriver::DataCallback");
889
890 #ifdef DEBUG
891 AutoInCallback aic(this);
892 #endif
893
894 uint32_t durationMS = aFrames * 1000 / mSampleRate;
895
896 // For now, simply average the duration with the previous
897 // duration so there is some damping against sudden changes.
898 if (!mIterationDurationMS) {
899 mIterationDurationMS = durationMS;
900 } else {
901 mIterationDurationMS = (mIterationDurationMS * 3) + durationMS;
902 mIterationDurationMS /= 4;
903 }
904
905 mBuffer.SetBuffer(aOutputBuffer, aFrames);
906 // fill part or all with leftover data from last iteration (since we
907 // align to Audio blocks)
908 uint32_t alreadyBuffered = mScratchBuffer.Empty(mBuffer);
909
910 // State computed time is decided by the audio callback's buffer length. We
911 // compute the iteration start and end from there, trying to keep the amount
912 // of buffering in the graph constant.
913 GraphTime nextStateComputedTime =
914 MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(mStateComputedTime +
915 mBuffer.Available());
916
917 mIterationStart = mIterationEnd;
918 // inGraph is the number of audio frames there is between the state time and
919 // the current time, i.e. the maximum theoretical length of the interval we
920 // could use as [mIterationStart; mIterationEnd].
921 GraphTime inGraph = mStateComputedTime - mIterationStart;
922 // We want the interval [mIterationStart; mIterationEnd] to be before the
923 // interval [mStateComputedTime; nextStateComputedTime]. We also want
924 // the distance between these intervals to be roughly equivalent each time, to
925 // ensure there is no clock drift between current time and state time. Since
926 // we can't act on the state time because we have to fill the audio buffer, we
927 // reclock the current time against the state time, here.
928 mIterationEnd = mIterationStart + 0.8 * inGraph;
929
930 LOG(LogLevel::Verbose,
931 ("%p: interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) "
932 "(duration ticks: %ld)",
933 Graph(), (long)mIterationStart, (long)mIterationEnd,
934 (long)mStateComputedTime, (long)nextStateComputedTime, (long)aFrames,
935 (uint32_t)durationMS,
936 (long)(nextStateComputedTime - mStateComputedTime)));
937
938 if (mStateComputedTime < mIterationEnd) {
939 LOG(LogLevel::Error, ("%p: Media graph global underrun detected", Graph()));
940 MOZ_ASSERT_UNREACHABLE("We should not underrun in full duplex");
941 mIterationEnd = mStateComputedTime;
942 }
943
944 // Process mic data if any/needed
945 if (aInputBuffer && mInputChannelCount > 0) {
946 Graph()->NotifyInputData(aInputBuffer, static_cast<size_t>(aFrames),
947 mSampleRate, mInputChannelCount, alreadyBuffered);
948 }
949
950 IterationResult result =
951 Graph()->OneIteration(nextStateComputedTime, mIterationEnd, &mMixer);
952
953 mStateComputedTime = nextStateComputedTime;
954
955 MOZ_ASSERT(mBuffer.Available() == 0,
956 "The graph should have filled the buffer");
957
958 mBuffer.BufferFilled();
959
960 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
961 // Prevent returning NaN to the OS mixer, and propagating NaN into the reverse
962 // stream of the AEC.
963 NaNToZeroInPlace(aOutputBuffer, aFrames * mOutputChannelCount);
964 #endif
965
966 // Callback any observers for the AEC speaker data. Note that one
967 // (maybe) of these will be full-duplex, the others will get their input
968 // data off separate cubeb callbacks. Take care with how stuff is
969 // removed/added to this list and TSAN issues, but input and output will
970 // use separate callback methods.
971 Graph()->NotifyOutputData(aOutputBuffer, static_cast<size_t>(aFrames),
972 mSampleRate, mOutputChannelCount);
973
974 #ifdef XP_MACOSX
975 // This only happens when the output is on a macbookpro's external speaker,
976 // that are stereo, but let's just be safe.
977 if (mNeedsPanning && mOutputChannelCount == 2) {
978 // hard pan to the right
979 for (uint32_t i = 0; i < aFrames * 2; i += 2) {
980 aOutputBuffer[i + 1] += aOutputBuffer[i];
981 aOutputBuffer[i] = 0.0;
982 }
983 }
984 #endif
985
986 // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable
987 if (aInputBuffer) {
988 mInputStreamFile.Write(static_cast<const AudioDataValue*>(aInputBuffer),
989 aFrames * mInputChannelCount);
990 }
991 mOutputStreamFile.Write(static_cast<const AudioDataValue*>(aOutputBuffer),
992 aFrames * mOutputChannelCount);
993
994 if (result.IsStop()) {
995 if (mInputDeviceID) {
996 mGraphInterface->NotifyInputStopped();
997 }
998 // Signal that we have stopped.
999 result.Stopped();
1000 // Update the flag before handing over the graph and going to drain.
1001 mAudioStreamState = AudioStreamState::Stopping;
1002 if (!mSandboxed) {
1003 CubebUtils::GetAudioThreadRegistry()->Unregister(mAudioThreadId);
1004 }
1005 return aFrames - 1;
1006 }
1007
1008 if (GraphDriver* nextDriver = result.NextDriver()) {
1009 LOG(LogLevel::Debug,
1010 ("%p: Switching to %s driver.", Graph(),
1011 nextDriver->AsAudioCallbackDriver() ? "audio" : "system"));
1012 if (mInputDeviceID) {
1013 mGraphInterface->NotifyInputStopped();
1014 }
1015 result.Switched();
1016 mAudioStreamState = AudioStreamState::Stopping;
1017 nextDriver->SetState(mIterationStart, mIterationEnd, mStateComputedTime);
1018 nextDriver->Start();
1019 if (!mSandboxed) {
1020 CubebUtils::GetAudioThreadRegistry()->Unregister(mAudioThreadId);
1021 }
1022 // Returning less than aFrames starts the draining and eventually stops the
1023 // audio thread. This function will never get called again.
1024 return aFrames - 1;
1025 }
1026
1027 MOZ_ASSERT(result.IsStillProcessing());
1028 return aFrames;
1029 }
1030
StateToString(cubeb_state aState)1031 static const char* StateToString(cubeb_state aState) {
1032 switch (aState) {
1033 case CUBEB_STATE_STARTED:
1034 return "STARTED";
1035 case CUBEB_STATE_STOPPED:
1036 return "STOPPED";
1037 case CUBEB_STATE_DRAINED:
1038 return "DRAINED";
1039 case CUBEB_STATE_ERROR:
1040 return "ERROR";
1041 default:
1042 MOZ_CRASH("Unexpected state!");
1043 }
1044 }
1045
StateCallback(cubeb_state aState)1046 void AudioCallbackDriver::StateCallback(cubeb_state aState) {
1047 MOZ_ASSERT(!InIteration());
1048 LOG(LogLevel::Debug,
1049 ("AudioCallbackDriver(%p) State: %s", this, StateToString(aState)));
1050
1051 AudioStreamState streamState = mAudioStreamState;
1052 if (aState != CUBEB_STATE_STARTED) {
1053 // Clear the flag for the not running states: stopped, drained, error.
1054 streamState = mAudioStreamState.exchange(AudioStreamState::None);
1055 }
1056
1057 if (aState == CUBEB_STATE_ERROR) {
1058 // About to hand over control of the graph. Do not start a new driver if
1059 // StateCallback() receives an error for this stream while the main thread
1060 // or another driver has control of the graph.
1061 if (streamState == AudioStreamState::Running) {
1062 MOZ_ASSERT(!ThreadRunning());
1063 mStarted = false;
1064 if (mFallbackDriverState == FallbackDriverState::None) {
1065 // Only switch to fallback if it's not already running. It could be
1066 // running with the callback driver having started but not seen a single
1067 // callback yet. I.e., handover from fallback to callback is not done.
1068 if (mInputDeviceID) {
1069 #ifdef DEBUG
1070 // No audio callback after an error. We're calling into the graph here
1071 // so we need to be regarded as "in iteration".
1072 AutoInCallback aic(this);
1073 #endif
1074 mGraphInterface->NotifyInputStopped();
1075 }
1076 FallbackToSystemClockDriver();
1077 }
1078 }
1079 } else if (aState == CUBEB_STATE_STOPPED) {
1080 MOZ_ASSERT(!ThreadRunning());
1081 mStarted = false;
1082 }
1083 }
1084
MixerCallback(AudioDataValue * aMixedBuffer,AudioSampleFormat aFormat,uint32_t aChannels,uint32_t aFrames,uint32_t aSampleRate)1085 void AudioCallbackDriver::MixerCallback(AudioDataValue* aMixedBuffer,
1086 AudioSampleFormat aFormat,
1087 uint32_t aChannels, uint32_t aFrames,
1088 uint32_t aSampleRate) {
1089 MOZ_ASSERT(InIteration());
1090 uint32_t toWrite = mBuffer.Available();
1091
1092 if (!mBuffer.Available() && aFrames > 0) {
1093 NS_WARNING("DataCallback buffer full, expect frame drops.");
1094 }
1095
1096 MOZ_ASSERT(mBuffer.Available() <= aFrames);
1097
1098 mBuffer.WriteFrames(aMixedBuffer, mBuffer.Available());
1099 MOZ_ASSERT(mBuffer.Available() == 0,
1100 "Missing frames to fill audio callback's buffer.");
1101
1102 DebugOnly<uint32_t> written = mScratchBuffer.Fill(
1103 aMixedBuffer + toWrite * aChannels, aFrames - toWrite);
1104 NS_WARNING_ASSERTION(written == aFrames - toWrite, "Dropping frames.");
1105 };
1106
PanOutputIfNeeded(bool aMicrophoneActive)1107 void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive) {
1108 #ifdef XP_MACOSX
1109 TRACE("AudioCallbackDriver::PanOutputIfNeeded");
1110 cubeb_device* out = nullptr;
1111 int rv;
1112 char name[128];
1113 size_t length = sizeof(name);
1114
1115 rv = sysctlbyname("hw.model", name, &length, NULL, 0);
1116 if (rv) {
1117 return;
1118 }
1119
1120 int major, minor;
1121 for (uint32_t i = 0; i < length; i++) {
1122 // skip the model name
1123 if (isalpha(name[i])) {
1124 continue;
1125 }
1126 sscanf(name + i, "%d,%d", &major, &minor);
1127 break;
1128 }
1129
1130 enum MacbookModel { MacBook, MacBookPro, MacBookAir, NotAMacbook };
1131
1132 MacbookModel model;
1133
1134 if (!strncmp(name, "MacBookPro", length)) {
1135 model = MacBookPro;
1136 } else if (strncmp(name, "MacBookAir", length)) {
1137 model = MacBookAir;
1138 } else if (strncmp(name, "MacBook", length)) {
1139 model = MacBook;
1140 } else {
1141 model = NotAMacbook;
1142 }
1143 // For macbook pro before 2016 model (change of chassis), hard pan the audio
1144 // to the right if the speakers are in use to avoid feedback.
1145 if (model == MacBookPro && major <= 12) {
1146 if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) {
1147 MOZ_ASSERT(out);
1148 // Check if we are currently outputing sound on external speakers.
1149 if (out->output_name && !strcmp(out->output_name, "ispk")) {
1150 // Pan everything to the right speaker.
1151 LOG(LogLevel::Debug, ("Using the built-in speakers, with%s audio input",
1152 aMicrophoneActive ? "" : "out"));
1153 mNeedsPanning = aMicrophoneActive;
1154 } else {
1155 LOG(LogLevel::Debug, ("Using an external output device"));
1156 mNeedsPanning = false;
1157 }
1158 cubeb_stream_device_destroy(mAudioStream, out);
1159 }
1160 }
1161 #endif
1162 }
1163
DeviceChangedCallback()1164 void AudioCallbackDriver::DeviceChangedCallback() {
1165 MOZ_ASSERT(!InIteration());
1166 // Tell the audio engine the device has changed, it might want to reset some
1167 // state.
1168 Graph()->DeviceChanged();
1169 #ifdef XP_MACOSX
1170 RefPtr<AudioCallbackDriver> self(this);
1171 bool hasInput = mInputChannelCount;
1172 NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1173 "PanOutputIfNeeded", [self{std::move(self)}, hasInput]() {
1174 self->PanOutputIfNeeded(hasInput);
1175 }));
1176 #endif
1177 }
1178
IterationDuration()1179 uint32_t AudioCallbackDriver::IterationDuration() {
1180 MOZ_ASSERT(InIteration());
1181 // The real fix would be to have an API in cubeb to give us the number. Short
1182 // of that, we approximate it here. bug 1019507
1183 return mIterationDurationMS;
1184 }
1185
EnsureNextIteration()1186 void AudioCallbackDriver::EnsureNextIteration() {
1187 if (mFallbackDriverState == FallbackDriverState::Running) {
1188 auto fallback = mFallback.Lock();
1189 if (fallback.ref()) {
1190 fallback.ref()->EnsureNextIteration();
1191 }
1192 }
1193 }
1194
IsStarted()1195 bool AudioCallbackDriver::IsStarted() { return mStarted; }
1196
AudioOutputLatency()1197 TimeDuration AudioCallbackDriver::AudioOutputLatency() {
1198 TRACE("AudioCallbackDriver::AudioOutputLatency");
1199 uint32_t latencyFrames;
1200 int rv = cubeb_stream_get_latency(mAudioStream, &latencyFrames);
1201 if (rv || mSampleRate == 0) {
1202 return TimeDuration::FromSeconds(0.0);
1203 }
1204
1205 return TimeDuration::FromSeconds(static_cast<double>(latencyFrames) /
1206 mSampleRate);
1207 }
1208
OnFallback() const1209 bool AudioCallbackDriver::OnFallback() const {
1210 MOZ_ASSERT(InIteration());
1211 return mFallbackDriverState == FallbackDriverState::Running;
1212 }
1213
FallbackToSystemClockDriver()1214 void AudioCallbackDriver::FallbackToSystemClockDriver() {
1215 MOZ_ASSERT(!ThreadRunning());
1216 MOZ_ASSERT(mAudioStreamState == AudioStreamState::None ||
1217 mAudioStreamState == AudioStreamState::Pending);
1218 MOZ_ASSERT(mFallbackDriverState == FallbackDriverState::None);
1219 LOG(LogLevel::Debug,
1220 ("%p: AudioCallbackDriver %p Falling back to SystemClockDriver.", Graph(),
1221 this));
1222 mFallbackDriverState = FallbackDriverState::Running;
1223 mNextReInitBackoffStep =
1224 TimeDuration::FromMilliseconds(AUDIO_INITIAL_FALLBACK_BACKOFF_STEP_MS);
1225 mNextReInitAttempt = TimeStamp::Now() + mNextReInitBackoffStep;
1226 auto fallback =
1227 MakeRefPtr<FallbackWrapper>(Graph(), this, mSampleRate, mIterationStart,
1228 mIterationEnd, mStateComputedTime);
1229 {
1230 auto driver = mFallback.Lock();
1231 driver.ref() = fallback;
1232 }
1233 fallback->Start();
1234 }
1235
FallbackDriverStopped(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime,FallbackDriverState aState)1236 void AudioCallbackDriver::FallbackDriverStopped(GraphTime aIterationStart,
1237 GraphTime aIterationEnd,
1238 GraphTime aStateComputedTime,
1239 FallbackDriverState aState) {
1240 mIterationStart = aIterationStart;
1241 mIterationEnd = aIterationEnd;
1242 mStateComputedTime = aStateComputedTime;
1243 mNextReInitAttempt = TimeStamp();
1244 mNextReInitBackoffStep = TimeDuration();
1245 {
1246 auto fallback = mFallback.Lock();
1247 MOZ_ASSERT(fallback.ref()->OnThread());
1248 fallback.ref() = nullptr;
1249 }
1250
1251 MOZ_ASSERT(aState == FallbackDriverState::None ||
1252 aState == FallbackDriverState::Stopped);
1253 MOZ_ASSERT_IF(aState == FallbackDriverState::None,
1254 mAudioStreamState == AudioStreamState::Running);
1255 mFallbackDriverState = aState;
1256 }
1257
MaybeStartAudioStream()1258 void AudioCallbackDriver::MaybeStartAudioStream() {
1259 AudioStreamState streamState = mAudioStreamState;
1260 if (streamState != AudioStreamState::None) {
1261 LOG(LogLevel::Verbose,
1262 ("%p: AudioCallbackDriver %p Cannot re-init.", Graph(), this));
1263 return;
1264 }
1265
1266 TimeStamp now = TimeStamp::Now();
1267 if (now < mNextReInitAttempt) {
1268 LOG(LogLevel::Verbose,
1269 ("%p: AudioCallbackDriver %p Not time to re-init yet. %.3fs left.",
1270 Graph(), this, (mNextReInitAttempt - now).ToSeconds()));
1271 return;
1272 }
1273
1274 LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p Attempting to re-init "
1275 "audio stream from fallback driver.",
1276 Graph(), this));
1277 mNextReInitBackoffStep = std::min(
1278 mNextReInitBackoffStep * 2,
1279 TimeDuration::FromMilliseconds(AUDIO_MAX_FALLBACK_BACKOFF_STEP_MS));
1280 mNextReInitAttempt = now + mNextReInitBackoffStep;
1281 Start();
1282 }
1283
1284 } // namespace mozilla
1285
1286 // avoid redefined macro in unified build
1287 #undef LOG
1288