1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5  * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 
7 #include "mozilla/dom/AudioContext.h"
8 #include "mozilla/dom/AudioDeviceInfo.h"
9 #include "mozilla/dom/BaseAudioContextBinding.h"
10 #include "mozilla/SchedulerGroup.h"
11 #include "mozilla/SharedThreadPool.h"
12 #include "mozilla/ClearOnShutdown.h"
13 #include "mozilla/Unused.h"
14 #include "mozilla/MathAlgorithms.h"
15 #include "CubebDeviceEnumerator.h"
16 #include "Tracing.h"
17 
18 #ifdef MOZ_WEBRTC
19 #  include "webrtc/MediaEngineWebRTC.h"
20 #endif
21 
22 #ifdef XP_MACOSX
23 #  include <sys/sysctl.h>
24 #endif
25 
26 extern mozilla::LazyLogModule gMediaTrackGraphLog;
27 #ifdef LOG
28 #  undef LOG
29 #endif  // LOG
30 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
31 
32 namespace mozilla {
33 
GraphDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)34 GraphDriver::GraphDriver(GraphInterface* aGraphInterface,
35                          GraphDriver* aPreviousDriver, uint32_t aSampleRate)
36     : mGraphInterface(aGraphInterface),
37       mSampleRate(aSampleRate),
38       mPreviousDriver(aPreviousDriver) {}
39 
SetState(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)40 void GraphDriver::SetState(GraphTime aIterationStart, GraphTime aIterationEnd,
41                            GraphTime aStateComputedTime) {
42   MOZ_ASSERT(InIteration() || !ThreadRunning());
43 
44   mIterationStart = aIterationStart;
45   mIterationEnd = aIterationEnd;
46   mStateComputedTime = aStateComputedTime;
47 }
48 
49 #ifdef DEBUG
InIteration()50 bool GraphDriver::InIteration() {
51   return OnThread() || Graph()->InDriverIteration(this);
52 }
53 #endif
54 
PreviousDriver()55 GraphDriver* GraphDriver::PreviousDriver() {
56   MOZ_ASSERT(InIteration() || !ThreadRunning());
57   return mPreviousDriver;
58 }
59 
SetPreviousDriver(GraphDriver * aPreviousDriver)60 void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver) {
61   MOZ_ASSERT(InIteration() || !ThreadRunning());
62   mPreviousDriver = aPreviousDriver;
63 }
64 
ThreadedDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)65 ThreadedDriver::ThreadedDriver(GraphInterface* aGraphInterface,
66                                GraphDriver* aPreviousDriver,
67                                uint32_t aSampleRate)
68     : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
69       mThreadRunning(false) {}
70 
71 class MediaTrackGraphShutdownThreadRunnable : public Runnable {
72  public:
MediaTrackGraphShutdownThreadRunnable(already_AddRefed<nsIThread> aThread)73   explicit MediaTrackGraphShutdownThreadRunnable(
74       already_AddRefed<nsIThread> aThread)
75       : Runnable("MediaTrackGraphShutdownThreadRunnable"), mThread(aThread) {}
Run()76   NS_IMETHOD Run() override {
77     MOZ_ASSERT(NS_IsMainThread());
78     MOZ_ASSERT(mThread);
79 
80     mThread->Shutdown();
81     mThread = nullptr;
82     return NS_OK;
83   }
84 
85  private:
86   nsCOMPtr<nsIThread> mThread;
87 };
88 
~ThreadedDriver()89 ThreadedDriver::~ThreadedDriver() {
90   if (mThread) {
91     nsCOMPtr<nsIRunnable> event =
92         new MediaTrackGraphShutdownThreadRunnable(mThread.forget());
93     SchedulerGroup::Dispatch(TaskCategory::Other, event.forget());
94   }
95 }
96 
97 class MediaTrackGraphInitThreadRunnable : public Runnable {
98  public:
MediaTrackGraphInitThreadRunnable(ThreadedDriver * aDriver)99   explicit MediaTrackGraphInitThreadRunnable(ThreadedDriver* aDriver)
100       : Runnable("MediaTrackGraphInitThreadRunnable"), mDriver(aDriver) {}
Run()101   NS_IMETHOD Run() override {
102     MOZ_ASSERT(!mDriver->ThreadRunning());
103     LOG(LogLevel::Debug, ("Starting a new system driver for graph %p",
104                           mDriver->mGraphInterface.get()));
105 
106     if (GraphDriver* previousDriver = mDriver->PreviousDriver()) {
107       LOG(LogLevel::Debug,
108           ("%p releasing an AudioCallbackDriver(%p), for graph %p",
109            mDriver.get(), previousDriver, mDriver->Graph()));
110       MOZ_ASSERT(!mDriver->AsAudioCallbackDriver());
111       RefPtr<AsyncCubebTask> releaseEvent =
112           new AsyncCubebTask(previousDriver->AsAudioCallbackDriver(),
113                              AsyncCubebOperation::SHUTDOWN);
114       releaseEvent->Dispatch();
115       mDriver->SetPreviousDriver(nullptr);
116     }
117 
118     mDriver->RunThread();
119     return NS_OK;
120   }
121 
122  private:
123   RefPtr<ThreadedDriver> mDriver;
124 };
125 
Start()126 void ThreadedDriver::Start() {
127   MOZ_ASSERT(!ThreadRunning());
128   LOG(LogLevel::Debug,
129       ("Starting thread for a SystemClockDriver  %p", mGraphInterface.get()));
130   Unused << NS_WARN_IF(mThread);
131   MOZ_ASSERT(!mThread);  // Ensure we haven't already started it
132 
133   nsCOMPtr<nsIRunnable> event = new MediaTrackGraphInitThreadRunnable(this);
134   // Note: mThread may be null during event->Run() if we pass to NewNamedThread!
135   // See AudioInitTask
136   nsresult rv = NS_NewNamedThread("MediaTrackGrph", getter_AddRefs(mThread));
137   if (NS_SUCCEEDED(rv)) {
138     mThread->EventTarget()->Dispatch(event.forget(), NS_DISPATCH_NORMAL);
139   }
140 }
141 
Shutdown()142 void ThreadedDriver::Shutdown() {
143   NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
144   // mGraph's thread is not running so it's OK to do whatever here
145   LOG(LogLevel::Debug, ("Stopping threads for MediaTrackGraph %p", this));
146 
147   if (mThread) {
148     LOG(LogLevel::Debug,
149         ("%p: Stopping ThreadedDriver's %p thread", Graph(), this));
150     mThread->Shutdown();
151     mThread = nullptr;
152   }
153 }
154 
SystemClockDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)155 SystemClockDriver::SystemClockDriver(GraphInterface* aGraphInterface,
156                                      GraphDriver* aPreviousDriver,
157                                      uint32_t aSampleRate)
158     : ThreadedDriver(aGraphInterface, aPreviousDriver, aSampleRate),
159       mInitialTimeStamp(TimeStamp::Now()),
160       mCurrentTimeStamp(TimeStamp::Now()),
161       mLastTimeStamp(TimeStamp::Now()) {}
162 
163 SystemClockDriver::~SystemClockDriver() = default;
164 
RunThread()165 void ThreadedDriver::RunThread() {
166   mThreadRunning = true;
167   while (true) {
168     mIterationStart = mIterationEnd;
169     mIterationEnd += GetIntervalForIteration();
170 
171     if (mStateComputedTime < mIterationEnd) {
172       LOG(LogLevel::Warning, ("%p: Global underrun detected", Graph()));
173       mIterationEnd = mStateComputedTime;
174     }
175 
176     if (mIterationStart >= mIterationEnd) {
177       NS_ASSERTION(mIterationStart == mIterationEnd,
178                    "Time can't go backwards!");
179       // This could happen due to low clock resolution, maybe?
180       LOG(LogLevel::Debug, ("%p: Time did not advance", Graph()));
181     }
182 
183     GraphTime nextStateComputedTime =
184         MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(
185             mIterationEnd + MillisecondsToMediaTime(AUDIO_TARGET_MS));
186     if (nextStateComputedTime < mStateComputedTime) {
187       // A previous driver may have been processing further ahead of
188       // iterationEnd.
189       LOG(LogLevel::Warning,
190           ("%p: Prevent state from going backwards. interval[%ld; %ld] "
191            "state[%ld; "
192            "%ld]",
193            Graph(), (long)mIterationStart, (long)mIterationEnd,
194            (long)mStateComputedTime, (long)nextStateComputedTime));
195       nextStateComputedTime = mStateComputedTime;
196     }
197     LOG(LogLevel::Verbose,
198         ("%p: interval[%ld; %ld] state[%ld; %ld]", Graph(),
199          (long)mIterationStart, (long)mIterationEnd, (long)mStateComputedTime,
200          (long)nextStateComputedTime));
201 
202     mStateComputedTime = nextStateComputedTime;
203     IterationResult result =
204         Graph()->OneIteration(mStateComputedTime, mIterationEnd, nullptr);
205 
206     if (result.IsStop()) {
207       // Signal that we're done stopping.
208       result.Stopped();
209       break;
210     }
211     WaitForNextIteration();
212     if (GraphDriver* nextDriver = result.NextDriver()) {
213       LOG(LogLevel::Debug, ("%p: Switching to AudioCallbackDriver", Graph()));
214       result.Switched();
215       nextDriver->SetState(mIterationStart, mIterationEnd, mStateComputedTime);
216       nextDriver->Start();
217       break;
218     }
219     MOZ_ASSERT(result.IsStillProcessing());
220   }
221   mThreadRunning = false;
222 }
223 
GetIntervalForIteration()224 MediaTime SystemClockDriver::GetIntervalForIteration() {
225   TimeStamp now = TimeStamp::Now();
226   MediaTime interval =
227       SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds());
228   mCurrentTimeStamp = now;
229 
230   MOZ_LOG(gMediaTrackGraphLog, LogLevel::Verbose,
231           ("%p: Updating current time to %f (real %f, StateComputedTime() %f)",
232            Graph(), MediaTimeToSeconds(mIterationEnd + interval),
233            (now - mInitialTimeStamp).ToSeconds(),
234            MediaTimeToSeconds(mStateComputedTime)));
235 
236   return interval;
237 }
238 
EnsureNextIteration()239 void ThreadedDriver::EnsureNextIteration() {
240   mWaitHelper.EnsureNextIteration();
241 }
242 
WaitForNextIteration()243 void ThreadedDriver::WaitForNextIteration() {
244   MOZ_ASSERT(mThread);
245   MOZ_ASSERT(OnThread());
246   mWaitHelper.WaitForNextIterationAtLeast(WaitInterval());
247 }
248 
WaitInterval()249 TimeDuration SystemClockDriver::WaitInterval() {
250   MOZ_ASSERT(mThread);
251   MOZ_ASSERT(OnThread());
252   TimeStamp now = TimeStamp::Now();
253   int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS -
254                       int64_t((now - mCurrentTimeStamp).ToMilliseconds());
255   // Make sure timeoutMS doesn't overflow 32 bits by waking up at
256   // least once a minute, if we need to wake up at all
257   timeoutMS = std::max<int64_t>(0, std::min<int64_t>(timeoutMS, 60 * 1000));
258   LOG(LogLevel::Verbose,
259       ("%p: Waiting for next iteration; at %f, timeout=%f", Graph(),
260        (now - mInitialTimeStamp).ToSeconds(), timeoutMS / 1000.0));
261 
262   return TimeDuration::FromMilliseconds(timeoutMS);
263 }
264 
OfflineClockDriver(GraphInterface * aGraphInterface,uint32_t aSampleRate,GraphTime aSlice)265 OfflineClockDriver::OfflineClockDriver(GraphInterface* aGraphInterface,
266                                        uint32_t aSampleRate, GraphTime aSlice)
267     : ThreadedDriver(aGraphInterface, nullptr, aSampleRate), mSlice(aSlice) {}
268 
269 OfflineClockDriver::~OfflineClockDriver() = default;
270 
RunThread()271 void OfflineClockDriver::RunThread() {
272   nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(mThread);
273   nsCOMPtr<nsIThreadObserver> observer = do_QueryInterface(Graph());
274   threadInternal->SetObserver(observer);
275 
276   ThreadedDriver::RunThread();
277 }
278 
GetIntervalForIteration()279 MediaTime OfflineClockDriver::GetIntervalForIteration() {
280   return MillisecondsToMediaTime(mSlice);
281 }
282 
AsyncCubebTask(AudioCallbackDriver * aDriver,AsyncCubebOperation aOperation)283 AsyncCubebTask::AsyncCubebTask(AudioCallbackDriver* aDriver,
284                                AsyncCubebOperation aOperation)
285     : Runnable("AsyncCubebTask"),
286       mDriver(aDriver),
287       mOperation(aOperation),
288       mShutdownGrip(aDriver->Graph()) {
289   NS_WARNING_ASSERTION(
290       mDriver->mAudioStream || aOperation == AsyncCubebOperation::INIT,
291       "No audio stream!");
292 }
293 
294 AsyncCubebTask::~AsyncCubebTask() = default;
295 
296 NS_IMETHODIMP
Run()297 AsyncCubebTask::Run() {
298   MOZ_ASSERT(mDriver);
299 
300   switch (mOperation) {
301     case AsyncCubebOperation::INIT: {
302       LOG(LogLevel::Debug, ("%p: AsyncCubebOperation::INIT driver=%p",
303                             mDriver->Graph(), mDriver.get()));
304       mDriver->Init();
305       mDriver->CompleteAudioContextOperations(mOperation);
306       break;
307     }
308     case AsyncCubebOperation::SHUTDOWN: {
309       LOG(LogLevel::Debug, ("%p: AsyncCubebOperation::SHUTDOWN driver=%p",
310                             mDriver->Graph(), mDriver.get()));
311       mDriver->Stop();
312       mDriver->CompleteAudioContextOperations(mOperation);
313       mDriver = nullptr;
314       mShutdownGrip = nullptr;
315       break;
316     }
317     default:
318       MOZ_CRASH("Operation not implemented.");
319   }
320 
321   // The thread will kill itself after a bit
322   return NS_OK;
323 }
324 
TrackAndPromiseForOperation(MediaTrack * aTrack,dom::AudioContextOperation aOperation,AbstractThread * aMainThread,MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise> && aHolder)325 TrackAndPromiseForOperation::TrackAndPromiseForOperation(
326     MediaTrack* aTrack, dom::AudioContextOperation aOperation,
327     AbstractThread* aMainThread,
328     MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise>&& aHolder)
329     : mTrack(aTrack),
330       mOperation(aOperation),
331       mMainThread(aMainThread),
332       mHolder(std::move(aHolder)) {}
333 
TrackAndPromiseForOperation(TrackAndPromiseForOperation && aOther)334 TrackAndPromiseForOperation::TrackAndPromiseForOperation(
335     TrackAndPromiseForOperation&& aOther) noexcept
336     : mTrack(std::move(aOther.mTrack)),
337       mOperation(aOther.mOperation),
338       mMainThread(std::move(aOther.mMainThread)),
339       mHolder(std::move(aOther.mHolder)) {}
340 
341 /* Helper to proxy the GraphInterface methods used by a running
342  * mFallbackDriver. */
343 class AudioCallbackDriver::FallbackWrapper : public GraphInterface {
344  public:
FallbackWrapper(RefPtr<GraphInterface> aGraph,RefPtr<AudioCallbackDriver> aOwner,uint32_t aSampleRate,GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)345   FallbackWrapper(RefPtr<GraphInterface> aGraph,
346                   RefPtr<AudioCallbackDriver> aOwner, uint32_t aSampleRate,
347                   GraphTime aIterationStart, GraphTime aIterationEnd,
348                   GraphTime aStateComputedTime)
349       : mGraph(std::move(aGraph)),
350         mOwner(std::move(aOwner)),
351         mFallbackDriver(
352             MakeRefPtr<SystemClockDriver>(this, nullptr, aSampleRate)),
353         mIterationStart(aIterationStart),
354         mIterationEnd(aIterationEnd),
355         mStateComputedTime(aStateComputedTime) {
356     mFallbackDriver->SetState(mIterationStart, mIterationEnd,
357                               mStateComputedTime);
358   }
359 
360   NS_DECL_THREADSAFE_ISUPPORTS
361 
362   /* Proxied SystemClockDriver methods */
SetState(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)363   void SetState(GraphTime aIterationStart, GraphTime aIterationEnd,
364                 GraphTime aStateComputedTime) {
365     mIterationStart = aIterationStart;
366     mIterationEnd = aIterationEnd;
367     mStateComputedTime = aStateComputedTime;
368     mFallbackDriver->SetState(aIterationStart, aIterationEnd,
369                               aStateComputedTime);
370   }
Start()371   void Start() { mFallbackDriver->Start(); }
Shutdown()372   MOZ_CAN_RUN_SCRIPT void Shutdown() {
373     RefPtr<SystemClockDriver> driver = mFallbackDriver;
374     driver->Shutdown();
375   }
EnsureNextIteration()376   void EnsureNextIteration() { mFallbackDriver->EnsureNextIteration(); }
377 #ifdef DEBUG
InIteration()378   bool InIteration() { return mFallbackDriver->InIteration(); }
379 #endif
OnThread()380   bool OnThread() { return mFallbackDriver->OnThread(); }
381 
382   /* GraphInterface methods */
NotifyOutputData(AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)383   void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
384                         TrackRate aRate, uint32_t aChannels) override {
385     MOZ_CRASH("Unexpected NotifyOutputData from fallback SystemClockDriver");
386   }
NotifyStarted()387   void NotifyStarted() override {
388     MOZ_CRASH("Unexpected NotifyStarted from fallback SystemClockDriver");
389   }
NotifyInputData(const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)390   void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
391                        TrackRate aRate, uint32_t aChannels) override {
392     MOZ_CRASH("Unexpected NotifyInputData from fallback SystemClockDriver");
393   }
DeviceChanged()394   void DeviceChanged() override {
395     MOZ_CRASH("Unexpected DeviceChanged from fallback SystemClockDriver");
396   }
397 #ifdef DEBUG
InDriverIteration(GraphDriver * aDriver)398   bool InDriverIteration(GraphDriver* aDriver) override {
399     return !mOwner->ThreadRunning() && mOwner->InIteration();
400   }
401 #endif
OneIteration(GraphTime aStateComputedEnd,GraphTime aIterationEnd,AudioMixer * aMixer)402   IterationResult OneIteration(GraphTime aStateComputedEnd,
403                                GraphTime aIterationEnd,
404                                AudioMixer* aMixer) override {
405     MOZ_ASSERT(!aMixer);
406 
407 #ifdef DEBUG
408     AutoInCallback aic(mOwner);
409 #endif
410 
411     mIterationStart = mIterationEnd;
412     mIterationEnd = aIterationEnd;
413     mStateComputedTime = aStateComputedEnd;
414     IterationResult result =
415         mGraph->OneIteration(aStateComputedEnd, aIterationEnd, aMixer);
416 
417     AudioStreamState audioState = mOwner->mAudioStreamState;
418 
419     MOZ_ASSERT(audioState != AudioStreamState::Stopping,
420                "The audio driver can only enter stopping if it iterated the "
421                "graph, which it can only do if there's no fallback driver");
422     if (audioState != AudioStreamState::Running && result.IsStillProcessing()) {
423       if (audioState != AudioStreamState::Errored) {
424         mOwner->MaybeStartAudioStream();
425       }
426       return result;
427     }
428 
429     MOZ_ASSERT(result.IsStillProcessing() || result.IsStop() ||
430                result.IsSwitchDriver());
431 
432     // Proxy the release of the fallback driver to a background thread, so it
433     // doesn't perform unexpected suicide.
434     IterationResult stopFallback =
435         IterationResult::CreateStop(NS_NewRunnableFunction(
436             "AudioCallbackDriver::FallbackDriverStopped",
437             [self = RefPtr<FallbackWrapper>(this), this,
438              result = std::move(result)]() mutable {
439               FallbackDriverState fallbackState =
440                   result.IsStillProcessing() ? FallbackDriverState::None
441                                              : FallbackDriverState::Stopped;
442               mOwner->FallbackDriverStopped(mIterationStart, mIterationEnd,
443                                             mStateComputedTime, fallbackState);
444 
445               if (fallbackState == FallbackDriverState::Stopped) {
446 #ifdef DEBUG
447                 // The AudioCallbackDriver may not iterate the graph, but we'll
448                 // call into it so we need to be regarded as "in iteration".
449                 AutoInCallback aic(mOwner);
450 #endif
451                 if (GraphDriver* nextDriver = result.NextDriver()) {
452                   LOG(LogLevel::Debug,
453                       ("%p: Switching from fallback to other driver.",
454                        mGraph.get()));
455                   result.Switched();
456                   nextDriver->SetState(mIterationStart, mIterationEnd,
457                                        mStateComputedTime);
458                   nextDriver->Start();
459                 } else if (result.IsStop()) {
460                   LOG(LogLevel::Debug,
461                       ("%p: Stopping fallback driver.", mGraph.get()));
462                   result.Stopped();
463                 }
464               }
465               mOwner = nullptr;
466               NS_DispatchBackgroundTask(NS_NewRunnableFunction(
467                   "AudioCallbackDriver::FallbackDriverStopped::Release",
468                   [fallback = std::move(self->mFallbackDriver)] {}));
469             }));
470 
471     return stopFallback;
472   }
473 
474  private:
475   virtual ~FallbackWrapper() = default;
476 
477   const RefPtr<GraphInterface> mGraph;
478   // Valid until mFallbackDriver has finished its last iteration.
479   RefPtr<AudioCallbackDriver> mOwner;
480   RefPtr<SystemClockDriver> mFallbackDriver;
481 
482   GraphTime mIterationStart;
483   GraphTime mIterationEnd;
484   GraphTime mStateComputedTime;
485 };
486 
NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)487 NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)
488 
489 AudioCallbackDriver::AudioCallbackDriver(
490     GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
491     uint32_t aSampleRate, uint32_t aOutputChannelCount,
492     uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
493     CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType)
494     : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
495       mOutputChannelCount(aOutputChannelCount),
496       mInputChannelCount(aInputChannelCount),
497       mOutputDeviceID(aOutputDeviceID),
498       mInputDeviceID(aInputDeviceID),
499       mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS),
500       mStarted(false),
501       mInitShutdownThread(
502           SharedThreadPool::Get(NS_LITERAL_CSTRING("CubebOperation"), 1)),
503       mPromisesForOperation("AudioCallbackDriver::mPromisesForOperation"),
504       mAudioThreadId(std::thread::id()),
505       mAudioStreamState(AudioStreamState::None),
506       mFallback("AudioCallbackDriver::mFallback") {
507   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver ctor", Graph()));
508 
509   NS_WARNING_ASSERTION(mOutputChannelCount != 0,
510                        "Invalid output channel count");
511   MOZ_ASSERT(mOutputChannelCount <= 8);
512 
513   const uint32_t kIdleThreadTimeoutMs = 2000;
514   mInitShutdownThread->SetIdleThreadTimeout(
515       PR_MillisecondsToInterval(kIdleThreadTimeoutMs));
516 
517 #if defined(XP_WIN)
518   if (XRE_IsContentProcess()) {
519     audio::AudioNotificationReceiver::Register(this);
520   }
521 #endif
522   if (aAudioInputType == AudioInputType::Voice) {
523     LOG(LogLevel::Debug, ("VOICE."));
524     mInputDevicePreference = CUBEB_DEVICE_PREF_VOICE;
525     CubebUtils::SetInCommunication(true);
526   } else {
527     mInputDevicePreference = CUBEB_DEVICE_PREF_ALL;
528   }
529 
530   mMixer.AddCallback(WrapNotNull(this));
531 }
532 
~AudioCallbackDriver()533 AudioCallbackDriver::~AudioCallbackDriver() {
534 #ifdef DEBUG
535   {
536     auto promises = mPromisesForOperation.Lock();
537     MOZ_ASSERT(promises->IsEmpty());
538   }
539 #endif
540 #if defined(XP_WIN)
541   if (XRE_IsContentProcess()) {
542     audio::AudioNotificationReceiver::Unregister(this);
543   }
544 #endif
545   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
546     CubebUtils::SetInCommunication(false);
547   }
548 }
549 
IsMacbookOrMacbookAir()550 bool IsMacbookOrMacbookAir() {
551 #ifdef XP_MACOSX
552   size_t len = 0;
553   sysctlbyname("hw.model", NULL, &len, NULL, 0);
554   if (len) {
555     UniquePtr<char[]> model(new char[len]);
556     // This string can be
557     // MacBook%d,%d for a normal MacBook
558     // MacBookPro%d,%d for a MacBook Pro
559     // MacBookAir%d,%d for a Macbook Air
560     sysctlbyname("hw.model", model.get(), &len, NULL, 0);
561     char* substring = strstr(model.get(), "MacBook");
562     if (substring) {
563       const size_t offset = strlen("MacBook");
564       if (!strncmp(model.get() + offset, "Air", 3) ||
565           isdigit(model[offset + 1])) {
566         return true;
567       }
568     }
569   }
570 #endif
571   return false;
572 }
573 
Init()574 void AudioCallbackDriver::Init() {
575   AUTO_PROFILER_LABEL("AudioCallbackDriver::Init", MEDIA_CUBEB);
576   MOZ_ASSERT(OnCubebOperationThread());
577   MOZ_ASSERT(mAudioStreamState == AudioStreamState::Pending);
578   FallbackDriverState fallbackState = mFallbackDriverState;
579   if (fallbackState == FallbackDriverState::Stopped) {
580     // The graph has already stopped us.
581     return;
582   }
583   bool fromFallback = fallbackState == FallbackDriverState::Running;
584   cubeb* cubebContext = CubebUtils::GetCubebContext();
585   if (!cubebContext) {
586     NS_WARNING("Could not get cubeb context.");
587     LOG(LogLevel::Warning, ("%s: Could not get cubeb context", __func__));
588     mAudioStreamState = AudioStreamState::None;
589     if (!fromFallback) {
590       CubebUtils::ReportCubebStreamInitFailure(true);
591       FallbackToSystemClockDriver();
592     }
593     return;
594   }
595 
596   cubeb_stream_params output;
597   cubeb_stream_params input;
598   bool firstStream = CubebUtils::GetFirstStream();
599 
600   MOZ_ASSERT(!NS_IsMainThread(),
601              "This is blocking and should never run on the main thread.");
602 
603   output.rate = mSampleRate;
604 
605   if (AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_S16) {
606     output.format = CUBEB_SAMPLE_S16NE;
607   } else {
608     output.format = CUBEB_SAMPLE_FLOAT32NE;
609   }
610 
611   if (!mOutputChannelCount) {
612     LOG(LogLevel::Warning, ("Output number of channels is 0."));
613     mAudioStreamState = AudioStreamState::None;
614     if (!fromFallback) {
615       CubebUtils::ReportCubebStreamInitFailure(firstStream);
616       FallbackToSystemClockDriver();
617     }
618     return;
619   }
620 
621   CubebUtils::AudioDeviceID forcedOutputDeviceId = nullptr;
622 
623   char* forcedOutputDeviceName = CubebUtils::GetForcedOutputDevice();
624   if (forcedOutputDeviceName) {
625     RefPtr<CubebDeviceEnumerator> enumerator = Enumerator::GetInstance();
626     RefPtr<AudioDeviceInfo> device = enumerator->DeviceInfoFromName(
627         NS_ConvertUTF8toUTF16(forcedOutputDeviceName), EnumeratorSide::OUTPUT);
628     if (device && device->DeviceID()) {
629       forcedOutputDeviceId = device->DeviceID();
630     }
631   }
632 
633   mBuffer = AudioCallbackBufferWrapper<AudioDataValue>(mOutputChannelCount);
634   mScratchBuffer =
635       SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2>(mOutputChannelCount);
636 
637   output.channels = mOutputChannelCount;
638   AudioConfig::ChannelLayout::ChannelMap channelMap =
639       AudioConfig::ChannelLayout(mOutputChannelCount).Map();
640 
641   output.layout = static_cast<uint32_t>(channelMap);
642   output.prefs = CubebUtils::GetDefaultStreamPrefs();
643 #if !defined(XP_WIN)
644   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE &&
645       CubebUtils::RouteOutputAsVoice()) {
646     output.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
647   }
648 #endif
649 
650   uint32_t latencyFrames = CubebUtils::GetCubebMTGLatencyInFrames(&output);
651 
652   // Macbook and MacBook air don't have enough CPU to run very low latency
653   // MediaTrackGraphs, cap the minimal latency to 512 frames int this case.
654   if (IsMacbookOrMacbookAir()) {
655     latencyFrames = std::max((uint32_t)512, latencyFrames);
656   }
657 
658   // On OSX, having a latency that is lower than 10ms is very common. It's
659   // not very useful when doing voice, because all the WebRTC code deal in 10ms
660   // chunks of audio.  Take the first power of two above 10ms at the current
661   // rate in this case. It's probably 512, for common rates.
662 #if defined(XP_MACOSX)
663   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
664     if (latencyFrames < mSampleRate / 100) {
665       latencyFrames = mozilla::RoundUpPow2(mSampleRate / 100);
666     }
667   }
668 #endif
669   LOG(LogLevel::Debug, ("Effective latency in frames: %d", latencyFrames));
670 
671   input = output;
672   input.channels = mInputChannelCount;
673   input.layout = CUBEB_LAYOUT_UNDEFINED;
674   input.prefs = CubebUtils::GetDefaultStreamPrefs();
675   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
676     input.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
677   }
678 
679   cubeb_stream* stream = nullptr;
680   bool inputWanted = mInputChannelCount > 0;
681   CubebUtils::AudioDeviceID outputId = mOutputDeviceID;
682   CubebUtils::AudioDeviceID inputId = mInputDeviceID;
683 
684   // XXX Only pass input input if we have an input listener.  Always
685   // set up output because it's easier, and it will just get silence.
686   if (cubeb_stream_init(cubebContext, &stream, "AudioCallbackDriver", inputId,
687                         inputWanted ? &input : nullptr,
688                         forcedOutputDeviceId ? forcedOutputDeviceId : outputId,
689                         &output, latencyFrames, DataCallback_s, StateCallback_s,
690                         this) == CUBEB_OK) {
691     mAudioStream.own(stream);
692     DebugOnly<int> rv =
693         cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale());
694     NS_WARNING_ASSERTION(
695         rv == CUBEB_OK,
696         "Could not set the audio stream volume in GraphDriver.cpp");
697     CubebUtils::ReportCubebBackendUsed();
698   } else {
699     NS_WARNING(
700         "Could not create a cubeb stream for MediaTrackGraph, falling "
701         "back to a SystemClockDriver");
702     mAudioStreamState = AudioStreamState::None;
703     // Only report failures when we're not coming from a driver that was
704     // created itself as a fallback driver because of a previous audio driver
705     // failure.
706     if (!fromFallback) {
707       CubebUtils::ReportCubebStreamInitFailure(firstStream);
708       FallbackToSystemClockDriver();
709     }
710     return;
711   }
712 
713 #ifdef XP_MACOSX
714   PanOutputIfNeeded(inputWanted);
715 #endif
716 
717   cubeb_stream_register_device_changed_callback(
718       mAudioStream, AudioCallbackDriver::DeviceChangedCallback_s);
719 
720   // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable. This
721   // is intended for diagnosing issues, and only works if the content sandbox is
722   // disabled.
723   mInputStreamFile.Open("GraphDriverInput", input.channels, input.rate);
724   mOutputStreamFile.Open("GraphDriverOutput", output.channels, output.rate);
725 
726   if (NS_WARN_IF(!StartStream())) {
727     LOG(LogLevel::Warning,
728         ("%p: AudioCallbackDriver couldn't start a cubeb stream.", Graph()));
729     return;
730   }
731 
732   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver started.", Graph()));
733 }
734 
Start()735 void AudioCallbackDriver::Start() {
736   MOZ_ASSERT(!IsStarted());
737   MOZ_ASSERT(mAudioStreamState == AudioStreamState::None);
738   MOZ_ASSERT_IF(PreviousDriver(), PreviousDriver()->InIteration());
739   mAudioStreamState = AudioStreamState::Pending;
740   mRanFirstIteration = false;
741 
742   if (mFallbackDriverState == FallbackDriverState::None) {
743     // Starting an audio driver could take a while. We start a system driver in
744     // the meantime so that the graph is kept running.
745     FallbackToSystemClockDriver();
746   }
747 
748   if (mPreviousDriver) {
749     if (mPreviousDriver->AsAudioCallbackDriver()) {
750       LOG(LogLevel::Debug, ("Releasing audio driver off main thread."));
751       RefPtr<AsyncCubebTask> releaseEvent =
752           new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(),
753                              AsyncCubebOperation::SHUTDOWN);
754       releaseEvent->Dispatch();
755     } else {
756       LOG(LogLevel::Debug,
757           ("Dropping driver reference for SystemClockDriver."));
758       MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver());
759     }
760     mPreviousDriver = nullptr;
761   }
762 
763   LOG(LogLevel::Debug, ("Starting new audio driver off main thread, "
764                         "to ensure it runs after previous shutdown."));
765   RefPtr<AsyncCubebTask> initEvent =
766       new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT);
767   initEvent->Dispatch();
768 }
769 
StartStream()770 bool AudioCallbackDriver::StartStream() {
771   AUTO_PROFILER_LABEL("AudioCallbackDriver::StartStream", MEDIA_CUBEB);
772   MOZ_ASSERT(!IsStarted() && OnCubebOperationThread());
773   // Set mStarted before cubeb_stream_start, since starting the cubeb stream can
774   // result in a callback (that may read mStarted) before mStarted would
775   // otherwise be set to true.
776   mStarted = true;
777   if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
778     NS_WARNING("Could not start cubeb stream for MTG.");
779     mStarted = false;
780     return false;
781   }
782 
783   return true;
784 }
785 
Stop()786 void AudioCallbackDriver::Stop() {
787   AUTO_PROFILER_LABEL("AudioCallbackDriver::Stop", MEDIA_CUBEB);
788   MOZ_ASSERT(OnCubebOperationThread());
789   if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) {
790     NS_WARNING("Could not stop cubeb stream for MTG.");
791   }
792   mStarted = false;
793 }
794 
Shutdown()795 void AudioCallbackDriver::Shutdown() {
796   MOZ_ASSERT(NS_IsMainThread());
797   RefPtr<FallbackWrapper> fallback;
798   {
799     auto fallbackLock = mFallback.Lock();
800     fallback = fallbackLock.ref();
801     fallbackLock.ref() = nullptr;
802   }
803   if (fallback) {
804     LOG(LogLevel::Debug,
805         ("%p: Releasing fallback driver %p.", Graph(), fallback.get()));
806     fallback->Shutdown();
807   }
808 
809   LOG(LogLevel::Debug,
810       ("%p: Releasing audio driver off main thread (GraphDriver::Shutdown).",
811        Graph()));
812   RefPtr<AsyncCubebTask> releaseEvent =
813       new AsyncCubebTask(this, AsyncCubebOperation::SHUTDOWN);
814   releaseEvent->Dispatch(NS_DISPATCH_SYNC);
815 }
816 
817 #if defined(XP_WIN)
ResetDefaultDevice()818 void AudioCallbackDriver::ResetDefaultDevice() {
819   AUTO_PROFILER_LABEL("AudioCallbackDriver::ResetDefaultDevice", MEDIA_CUBEB);
820   if (cubeb_stream_reset_default_device(mAudioStream) != CUBEB_OK) {
821     NS_WARNING("Could not reset cubeb stream to default output device.");
822   }
823 }
824 #endif
825 
826 /* static */
DataCallback_s(cubeb_stream * aStream,void * aUser,const void * aInputBuffer,void * aOutputBuffer,long aFrames)827 long AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream, void* aUser,
828                                          const void* aInputBuffer,
829                                          void* aOutputBuffer, long aFrames) {
830   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
831   return driver->DataCallback(static_cast<const AudioDataValue*>(aInputBuffer),
832                               static_cast<AudioDataValue*>(aOutputBuffer),
833                               aFrames);
834 }
835 
836 /* static */
StateCallback_s(cubeb_stream * aStream,void * aUser,cubeb_state aState)837 void AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void* aUser,
838                                           cubeb_state aState) {
839   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
840   driver->StateCallback(aState);
841 }
842 
843 /* static */
DeviceChangedCallback_s(void * aUser)844 void AudioCallbackDriver::DeviceChangedCallback_s(void* aUser) {
845   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
846   driver->DeviceChangedCallback();
847 }
848 
AutoInCallback(AudioCallbackDriver * aDriver)849 AudioCallbackDriver::AutoInCallback::AutoInCallback(
850     AudioCallbackDriver* aDriver)
851     : mDriver(aDriver) {
852   MOZ_ASSERT(mDriver->mAudioThreadId == std::thread::id());
853   mDriver->mAudioThreadId = std::this_thread::get_id();
854 }
855 
~AutoInCallback()856 AudioCallbackDriver::AutoInCallback::~AutoInCallback() {
857   MOZ_ASSERT(mDriver->mAudioThreadId == std::this_thread::get_id());
858   mDriver->mAudioThreadId = std::thread::id();
859 }
860 
DataCallback(const AudioDataValue * aInputBuffer,AudioDataValue * aOutputBuffer,long aFrames)861 long AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer,
862                                        AudioDataValue* aOutputBuffer,
863                                        long aFrames) {
864   FallbackDriverState fallbackState = mFallbackDriverState;
865   if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Running)) {
866     // Wait for the fallback driver to stop. Wake it up so it can stop if it's
867     // sleeping.
868     EnsureNextIteration();
869     PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
870     return aFrames;
871   }
872 
873   if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Stopped)) {
874     // We're supposed to stop.
875     PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
876     return aFrames - 1;
877   }
878 
879   MOZ_ASSERT(ThreadRunning());
880   TRACE_AUDIO_CALLBACK_BUDGET(aFrames, mSampleRate);
881   TRACE();
882 
883 #ifdef DEBUG
884   AutoInCallback aic(this);
885 #endif
886 
887   if (!mRanFirstIteration) {
888     Graph()->NotifyStarted();
889     mRanFirstIteration = true;
890   }
891 
892   uint32_t durationMS = aFrames * 1000 / mSampleRate;
893 
894   // For now, simply average the duration with the previous
895   // duration so there is some damping against sudden changes.
896   if (!mIterationDurationMS) {
897     mIterationDurationMS = durationMS;
898   } else {
899     mIterationDurationMS = (mIterationDurationMS * 3) + durationMS;
900     mIterationDurationMS /= 4;
901   }
902 
903   mBuffer.SetBuffer(aOutputBuffer, aFrames);
904   // fill part or all with leftover data from last iteration (since we
905   // align to Audio blocks)
906   mScratchBuffer.Empty(mBuffer);
907 
908   // State computed time is decided by the audio callback's buffer length. We
909   // compute the iteration start and end from there, trying to keep the amount
910   // of buffering in the graph constant.
911   GraphTime nextStateComputedTime =
912       MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(mStateComputedTime +
913                                                     mBuffer.Available());
914 
915   mIterationStart = mIterationEnd;
916   // inGraph is the number of audio frames there is between the state time and
917   // the current time, i.e. the maximum theoretical length of the interval we
918   // could use as [mIterationStart; mIterationEnd].
919   GraphTime inGraph = mStateComputedTime - mIterationStart;
920   // We want the interval [mIterationStart; mIterationEnd] to be before the
921   // interval [mStateComputedTime; nextStateComputedTime]. We also want
922   // the distance between these intervals to be roughly equivalent each time, to
923   // ensure there is no clock drift between current time and state time. Since
924   // we can't act on the state time because we have to fill the audio buffer, we
925   // reclock the current time against the state time, here.
926   mIterationEnd = mIterationStart + 0.8 * inGraph;
927 
928   LOG(LogLevel::Verbose,
929       ("%p: interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) "
930        "(duration ticks: %ld)",
931        Graph(), (long)mIterationStart, (long)mIterationEnd,
932        (long)mStateComputedTime, (long)nextStateComputedTime, (long)aFrames,
933        (uint32_t)durationMS,
934        (long)(nextStateComputedTime - mStateComputedTime)));
935 
936   if (mStateComputedTime < mIterationEnd) {
937     LOG(LogLevel::Error, ("%p: Media graph global underrun detected", Graph()));
938     MOZ_ASSERT_UNREACHABLE("We should not underrun in full duplex");
939     mIterationEnd = mStateComputedTime;
940   }
941 
942   // Process mic data if any/needed
943   if (aInputBuffer && mInputChannelCount > 0) {
944     Graph()->NotifyInputData(aInputBuffer, static_cast<size_t>(aFrames),
945                              mSampleRate, mInputChannelCount);
946   }
947 
948   bool iterate = mBuffer.Available();
949   IterationResult result =
950       iterate
951           ? Graph()->OneIteration(nextStateComputedTime, mIterationEnd, &mMixer)
952           : IterationResult::CreateStillProcessing();
953   if (iterate) {
954     // We totally filled the buffer (and mScratchBuffer isn't empty).
955     // We don't need to run an iteration and if we do so we may overflow.
956     mStateComputedTime = nextStateComputedTime;
957   } else {
958     LOG(LogLevel::Verbose,
959         ("%p: DataCallback buffer filled entirely from scratch "
960          "buffer, skipping iteration.",
961          Graph()));
962     result = IterationResult::CreateStillProcessing();
963   }
964 
965   mBuffer.BufferFilled();
966 
967 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
968   // Prevent returning NaN to the OS mixer, and propagating NaN into the reverse
969   // stream of the AEC.
970   NaNToZeroInPlace(aOutputBuffer, aFrames * mOutputChannelCount);
971 #endif
972 
973   // Callback any observers for the AEC speaker data.  Note that one
974   // (maybe) of these will be full-duplex, the others will get their input
975   // data off separate cubeb callbacks.  Take care with how stuff is
976   // removed/added to this list and TSAN issues, but input and output will
977   // use separate callback methods.
978   Graph()->NotifyOutputData(aOutputBuffer, static_cast<size_t>(aFrames),
979                             mSampleRate, mOutputChannelCount);
980 
981 #ifdef XP_MACOSX
982   // This only happens when the output is on a macbookpro's external speaker,
983   // that are stereo, but let's just be safe.
984   if (mNeedsPanning && mOutputChannelCount == 2) {
985     // hard pan to the right
986     for (uint32_t i = 0; i < aFrames * 2; i += 2) {
987       aOutputBuffer[i + 1] += aOutputBuffer[i];
988       aOutputBuffer[i] = 0.0;
989     }
990   }
991 #endif
992 
993   // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable
994   if (aInputBuffer) {
995     mInputStreamFile.Write(static_cast<const AudioDataValue*>(aInputBuffer),
996                            aFrames * mInputChannelCount);
997   }
998   mOutputStreamFile.Write(static_cast<const AudioDataValue*>(aOutputBuffer),
999                           aFrames * mOutputChannelCount);
1000 
1001   if (result.IsStop()) {
1002     // Signal that we have stopped.
1003     result.Stopped();
1004     // Update the flag before handing over the graph and going to drain.
1005     mAudioStreamState = AudioStreamState::Stopping;
1006     return aFrames - 1;
1007   }
1008 
1009   if (GraphDriver* nextDriver = result.NextDriver()) {
1010     LOG(LogLevel::Debug,
1011         ("%p: Switching to %s driver.", Graph(),
1012          nextDriver->AsAudioCallbackDriver() ? "audio" : "system"));
1013     result.Switched();
1014     mAudioStreamState = AudioStreamState::Stopping;
1015     nextDriver->SetState(mIterationStart, mIterationEnd, mStateComputedTime);
1016     nextDriver->Start();
1017     // Returning less than aFrames starts the draining and eventually stops the
1018     // audio thread. This function will never get called again.
1019     return aFrames - 1;
1020   }
1021 
1022   MOZ_ASSERT(result.IsStillProcessing());
1023   return aFrames;
1024 }
1025 
StateToString(cubeb_state aState)1026 static const char* StateToString(cubeb_state aState) {
1027   switch (aState) {
1028     case CUBEB_STATE_STARTED:
1029       return "STARTED";
1030     case CUBEB_STATE_STOPPED:
1031       return "STOPPED";
1032     case CUBEB_STATE_DRAINED:
1033       return "DRAINED";
1034     case CUBEB_STATE_ERROR:
1035       return "ERROR";
1036     default:
1037       MOZ_CRASH("Unexpected state!");
1038   }
1039 }
1040 
StateCallback(cubeb_state aState)1041 void AudioCallbackDriver::StateCallback(cubeb_state aState) {
1042   MOZ_ASSERT(!InIteration());
1043   LOG(LogLevel::Debug,
1044       ("AudioCallbackDriver(%p) State: %s", this, StateToString(aState)));
1045 
1046   // Clear the flag for the not running and error states (stopped, drained)
1047   AudioStreamState streamState = mAudioStreamState.exchange(
1048       aState == CUBEB_STATE_STARTED
1049           ? AudioStreamState::Running
1050           : aState == CUBEB_STATE_ERROR ? AudioStreamState::Errored
1051                                         : AudioStreamState::None);
1052 
1053   if (aState == CUBEB_STATE_ERROR) {
1054     // About to hand over control of the graph.  Do not start a new driver if
1055     // StateCallback() receives an error for this stream while the main thread
1056     // or another driver has control of the graph.
1057     if (streamState == AudioStreamState::Running) {
1058       MOZ_ASSERT(!ThreadRunning());
1059       FallbackToSystemClockDriver();
1060     }
1061   } else if (aState == CUBEB_STATE_STOPPED) {
1062     MOZ_ASSERT(!ThreadRunning());
1063   }
1064 }
1065 
MixerCallback(AudioDataValue * aMixedBuffer,AudioSampleFormat aFormat,uint32_t aChannels,uint32_t aFrames,uint32_t aSampleRate)1066 void AudioCallbackDriver::MixerCallback(AudioDataValue* aMixedBuffer,
1067                                         AudioSampleFormat aFormat,
1068                                         uint32_t aChannels, uint32_t aFrames,
1069                                         uint32_t aSampleRate) {
1070   MOZ_ASSERT(InIteration());
1071   uint32_t toWrite = mBuffer.Available();
1072 
1073   if (!mBuffer.Available()) {
1074     NS_WARNING("DataCallback buffer full, expect frame drops.");
1075   }
1076 
1077   MOZ_ASSERT(mBuffer.Available() <= aFrames);
1078 
1079   mBuffer.WriteFrames(aMixedBuffer, mBuffer.Available());
1080   MOZ_ASSERT(mBuffer.Available() == 0,
1081              "Missing frames to fill audio callback's buffer.");
1082 
1083   DebugOnly<uint32_t> written = mScratchBuffer.Fill(
1084       aMixedBuffer + toWrite * aChannels, aFrames - toWrite);
1085   NS_WARNING_ASSERTION(written == aFrames - toWrite, "Dropping frames.");
1086 };
1087 
PanOutputIfNeeded(bool aMicrophoneActive)1088 void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive) {
1089 #ifdef XP_MACOSX
1090   AUTO_PROFILER_LABEL("AudioCallbackDriver::PanOutputIfNeeded", MEDIA_CUBEB);
1091   cubeb_device* out = nullptr;
1092   int rv;
1093   char name[128];
1094   size_t length = sizeof(name);
1095 
1096   rv = sysctlbyname("hw.model", name, &length, NULL, 0);
1097   if (rv) {
1098     return;
1099   }
1100 
1101   int major, minor;
1102   for (uint32_t i = 0; i < length; i++) {
1103     // skip the model name
1104     if (isalpha(name[i])) {
1105       continue;
1106     }
1107     sscanf(name + i, "%d,%d", &major, &minor);
1108     break;
1109   }
1110 
1111   enum MacbookModel { MacBook, MacBookPro, MacBookAir, NotAMacbook };
1112 
1113   MacbookModel model;
1114 
1115   if (!strncmp(name, "MacBookPro", length)) {
1116     model = MacBookPro;
1117   } else if (strncmp(name, "MacBookAir", length)) {
1118     model = MacBookAir;
1119   } else if (strncmp(name, "MacBook", length)) {
1120     model = MacBook;
1121   } else {
1122     model = NotAMacbook;
1123   }
1124   // For macbook pro before 2016 model (change of chassis), hard pan the audio
1125   // to the right if the speakers are in use to avoid feedback.
1126   if (model == MacBookPro && major <= 12) {
1127     if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) {
1128       MOZ_ASSERT(out);
1129       // Check if we are currently outputing sound on external speakers.
1130       if (out->output_name && !strcmp(out->output_name, "ispk")) {
1131         // Pan everything to the right speaker.
1132         LOG(LogLevel::Debug, ("Using the built-in speakers, with%s audio input",
1133                               aMicrophoneActive ? "" : "out"));
1134         mNeedsPanning = aMicrophoneActive;
1135       } else {
1136         LOG(LogLevel::Debug, ("Using an external output device"));
1137         mNeedsPanning = false;
1138       }
1139       cubeb_stream_device_destroy(mAudioStream, out);
1140     }
1141   }
1142 #endif
1143 }
1144 
DeviceChangedCallback()1145 void AudioCallbackDriver::DeviceChangedCallback() {
1146   MOZ_ASSERT(!InIteration());
1147   // Tell the audio engine the device has changed, it might want to reset some
1148   // state.
1149   Graph()->DeviceChanged();
1150 #ifdef XP_MACOSX
1151   RefPtr<AudioCallbackDriver> self(this);
1152   bool hasInput = mInputChannelCount;
1153   NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1154       "PanOutputIfNeeded", [self{std::move(self)}, hasInput]() {
1155         self->PanOutputIfNeeded(hasInput);
1156       }));
1157 #endif
1158 }
1159 
IterationDuration()1160 uint32_t AudioCallbackDriver::IterationDuration() {
1161   MOZ_ASSERT(InIteration());
1162   // The real fix would be to have an API in cubeb to give us the number. Short
1163   // of that, we approximate it here. bug 1019507
1164   return mIterationDurationMS;
1165 }
1166 
EnsureNextIteration()1167 void AudioCallbackDriver::EnsureNextIteration() {
1168   if (mFallbackDriverState == FallbackDriverState::Running) {
1169     auto fallback = mFallback.Lock();
1170     if (fallback.ref()) {
1171       fallback.ref()->EnsureNextIteration();
1172     }
1173   }
1174 }
1175 
IsStarted()1176 bool AudioCallbackDriver::IsStarted() { return mStarted; }
1177 
EnqueueTrackAndPromiseForOperation(MediaTrack * aTrack,dom::AudioContextOperation aOperation,AbstractThread * aMainThread,MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise> && aHolder)1178 void AudioCallbackDriver::EnqueueTrackAndPromiseForOperation(
1179     MediaTrack* aTrack, dom::AudioContextOperation aOperation,
1180     AbstractThread* aMainThread,
1181     MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise>&& aHolder) {
1182   MOZ_ASSERT(InIteration() || !ThreadRunning());
1183   auto promises = mPromisesForOperation.Lock();
1184   promises->AppendElement(TrackAndPromiseForOperation(
1185       aTrack, aOperation, aMainThread, std::move(aHolder)));
1186 }
1187 
CompleteAudioContextOperations(AsyncCubebOperation aOperation)1188 void AudioCallbackDriver::CompleteAudioContextOperations(
1189     AsyncCubebOperation aOperation) {
1190   MOZ_ASSERT(OnCubebOperationThread());
1191   auto promises = mPromisesForOperation.Lock();
1192   for (uint32_t i = 0; i < promises->Length(); i++) {
1193     TrackAndPromiseForOperation& s = promises.ref()[i];
1194     if ((aOperation == AsyncCubebOperation::INIT &&
1195          s.mOperation == dom::AudioContextOperation::Resume) ||
1196         (aOperation == AsyncCubebOperation::SHUTDOWN &&
1197          s.mOperation != dom::AudioContextOperation::Resume)) {
1198       AudioContextState state;
1199       switch (s.mOperation) {
1200         case dom::AudioContextOperation::Resume:
1201           state = dom::AudioContextState::Running;
1202           break;
1203         case dom::AudioContextOperation::Suspend:
1204           state = dom::AudioContextState::Suspended;
1205           break;
1206         case dom::AudioContextOperation::Close:
1207           state = dom::AudioContextState::Closed;
1208           break;
1209         default:
1210           MOZ_CRASH("Unexpected operation");
1211       }
1212       s.mMainThread->Dispatch(NS_NewRunnableFunction(
1213           "AudioContextOperation::Resolve",
1214           [holder = std::move(s.mHolder), state]() mutable {
1215             holder.Resolve(state, __func__);
1216           }));
1217       promises->RemoveElementAt(i);
1218       i--;
1219     }
1220   }
1221 }
1222 
AudioOutputLatency()1223 TimeDuration AudioCallbackDriver::AudioOutputLatency() {
1224   AUTO_PROFILER_LABEL("AudioCallbackDriver::AudioOutputLatency", MEDIA_CUBEB);
1225   uint32_t latencyFrames;
1226   int rv = cubeb_stream_get_latency(mAudioStream, &latencyFrames);
1227   if (rv || mSampleRate == 0) {
1228     return TimeDuration::FromSeconds(0.0);
1229   }
1230 
1231   return TimeDuration::FromSeconds(static_cast<double>(latencyFrames) /
1232                                    mSampleRate);
1233 }
1234 
AudioInputLatency()1235 TimeDuration AudioCallbackDriver::AudioInputLatency() {
1236   uint32_t latencyFrames;
1237   int rv = cubeb_stream_get_input_latency(mAudioStream, &latencyFrames);
1238   if (rv || mSampleRate == 0) {
1239     return TimeDuration::FromSeconds(0.0);
1240   }
1241 
1242   return TimeDuration::FromSeconds(static_cast<double>(latencyFrames) /
1243                                    mSampleRate);
1244 }
1245 
FallbackToSystemClockDriver()1246 void AudioCallbackDriver::FallbackToSystemClockDriver() {
1247   MOZ_ASSERT(!ThreadRunning());
1248   MOZ_ASSERT(mAudioStreamState == AudioStreamState::None ||
1249              mAudioStreamState == AudioStreamState::Errored ||
1250              mAudioStreamState == AudioStreamState::Pending);
1251   MOZ_ASSERT(mFallbackDriverState == FallbackDriverState::None);
1252   LOG(LogLevel::Debug,
1253       ("%p: AudioCallbackDriver %p Falling back to SystemClockDriver.", Graph(),
1254        this));
1255   mFallbackDriverState = FallbackDriverState::Running;
1256   mNextReInitBackoffStep =
1257       TimeDuration::FromMilliseconds(AUDIO_INITIAL_FALLBACK_BACKOFF_STEP_MS);
1258   mNextReInitAttempt = TimeStamp::Now() + mNextReInitBackoffStep;
1259   auto fallback =
1260       MakeRefPtr<FallbackWrapper>(Graph(), this, mSampleRate, mIterationStart,
1261                                   mIterationEnd, mStateComputedTime);
1262   {
1263     auto driver = mFallback.Lock();
1264     driver.ref() = fallback;
1265   }
1266   fallback->Start();
1267 }
1268 
FallbackDriverStopped(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime,FallbackDriverState aState)1269 void AudioCallbackDriver::FallbackDriverStopped(GraphTime aIterationStart,
1270                                                 GraphTime aIterationEnd,
1271                                                 GraphTime aStateComputedTime,
1272                                                 FallbackDriverState aState) {
1273   mIterationStart = aIterationStart;
1274   mIterationEnd = aIterationEnd;
1275   mStateComputedTime = aStateComputedTime;
1276   mNextReInitAttempt = TimeStamp();
1277   mNextReInitBackoffStep = TimeDuration();
1278   {
1279     auto fallback = mFallback.Lock();
1280     MOZ_ASSERT(fallback.ref()->OnThread());
1281     fallback.ref() = nullptr;
1282   }
1283 
1284   MOZ_ASSERT(aState == FallbackDriverState::None ||
1285              aState == FallbackDriverState::Stopped);
1286   MOZ_ASSERT_IF(aState == FallbackDriverState::None,
1287                 mAudioStreamState == AudioStreamState::Running);
1288   mFallbackDriverState = aState;
1289 }
1290 
MaybeStartAudioStream()1291 void AudioCallbackDriver::MaybeStartAudioStream() {
1292   AudioStreamState streamState = mAudioStreamState;
1293   MOZ_ASSERT(
1294       streamState != AudioStreamState::Errored,
1295       "An errored stream must not attempted to be re-started, an error stream"
1296       " has already beed started once");
1297   if (streamState != AudioStreamState::None) {
1298     LOG(LogLevel::Verbose,
1299         ("%p: AudioCallbackDriver %p Cannot re-init.", Graph(), this));
1300     return;
1301   }
1302 
1303   TimeStamp now = TimeStamp::Now();
1304   if (now < mNextReInitAttempt) {
1305     LOG(LogLevel::Verbose,
1306         ("%p: AudioCallbackDriver %p Not time to re-init yet. %.3fs left.",
1307          Graph(), this, (mNextReInitAttempt - now).ToSeconds()));
1308     return;
1309   }
1310 
1311   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p Attempting to re-init "
1312                         "audio stream from fallback driver.",
1313                         Graph(), this));
1314   mNextReInitBackoffStep = std::min(
1315       mNextReInitBackoffStep * 2,
1316       TimeDuration::FromMilliseconds(AUDIO_MAX_FALLBACK_BACKOFF_STEP_MS));
1317   mNextReInitAttempt = now + mNextReInitBackoffStep;
1318   Start();
1319 }
1320 
1321 }  // namespace mozilla
1322 
1323 // avoid redefined macro in unified build
1324 #undef LOG
1325