1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5  * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 
7 #include "GraphDriver.h"
8 
9 #include "AudioNodeEngine.h"
10 #include "mozilla/dom/AudioContext.h"
11 #include "mozilla/dom/AudioDeviceInfo.h"
12 #include "mozilla/dom/BaseAudioContextBinding.h"
13 #include "mozilla/SchedulerGroup.h"
14 #include "mozilla/SharedThreadPool.h"
15 #include "mozilla/ClearOnShutdown.h"
16 #include "mozilla/Unused.h"
17 #include "mozilla/MathAlgorithms.h"
18 #include "CubebDeviceEnumerator.h"
19 #include "MediaTrackGraphImpl.h"
20 #include "AudioThreadRegistry.h"
21 #include "Tracing.h"
22 
23 #ifdef MOZ_WEBRTC
24 #  include "webrtc/MediaEngineWebRTC.h"
25 #endif
26 
27 #ifdef XP_MACOSX
28 #  include <sys/sysctl.h>
29 #endif
30 
31 extern mozilla::LazyLogModule gMediaTrackGraphLog;
32 #ifdef LOG
33 #  undef LOG
34 #endif  // LOG
35 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
36 
37 namespace mozilla {
38 
GraphDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)39 GraphDriver::GraphDriver(GraphInterface* aGraphInterface,
40                          GraphDriver* aPreviousDriver, uint32_t aSampleRate)
41     : mGraphInterface(aGraphInterface),
42       mSampleRate(aSampleRate),
43       mPreviousDriver(aPreviousDriver) {}
44 
SetState(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)45 void GraphDriver::SetState(GraphTime aIterationStart, GraphTime aIterationEnd,
46                            GraphTime aStateComputedTime) {
47   MOZ_ASSERT(InIteration() || !ThreadRunning());
48 
49   mIterationStart = aIterationStart;
50   mIterationEnd = aIterationEnd;
51   mStateComputedTime = aStateComputedTime;
52 }
53 
54 #ifdef DEBUG
InIteration() const55 bool GraphDriver::InIteration() const {
56   return OnThread() || Graph()->InDriverIteration(this);
57 }
58 #endif
59 
PreviousDriver()60 GraphDriver* GraphDriver::PreviousDriver() {
61   MOZ_ASSERT(InIteration() || !ThreadRunning());
62   return mPreviousDriver;
63 }
64 
SetPreviousDriver(GraphDriver * aPreviousDriver)65 void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver) {
66   MOZ_ASSERT(InIteration() || !ThreadRunning());
67   mPreviousDriver = aPreviousDriver;
68 }
69 
ThreadedDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)70 ThreadedDriver::ThreadedDriver(GraphInterface* aGraphInterface,
71                                GraphDriver* aPreviousDriver,
72                                uint32_t aSampleRate)
73     : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
74       mThreadRunning(false) {}
75 
76 class MediaTrackGraphShutdownThreadRunnable : public Runnable {
77  public:
MediaTrackGraphShutdownThreadRunnable(already_AddRefed<nsIThread> aThread)78   explicit MediaTrackGraphShutdownThreadRunnable(
79       already_AddRefed<nsIThread> aThread)
80       : Runnable("MediaTrackGraphShutdownThreadRunnable"), mThread(aThread) {}
Run()81   NS_IMETHOD Run() override {
82     TRACE("MediaTrackGraphShutdownThreadRunnable");
83     MOZ_ASSERT(NS_IsMainThread());
84     MOZ_ASSERT(mThread);
85 
86     mThread->Shutdown();
87     mThread = nullptr;
88     return NS_OK;
89   }
90 
91  private:
92   nsCOMPtr<nsIThread> mThread;
93 };
94 
~ThreadedDriver()95 ThreadedDriver::~ThreadedDriver() {
96   if (mThread) {
97     nsCOMPtr<nsIRunnable> event =
98         new MediaTrackGraphShutdownThreadRunnable(mThread.forget());
99     SchedulerGroup::Dispatch(TaskCategory::Other, event.forget());
100   }
101 }
102 
103 class MediaTrackGraphInitThreadRunnable : public Runnable {
104  public:
MediaTrackGraphInitThreadRunnable(ThreadedDriver * aDriver)105   explicit MediaTrackGraphInitThreadRunnable(ThreadedDriver* aDriver)
106       : Runnable("MediaTrackGraphInitThreadRunnable"), mDriver(aDriver) {}
Run()107   NS_IMETHOD Run() override {
108     TRACE("MediaTrackGraphInitThreadRunnable");
109     MOZ_ASSERT(!mDriver->ThreadRunning());
110     LOG(LogLevel::Debug, ("Starting a new system driver for graph %p",
111                           mDriver->mGraphInterface.get()));
112 
113     if (GraphDriver* previousDriver = mDriver->PreviousDriver()) {
114       LOG(LogLevel::Debug,
115           ("%p releasing an AudioCallbackDriver(%p), for graph %p",
116            mDriver.get(), previousDriver, mDriver->Graph()));
117       MOZ_ASSERT(!mDriver->AsAudioCallbackDriver());
118       RefPtr<AsyncCubebTask> releaseEvent =
119           new AsyncCubebTask(previousDriver->AsAudioCallbackDriver(),
120                              AsyncCubebOperation::SHUTDOWN);
121       releaseEvent->Dispatch();
122       mDriver->SetPreviousDriver(nullptr);
123     }
124 
125     mDriver->RunThread();
126     return NS_OK;
127   }
128 
129  private:
130   RefPtr<ThreadedDriver> mDriver;
131 };
132 
Start()133 void ThreadedDriver::Start() {
134   MOZ_ASSERT(!ThreadRunning());
135   LOG(LogLevel::Debug,
136       ("Starting thread for a SystemClockDriver  %p", mGraphInterface.get()));
137   Unused << NS_WARN_IF(mThread);
138   MOZ_ASSERT(!mThread);  // Ensure we haven't already started it
139 
140   nsCOMPtr<nsIRunnable> event = new MediaTrackGraphInitThreadRunnable(this);
141   // Note: mThread may be null during event->Run() if we pass to NewNamedThread!
142   // See AudioInitTask
143   nsresult rv = NS_NewNamedThread("MediaTrackGrph", getter_AddRefs(mThread));
144   if (NS_SUCCEEDED(rv)) {
145     mThread->EventTarget()->Dispatch(event.forget(), NS_DISPATCH_NORMAL);
146   }
147 }
148 
Shutdown()149 void ThreadedDriver::Shutdown() {
150   NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
151   // mGraph's thread is not running so it's OK to do whatever here
152   LOG(LogLevel::Debug, ("Stopping threads for MediaTrackGraph %p", this));
153 
154   if (mThread) {
155     LOG(LogLevel::Debug,
156         ("%p: Stopping ThreadedDriver's %p thread", Graph(), this));
157     mThread->Shutdown();
158     mThread = nullptr;
159   }
160 }
161 
SystemClockDriver(GraphInterface * aGraphInterface,GraphDriver * aPreviousDriver,uint32_t aSampleRate)162 SystemClockDriver::SystemClockDriver(GraphInterface* aGraphInterface,
163                                      GraphDriver* aPreviousDriver,
164                                      uint32_t aSampleRate)
165     : ThreadedDriver(aGraphInterface, aPreviousDriver, aSampleRate),
166       mInitialTimeStamp(TimeStamp::Now()),
167       mCurrentTimeStamp(TimeStamp::Now()),
168       mLastTimeStamp(TimeStamp::Now()) {}
169 
170 SystemClockDriver::~SystemClockDriver() = default;
171 
RunThread()172 void ThreadedDriver::RunThread() {
173   mThreadRunning = true;
174   while (true) {
175     mIterationStart = mIterationEnd;
176     mIterationEnd += GetIntervalForIteration();
177 
178     if (mStateComputedTime < mIterationEnd) {
179       LOG(LogLevel::Warning, ("%p: Global underrun detected", Graph()));
180       mIterationEnd = mStateComputedTime;
181     }
182 
183     if (mIterationStart >= mIterationEnd) {
184       NS_ASSERTION(mIterationStart == mIterationEnd,
185                    "Time can't go backwards!");
186       // This could happen due to low clock resolution, maybe?
187       LOG(LogLevel::Debug, ("%p: Time did not advance", Graph()));
188     }
189 
190     GraphTime nextStateComputedTime =
191         MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(
192             mIterationEnd + MillisecondsToMediaTime(AUDIO_TARGET_MS));
193     if (nextStateComputedTime < mStateComputedTime) {
194       // A previous driver may have been processing further ahead of
195       // iterationEnd.
196       LOG(LogLevel::Warning,
197           ("%p: Prevent state from going backwards. interval[%ld; %ld] "
198            "state[%ld; "
199            "%ld]",
200            Graph(), (long)mIterationStart, (long)mIterationEnd,
201            (long)mStateComputedTime, (long)nextStateComputedTime));
202       nextStateComputedTime = mStateComputedTime;
203     }
204     LOG(LogLevel::Verbose,
205         ("%p: interval[%ld; %ld] state[%ld; %ld]", Graph(),
206          (long)mIterationStart, (long)mIterationEnd, (long)mStateComputedTime,
207          (long)nextStateComputedTime));
208 
209     mStateComputedTime = nextStateComputedTime;
210     IterationResult result =
211         Graph()->OneIteration(mStateComputedTime, mIterationEnd, nullptr);
212 
213     if (result.IsStop()) {
214       // Signal that we're done stopping.
215       result.Stopped();
216       break;
217     }
218     WaitForNextIteration();
219     if (GraphDriver* nextDriver = result.NextDriver()) {
220       LOG(LogLevel::Debug, ("%p: Switching to AudioCallbackDriver", Graph()));
221       result.Switched();
222       nextDriver->SetState(mIterationStart, mIterationEnd, mStateComputedTime);
223       nextDriver->Start();
224       break;
225     }
226     MOZ_ASSERT(result.IsStillProcessing());
227   }
228   mThreadRunning = false;
229 }
230 
GetIntervalForIteration()231 MediaTime SystemClockDriver::GetIntervalForIteration() {
232   TimeStamp now = TimeStamp::Now();
233   MediaTime interval =
234       SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds());
235   mCurrentTimeStamp = now;
236 
237   MOZ_LOG(gMediaTrackGraphLog, LogLevel::Verbose,
238           ("%p: Updating current time to %f (real %f, StateComputedTime() %f)",
239            Graph(), MediaTimeToSeconds(mIterationEnd + interval),
240            (now - mInitialTimeStamp).ToSeconds(),
241            MediaTimeToSeconds(mStateComputedTime)));
242 
243   return interval;
244 }
245 
EnsureNextIteration()246 void ThreadedDriver::EnsureNextIteration() {
247   mWaitHelper.EnsureNextIteration();
248 }
249 
WaitForNextIteration()250 void ThreadedDriver::WaitForNextIteration() {
251   MOZ_ASSERT(mThread);
252   MOZ_ASSERT(OnThread());
253   mWaitHelper.WaitForNextIterationAtLeast(WaitInterval());
254 }
255 
WaitInterval()256 TimeDuration SystemClockDriver::WaitInterval() {
257   MOZ_ASSERT(mThread);
258   MOZ_ASSERT(OnThread());
259   TimeStamp now = TimeStamp::Now();
260   int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS -
261                       int64_t((now - mCurrentTimeStamp).ToMilliseconds());
262   // Make sure timeoutMS doesn't overflow 32 bits by waking up at
263   // least once a minute, if we need to wake up at all
264   timeoutMS = std::max<int64_t>(0, std::min<int64_t>(timeoutMS, 60 * 1000));
265   LOG(LogLevel::Verbose,
266       ("%p: Waiting for next iteration; at %f, timeout=%f", Graph(),
267        (now - mInitialTimeStamp).ToSeconds(), timeoutMS / 1000.0));
268 
269   return TimeDuration::FromMilliseconds(timeoutMS);
270 }
271 
OfflineClockDriver(GraphInterface * aGraphInterface,uint32_t aSampleRate,GraphTime aSlice)272 OfflineClockDriver::OfflineClockDriver(GraphInterface* aGraphInterface,
273                                        uint32_t aSampleRate, GraphTime aSlice)
274     : ThreadedDriver(aGraphInterface, nullptr, aSampleRate), mSlice(aSlice) {}
275 
276 OfflineClockDriver::~OfflineClockDriver() = default;
277 
RunThread()278 void OfflineClockDriver::RunThread() {
279   nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(mThread);
280   nsCOMPtr<nsIThreadObserver> observer = do_QueryInterface(Graph());
281   threadInternal->SetObserver(observer);
282 
283   ThreadedDriver::RunThread();
284 }
285 
GetIntervalForIteration()286 MediaTime OfflineClockDriver::GetIntervalForIteration() {
287   return MillisecondsToMediaTime(mSlice);
288 }
289 
AsyncCubebTask(AudioCallbackDriver * aDriver,AsyncCubebOperation aOperation)290 AsyncCubebTask::AsyncCubebTask(AudioCallbackDriver* aDriver,
291                                AsyncCubebOperation aOperation)
292     : Runnable("AsyncCubebTask"),
293       mDriver(aDriver),
294       mOperation(aOperation),
295       mShutdownGrip(aDriver->Graph()) {
296   NS_WARNING_ASSERTION(
297       mDriver->mAudioStream || aOperation == AsyncCubebOperation::INIT,
298       "No audio stream!");
299 }
300 
301 AsyncCubebTask::~AsyncCubebTask() = default;
302 
303 NS_IMETHODIMP
Run()304 AsyncCubebTask::Run() {
305   MOZ_ASSERT(mDriver);
306 
307   switch (mOperation) {
308     case AsyncCubebOperation::INIT: {
309       LOG(LogLevel::Debug, ("%p: AsyncCubebOperation::INIT driver=%p",
310                             mDriver->Graph(), mDriver.get()));
311       mDriver->Init();
312       break;
313     }
314     case AsyncCubebOperation::SHUTDOWN: {
315       LOG(LogLevel::Debug, ("%p: AsyncCubebOperation::SHUTDOWN driver=%p",
316                             mDriver->Graph(), mDriver.get()));
317       mDriver->Stop();
318       mDriver = nullptr;
319       mShutdownGrip = nullptr;
320       break;
321     }
322     default:
323       MOZ_CRASH("Operation not implemented.");
324   }
325 
326   // The thread will kill itself after a bit
327   return NS_OK;
328 }
329 
TrackAndPromiseForOperation(MediaTrack * aTrack,dom::AudioContextOperation aOperation,AbstractThread * aMainThread,MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise> && aHolder)330 TrackAndPromiseForOperation::TrackAndPromiseForOperation(
331     MediaTrack* aTrack, dom::AudioContextOperation aOperation,
332     AbstractThread* aMainThread,
333     MozPromiseHolder<MediaTrackGraph::AudioContextOperationPromise>&& aHolder)
334     : mTrack(aTrack),
335       mOperation(aOperation),
336       mMainThread(aMainThread),
337       mHolder(std::move(aHolder)) {}
338 
TrackAndPromiseForOperation(TrackAndPromiseForOperation && aOther)339 TrackAndPromiseForOperation::TrackAndPromiseForOperation(
340     TrackAndPromiseForOperation&& aOther) noexcept
341     : mTrack(std::move(aOther.mTrack)),
342       mOperation(aOther.mOperation),
343       mMainThread(std::move(aOther.mMainThread)),
344       mHolder(std::move(aOther.mHolder)) {}
345 
346 /* Helper to proxy the GraphInterface methods used by a running
347  * mFallbackDriver. */
348 class AudioCallbackDriver::FallbackWrapper : public GraphInterface {
349  public:
FallbackWrapper(RefPtr<GraphInterface> aGraph,RefPtr<AudioCallbackDriver> aOwner,uint32_t aSampleRate,GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)350   FallbackWrapper(RefPtr<GraphInterface> aGraph,
351                   RefPtr<AudioCallbackDriver> aOwner, uint32_t aSampleRate,
352                   GraphTime aIterationStart, GraphTime aIterationEnd,
353                   GraphTime aStateComputedTime)
354       : mGraph(std::move(aGraph)),
355         mOwner(std::move(aOwner)),
356         mFallbackDriver(
357             MakeRefPtr<SystemClockDriver>(this, nullptr, aSampleRate)),
358         mIterationStart(aIterationStart),
359         mIterationEnd(aIterationEnd),
360         mStateComputedTime(aStateComputedTime) {
361     mFallbackDriver->SetState(mIterationStart, mIterationEnd,
362                               mStateComputedTime);
363   }
364 
365   NS_DECL_THREADSAFE_ISUPPORTS
366 
367   /* Proxied SystemClockDriver methods */
SetState(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime)368   void SetState(GraphTime aIterationStart, GraphTime aIterationEnd,
369                 GraphTime aStateComputedTime) {
370     mIterationStart = aIterationStart;
371     mIterationEnd = aIterationEnd;
372     mStateComputedTime = aStateComputedTime;
373     mFallbackDriver->SetState(aIterationStart, aIterationEnd,
374                               aStateComputedTime);
375   }
Start()376   void Start() { mFallbackDriver->Start(); }
Shutdown()377   MOZ_CAN_RUN_SCRIPT void Shutdown() {
378     RefPtr<SystemClockDriver> driver = mFallbackDriver;
379     driver->Shutdown();
380   }
EnsureNextIteration()381   void EnsureNextIteration() { mFallbackDriver->EnsureNextIteration(); }
382 #ifdef DEBUG
InIteration()383   bool InIteration() { return mFallbackDriver->InIteration(); }
384 #endif
OnThread()385   bool OnThread() { return mFallbackDriver->OnThread(); }
386 
387   /* GraphInterface methods */
NotifyOutputData(AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)388   void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
389                         TrackRate aRate, uint32_t aChannels) override {
390     MOZ_CRASH("Unexpected NotifyOutputData from fallback SystemClockDriver");
391   }
NotifyInputStopped()392   void NotifyInputStopped() override {
393     MOZ_CRASH("Unexpected NotifyInputStopped from fallback SystemClockDriver");
394   }
NotifyInputData(const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels,uint32_t aAlreadyBuffered)395   void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
396                        TrackRate aRate, uint32_t aChannels,
397                        uint32_t aAlreadyBuffered) override {
398     MOZ_CRASH("Unexpected NotifyInputData from fallback SystemClockDriver");
399   }
DeviceChanged()400   void DeviceChanged() override {
401     MOZ_CRASH("Unexpected DeviceChanged from fallback SystemClockDriver");
402   }
403 #ifdef DEBUG
InDriverIteration(const GraphDriver * aDriver) const404   bool InDriverIteration(const GraphDriver* aDriver) const override {
405     return !mOwner->ThreadRunning() && mOwner->InIteration();
406   }
407 #endif
OneIteration(GraphTime aStateComputedEnd,GraphTime aIterationEnd,AudioMixer * aMixer)408   IterationResult OneIteration(GraphTime aStateComputedEnd,
409                                GraphTime aIterationEnd,
410                                AudioMixer* aMixer) override {
411     MOZ_ASSERT(!aMixer);
412 
413 #ifdef DEBUG
414     AutoInCallback aic(mOwner);
415 #endif
416 
417     mIterationStart = mIterationEnd;
418     mIterationEnd = aIterationEnd;
419     mStateComputedTime = aStateComputedEnd;
420     IterationResult result =
421         mGraph->OneIteration(aStateComputedEnd, aIterationEnd, aMixer);
422 
423     AudioStreamState audioState = mOwner->mAudioStreamState;
424 
425     MOZ_ASSERT(audioState != AudioStreamState::Stopping,
426                "The audio driver can only enter stopping if it iterated the "
427                "graph, which it can only do if there's no fallback driver");
428     if (audioState != AudioStreamState::Running && result.IsStillProcessing()) {
429       mOwner->MaybeStartAudioStream();
430       return result;
431     }
432 
433     MOZ_ASSERT(result.IsStillProcessing() || result.IsStop() ||
434                result.IsSwitchDriver());
435 
436     // Proxy the release of the fallback driver to a background thread, so it
437     // doesn't perform unexpected suicide.
438     IterationResult stopFallback =
439         IterationResult::CreateStop(NS_NewRunnableFunction(
440             "AudioCallbackDriver::FallbackDriverStopped",
441             [self = RefPtr<FallbackWrapper>(this), this,
442              result = std::move(result)]() mutable {
443               FallbackDriverState fallbackState =
444                   result.IsStillProcessing() ? FallbackDriverState::None
445                                              : FallbackDriverState::Stopped;
446               mOwner->FallbackDriverStopped(mIterationStart, mIterationEnd,
447                                             mStateComputedTime, fallbackState);
448 
449               if (fallbackState == FallbackDriverState::Stopped) {
450 #ifdef DEBUG
451                 // The AudioCallbackDriver may not iterate the graph, but we'll
452                 // call into it so we need to be regarded as "in iteration".
453                 AutoInCallback aic(mOwner);
454 #endif
455                 if (GraphDriver* nextDriver = result.NextDriver()) {
456                   LOG(LogLevel::Debug,
457                       ("%p: Switching from fallback to other driver.",
458                        mGraph.get()));
459                   result.Switched();
460                   nextDriver->SetState(mIterationStart, mIterationEnd,
461                                        mStateComputedTime);
462                   nextDriver->Start();
463                 } else if (result.IsStop()) {
464                   LOG(LogLevel::Debug,
465                       ("%p: Stopping fallback driver.", mGraph.get()));
466                   result.Stopped();
467                 }
468               }
469               mOwner = nullptr;
470               NS_DispatchBackgroundTask(NS_NewRunnableFunction(
471                   "AudioCallbackDriver::FallbackDriverStopped::Release",
472                   [fallback = std::move(self->mFallbackDriver)] {}));
473             }));
474 
475     return stopFallback;
476   }
477 
478  private:
479   virtual ~FallbackWrapper() = default;
480 
481   const RefPtr<GraphInterface> mGraph;
482   // Valid until mFallbackDriver has finished its last iteration.
483   RefPtr<AudioCallbackDriver> mOwner;
484   RefPtr<SystemClockDriver> mFallbackDriver;
485 
486   GraphTime mIterationStart;
487   GraphTime mIterationEnd;
488   GraphTime mStateComputedTime;
489 };
490 
NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)491 NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)
492 
493 AudioCallbackDriver::AudioCallbackDriver(
494     GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
495     uint32_t aSampleRate, uint32_t aOutputChannelCount,
496     uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
497     CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType)
498     : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
499       mOutputChannelCount(aOutputChannelCount),
500       mInputChannelCount(aInputChannelCount),
501       mOutputDeviceID(aOutputDeviceID),
502       mInputDeviceID(aInputDeviceID),
503       mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS),
504       mStarted(false),
505       mInitShutdownThread(SharedThreadPool::Get("CubebOperation"_ns, 1)),
506       mAudioThreadId(ProfilerThreadId{}),
507       mAudioThreadIdInCb(std::thread::id()),
508       mAudioStreamState(AudioStreamState::None),
509       mFallback("AudioCallbackDriver::mFallback"),
510       mSandboxed(CubebUtils::SandboxEnabled()) {
511   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p ctor - input: device %p, "
512                         "channel %d, output: device %p, channel %d",
513                         Graph(), this, mInputDeviceID, mInputChannelCount,
514                         mOutputDeviceID, mOutputChannelCount));
515 
516   NS_WARNING_ASSERTION(mOutputChannelCount != 0,
517                        "Invalid output channel count");
518   MOZ_ASSERT(mOutputChannelCount <= 8);
519 
520   const uint32_t kIdleThreadTimeoutMs = 2000;
521   mInitShutdownThread->SetIdleThreadTimeout(
522       PR_MillisecondsToInterval(kIdleThreadTimeoutMs));
523 
524   if (aAudioInputType == AudioInputType::Voice) {
525     LOG(LogLevel::Debug, ("VOICE."));
526     mInputDevicePreference = CUBEB_DEVICE_PREF_VOICE;
527     CubebUtils::SetInCommunication(true);
528   } else {
529     mInputDevicePreference = CUBEB_DEVICE_PREF_ALL;
530   }
531 
532   mMixer.AddCallback(WrapNotNull(this));
533 }
534 
~AudioCallbackDriver()535 AudioCallbackDriver::~AudioCallbackDriver() {
536   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
537     CubebUtils::SetInCommunication(false);
538   }
539 }
540 
IsMacbookOrMacbookAir()541 bool IsMacbookOrMacbookAir() {
542 #ifdef XP_MACOSX
543   size_t len = 0;
544   sysctlbyname("hw.model", NULL, &len, NULL, 0);
545   if (len) {
546     UniquePtr<char[]> model(new char[len]);
547     // This string can be
548     // MacBook%d,%d for a normal MacBook
549     // MacBookAir%d,%d for a Macbook Air
550     sysctlbyname("hw.model", model.get(), &len, NULL, 0);
551     char* substring = strstr(model.get(), "MacBook");
552     if (substring) {
553       const size_t offset = strlen("MacBook");
554       if (!strncmp(model.get() + offset, "Air", 3) ||
555           isdigit(model[offset + 1])) {
556         return true;
557       }
558     }
559   }
560 #endif
561   return false;
562 }
563 
Init()564 void AudioCallbackDriver::Init() {
565   TRACE("AudioCallbackDriver::Init");
566   MOZ_ASSERT(OnCubebOperationThread());
567   MOZ_ASSERT(mAudioStreamState == AudioStreamState::Pending);
568   FallbackDriverState fallbackState = mFallbackDriverState;
569   if (fallbackState == FallbackDriverState::Stopped) {
570     // The graph has already stopped us.
571     return;
572   }
573   bool fromFallback = fallbackState == FallbackDriverState::Running;
574   cubeb* cubebContext = CubebUtils::GetCubebContext();
575   if (!cubebContext) {
576     NS_WARNING("Could not get cubeb context.");
577     LOG(LogLevel::Warning, ("%s: Could not get cubeb context", __func__));
578     mAudioStreamState = AudioStreamState::None;
579     if (!fromFallback) {
580       CubebUtils::ReportCubebStreamInitFailure(true);
581       FallbackToSystemClockDriver();
582     }
583     return;
584   }
585 
586   cubeb_stream_params output;
587   cubeb_stream_params input;
588   bool firstStream = CubebUtils::GetFirstStream();
589 
590   MOZ_ASSERT(!NS_IsMainThread(),
591              "This is blocking and should never run on the main thread.");
592 
593   output.rate = mSampleRate;
594 
595 #ifdef MOZ_SAMPLE_TYPE_S16
596   MOZ_ASSERT(AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_S16);
597   output.format = CUBEB_SAMPLE_S16NE;
598 #else
599   MOZ_ASSERT(AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_FLOAT32);
600   output.format = CUBEB_SAMPLE_FLOAT32NE;
601 #endif
602 
603   if (!mOutputChannelCount) {
604     LOG(LogLevel::Warning, ("Output number of channels is 0."));
605     mAudioStreamState = AudioStreamState::None;
606     if (!fromFallback) {
607       CubebUtils::ReportCubebStreamInitFailure(firstStream);
608       FallbackToSystemClockDriver();
609     }
610     return;
611   }
612 
613   CubebUtils::AudioDeviceID forcedOutputDeviceId = nullptr;
614 
615   char* forcedOutputDeviceName = CubebUtils::GetForcedOutputDevice();
616   if (forcedOutputDeviceName) {
617     RefPtr<CubebDeviceEnumerator> enumerator = Enumerator::GetInstance();
618     RefPtr<AudioDeviceInfo> device = enumerator->DeviceInfoFromName(
619         NS_ConvertUTF8toUTF16(forcedOutputDeviceName), EnumeratorSide::OUTPUT);
620     if (device && device->DeviceID()) {
621       forcedOutputDeviceId = device->DeviceID();
622     }
623   }
624 
625   mBuffer = AudioCallbackBufferWrapper<AudioDataValue>(mOutputChannelCount);
626   mScratchBuffer =
627       SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2>(mOutputChannelCount);
628 
629   output.channels = mOutputChannelCount;
630   AudioConfig::ChannelLayout::ChannelMap channelMap =
631       AudioConfig::ChannelLayout(mOutputChannelCount).Map();
632 
633   output.layout = static_cast<uint32_t>(channelMap);
634   output.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_OUTPUT);
635   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE &&
636       CubebUtils::RouteOutputAsVoice()) {
637     output.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
638   }
639 
640   uint32_t latencyFrames = CubebUtils::GetCubebMTGLatencyInFrames(&output);
641 
642   // Macbook and MacBook air don't have enough CPU to run very low latency
643   // MediaTrackGraphs, cap the minimal latency to 512 frames int this case.
644   if (IsMacbookOrMacbookAir()) {
645     latencyFrames = std::max((uint32_t)512, latencyFrames);
646   }
647 
648   // On OSX, having a latency that is lower than 10ms is very common. It's
649   // not very useful when doing voice, because all the WebRTC code deal in 10ms
650   // chunks of audio.  Take the first power of two above 10ms at the current
651   // rate in this case. It's probably 512, for common rates.
652 #if defined(XP_MACOSX)
653   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
654     if (latencyFrames < mSampleRate / 100) {
655       latencyFrames = mozilla::RoundUpPow2(mSampleRate / 100);
656     }
657   }
658 #endif
659   LOG(LogLevel::Debug, ("Effective latency in frames: %d", latencyFrames));
660 
661   input = output;
662   input.channels = mInputChannelCount;
663   input.layout = CUBEB_LAYOUT_UNDEFINED;
664   input.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_INPUT);
665   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
666     input.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
667   }
668 
669   cubeb_stream* stream = nullptr;
670   bool inputWanted = mInputChannelCount > 0;
671   CubebUtils::AudioDeviceID outputId = mOutputDeviceID;
672   CubebUtils::AudioDeviceID inputId = mInputDeviceID;
673 
674   // XXX Only pass input input if we have an input listener.  Always
675   // set up output because it's easier, and it will just get silence.
676   if (cubeb_stream_init(cubebContext, &stream, "AudioCallbackDriver", inputId,
677                         inputWanted ? &input : nullptr,
678                         forcedOutputDeviceId ? forcedOutputDeviceId : outputId,
679                         &output, latencyFrames, DataCallback_s, StateCallback_s,
680                         this) == CUBEB_OK) {
681     mAudioStream.own(stream);
682     DebugOnly<int> rv =
683         cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale());
684     NS_WARNING_ASSERTION(
685         rv == CUBEB_OK,
686         "Could not set the audio stream volume in GraphDriver.cpp");
687     CubebUtils::ReportCubebBackendUsed();
688   } else {
689     NS_WARNING(
690         "Could not create a cubeb stream for MediaTrackGraph, falling "
691         "back to a SystemClockDriver");
692     mAudioStreamState = AudioStreamState::None;
693     // Only report failures when we're not coming from a driver that was
694     // created itself as a fallback driver because of a previous audio driver
695     // failure.
696     if (!fromFallback) {
697       CubebUtils::ReportCubebStreamInitFailure(firstStream);
698       FallbackToSystemClockDriver();
699     }
700     return;
701   }
702 
703 #ifdef XP_MACOSX
704   PanOutputIfNeeded(inputWanted);
705 #endif
706 
707   cubeb_stream_register_device_changed_callback(
708       mAudioStream, AudioCallbackDriver::DeviceChangedCallback_s);
709 
710   // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable. This
711   // is intended for diagnosing issues, and only works if the content sandbox is
712   // disabled.
713   mInputStreamFile.Open("GraphDriverInput", input.channels, input.rate);
714   mOutputStreamFile.Open("GraphDriverOutput", output.channels, output.rate);
715 
716   if (NS_WARN_IF(!StartStream())) {
717     LOG(LogLevel::Warning,
718         ("%p: AudioCallbackDriver couldn't start a cubeb stream.", Graph()));
719     return;
720   }
721 
722   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver started.", Graph()));
723 }
724 
Start()725 void AudioCallbackDriver::Start() {
726   MOZ_ASSERT(!IsStarted());
727   MOZ_ASSERT(mAudioStreamState == AudioStreamState::None);
728   MOZ_ASSERT_IF(PreviousDriver(), PreviousDriver()->InIteration());
729   mAudioStreamState = AudioStreamState::Pending;
730 
731   if (mFallbackDriverState == FallbackDriverState::None) {
732     // Starting an audio driver could take a while. We start a system driver in
733     // the meantime so that the graph is kept running.
734     FallbackToSystemClockDriver();
735   }
736 
737   if (mPreviousDriver) {
738     if (mPreviousDriver->AsAudioCallbackDriver()) {
739       LOG(LogLevel::Debug, ("Releasing audio driver off main thread."));
740       RefPtr<AsyncCubebTask> releaseEvent =
741           new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(),
742                              AsyncCubebOperation::SHUTDOWN);
743       releaseEvent->Dispatch();
744     } else {
745       LOG(LogLevel::Debug,
746           ("Dropping driver reference for SystemClockDriver."));
747       MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver());
748     }
749     mPreviousDriver = nullptr;
750   }
751 
752   LOG(LogLevel::Debug, ("Starting new audio driver off main thread, "
753                         "to ensure it runs after previous shutdown."));
754   RefPtr<AsyncCubebTask> initEvent =
755       new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT);
756   initEvent->Dispatch();
757 }
758 
StartStream()759 bool AudioCallbackDriver::StartStream() {
760   TRACE("AudioCallbackDriver::StartStream");
761   MOZ_ASSERT(!IsStarted() && OnCubebOperationThread());
762   // Set mStarted before cubeb_stream_start, since starting the cubeb stream can
763   // result in a callback (that may read mStarted) before mStarted would
764   // otherwise be set to true.
765   mStarted = true;
766   if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
767     NS_WARNING("Could not start cubeb stream for MTG.");
768     mStarted = false;
769     return false;
770   }
771 
772   return true;
773 }
774 
Stop()775 void AudioCallbackDriver::Stop() {
776   TRACE("AudioCallbackDriver::Stop");
777   MOZ_ASSERT(OnCubebOperationThread());
778   cubeb_stream_register_device_changed_callback(mAudioStream, nullptr);
779   if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) {
780     NS_WARNING("Could not stop cubeb stream for MTG.");
781   }
782 }
783 
Shutdown()784 void AudioCallbackDriver::Shutdown() {
785   MOZ_ASSERT(NS_IsMainThread());
786   RefPtr<FallbackWrapper> fallback;
787   {
788     auto fallbackLock = mFallback.Lock();
789     fallback = fallbackLock.ref();
790     fallbackLock.ref() = nullptr;
791   }
792   if (fallback) {
793     LOG(LogLevel::Debug,
794         ("%p: Releasing fallback driver %p.", Graph(), fallback.get()));
795     fallback->Shutdown();
796   }
797 
798   LOG(LogLevel::Debug,
799       ("%p: Releasing audio driver off main thread (GraphDriver::Shutdown).",
800        Graph()));
801 
802   RefPtr<AsyncCubebTask> releaseEvent =
803       new AsyncCubebTask(this, AsyncCubebOperation::SHUTDOWN);
804   releaseEvent->Dispatch(NS_DISPATCH_SYNC);
805 }
806 
807 /* static */
DataCallback_s(cubeb_stream * aStream,void * aUser,const void * aInputBuffer,void * aOutputBuffer,long aFrames)808 long AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream, void* aUser,
809                                          const void* aInputBuffer,
810                                          void* aOutputBuffer, long aFrames) {
811   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
812   return driver->DataCallback(static_cast<const AudioDataValue*>(aInputBuffer),
813                               static_cast<AudioDataValue*>(aOutputBuffer),
814                               aFrames);
815 }
816 
817 /* static */
StateCallback_s(cubeb_stream * aStream,void * aUser,cubeb_state aState)818 void AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void* aUser,
819                                           cubeb_state aState) {
820   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
821   driver->StateCallback(aState);
822 }
823 
824 /* static */
DeviceChangedCallback_s(void * aUser)825 void AudioCallbackDriver::DeviceChangedCallback_s(void* aUser) {
826   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
827   driver->DeviceChangedCallback();
828 }
829 
AutoInCallback(AudioCallbackDriver * aDriver)830 AudioCallbackDriver::AutoInCallback::AutoInCallback(
831     AudioCallbackDriver* aDriver)
832     : mDriver(aDriver) {
833   MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::thread::id());
834   mDriver->mAudioThreadIdInCb = std::this_thread::get_id();
835 }
836 
~AutoInCallback()837 AudioCallbackDriver::AutoInCallback::~AutoInCallback() {
838   MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::this_thread::get_id());
839   mDriver->mAudioThreadIdInCb = std::thread::id();
840 }
841 
CheckThreadIdChanged()842 bool AudioCallbackDriver::CheckThreadIdChanged() {
843   ProfilerThreadId id = profiler_current_thread_id();
844   if (id != mAudioThreadId) {
845     mAudioThreadId = id;
846     return true;
847   }
848   return false;
849 }
850 
DataCallback(const AudioDataValue * aInputBuffer,AudioDataValue * aOutputBuffer,long aFrames)851 long AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer,
852                                        AudioDataValue* aOutputBuffer,
853                                        long aFrames) {
854   if (!mSandboxed && CheckThreadIdChanged()) {
855     CubebUtils::GetAudioThreadRegistry()->Register(mAudioThreadId);
856   }
857 
858   if (mAudioStreamState.compareExchange(AudioStreamState::Pending,
859                                         AudioStreamState::Running)) {
860     LOG(LogLevel::Verbose, ("%p: AudioCallbackDriver %p First audio callback "
861                             "close the Fallback driver",
862                             Graph(), this));
863   }
864 
865   FallbackDriverState fallbackState = mFallbackDriverState;
866   if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Running)) {
867     // Wait for the fallback driver to stop. Wake it up so it can stop if it's
868     // sleeping.
869     LOG(LogLevel::Verbose,
870         ("%p: AudioCallbackDriver %p Waiting for the Fallback driver to stop",
871          Graph(), this));
872     EnsureNextIteration();
873     PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
874     return aFrames;
875   }
876 
877   if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Stopped)) {
878     // We're supposed to stop.
879     PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
880     if (!mSandboxed) {
881       CubebUtils::GetAudioThreadRegistry()->Unregister(mAudioThreadId);
882     }
883     return aFrames - 1;
884   }
885 
886   MOZ_ASSERT(ThreadRunning());
887   TRACE_AUDIO_CALLBACK_BUDGET(aFrames, mSampleRate);
888   TRACE("AudioCallbackDriver::DataCallback");
889 
890 #ifdef DEBUG
891   AutoInCallback aic(this);
892 #endif
893 
894   uint32_t durationMS = aFrames * 1000 / mSampleRate;
895 
896   // For now, simply average the duration with the previous
897   // duration so there is some damping against sudden changes.
898   if (!mIterationDurationMS) {
899     mIterationDurationMS = durationMS;
900   } else {
901     mIterationDurationMS = (mIterationDurationMS * 3) + durationMS;
902     mIterationDurationMS /= 4;
903   }
904 
905   mBuffer.SetBuffer(aOutputBuffer, aFrames);
906   // fill part or all with leftover data from last iteration (since we
907   // align to Audio blocks)
908   uint32_t alreadyBuffered = mScratchBuffer.Empty(mBuffer);
909 
910   // State computed time is decided by the audio callback's buffer length. We
911   // compute the iteration start and end from there, trying to keep the amount
912   // of buffering in the graph constant.
913   GraphTime nextStateComputedTime =
914       MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(mStateComputedTime +
915                                                     mBuffer.Available());
916 
917   mIterationStart = mIterationEnd;
918   // inGraph is the number of audio frames there is between the state time and
919   // the current time, i.e. the maximum theoretical length of the interval we
920   // could use as [mIterationStart; mIterationEnd].
921   GraphTime inGraph = mStateComputedTime - mIterationStart;
922   // We want the interval [mIterationStart; mIterationEnd] to be before the
923   // interval [mStateComputedTime; nextStateComputedTime]. We also want
924   // the distance between these intervals to be roughly equivalent each time, to
925   // ensure there is no clock drift between current time and state time. Since
926   // we can't act on the state time because we have to fill the audio buffer, we
927   // reclock the current time against the state time, here.
928   mIterationEnd = mIterationStart + 0.8 * inGraph;
929 
930   LOG(LogLevel::Verbose,
931       ("%p: interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) "
932        "(duration ticks: %ld)",
933        Graph(), (long)mIterationStart, (long)mIterationEnd,
934        (long)mStateComputedTime, (long)nextStateComputedTime, (long)aFrames,
935        (uint32_t)durationMS,
936        (long)(nextStateComputedTime - mStateComputedTime)));
937 
938   if (mStateComputedTime < mIterationEnd) {
939     LOG(LogLevel::Error, ("%p: Media graph global underrun detected", Graph()));
940     MOZ_ASSERT_UNREACHABLE("We should not underrun in full duplex");
941     mIterationEnd = mStateComputedTime;
942   }
943 
944   // Process mic data if any/needed
945   if (aInputBuffer && mInputChannelCount > 0) {
946     Graph()->NotifyInputData(aInputBuffer, static_cast<size_t>(aFrames),
947                              mSampleRate, mInputChannelCount, alreadyBuffered);
948   }
949 
950   IterationResult result =
951       Graph()->OneIteration(nextStateComputedTime, mIterationEnd, &mMixer);
952 
953   mStateComputedTime = nextStateComputedTime;
954 
955   MOZ_ASSERT(mBuffer.Available() == 0,
956              "The graph should have filled the buffer");
957 
958   mBuffer.BufferFilled();
959 
960 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
961   // Prevent returning NaN to the OS mixer, and propagating NaN into the reverse
962   // stream of the AEC.
963   NaNToZeroInPlace(aOutputBuffer, aFrames * mOutputChannelCount);
964 #endif
965 
966   // Callback any observers for the AEC speaker data.  Note that one
967   // (maybe) of these will be full-duplex, the others will get their input
968   // data off separate cubeb callbacks.  Take care with how stuff is
969   // removed/added to this list and TSAN issues, but input and output will
970   // use separate callback methods.
971   Graph()->NotifyOutputData(aOutputBuffer, static_cast<size_t>(aFrames),
972                             mSampleRate, mOutputChannelCount);
973 
974 #ifdef XP_MACOSX
975   // This only happens when the output is on a macbookpro's external speaker,
976   // that are stereo, but let's just be safe.
977   if (mNeedsPanning && mOutputChannelCount == 2) {
978     // hard pan to the right
979     for (uint32_t i = 0; i < aFrames * 2; i += 2) {
980       aOutputBuffer[i + 1] += aOutputBuffer[i];
981       aOutputBuffer[i] = 0.0;
982     }
983   }
984 #endif
985 
986   // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable
987   if (aInputBuffer) {
988     mInputStreamFile.Write(static_cast<const AudioDataValue*>(aInputBuffer),
989                            aFrames * mInputChannelCount);
990   }
991   mOutputStreamFile.Write(static_cast<const AudioDataValue*>(aOutputBuffer),
992                           aFrames * mOutputChannelCount);
993 
994   if (result.IsStop()) {
995     if (mInputDeviceID) {
996       mGraphInterface->NotifyInputStopped();
997     }
998     // Signal that we have stopped.
999     result.Stopped();
1000     // Update the flag before handing over the graph and going to drain.
1001     mAudioStreamState = AudioStreamState::Stopping;
1002     if (!mSandboxed) {
1003       CubebUtils::GetAudioThreadRegistry()->Unregister(mAudioThreadId);
1004     }
1005     return aFrames - 1;
1006   }
1007 
1008   if (GraphDriver* nextDriver = result.NextDriver()) {
1009     LOG(LogLevel::Debug,
1010         ("%p: Switching to %s driver.", Graph(),
1011          nextDriver->AsAudioCallbackDriver() ? "audio" : "system"));
1012     if (mInputDeviceID) {
1013       mGraphInterface->NotifyInputStopped();
1014     }
1015     result.Switched();
1016     mAudioStreamState = AudioStreamState::Stopping;
1017     nextDriver->SetState(mIterationStart, mIterationEnd, mStateComputedTime);
1018     nextDriver->Start();
1019     if (!mSandboxed) {
1020       CubebUtils::GetAudioThreadRegistry()->Unregister(mAudioThreadId);
1021     }
1022     // Returning less than aFrames starts the draining and eventually stops the
1023     // audio thread. This function will never get called again.
1024     return aFrames - 1;
1025   }
1026 
1027   MOZ_ASSERT(result.IsStillProcessing());
1028   return aFrames;
1029 }
1030 
StateToString(cubeb_state aState)1031 static const char* StateToString(cubeb_state aState) {
1032   switch (aState) {
1033     case CUBEB_STATE_STARTED:
1034       return "STARTED";
1035     case CUBEB_STATE_STOPPED:
1036       return "STOPPED";
1037     case CUBEB_STATE_DRAINED:
1038       return "DRAINED";
1039     case CUBEB_STATE_ERROR:
1040       return "ERROR";
1041     default:
1042       MOZ_CRASH("Unexpected state!");
1043   }
1044 }
1045 
StateCallback(cubeb_state aState)1046 void AudioCallbackDriver::StateCallback(cubeb_state aState) {
1047   MOZ_ASSERT(!InIteration());
1048   LOG(LogLevel::Debug,
1049       ("AudioCallbackDriver(%p) State: %s", this, StateToString(aState)));
1050 
1051   AudioStreamState streamState = mAudioStreamState;
1052   if (aState != CUBEB_STATE_STARTED) {
1053     // Clear the flag for the not running states: stopped, drained, error.
1054     streamState = mAudioStreamState.exchange(AudioStreamState::None);
1055   }
1056 
1057   if (aState == CUBEB_STATE_ERROR) {
1058     // About to hand over control of the graph.  Do not start a new driver if
1059     // StateCallback() receives an error for this stream while the main thread
1060     // or another driver has control of the graph.
1061     if (streamState == AudioStreamState::Running) {
1062       MOZ_ASSERT(!ThreadRunning());
1063       mStarted = false;
1064       if (mFallbackDriverState == FallbackDriverState::None) {
1065         // Only switch to fallback if it's not already running. It could be
1066         // running with the callback driver having started but not seen a single
1067         // callback yet. I.e., handover from fallback to callback is not done.
1068         if (mInputDeviceID) {
1069 #ifdef DEBUG
1070           // No audio callback after an error. We're calling into the graph here
1071           // so we need to be regarded as "in iteration".
1072           AutoInCallback aic(this);
1073 #endif
1074           mGraphInterface->NotifyInputStopped();
1075         }
1076         FallbackToSystemClockDriver();
1077       }
1078     }
1079   } else if (aState == CUBEB_STATE_STOPPED) {
1080     MOZ_ASSERT(!ThreadRunning());
1081     mStarted = false;
1082   }
1083 }
1084 
MixerCallback(AudioDataValue * aMixedBuffer,AudioSampleFormat aFormat,uint32_t aChannels,uint32_t aFrames,uint32_t aSampleRate)1085 void AudioCallbackDriver::MixerCallback(AudioDataValue* aMixedBuffer,
1086                                         AudioSampleFormat aFormat,
1087                                         uint32_t aChannels, uint32_t aFrames,
1088                                         uint32_t aSampleRate) {
1089   MOZ_ASSERT(InIteration());
1090   uint32_t toWrite = mBuffer.Available();
1091 
1092   if (!mBuffer.Available() && aFrames > 0) {
1093     NS_WARNING("DataCallback buffer full, expect frame drops.");
1094   }
1095 
1096   MOZ_ASSERT(mBuffer.Available() <= aFrames);
1097 
1098   mBuffer.WriteFrames(aMixedBuffer, mBuffer.Available());
1099   MOZ_ASSERT(mBuffer.Available() == 0,
1100              "Missing frames to fill audio callback's buffer.");
1101 
1102   DebugOnly<uint32_t> written = mScratchBuffer.Fill(
1103       aMixedBuffer + toWrite * aChannels, aFrames - toWrite);
1104   NS_WARNING_ASSERTION(written == aFrames - toWrite, "Dropping frames.");
1105 };
1106 
PanOutputIfNeeded(bool aMicrophoneActive)1107 void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive) {
1108 #ifdef XP_MACOSX
1109   TRACE("AudioCallbackDriver::PanOutputIfNeeded");
1110   cubeb_device* out = nullptr;
1111   int rv;
1112   char name[128];
1113   size_t length = sizeof(name);
1114 
1115   rv = sysctlbyname("hw.model", name, &length, NULL, 0);
1116   if (rv) {
1117     return;
1118   }
1119 
1120   int major, minor;
1121   for (uint32_t i = 0; i < length; i++) {
1122     // skip the model name
1123     if (isalpha(name[i])) {
1124       continue;
1125     }
1126     sscanf(name + i, "%d,%d", &major, &minor);
1127     break;
1128   }
1129 
1130   enum MacbookModel { MacBook, MacBookPro, MacBookAir, NotAMacbook };
1131 
1132   MacbookModel model;
1133 
1134   if (!strncmp(name, "MacBookPro", length)) {
1135     model = MacBookPro;
1136   } else if (strncmp(name, "MacBookAir", length)) {
1137     model = MacBookAir;
1138   } else if (strncmp(name, "MacBook", length)) {
1139     model = MacBook;
1140   } else {
1141     model = NotAMacbook;
1142   }
1143   // For macbook pro before 2016 model (change of chassis), hard pan the audio
1144   // to the right if the speakers are in use to avoid feedback.
1145   if (model == MacBookPro && major <= 12) {
1146     if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) {
1147       MOZ_ASSERT(out);
1148       // Check if we are currently outputing sound on external speakers.
1149       if (out->output_name && !strcmp(out->output_name, "ispk")) {
1150         // Pan everything to the right speaker.
1151         LOG(LogLevel::Debug, ("Using the built-in speakers, with%s audio input",
1152                               aMicrophoneActive ? "" : "out"));
1153         mNeedsPanning = aMicrophoneActive;
1154       } else {
1155         LOG(LogLevel::Debug, ("Using an external output device"));
1156         mNeedsPanning = false;
1157       }
1158       cubeb_stream_device_destroy(mAudioStream, out);
1159     }
1160   }
1161 #endif
1162 }
1163 
DeviceChangedCallback()1164 void AudioCallbackDriver::DeviceChangedCallback() {
1165   MOZ_ASSERT(!InIteration());
1166   // Tell the audio engine the device has changed, it might want to reset some
1167   // state.
1168   Graph()->DeviceChanged();
1169 #ifdef XP_MACOSX
1170   RefPtr<AudioCallbackDriver> self(this);
1171   bool hasInput = mInputChannelCount;
1172   NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1173       "PanOutputIfNeeded", [self{std::move(self)}, hasInput]() {
1174         self->PanOutputIfNeeded(hasInput);
1175       }));
1176 #endif
1177 }
1178 
IterationDuration()1179 uint32_t AudioCallbackDriver::IterationDuration() {
1180   MOZ_ASSERT(InIteration());
1181   // The real fix would be to have an API in cubeb to give us the number. Short
1182   // of that, we approximate it here. bug 1019507
1183   return mIterationDurationMS;
1184 }
1185 
EnsureNextIteration()1186 void AudioCallbackDriver::EnsureNextIteration() {
1187   if (mFallbackDriverState == FallbackDriverState::Running) {
1188     auto fallback = mFallback.Lock();
1189     if (fallback.ref()) {
1190       fallback.ref()->EnsureNextIteration();
1191     }
1192   }
1193 }
1194 
IsStarted()1195 bool AudioCallbackDriver::IsStarted() { return mStarted; }
1196 
AudioOutputLatency()1197 TimeDuration AudioCallbackDriver::AudioOutputLatency() {
1198   TRACE("AudioCallbackDriver::AudioOutputLatency");
1199   uint32_t latencyFrames;
1200   int rv = cubeb_stream_get_latency(mAudioStream, &latencyFrames);
1201   if (rv || mSampleRate == 0) {
1202     return TimeDuration::FromSeconds(0.0);
1203   }
1204 
1205   return TimeDuration::FromSeconds(static_cast<double>(latencyFrames) /
1206                                    mSampleRate);
1207 }
1208 
OnFallback() const1209 bool AudioCallbackDriver::OnFallback() const {
1210   MOZ_ASSERT(InIteration());
1211   return mFallbackDriverState == FallbackDriverState::Running;
1212 }
1213 
FallbackToSystemClockDriver()1214 void AudioCallbackDriver::FallbackToSystemClockDriver() {
1215   MOZ_ASSERT(!ThreadRunning());
1216   MOZ_ASSERT(mAudioStreamState == AudioStreamState::None ||
1217              mAudioStreamState == AudioStreamState::Pending);
1218   MOZ_ASSERT(mFallbackDriverState == FallbackDriverState::None);
1219   LOG(LogLevel::Debug,
1220       ("%p: AudioCallbackDriver %p Falling back to SystemClockDriver.", Graph(),
1221        this));
1222   mFallbackDriverState = FallbackDriverState::Running;
1223   mNextReInitBackoffStep =
1224       TimeDuration::FromMilliseconds(AUDIO_INITIAL_FALLBACK_BACKOFF_STEP_MS);
1225   mNextReInitAttempt = TimeStamp::Now() + mNextReInitBackoffStep;
1226   auto fallback =
1227       MakeRefPtr<FallbackWrapper>(Graph(), this, mSampleRate, mIterationStart,
1228                                   mIterationEnd, mStateComputedTime);
1229   {
1230     auto driver = mFallback.Lock();
1231     driver.ref() = fallback;
1232   }
1233   fallback->Start();
1234 }
1235 
FallbackDriverStopped(GraphTime aIterationStart,GraphTime aIterationEnd,GraphTime aStateComputedTime,FallbackDriverState aState)1236 void AudioCallbackDriver::FallbackDriverStopped(GraphTime aIterationStart,
1237                                                 GraphTime aIterationEnd,
1238                                                 GraphTime aStateComputedTime,
1239                                                 FallbackDriverState aState) {
1240   mIterationStart = aIterationStart;
1241   mIterationEnd = aIterationEnd;
1242   mStateComputedTime = aStateComputedTime;
1243   mNextReInitAttempt = TimeStamp();
1244   mNextReInitBackoffStep = TimeDuration();
1245   {
1246     auto fallback = mFallback.Lock();
1247     MOZ_ASSERT(fallback.ref()->OnThread());
1248     fallback.ref() = nullptr;
1249   }
1250 
1251   MOZ_ASSERT(aState == FallbackDriverState::None ||
1252              aState == FallbackDriverState::Stopped);
1253   MOZ_ASSERT_IF(aState == FallbackDriverState::None,
1254                 mAudioStreamState == AudioStreamState::Running);
1255   mFallbackDriverState = aState;
1256 }
1257 
MaybeStartAudioStream()1258 void AudioCallbackDriver::MaybeStartAudioStream() {
1259   AudioStreamState streamState = mAudioStreamState;
1260   if (streamState != AudioStreamState::None) {
1261     LOG(LogLevel::Verbose,
1262         ("%p: AudioCallbackDriver %p Cannot re-init.", Graph(), this));
1263     return;
1264   }
1265 
1266   TimeStamp now = TimeStamp::Now();
1267   if (now < mNextReInitAttempt) {
1268     LOG(LogLevel::Verbose,
1269         ("%p: AudioCallbackDriver %p Not time to re-init yet. %.3fs left.",
1270          Graph(), this, (mNextReInitAttempt - now).ToSeconds()));
1271     return;
1272   }
1273 
1274   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p Attempting to re-init "
1275                         "audio stream from fallback driver.",
1276                         Graph(), this));
1277   mNextReInitBackoffStep = std::min(
1278       mNextReInitBackoffStep * 2,
1279       TimeDuration::FromMilliseconds(AUDIO_MAX_FALLBACK_BACKOFF_STEP_MS));
1280   mNextReInitAttempt = now + mNextReInitBackoffStep;
1281   Start();
1282 }
1283 
1284 }  // namespace mozilla
1285 
1286 // avoid redefined macro in unified build
1287 #undef LOG
1288