1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ 2 /* This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 #ifndef MOZILLA_MEDIATRACKGRAPHIMPL_H_ 7 #define MOZILLA_MEDIATRACKGRAPHIMPL_H_ 8 9 #include "MediaTrackGraph.h" 10 11 #include "AudioMixer.h" 12 #include "GraphDriver.h" 13 #include "mozilla/Atomics.h" 14 #include "mozilla/Maybe.h" 15 #include "mozilla/Monitor.h" 16 #include "mozilla/TimeStamp.h" 17 #include "mozilla/UniquePtr.h" 18 #include "mozilla/WeakPtr.h" 19 #include "nsClassHashtable.h" 20 #include "nsIMemoryReporter.h" 21 #include "nsINamed.h" 22 #include "nsIRunnable.h" 23 #include "nsIThreadInternal.h" 24 #include "nsITimer.h" 25 #include "AsyncLogger.h" 26 27 namespace mozilla { 28 29 namespace media { 30 class ShutdownBlocker; 31 } 32 33 class AudioContextOperationControlMessage; 34 template <typename T> 35 class LinkedList; 36 class GraphRunner; 37 38 // MediaTrack subclass storing the raw audio data from microphone. 39 class NativeInputTrack : public ProcessedMediaTrack { 40 ~NativeInputTrack() = default; NativeInputTrack(TrackRate aSampleRate)41 explicit NativeInputTrack(TrackRate aSampleRate) 42 : ProcessedMediaTrack(aSampleRate, MediaSegment::AUDIO, 43 new AudioSegment()) {} 44 45 public: 46 // Main Thread API 47 static NativeInputTrack* Create(MediaTrackGraphImpl* aGraph); 48 49 size_t AddUser(); 50 size_t RemoveUser(); 51 52 // Graph Thread APIs, for ProcessedMediaTrack 53 void DestroyImpl() override; 54 void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override; 55 uint32_t NumberOfChannels() const override; 56 57 // Graph thread APIs: Redirect calls from GraphDriver to mDataUsers 58 void NotifyOutputData(MediaTrackGraphImpl* aGraph, AudioDataValue* aBuffer, 59 size_t aFrames, TrackRate aRate, uint32_t aChannels); 60 void NotifyInputStopped(MediaTrackGraphImpl* aGraph); 61 void NotifyInputData(MediaTrackGraphImpl* aGraph, 62 const AudioDataValue* aBuffer, size_t aFrames, 63 TrackRate aRate, uint32_t aChannels, 64 uint32_t aAlreadyBuffered); 65 void DeviceChanged(MediaTrackGraphImpl* aGraph); 66 67 // Other Graph Thread APIs 68 void InitDataHolderIfNeeded(); 69 70 // Any thread AsNativeInputTrack()71 NativeInputTrack* AsNativeInputTrack() override { return this; } 72 73 public: 74 // Only accessed on the graph thread. 75 nsTArray<RefPtr<AudioDataListener>> mDataUsers; 76 77 private: 78 class AudioDataBuffers { 79 public: 80 AudioDataBuffers() = default; 81 void SetOutputData(AudioDataValue* aBuffer, size_t aFrames, 82 uint32_t aChannels, TrackRate aRate); 83 void SetInputData(AudioDataValue* aBuffer, size_t aFrames, 84 uint32_t aChannels, TrackRate aRate); 85 86 enum Scope : unsigned char { 87 Input = 0x01, 88 Output = 0x02, 89 }; 90 void Clear(Scope aScope); 91 92 typedef AudioDataListenerInterface::BufferInfo BufferInfo; 93 // Storing the audio output data coming from NotifyOutputData 94 Maybe<BufferInfo> mOutputData; 95 // Storing the audio input data coming from NotifyInputData 96 Maybe<BufferInfo> mInputData; 97 }; 98 99 // Only accessed on the graph thread. 100 // Storing the audio data coming from GraphDriver directly. 101 Maybe<AudioDataBuffers> mDataHolder; 102 103 // Only accessed on the graph thread. 104 uint32_t mInputChannels = 0; 105 106 // Only accessed on the main thread. 107 // When this becomes zero, this NativeInputTrack is no longer needed. 108 int32_t mUserCount = 0; 109 }; 110 111 /** 112 * A per-track update message passed from the media graph thread to the 113 * main thread. 114 */ 115 struct TrackUpdate { 116 RefPtr<MediaTrack> mTrack; 117 TrackTime mNextMainThreadCurrentTime; 118 bool mNextMainThreadEnded; 119 }; 120 121 /** 122 * This represents a message run on the graph thread to modify track or graph 123 * state. These are passed from main thread to graph thread through 124 * AppendMessage(), or scheduled on the graph thread with 125 * RunMessageAfterProcessing(). A ControlMessage 126 * always has a weak reference to a particular affected track. 127 */ 128 class ControlMessage { 129 public: ControlMessage(MediaTrack * aTrack)130 explicit ControlMessage(MediaTrack* aTrack) : mTrack(aTrack) { 131 MOZ_COUNT_CTOR(ControlMessage); 132 } 133 // All these run on the graph thread 134 MOZ_COUNTED_DTOR_VIRTUAL(ControlMessage) 135 // Do the action of this message on the MediaTrackGraph thread. Any actions 136 // affecting graph processing should take effect at mProcessedTime. 137 // All track data for times < mProcessedTime has already been 138 // computed. 139 virtual void Run() = 0; 140 // RunDuringShutdown() is only relevant to messages generated on the main 141 // thread (for AppendMessage()). 142 // When we're shutting down the application, most messages are ignored but 143 // some cleanup messages should still be processed (on the main thread). 144 // This must not add new control messages to the graph. RunDuringShutdown()145 virtual void RunDuringShutdown() {} GetTrack()146 MediaTrack* GetTrack() { return mTrack; } 147 148 protected: 149 // We do not hold a reference to mTrack. The graph will be holding a reference 150 // to the track until the Destroy message is processed. The last message 151 // referencing a track is the Destroy message for that track. 152 MediaTrack* mTrack; 153 }; 154 155 class MessageBlock { 156 public: 157 nsTArray<UniquePtr<ControlMessage>> mMessages; 158 }; 159 160 /** 161 * The implementation of a media track graph. This class is private to this 162 * file. It's not in the anonymous namespace because MediaTrack needs to 163 * be able to friend it. 164 * 165 * There can be multiple MediaTrackGraph per process: one per document. 166 * Additionaly, each OfflineAudioContext object creates its own MediaTrackGraph 167 * object too. 168 */ 169 class MediaTrackGraphImpl : public MediaTrackGraph, 170 public GraphInterface, 171 public nsIMemoryReporter, 172 public nsIThreadObserver, 173 public nsITimerCallback, 174 public nsINamed { 175 public: 176 NS_DECL_THREADSAFE_ISUPPORTS 177 NS_DECL_NSIMEMORYREPORTER 178 NS_DECL_NSITHREADOBSERVER 179 NS_DECL_NSITIMERCALLBACK 180 NS_DECL_NSINAMED 181 182 /** 183 * Use aGraphDriverRequested with SYSTEM_THREAD_DRIVER or AUDIO_THREAD_DRIVER 184 * to create a MediaTrackGraph which provides support for real-time audio 185 * and/or video. Set it to OFFLINE_THREAD_DRIVER in order to create a 186 * non-realtime instance which just churns through its inputs and produces 187 * output. Those objects currently only support audio, and are used to 188 * implement OfflineAudioContext. They do not support MediaTrack inputs. 189 */ 190 explicit MediaTrackGraphImpl(GraphDriverType aGraphDriverRequested, 191 GraphRunType aRunTypeRequested, 192 TrackRate aSampleRate, uint32_t aChannelCount, 193 CubebUtils::AudioDeviceID aOutputDeviceID, 194 nsISerialEventTarget* aWindow); 195 196 // Intended only for assertions, either on graph thread or not running (in 197 // which case we must be on the main thread). 198 bool OnGraphThreadOrNotRunning() const override; 199 bool OnGraphThread() const override; 200 201 bool Destroyed() const override; 202 203 #ifdef DEBUG 204 /** 205 * True if we're on aDriver's thread, or if we're on mGraphRunner's thread 206 * and mGraphRunner is currently run by aDriver. 207 */ 208 bool InDriverIteration(const GraphDriver* aDriver) const override; 209 #endif 210 211 /** 212 * Unregisters memory reporting and deletes this instance. This should be 213 * called instead of calling the destructor directly. 214 */ 215 void Destroy(); 216 217 // Main thread only. 218 /** 219 * This runs every time we need to sync state from the media graph thread 220 * to the main thread while the main thread is not in the middle 221 * of a script. It runs during a "stable state" (per HTML5) or during 222 * an event posted to the main thread. 223 * The boolean affects which boolean controlling runnable dispatch is cleared 224 */ 225 void RunInStableState(bool aSourceIsMTG); 226 /** 227 * Ensure a runnable to run RunInStableState is posted to the appshell to 228 * run at the next stable state (per HTML5). 229 * See EnsureStableStateEventPosted. 230 */ 231 void EnsureRunInStableState(); 232 /** 233 * Called to apply a TrackUpdate to its track. 234 */ 235 void ApplyTrackUpdate(TrackUpdate* aUpdate); 236 /** 237 * Append a ControlMessage to the message queue. This queue is drained 238 * during RunInStableState; the messages will run on the graph thread. 239 */ 240 virtual void AppendMessage(UniquePtr<ControlMessage> aMessage); 241 242 /** 243 * Dispatches a runnable from any thread to the correct main thread for this 244 * MediaTrackGraph. 245 */ 246 void Dispatch(already_AddRefed<nsIRunnable>&& aRunnable); 247 248 /** 249 * Make this MediaTrackGraph enter forced-shutdown state. This state 250 * will be noticed by the media graph thread, which will shut down all tracks 251 * and other state controlled by the media graph thread. 252 * This is called during application shutdown, and on document unload if an 253 * AudioContext is using the graph. 254 */ 255 void ForceShutDown(); 256 257 /** 258 * Sets mShutdownBlocker and makes it block shutdown. 259 * Main thread only. Not idempotent. 260 */ 261 void AddShutdownBlocker(); 262 263 /** 264 * Removes mShutdownBlocker and unblocks shutdown. 265 * Main thread only. Idempotent. 266 */ 267 void RemoveShutdownBlocker(); 268 269 /** 270 * Called before the thread runs. 271 */ 272 void Init(); 273 274 /** 275 * Respond to CollectReports with sizes collected on the graph thread. 276 */ 277 static void FinishCollectReports( 278 nsIHandleReportCallback* aHandleReport, nsISupports* aData, 279 const nsTArray<AudioNodeSizes>& aAudioTrackSizes); 280 281 // The following methods run on the graph thread (or possibly the main thread 282 // if mLifecycleState > LIFECYCLE_RUNNING) 283 void CollectSizesForMemoryReport( 284 already_AddRefed<nsIHandleReportCallback> aHandleReport, 285 already_AddRefed<nsISupports> aHandlerData); 286 287 /** 288 * Returns true if this MediaTrackGraph should keep running 289 */ 290 bool UpdateMainThreadState(); 291 292 /** 293 * Proxy method called by GraphDriver to iterate the graph. 294 * If this graph was created with GraphRunType SINGLE_THREAD, mGraphRunner 295 * will take care of calling OneIterationImpl from its thread. Otherwise, 296 * OneIterationImpl is called directly. Output from the graph gets mixed into 297 * aMixer, if it is non-null. 298 */ 299 IterationResult OneIteration(GraphTime aStateTime, GraphTime aIterationEnd, 300 AudioMixer* aMixer) override; 301 302 /** 303 * Returns true if this MediaTrackGraph should keep running 304 */ 305 IterationResult OneIterationImpl(GraphTime aStateTime, 306 GraphTime aIterationEnd, AudioMixer* aMixer); 307 308 /** 309 * Called from the driver, when the graph thread is about to stop, to tell 310 * the main thread to attempt to begin cleanup. The main thread may either 311 * shutdown or revive the graph depending on whether it receives new 312 * messages. 313 */ 314 void SignalMainThreadCleanup(); 315 316 /* This is the end of the current iteration, that is, the current time of the 317 * graph. */ 318 GraphTime IterationEnd() const; 319 320 /** 321 * Ensure there is an event posted to the main thread to run RunInStableState. 322 * mMonitor must be held. 323 * See EnsureRunInStableState 324 */ 325 void EnsureStableStateEventPosted(); 326 /** 327 * Generate messages to the main thread to update it for all state changes. 328 * mMonitor must be held. 329 */ 330 void PrepareUpdatesToMainThreadState(bool aFinalUpdate); 331 /** 332 * If we are rendering in non-realtime mode, we don't want to send messages to 333 * the main thread at each iteration for performance reasons. We instead 334 * notify the main thread at the same rate 335 */ 336 bool ShouldUpdateMainThread(); 337 // The following methods are the various stages of RunThread processing. 338 /** 339 * Advance all track state to mStateComputedTime. 340 */ 341 void UpdateCurrentTimeForTracks(GraphTime aPrevCurrentTime); 342 /** 343 * Process chunks for all tracks and raise events for properties that have 344 * changed, such as principalId. 345 */ 346 void ProcessChunkMetadata(GraphTime aPrevCurrentTime); 347 /** 348 * Process chunks for the given track and interval, and raise events for 349 * properties that have changed, such as principalHandle. 350 */ 351 template <typename C, typename Chunk> 352 void ProcessChunkMetadataForInterval(MediaTrack* aTrack, C& aSegment, 353 TrackTime aStart, TrackTime aEnd); 354 /** 355 * Process graph messages in mFrontMessageQueue. 356 */ 357 void RunMessagesInQueue(); 358 /** 359 * Update track processing order and recompute track blocking until 360 * aEndBlockingDecisions. 361 */ 362 void UpdateGraph(GraphTime aEndBlockingDecisions); 363 SwapMessageQueues()364 void SwapMessageQueues() { 365 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 366 mMonitor.AssertCurrentThreadOwns(); 367 MOZ_ASSERT(mFrontMessageQueue.IsEmpty()); 368 mFrontMessageQueue.SwapElements(mBackMessageQueue); 369 if (!mFrontMessageQueue.IsEmpty()) { 370 EnsureNextIteration(); 371 } 372 } 373 /** 374 * Do all the processing and play the audio and video, from 375 * mProcessedTime to mStateComputedTime. 376 */ 377 void Process(AudioMixer* aMixer); 378 379 /** 380 * For use during ProcessedMediaTrack::ProcessInput() or 381 * MediaTrackListener callbacks, when graph state cannot be changed. 382 * Schedules |aMessage| to run after processing, at a time when graph state 383 * can be changed. Graph thread. 384 */ 385 void RunMessageAfterProcessing(UniquePtr<ControlMessage> aMessage); 386 387 /** 388 * Resolve the GraphStartedPromise when the driver has started processing on 389 * the audio thread after the device has started. 390 */ 391 void NotifyWhenGraphStarted(RefPtr<MediaTrack> aTrack, 392 MozPromiseHolder<GraphStartedPromise>&& aHolder); 393 394 /** 395 * Apply an AudioContext operation (suspend/resume/close), on the graph 396 * thread. 397 */ 398 void ApplyAudioContextOperationImpl( 399 AudioContextOperationControlMessage* aMessage); 400 401 /** 402 * Determine if we have any audio tracks, or are about to add any audiotracks. 403 */ 404 bool AudioTrackPresent(); 405 406 /** 407 * Schedules a replacement GraphDriver in mNextDriver, if necessary. 408 */ 409 void CheckDriver(); 410 411 /** 412 * Sort mTracks so that every track not in a cycle is after any tracks 413 * it depends on, and every track in a cycle is marked as being in a cycle. 414 */ 415 void UpdateTrackOrder(); 416 417 /** 418 * Returns smallest value of t such that t is a multiple of 419 * WEBAUDIO_BLOCK_SIZE and t >= aTime. 420 */ 421 static GraphTime RoundUpToEndOfAudioBlock(GraphTime aTime); 422 /** 423 * Returns smallest value of t such that t is a multiple of 424 * WEBAUDIO_BLOCK_SIZE and t > aTime. 425 */ 426 static GraphTime RoundUpToNextAudioBlock(GraphTime aTime); 427 /** 428 * Produce data for all tracks >= aTrackIndex for the current time interval. 429 * Advances block by block, each iteration producing data for all tracks 430 * for a single block. 431 * This is called whenever we have an AudioNodeTrack in the graph. 432 */ 433 void ProduceDataForTracksBlockByBlock(uint32_t aTrackIndex, 434 TrackRate aSampleRate); 435 /** 436 * If aTrack will underrun between aTime, and aEndBlockingDecisions, returns 437 * the time at which the underrun will start. Otherwise return 438 * aEndBlockingDecisions. 439 */ 440 GraphTime WillUnderrun(MediaTrack* aTrack, GraphTime aEndBlockingDecisions); 441 442 /** 443 * Given a graph time aTime, convert it to a track time taking into 444 * account the time during which aTrack is scheduled to be blocked. 445 */ 446 TrackTime GraphTimeToTrackTimeWithBlocking(const MediaTrack* aTrack, 447 GraphTime aTime) const; 448 449 /** 450 * If aTrack needs an audio track but doesn't have one, create it. 451 * If aTrack doesn't need an audio track but has one, destroy it. 452 */ 453 void CreateOrDestroyAudioTracks(MediaTrack* aTrack); 454 /** 455 * Queue audio (mix of track audio and silence for blocked intervals) 456 * to the audio output track. Returns the number of frames played. 457 */ 458 struct TrackKeyAndVolume { 459 MediaTrack* mTrack; 460 void* mKey; 461 float mVolume; 462 }; 463 TrackTime PlayAudio(AudioMixer* aMixer, const TrackKeyAndVolume& aTkv, 464 GraphTime aPlayedTime); 465 466 /* Called on the main thread when AudioInputTrack requests audio data from an 467 * input device aID. */ 468 ProcessedMediaTrack* GetDeviceTrack(CubebUtils::AudioDeviceID aID); 469 470 /* Runs off a message on the graph thread when something requests audio from 471 * an input audio device of ID aID, and delivers the input audio frames to 472 * aListener. */ 473 void OpenAudioInputImpl(CubebUtils::AudioDeviceID aID, 474 AudioDataListener* aListener, 475 NativeInputTrack* aInputTrack); 476 /* Called on the main thread when something requests audio from an input 477 * audio device aID. */ 478 virtual nsresult OpenAudioInput(CubebUtils::AudioDeviceID aID, 479 AudioDataListener* aListener) override; 480 481 /* Runs off a message on the graph when input audio from aID is not needed 482 * anymore, for a particular track. It can be that other tracks still need 483 * audio from this audio input device. */ 484 void CloseAudioInputImpl(CubebUtils::AudioDeviceID aID, 485 AudioDataListener* aListener, 486 NativeInputTrack* aInputTrack); 487 /* Called on the main thread when input audio from aID is not needed 488 * anymore, for a particular track. It can be that other tracks still need 489 * audio from this audio input device. */ 490 virtual void CloseAudioInput(CubebUtils::AudioDeviceID aID, 491 AudioDataListener* aListener) override; 492 493 /* Add or remove an audio output for this track. All tracks that have an 494 * audio output are mixed and written to a single audio output stream. */ 495 void RegisterAudioOutput(MediaTrack* aTrack, void* aKey); 496 void UnregisterAudioOutput(MediaTrack* aTrack, void* aKey); 497 void UnregisterAllAudioOutputs(MediaTrack* aTrack); 498 void SetAudioOutputVolume(MediaTrack* aTrack, void* aKey, float aVolume); 499 500 /* Called on the graph thread when the input device settings should be 501 * reevaluated, for example, if the channel count of the input track should 502 * be changed. */ 503 void ReevaluateInputDevice(); 504 505 /* Called on the graph thread when there is new output data for listeners. 506 * This is the mixed audio output of this MediaTrackGraph. */ 507 void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames, 508 TrackRate aRate, uint32_t aChannels) override; 509 /* Called on the graph thread after an AudioCallbackDriver with an input 510 * stream has stopped. */ 511 void NotifyInputStopped() override; 512 /* Called on the graph thread when there is new input data for listeners. This 513 * is the raw audio input for this MediaTrackGraph. */ 514 void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames, 515 TrackRate aRate, uint32_t aChannels, 516 uint32_t aAlreadyBuffered) override; 517 /* Called every time there are changes to input/output audio devices like 518 * plug/unplug etc. This can be called on any thread, and posts a message to 519 * the main thread so that it can post a message to the graph thread. */ 520 void DeviceChanged() override; 521 /* Called every time there are changes to input/output audio devices. This is 522 * called on the graph thread. */ 523 void DeviceChangedImpl(); 524 525 /** 526 * Compute how much track data we would like to buffer for aTrack. 527 */ 528 TrackTime GetDesiredBufferEnd(MediaTrack* aTrack); 529 /** 530 * Returns true when there are no active tracks. 531 */ IsEmpty()532 bool IsEmpty() const { 533 MOZ_ASSERT( 534 OnGraphThreadOrNotRunning() || 535 (NS_IsMainThread() && 536 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP)); 537 return mTracks.IsEmpty() && mSuspendedTracks.IsEmpty() && mPortCount == 0; 538 } 539 540 /** 541 * Add aTrack to the graph and initializes its graph-specific state. 542 */ 543 void AddTrackGraphThread(MediaTrack* aTrack); 544 /** 545 * Remove aTrack from the graph. Ensures that pending messages about the 546 * track back to the main thread are flushed. 547 */ 548 void RemoveTrackGraphThread(MediaTrack* aTrack); 549 /** 550 * Remove a track from the graph. Main thread. 551 */ 552 void RemoveTrack(MediaTrack* aTrack); 553 /** 554 * Remove aPort from the graph and release it. 555 */ 556 void DestroyPort(MediaInputPort* aPort); 557 /** 558 * Mark the media track order as dirty. 559 */ SetTrackOrderDirty()560 void SetTrackOrderDirty() { 561 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 562 mTrackOrderDirty = true; 563 } 564 565 // Get the current maximum channel count. Graph thread only. 566 uint32_t AudioOutputChannelCount() const; 567 // Set a new maximum channel count. Graph thread only. 568 void SetMaxOutputChannelCount(uint32_t aMaxChannelCount); 569 570 double AudioOutputLatency(); 571 572 /** 573 * The audio input channel count for a MediaTrackGraph is the max of all the 574 * channel counts requested by the listeners. The max channel count is 575 * delivered to the listeners themselves, and they take care of downmixing. 576 */ AudioInputChannelCount()577 uint32_t AudioInputChannelCount() { 578 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 579 580 #ifdef ANDROID 581 if (!mDeviceTrackMap.Contains(mInputDeviceID)) { 582 return 0; 583 } 584 #else 585 if (!mInputDeviceID) { 586 MOZ_ASSERT(mDeviceTrackMap.Count() == 0, 587 "If running on a platform other than android," 588 "an explicit device id should be present"); 589 return 0; 590 } 591 #endif 592 uint32_t maxInputChannels = 0; 593 // When/if we decide to support multiple input device per graph, this needs 594 // loop over them. 595 auto result = mDeviceTrackMap.Lookup(mInputDeviceID); 596 MOZ_ASSERT(result); 597 if (!result) { 598 return maxInputChannels; 599 } 600 for (const auto& listener : result.Data()->mDataUsers) { 601 maxInputChannels = std::max(maxInputChannels, 602 listener->RequestedInputChannelCount(this)); 603 } 604 return maxInputChannels; 605 } 606 AudioInputDevicePreference()607 AudioInputType AudioInputDevicePreference() { 608 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 609 610 auto result = mDeviceTrackMap.Lookup(mInputDeviceID); 611 if (!result) { 612 return AudioInputType::Unknown; 613 } 614 bool voiceInput = false; 615 // When/if we decide to support multiple input device per graph, this needs 616 // loop over them. 617 618 // If at least one track is considered to be voice, 619 // XXX This could use short-circuit evaluation resp. std::any_of. 620 for (const auto& listener : result.Data()->mDataUsers) { 621 voiceInput |= listener->IsVoiceInput(this); 622 } 623 if (voiceInput) { 624 return AudioInputType::Voice; 625 } 626 return AudioInputType::Unknown; 627 } 628 InputDeviceID()629 CubebUtils::AudioDeviceID InputDeviceID() { return mInputDeviceID; } 630 MediaTimeToSeconds(GraphTime aTime)631 double MediaTimeToSeconds(GraphTime aTime) const { 632 NS_ASSERTION(aTime > -TRACK_TIME_MAX && aTime <= TRACK_TIME_MAX, 633 "Bad time"); 634 return static_cast<double>(aTime) / GraphRate(); 635 } 636 637 /** 638 * Signal to the graph that the thread has paused indefinitly, 639 * or resumed. 640 */ 641 void PausedIndefinitly(); 642 void ResumedFromPaused(); 643 644 /** 645 * Not safe to call off the MediaTrackGraph thread unless monitor is held! 646 */ CurrentDriver()647 GraphDriver* CurrentDriver() const { 648 #ifdef DEBUG 649 if (!OnGraphThreadOrNotRunning()) { 650 mMonitor.AssertCurrentThreadOwns(); 651 } 652 #endif 653 return mDriver; 654 } 655 656 /** 657 * Effectively set the new driver, while we are switching. 658 * It is only safe to call this at the very end of an iteration, when there 659 * has been a SwitchAtNextIteration call during the iteration. The driver 660 * should return and pass the control to the new driver shortly after. 661 * Monitor must be held. 662 */ SetCurrentDriver(GraphDriver * aDriver)663 void SetCurrentDriver(GraphDriver* aDriver) { 664 MOZ_ASSERT_IF(mGraphDriverRunning, InDriverIteration(mDriver)); 665 MOZ_ASSERT_IF(!mGraphDriverRunning, NS_IsMainThread()); 666 MonitorAutoLock lock(GetMonitor()); 667 mDriver = aDriver; 668 } 669 NextDriver()670 GraphDriver* NextDriver() const { 671 MOZ_ASSERT(OnGraphThread()); 672 return mNextDriver; 673 } 674 Switching()675 bool Switching() const { return NextDriver(); } 676 677 void SwitchAtNextIteration(GraphDriver* aNextDriver); 678 GetMonitor()679 Monitor& GetMonitor() { return mMonitor; } 680 EnsureNextIteration()681 void EnsureNextIteration() { CurrentDriver()->EnsureNextIteration(); } 682 683 // Capture API. This allows to get a mixed-down output for a window. 684 void RegisterCaptureTrackForWindow(uint64_t aWindowId, 685 ProcessedMediaTrack* aCaptureTrack); 686 void UnregisterCaptureTrackForWindow(uint64_t aWindowId); 687 already_AddRefed<MediaInputPort> ConnectToCaptureTrack( 688 uint64_t aWindowId, MediaTrack* aMediaTrack); 689 690 Watchable<GraphTime>& CurrentTime() override; 691 692 /** 693 * Interrupt any JS running on the graph thread. 694 * Called on the main thread when shutting down the graph. 695 */ 696 void InterruptJS(); 697 698 class TrackSet { 699 public: 700 class iterator { 701 public: iterator(MediaTrackGraphImpl & aGraph)702 explicit iterator(MediaTrackGraphImpl& aGraph) 703 : mGraph(&aGraph), mArrayNum(-1), mArrayIndex(0) { 704 ++(*this); 705 } iterator()706 iterator() : mGraph(nullptr), mArrayNum(2), mArrayIndex(0) {} 707 MediaTrack* operator*() { return Array()->ElementAt(mArrayIndex); } 708 iterator operator++() { 709 ++mArrayIndex; 710 while (mArrayNum < 2 && 711 (mArrayNum < 0 || mArrayIndex >= Array()->Length())) { 712 ++mArrayNum; 713 mArrayIndex = 0; 714 } 715 return *this; 716 } 717 bool operator==(const iterator& aOther) const { 718 return mArrayNum == aOther.mArrayNum && 719 mArrayIndex == aOther.mArrayIndex; 720 } 721 bool operator!=(const iterator& aOther) const { 722 return !(*this == aOther); 723 } 724 725 private: Array()726 nsTArray<MediaTrack*>* Array() { 727 return mArrayNum == 0 ? &mGraph->mTracks : &mGraph->mSuspendedTracks; 728 } 729 MediaTrackGraphImpl* mGraph; 730 int mArrayNum; 731 uint32_t mArrayIndex; 732 }; 733 TrackSet(MediaTrackGraphImpl & aGraph)734 explicit TrackSet(MediaTrackGraphImpl& aGraph) : mGraph(aGraph) {} begin()735 iterator begin() { return iterator(mGraph); } end()736 iterator end() { return iterator(); } 737 738 private: 739 MediaTrackGraphImpl& mGraph; 740 }; AllTracks()741 TrackSet AllTracks() { return TrackSet(*this); } 742 743 // Data members 744 745 /* 746 * If set, the GraphRunner class handles handing over data from audio 747 * callbacks to a common single thread, shared across GraphDrivers. 748 */ 749 const RefPtr<GraphRunner> mGraphRunner; 750 751 /** 752 * Main-thread view of the number of tracks in this graph, for lifetime 753 * management. 754 * 755 * When this becomes zero, the graph is marked as forbidden to add more 756 * tracks to. It will be shut down shortly after. 757 */ 758 size_t mMainThreadTrackCount = 0; 759 760 /** 761 * Main-thread view of the number of ports in this graph, to catch bugs. 762 * 763 * When this becomes zero, and mMainThreadTrackCount is 0, the graph is 764 * marked as forbidden to add more ControlMessages to. It will be shut down 765 * shortly after. 766 */ 767 size_t mMainThreadPortCount = 0; 768 769 /** 770 * Graphs own owning references to their driver, until shutdown. When a driver 771 * switch occur, previous driver is either deleted, or it's ownership is 772 * passed to a event that will take care of the asynchronous cleanup, as 773 * audio track can take some time to shut down. 774 * Accessed on both the main thread and the graph thread; both read and write. 775 * Must hold monitor to access it. 776 */ 777 RefPtr<GraphDriver> mDriver; 778 779 // Set during an iteration to switch driver after the iteration has finished. 780 // Should the current iteration be the last iteration, the next driver will be 781 // discarded. Access through SwitchAtNextIteration()/NextDriver(). Graph 782 // thread only. 783 RefPtr<GraphDriver> mNextDriver; 784 785 // The following state is managed on the graph thread only, unless 786 // mLifecycleState > LIFECYCLE_RUNNING in which case the graph thread 787 // is not running and this state can be used from the main thread. 788 789 /** 790 * The graph keeps a reference to each track. 791 * References are maintained manually to simplify reordering without 792 * unnecessary thread-safe refcount changes. 793 * Must satisfy OnGraphThreadOrNotRunning(). 794 */ 795 nsTArray<MediaTrack*> mTracks; 796 /** 797 * This stores MediaTracks that are part of suspended AudioContexts. 798 * mTracks and mSuspendTracks are disjoint sets: a track is either suspended 799 * or not suspended. Suspended tracks are not ordered in UpdateTrackOrder, 800 * and are therefore not doing any processing. 801 * Must satisfy OnGraphThreadOrNotRunning(). 802 */ 803 nsTArray<MediaTrack*> mSuspendedTracks; 804 /** 805 * Tracks from mFirstCycleBreaker to the end of mTracks produce output before 806 * they receive input. They correspond to DelayNodes that are in cycles. 807 */ 808 uint32_t mFirstCycleBreaker; 809 /** 810 * Blocking decisions have been computed up to this time. 811 * Between each iteration, this is the same as mProcessedTime. 812 */ 813 GraphTime mStateComputedTime = 0; 814 /** 815 * All track contents have been computed up to this time. 816 * The next batch of updates from the main thread will be processed 817 * at this time. This is behind mStateComputedTime during processing. 818 */ 819 GraphTime mProcessedTime = 0; 820 /** 821 * The end of the current iteration. Only access on the graph thread. 822 */ 823 GraphTime mIterationEndTime = 0; 824 /** 825 * The graph should stop processing at this time. 826 */ 827 GraphTime mEndTime; 828 /** 829 * Date of the last time we updated the main thread with the graph state. 830 */ 831 TimeStamp mLastMainThreadUpdate; 832 /** 833 * Number of active MediaInputPorts 834 */ 835 int32_t mPortCount; 836 /** 837 * Runnables to run after the next update to main thread state, but that are 838 * still waiting for the next iteration to finish. 839 */ 840 nsTArray<nsCOMPtr<nsIRunnable>> mPendingUpdateRunnables; 841 842 /** 843 * Devices to use for cubeb input & output, or nullptr for default device. 844 * A MediaTrackGraph always has an output (even if silent). 845 * If `mDeviceTrackMap.Count() != 0`, this MediaTrackGraph wants audio 846 * input. 847 * 848 * All mInputDeviceID access is on the graph thread except for reads via 849 * InputDeviceID(), which are racy but used only for comparison. 850 * 851 * In any case, the number of channels to use can be queried (on the graph 852 * thread) by AudioInputChannelCount() and AudioOutputChannelCount(). 853 */ 854 std::atomic<CubebUtils::AudioDeviceID> mInputDeviceID; 855 CubebUtils::AudioDeviceID mOutputDeviceID; 856 857 // Maps AudioDeviceID to a device track that delivers audio input/output 858 // data and send device-changed signals to its listeners. This is only 859 // touched on the graph thread. The NativeInputTrack* here is used for 860 // for bookkeeping on the graph thread. The owner of the NativeInputTrack is 861 // mDeviceTracks, which is only touched by main thread. 862 nsTHashMap<CubebUtils::AudioDeviceID, NativeInputTrack*> mDeviceTrackMap; 863 864 /** 865 * List of resume operations waiting for a switch to an AudioCallbackDriver. 866 */ 867 class PendingResumeOperation { 868 public: 869 explicit PendingResumeOperation( 870 AudioContextOperationControlMessage* aMessage); 871 void Apply(MediaTrackGraphImpl* aGraph); 872 void Abort(); DestinationTrack()873 MediaTrack* DestinationTrack() const { return mDestinationTrack; } 874 875 private: 876 RefPtr<MediaTrack> mDestinationTrack; 877 nsTArray<RefPtr<MediaTrack>> mTracks; 878 MozPromiseHolder<AudioContextOperationPromise> mHolder; 879 }; 880 AutoTArray<PendingResumeOperation, 1> mPendingResumeOperations; 881 882 // mMonitor guards the data below. 883 // MediaTrackGraph normally does its work without holding mMonitor, so it is 884 // not safe to just grab mMonitor from some thread and start monkeying with 885 // the graph. Instead, communicate with the graph thread using provided 886 // mechanisms such as the ControlMessage queue. 887 Monitor mMonitor; 888 889 // Data guarded by mMonitor (must always be accessed with mMonitor held, 890 // regardless of the value of mLifecycleState). 891 892 /** 893 * State to copy to main thread 894 */ 895 nsTArray<TrackUpdate> mTrackUpdates; 896 /** 897 * Runnables to run after the next update to main thread state. 898 */ 899 nsTArray<nsCOMPtr<nsIRunnable>> mUpdateRunnables; 900 /** 901 * A list of batches of messages to process. Each batch is processed 902 * as an atomic unit. 903 */ 904 /* 905 * Message queue processed by the MTG thread during an iteration. 906 * Accessed on graph thread only. 907 */ 908 nsTArray<MessageBlock> mFrontMessageQueue; 909 /* 910 * Message queue in which the main thread appends messages. 911 * Access guarded by mMonitor. 912 */ 913 nsTArray<MessageBlock> mBackMessageQueue; 914 915 /* True if there will messages to process if we swap the message queues. */ MessagesQueued()916 bool MessagesQueued() const { 917 mMonitor.AssertCurrentThreadOwns(); 918 return !mBackMessageQueue.IsEmpty(); 919 } 920 /** 921 * This enum specifies where this graph is in its lifecycle. This is used 922 * to control shutdown. 923 * Shutdown is tricky because it can happen in two different ways: 924 * 1) Shutdown due to inactivity. RunThread() detects that it has no 925 * pending messages and no tracks, and exits. The next RunInStableState() 926 * checks if there are new pending messages from the main thread (true only 927 * if new track creation raced with shutdown); if there are, it revives 928 * RunThread(), otherwise it commits to shutting down the graph. New track 929 * creation after this point will create a new graph. An async event is 930 * dispatched to Shutdown() the graph's threads and then delete the graph 931 * object. 932 * 2) Forced shutdown at application shutdown, completion of a non-realtime 933 * graph, or document unload. A flag is set, RunThread() detects the flag 934 * and exits, the next RunInStableState() detects the flag, and dispatches 935 * the async event to Shutdown() the graph's threads. However the graph 936 * object is not deleted. New messages for the graph are processed 937 * synchronously on the main thread if necessary. When the last track is 938 * destroyed, the graph object is deleted. 939 * 940 * This should be kept in sync with the LifecycleState_str array in 941 * MediaTrackGraph.cpp 942 */ 943 enum LifecycleState { 944 // The graph thread hasn't started yet. 945 LIFECYCLE_THREAD_NOT_STARTED, 946 // RunThread() is running normally. 947 LIFECYCLE_RUNNING, 948 // In the following states, the graph thread is not running so 949 // all "graph thread only" state in this class can be used safely 950 // on the main thread. 951 // RunThread() has exited and we're waiting for the next 952 // RunInStableState(), at which point we can clean up the main-thread 953 // side of the graph. 954 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP, 955 // RunInStableState() posted a ShutdownRunnable, and we're waiting for it 956 // to shut down the graph thread(s). 957 LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN, 958 // Graph threads have shut down but we're waiting for remaining tracks 959 // to be destroyed. Only happens during application shutdown and on 960 // completed non-realtime graphs, since normally we'd only shut down a 961 // realtime graph when it has no tracks. 962 LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION 963 }; 964 965 /** 966 * Modified only in mMonitor. Transitions to 967 * LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP occur on the graph thread at 968 * the end of an iteration. All other transitions occur on the main thread. 969 */ 970 LifecycleState mLifecycleState; LifecycleStateRef()971 LifecycleState& LifecycleStateRef() { 972 #if DEBUG 973 if (mGraphDriverRunning) { 974 mMonitor.AssertCurrentThreadOwns(); 975 } else { 976 MOZ_ASSERT(NS_IsMainThread()); 977 } 978 #endif 979 return mLifecycleState; 980 } LifecycleStateRef()981 const LifecycleState& LifecycleStateRef() const { 982 #if DEBUG 983 if (mGraphDriverRunning) { 984 mMonitor.AssertCurrentThreadOwns(); 985 } else { 986 MOZ_ASSERT(NS_IsMainThread()); 987 } 988 #endif 989 return mLifecycleState; 990 } 991 992 /** 993 * True once the graph thread has received the message from ForceShutDown(). 994 * This is checked in the decision to shut down the 995 * graph thread so that control messages dispatched before forced shutdown 996 * are processed on the graph thread. 997 * Only set on the graph thread. 998 * Can be read safely on the thread currently owning the graph, as indicated 999 * by mLifecycleState. 1000 */ 1001 bool mForceShutDownReceived = false; 1002 /** 1003 * true when InterruptJS() has been called, because shutdown (normal or 1004 * forced) has commenced. Set on the main thread under mMonitor and read on 1005 * the graph thread under mMonitor. 1006 **/ 1007 bool mInterruptJSCalled = false; 1008 1009 /** 1010 * Remove this blocker to unblock shutdown. 1011 * Only accessed on the main thread. 1012 **/ 1013 RefPtr<media::ShutdownBlocker> mShutdownBlocker; 1014 1015 /** 1016 * True when we have posted an event to the main thread to run 1017 * RunInStableState() and the event hasn't run yet. 1018 * Accessed on both main and MTG thread, mMonitor must be held. 1019 */ 1020 bool mPostedRunInStableStateEvent; 1021 1022 /** 1023 * The JSContext of the graph thread. Set under mMonitor on only the graph 1024 * or GraphRunner thread. Once set this does not change until reset when 1025 * the thread is about to exit. Read under mMonitor on the main thread to 1026 * interrupt running JS for forced shutdown. 1027 **/ 1028 JSContext* mJSContext = nullptr; 1029 1030 // Main thread only 1031 1032 /** 1033 * Messages posted by the current event loop task. These are forwarded to 1034 * the media graph thread during RunInStableState. We can't forward them 1035 * immediately because we want all messages between stable states to be 1036 * processed as an atomic batch. 1037 */ 1038 nsTArray<UniquePtr<ControlMessage>> mCurrentTaskMessageQueue; 1039 /** 1040 * True from when RunInStableState sets mLifecycleState to LIFECYCLE_RUNNING, 1041 * until RunInStableState has determined that mLifecycleState is > 1042 * LIFECYCLE_RUNNING. 1043 */ 1044 Atomic<bool> mGraphDriverRunning; 1045 /** 1046 * True when a stable state runner has been posted to the appshell to run 1047 * RunInStableState at the next stable state. 1048 * Only accessed on the main thread. 1049 */ 1050 bool mPostedRunInStableState; 1051 /** 1052 * True when processing real-time audio/video. False when processing 1053 * non-realtime audio. 1054 */ 1055 const bool mRealtime; 1056 /** 1057 * True when a change has happened which requires us to recompute the track 1058 * blocking order. 1059 */ 1060 bool mTrackOrderDirty; 1061 const RefPtr<nsISerialEventTarget> mMainThread; 1062 1063 // used to limit graph shutdown time 1064 // Only accessed on the main thread. 1065 nsCOMPtr<nsITimer> mShutdownTimer; 1066 1067 protected: 1068 virtual ~MediaTrackGraphImpl(); 1069 1070 private: 1071 MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf) 1072 1073 /** 1074 * This class uses manual memory management, and all pointers to it are raw 1075 * pointers. However, in order for it to implement nsIMemoryReporter, it needs 1076 * to implement nsISupports and so be ref-counted. So it maintains a single 1077 * nsRefPtr to itself, giving it a ref-count of 1 during its entire lifetime, 1078 * and Destroy() nulls this self-reference in order to trigger self-deletion. 1079 */ 1080 RefPtr<MediaTrackGraphImpl> mSelfRef; 1081 1082 struct WindowAndTrack { 1083 uint64_t mWindowId; 1084 RefPtr<ProcessedMediaTrack> mCaptureTrackSink; 1085 }; 1086 /** 1087 * Track for window audio capture. 1088 */ 1089 nsTArray<WindowAndTrack> mWindowCaptureTracks; 1090 /** 1091 * Tracks that have their audio output mixed and written to an audio output 1092 * device. 1093 */ 1094 nsTArray<TrackKeyAndVolume> mAudioOutputs; 1095 1096 /** 1097 * Global volume scale. Used when running tests so that the output is not too 1098 * loud. 1099 */ 1100 const float mGlobalVolume; 1101 1102 #ifdef DEBUG 1103 /** 1104 * Used to assert when AppendMessage() runs ControlMessages synchronously. 1105 */ 1106 bool mCanRunMessagesSynchronously; 1107 #endif 1108 1109 /** 1110 * The graph's main-thread observable graph time. 1111 * Updated by the stable state runnable after each iteration. 1112 */ 1113 Watchable<GraphTime> mMainThreadGraphTime; 1114 1115 /** 1116 * Set based on mProcessedTime at end of iteration. 1117 * Read by stable state runnable on main thread. Protected by mMonitor. 1118 */ 1119 GraphTime mNextMainThreadGraphTime = 0; 1120 1121 /** 1122 * Cached audio output latency, in seconds. Main thread only. This is reset 1123 * whenever the audio device running this MediaTrackGraph changes. 1124 */ 1125 double mAudioOutputLatency; 1126 1127 /** 1128 * The max audio output channel count the default audio output device 1129 * supports. This is cached here because it can be expensive to query. The 1130 * cache is invalidated when the device is changed. This is initialized in the 1131 * ctor, and the read/write only on the graph thread. 1132 */ 1133 uint32_t mMaxOutputChannelCount; 1134 1135 /* 1136 * Hold the NativeInputTrack for a certain device 1137 */ 1138 nsTHashMap<CubebUtils::AudioDeviceID, RefPtr<NativeInputTrack>> mDeviceTracks; 1139 }; 1140 1141 } // namespace mozilla 1142 1143 #endif /* MEDIATRACKGRAPHIMPL_H_ */ 1144