1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ 2 /* This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 #ifndef MOZILLA_MEDIATRACKGRAPH_H_ 7 #define MOZILLA_MEDIATRACKGRAPH_H_ 8 9 #include "AudioSampleFormat.h" 10 #include "CubebUtils.h" 11 #include "MainThreadUtils.h" 12 #include "MediaSegment.h" 13 #include "mozilla/LinkedList.h" 14 #include "mozilla/Maybe.h" 15 #include "mozilla/Mutex.h" 16 #include "mozilla/StateWatching.h" 17 #include "mozilla/TaskQueue.h" 18 #include "nsAutoRef.h" 19 #include "nsIRunnable.h" 20 #include "nsTArray.h" 21 #include <speex/speex_resampler.h> 22 23 class nsIRunnable; 24 class nsIGlobalObject; 25 class nsPIDOMWindowInner; 26 27 namespace mozilla { 28 class AsyncLogger; 29 class AudioCaptureTrack; 30 class CrossGraphTransmitter; 31 class CrossGraphReceiver; 32 }; // namespace mozilla 33 34 extern mozilla::AsyncLogger gMTGTraceLogger; 35 36 template <> 37 class nsAutoRefTraits<SpeexResamplerState> 38 : public nsPointerRefTraits<SpeexResamplerState> { 39 public: Release(SpeexResamplerState * aState)40 static void Release(SpeexResamplerState* aState) { 41 speex_resampler_destroy(aState); 42 } 43 }; 44 45 namespace mozilla { 46 47 extern LazyLogModule gMediaTrackGraphLog; 48 49 namespace dom { 50 enum class AudioContextOperation; 51 enum class AudioContextOperationFlags; 52 enum class AudioContextState : uint8_t; 53 } // namespace dom 54 55 /* 56 * MediaTrackGraph is a framework for synchronized audio/video processing 57 * and playback. It is designed to be used by other browser components such as 58 * HTML media elements, media capture APIs, real-time media streaming APIs, 59 * multitrack media APIs, and advanced audio APIs. 60 * 61 * The MediaTrackGraph uses a dedicated thread to process media --- the media 62 * graph thread. This ensures that we can process media through the graph 63 * without blocking on main-thread activity. The media graph is only modified 64 * on the media graph thread, to ensure graph changes can be processed without 65 * interfering with media processing. All interaction with the media graph 66 * thread is done with message passing. 67 * 68 * APIs that modify the graph or its properties are described as "control APIs". 69 * These APIs are asynchronous; they queue graph changes internally and 70 * those changes are processed all-at-once by the MediaTrackGraph. The 71 * MediaTrackGraph monitors the main thread event loop via 72 * nsIAppShell::RunInStableState to ensure that graph changes from a single 73 * event loop task are always processed all together. Control APIs should only 74 * be used on the main thread, currently; we may be able to relax that later. 75 * 76 * To allow precise synchronization of times in the control API, the 77 * MediaTrackGraph maintains a "media timeline". Control APIs that take or 78 * return times use that timeline. Those times never advance during 79 * an event loop task. This time is returned by 80 * MediaTrackGraph::GetCurrentTime(). 81 * 82 * Media decoding, audio processing and media playback use thread-safe APIs to 83 * the media graph to ensure they can continue while the main thread is blocked. 84 * 85 * When the graph is changed, we may need to throw out buffered data and 86 * reprocess it. This is triggered automatically by the MediaTrackGraph. 87 */ 88 89 class AudioInputTrack; 90 class AudioNodeEngine; 91 class AudioNodeExternalInputTrack; 92 class AudioNodeTrack; 93 class DirectMediaTrackListener; 94 class ForwardedInputTrack; 95 class MediaInputPort; 96 class MediaTrack; 97 class MediaTrackGraph; 98 class MediaTrackGraphImpl; 99 class MediaTrackListener; 100 class NativeInputTrack; 101 class ProcessedMediaTrack; 102 class SourceMediaTrack; 103 104 class AudioDataListenerInterface { 105 protected: 106 // Protected destructor, to discourage deletion outside of Release(): 107 virtual ~AudioDataListenerInterface() = default; 108 109 public: 110 // Information for the interleaved buffer coming from the audio callbacks 111 struct BufferInfo { 112 AudioDataValue* mBuffer = nullptr; 113 size_t mFrames = 0; 114 uint32_t mChannels = 0; 115 TrackRate mRate = 0; 116 }; 117 118 /* These are for cubeb audio input & output streams: */ 119 /** 120 * Output data to speakers, for use as the "far-end" data for echo 121 * cancellation. This is not guaranteed to be in any particular size 122 * chunks. 123 */ 124 virtual void NotifyOutputData(MediaTrackGraphImpl* aGraph, 125 BufferInfo aInfo) = 0; 126 /** 127 * An AudioCallbackDriver with an input stream signaling that it has stopped 128 * for any reason and the AudioDataListener will not be notified of input data 129 * until the driver is restarted or another driver has started. 130 */ 131 virtual void NotifyInputStopped(MediaTrackGraphImpl* aGraph) = 0; 132 /** 133 * Input data from a microphone (or other audio source. This is not 134 * guaranteed to be in any particular size chunks. 135 */ 136 virtual void NotifyInputData(MediaTrackGraphImpl* aGraph, 137 const BufferInfo aInfo, 138 uint32_t aAlreadyBuffered) = 0; 139 140 /** 141 * Number of audio input channels. 142 */ 143 virtual uint32_t RequestedInputChannelCount(MediaTrackGraphImpl* aGraph) = 0; 144 145 /** 146 * Whether the underlying audio device is used for voice input. 147 */ 148 virtual bool IsVoiceInput(MediaTrackGraphImpl* aGraph) const = 0; 149 /** 150 * Called when the underlying audio device has changed. 151 */ 152 virtual void DeviceChanged(MediaTrackGraphImpl* aGraph) = 0; 153 154 /** 155 * Called when the underlying audio device is being closed. 156 */ 157 virtual void Disconnect(MediaTrackGraphImpl* aGraph) = 0; 158 }; 159 160 class AudioDataListener : public AudioDataListenerInterface { 161 protected: 162 // Protected destructor, to discourage deletion outside of Release(): 163 virtual ~AudioDataListener() = default; 164 165 public: 166 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataListener) 167 }; 168 169 /** 170 * This is a base class for main-thread listener callbacks. 171 * This callback is invoked on the main thread when the main-thread-visible 172 * state of a track has changed. 173 * 174 * These methods are called with the media graph monitor held, so 175 * reentry into general media graph methods is not possible. 176 * You should do something non-blocking and non-reentrant (e.g. dispatch an 177 * event) and return. NS_DispatchToCurrentThread would be a good choice. 178 * The listener is allowed to synchronously remove itself from the track, but 179 * not add or remove any other listeners. 180 */ 181 class MainThreadMediaTrackListener { 182 public: 183 virtual void NotifyMainThreadTrackEnded() = 0; 184 }; 185 186 /** 187 * Helper struct used to keep track of memory usage by AudioNodes. 188 */ 189 struct AudioNodeSizes { AudioNodeSizesAudioNodeSizes190 AudioNodeSizes() : mTrack(0), mEngine(0), mNodeType() {} 191 size_t mTrack; 192 size_t mEngine; 193 const char* mNodeType; 194 }; 195 196 /** 197 * Describes how a track should be disabled. 198 * 199 * ENABLED Not disabled. 200 * SILENCE_BLACK Audio data is turned into silence, video frames are made 201 * black. 202 * SILENCE_FREEZE Audio data is turned into silence, video freezes at 203 * last frame. 204 */ 205 enum class DisabledTrackMode { ENABLED, SILENCE_BLACK, SILENCE_FREEZE }; 206 207 /** 208 * A track of audio or video data. The media type must be known at construction 209 * and cannot change. All tracks progress at the same rate --- "real time". 210 * Tracks cannot seek. The only operation readers can perform on a track is to 211 * read the next data. 212 * 213 * Consumers of a track can be reading from it at different offsets, but that 214 * should only happen due to the order in which consumers are being run. 215 * Those offsets must not diverge in the long term, otherwise we would require 216 * unbounded buffering. 217 * 218 * (DEPRECATED to be removed in bug 1581074) 219 * Tracks can be in a "blocked" state. While blocked, a track does not 220 * produce data. A track can be explicitly blocked via the control API, 221 * or implicitly blocked by whatever's generating it (e.g. an underrun in the 222 * source resource), or implicitly blocked because something consuming it 223 * blocks, or implicitly because it has ended. 224 * 225 * A track can be in an "ended" state. "Ended" tracks are permanently blocked. 226 * The "ended" state is terminal. 227 * 228 * Transitions into and out of the "blocked" and "ended" states are managed 229 * by the MediaTrackGraph on the media graph thread. 230 * 231 * We buffer media data ahead of the consumers' reading offsets. It is possible 232 * to have buffered data but still be blocked. 233 * 234 * Any track can have its audio or video playing when requested. The media 235 * track graph plays audio by constructing audio output tracks as necessary. 236 * Video is played through a DirectMediaTrackListener managed by 237 * VideoStreamTrack. 238 * 239 * The data in a track is managed by mSegment. The segment starts at GraphTime 240 * mStartTime and encodes its own TrackTime duration. 241 * 242 * Tracks are explicitly managed. The client creates them via 243 * MediaTrackGraph::Create{Source|ForwardedInput}Track, and releases them by 244 * calling Destroy() when no longer needed (actual destruction will be 245 * deferred). The actual object is owned by the MediaTrackGraph. The basic idea 246 * is that main thread objects will keep Tracks alive as long as necessary 247 * (using the cycle collector to clean up whenever needed). 248 * 249 * We make them refcounted only so that track-related messages with 250 * MediaTrack* pointers can be sent to the main thread safely. 251 * 252 * The lifetimes of MediaTracks are controlled from the main thread. 253 * For MediaTracks exposed to the DOM, the lifetime is controlled by the DOM 254 * wrapper; the DOM wrappers own their associated MediaTracks. When a DOM 255 * wrapper is destroyed, it sends a Destroy message for the associated 256 * MediaTrack and clears its reference (the last main-thread reference to 257 * the object). When the Destroy message is processed on the graph thread we 258 * immediately release the affected objects (disentangling them from other 259 * objects as necessary). 260 * 261 * This could cause problems for media processing if a MediaTrack is destroyed 262 * while a downstream MediaTrack is still using it. Therefore the DOM wrappers 263 * must keep upstream MediaTracks alive as long as they could be used in the 264 * media graph. 265 * 266 * At any time, however, a set of MediaTrack wrappers could be collected via 267 * cycle collection. Destroy messages will be sent for those objects in 268 * arbitrary order and the MediaTrackGraph has to be able to handle this. 269 */ 270 class MediaTrack : public mozilla::LinkedListElement<MediaTrack> { 271 public: 272 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaTrack) 273 274 MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType, 275 MediaSegment* aSegment); 276 277 // The sample rate of the graph. 278 const TrackRate mSampleRate; 279 const MediaSegment::Type mType; 280 281 protected: 282 // Protected destructor, to discourage deletion outside of Release(): 283 virtual ~MediaTrack(); 284 285 public: 286 /** 287 * Returns the graph that owns this track. 288 */ 289 MediaTrackGraphImpl* GraphImpl(); 290 const MediaTrackGraphImpl* GraphImpl() const; 291 MediaTrackGraph* Graph(); 292 const MediaTrackGraph* Graph() const; 293 /** 294 * Sets the graph that owns this track. Should only be called once. 295 */ 296 void SetGraphImpl(MediaTrackGraphImpl* aGraph); 297 void SetGraphImpl(MediaTrackGraph* aGraph); 298 299 // Control API. 300 virtual void AddAudioOutput(void* aKey); 301 virtual void SetAudioOutputVolume(void* aKey, float aVolume); 302 virtual void RemoveAudioOutput(void* aKey); 303 // Explicitly suspend. Useful for example if a media element is pausing 304 // and we need to stop its track emitting its buffered data. As soon as the 305 // Suspend message reaches the graph, the track stops processing. It 306 // ignores its inputs and produces silence/no video until Resumed. Its 307 // current time does not advance. 308 virtual void Suspend(); 309 virtual void Resume(); 310 // Events will be dispatched by calling methods of aListener. 311 virtual void AddListener(MediaTrackListener* aListener); 312 virtual RefPtr<GenericPromise> RemoveListener(MediaTrackListener* aListener); 313 314 /** 315 * Adds aListener to the source track of this track. 316 * When the MediaTrackGraph processes the added listener, it will traverse 317 * the graph and add it to the track's source track. 318 * Note that the listener will be notified on the MediaTrackGraph thread 319 * with whether the installation of it at the source was successful or not. 320 */ 321 virtual void AddDirectListener(DirectMediaTrackListener* aListener); 322 323 /** 324 * Removes aListener from the source track of this track. 325 * Note that the listener has already been removed if the link between the 326 * source and this track has been broken. The caller doesn't have to care 327 * about this, removing when the source cannot be found, or when the listener 328 * had already been removed does nothing. 329 */ 330 virtual void RemoveDirectListener(DirectMediaTrackListener* aListener); 331 332 // A disabled track has video replaced by black, and audio replaced by 333 // silence. 334 void SetDisabledTrackMode(DisabledTrackMode aMode); 335 336 // End event will be notified by calling methods of aListener. It is the 337 // responsibility of the caller to remove aListener before it is destroyed. 338 void AddMainThreadListener(MainThreadMediaTrackListener* aListener); 339 // It's safe to call this even if aListener is not currently a listener; 340 // the call will be ignored. RemoveMainThreadListener(MainThreadMediaTrackListener * aListener)341 void RemoveMainThreadListener(MainThreadMediaTrackListener* aListener) { 342 MOZ_ASSERT(NS_IsMainThread()); 343 MOZ_ASSERT(aListener); 344 mMainThreadListeners.RemoveElement(aListener); 345 } 346 347 /** 348 * Ensure a runnable will run on the main thread after running all pending 349 * updates that were sent from the graph thread or will be sent before the 350 * graph thread receives the next graph update. 351 * 352 * If the graph has been shut down or destroyed, then the runnable will be 353 * dispatched to the event queue immediately. (There are no pending updates 354 * in this situation.) 355 * 356 * Main thread only. 357 */ 358 void RunAfterPendingUpdates(already_AddRefed<nsIRunnable> aRunnable); 359 360 // Signal that the client is done with this MediaTrack. It will be deleted 361 // later. 362 virtual void Destroy(); 363 364 // Returns the main-thread's view of how much data has been processed by 365 // this track. GetCurrentTime()366 TrackTime GetCurrentTime() const { 367 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); 368 return mMainThreadCurrentTime; 369 } 370 // Return the main thread's view of whether this track has ended. IsEnded()371 bool IsEnded() const { 372 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); 373 return mMainThreadEnded; 374 } 375 IsDestroyed()376 bool IsDestroyed() const { 377 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); 378 return mMainThreadDestroyed; 379 } 380 381 friend class MediaTrackGraphImpl; 382 friend class MediaInputPort; 383 friend class AudioNodeExternalInputTrack; 384 AsAudioInputTrack()385 virtual AudioInputTrack* AsAudioInputTrack() { return nullptr; } AsSourceTrack()386 virtual SourceMediaTrack* AsSourceTrack() { return nullptr; } AsProcessedTrack()387 virtual ProcessedMediaTrack* AsProcessedTrack() { return nullptr; } AsAudioNodeTrack()388 virtual AudioNodeTrack* AsAudioNodeTrack() { return nullptr; } AsForwardedInputTrack()389 virtual ForwardedInputTrack* AsForwardedInputTrack() { return nullptr; } AsCrossGraphTransmitter()390 virtual CrossGraphTransmitter* AsCrossGraphTransmitter() { return nullptr; } AsCrossGraphReceiver()391 virtual CrossGraphReceiver* AsCrossGraphReceiver() { return nullptr; } AsNativeInputTrack()392 virtual NativeInputTrack* AsNativeInputTrack() { return nullptr; } 393 394 // These Impl methods perform the core functionality of the control methods 395 // above, on the media graph thread. 396 /** 397 * Stop all track activity and disconnect it from all inputs and outputs. 398 * This must be idempotent. 399 */ 400 virtual void DestroyImpl(); 401 TrackTime GetEnd() const; 402 void SetAudioOutputVolumeImpl(void* aKey, float aVolume); 403 void AddAudioOutputImpl(void* aKey); 404 void RemoveAudioOutputImpl(void* aKey); 405 406 /** 407 * Removes all direct listeners and signals to them that they have been 408 * uninstalled. 409 */ RemoveAllDirectListenersImpl()410 virtual void RemoveAllDirectListenersImpl() {} 411 void RemoveAllResourcesAndListenersImpl(); 412 413 virtual void AddListenerImpl(already_AddRefed<MediaTrackListener> aListener); 414 virtual void RemoveListenerImpl(MediaTrackListener* aListener); 415 virtual void AddDirectListenerImpl( 416 already_AddRefed<DirectMediaTrackListener> aListener); 417 virtual void RemoveDirectListenerImpl(DirectMediaTrackListener* aListener); 418 virtual void SetDisabledTrackModeImpl(DisabledTrackMode aMode); 419 AddConsumer(MediaInputPort * aPort)420 void AddConsumer(MediaInputPort* aPort) { mConsumers.AppendElement(aPort); } RemoveConsumer(MediaInputPort * aPort)421 void RemoveConsumer(MediaInputPort* aPort) { 422 mConsumers.RemoveElement(aPort); 423 } StartTime()424 GraphTime StartTime() const { return mStartTime; } Ended()425 bool Ended() const { return mEnded; } 426 427 // Returns the current number of channels this track contains if it's an audio 428 // track. Calling this on a video track will trip assertions. Graph thread 429 // only. 430 virtual uint32_t NumberOfChannels() const = 0; 431 432 // The DisabledTrackMode after combining the explicit mode and that of the 433 // input, if any. CombinedDisabledMode()434 virtual DisabledTrackMode CombinedDisabledMode() const { 435 return mDisabledMode; 436 } 437 438 template <class SegmentType> GetData()439 SegmentType* GetData() const { 440 if (!mSegment) { 441 return nullptr; 442 } 443 if (mSegment->GetType() != SegmentType::StaticType()) { 444 return nullptr; 445 } 446 return static_cast<SegmentType*>(mSegment.get()); 447 } GetData()448 MediaSegment* GetData() const { return mSegment.get(); } 449 TrackTimeToSeconds(TrackTime aTime)450 double TrackTimeToSeconds(TrackTime aTime) const { 451 NS_ASSERTION(0 <= aTime && aTime <= TRACK_TIME_MAX, "Bad time"); 452 return static_cast<double>(aTime) / mSampleRate; 453 } TrackTimeToMicroseconds(TrackTime aTime)454 int64_t TrackTimeToMicroseconds(TrackTime aTime) const { 455 NS_ASSERTION(0 <= aTime && aTime <= TRACK_TIME_MAX, "Bad time"); 456 return (aTime * 1000000) / mSampleRate; 457 } SecondsToNearestTrackTime(double aSeconds)458 TrackTime SecondsToNearestTrackTime(double aSeconds) const { 459 NS_ASSERTION(0 <= aSeconds && aSeconds <= TRACK_TICKS_MAX / TRACK_RATE_MAX, 460 "Bad seconds"); 461 return mSampleRate * aSeconds + 0.5; 462 } MicrosecondsToTrackTimeRoundDown(int64_t aMicroseconds)463 TrackTime MicrosecondsToTrackTimeRoundDown(int64_t aMicroseconds) const { 464 return (aMicroseconds * mSampleRate) / 1000000; 465 } 466 TimeToTicksRoundUp(TrackRate aRate,TrackTime aTime)467 TrackTicks TimeToTicksRoundUp(TrackRate aRate, TrackTime aTime) const { 468 return RateConvertTicksRoundUp(aRate, mSampleRate, aTime); 469 } TicksToTimeRoundDown(TrackRate aRate,TrackTicks aTicks)470 TrackTime TicksToTimeRoundDown(TrackRate aRate, TrackTicks aTicks) const { 471 return RateConvertTicksRoundDown(mSampleRate, aRate, aTicks); 472 } 473 /** 474 * Convert graph time to track time. aTime must be <= mStateComputedTime 475 * to ensure we know exactly how much time this track will be blocked during 476 * the interval. 477 */ 478 TrackTime GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const; 479 /** 480 * Convert graph time to track time. This assumes there is no blocking time 481 * to take account of, which is always true except between a track 482 * having its blocking time calculated in UpdateGraph and its blocking time 483 * taken account of in UpdateCurrentTimeForTracks. 484 */ 485 TrackTime GraphTimeToTrackTime(GraphTime aTime) const; 486 /** 487 * Convert track time to graph time. This assumes there is no blocking time 488 * to take account of, which is always true except between a track 489 * having its blocking time calculated in UpdateGraph and its blocking time 490 * taken account of in UpdateCurrentTimeForTracks. 491 */ 492 GraphTime TrackTimeToGraphTime(TrackTime aTime) const; 493 494 virtual void ApplyTrackDisabling(MediaSegment* aSegment, 495 MediaSegment* aRawSegment = nullptr); 496 497 // Return true if the main thread needs to observe updates from this track. MainThreadNeedsUpdates()498 virtual bool MainThreadNeedsUpdates() const { return true; } 499 500 virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const; 501 virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const; 502 IsSuspended()503 bool IsSuspended() const { return mSuspendedCount > 0; } 504 /** 505 * Increment suspend count and move it to mGraph->mSuspendedTracks if 506 * necessary. Graph thread. 507 */ 508 void IncrementSuspendCount(); 509 /** 510 * Increment suspend count on aTrack and move it to mGraph->mTracks if 511 * necessary. GraphThread. 512 */ 513 virtual void DecrementSuspendCount(); 514 515 protected: 516 // Called on graph thread either during destroy handling or before handing 517 // graph control to the main thread to release tracks. OnGraphThreadDone()518 virtual void OnGraphThreadDone() {} 519 520 // |AdvanceTimeVaryingValuesToCurrentTime| will be override in 521 // SourceMediaTrack. 522 virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, 523 GraphTime aBlockedTime); 524 NotifyMainThreadListeners()525 void NotifyMainThreadListeners() { 526 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); 527 528 for (int32_t i = mMainThreadListeners.Length() - 1; i >= 0; --i) { 529 mMainThreadListeners[i]->NotifyMainThreadTrackEnded(); 530 } 531 mMainThreadListeners.Clear(); 532 } 533 ShouldNotifyTrackEnded()534 bool ShouldNotifyTrackEnded() { 535 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); 536 if (!mMainThreadEnded || mEndedNotificationSent) { 537 return false; 538 } 539 540 mEndedNotificationSent = true; 541 return true; 542 } 543 544 // Notifies listeners and consumers of the change in disabled mode when the 545 // current combined mode is different from aMode. 546 void NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode); 547 548 // This state is all initialized on the main thread but 549 // otherwise modified only on the media graph thread. 550 551 // Buffered data. The start of the buffer corresponds to mStartTime. 552 // Conceptually the buffer contains everything this track has ever played, 553 // but we forget some prefix of the buffered data to bound the space usage. 554 // Note that this may be null for tracks that never contain data, like 555 // non-external AudioNodeTracks. 556 const UniquePtr<MediaSegment> mSegment; 557 558 // The time when the buffered data could be considered to have started 559 // playing. This increases over time to account for time the track was 560 // blocked before mCurrentTime. 561 GraphTime mStartTime; 562 563 // The time until which we last called mSegment->ForgetUpTo(). 564 TrackTime mForgottenTime; 565 566 // True once we've processed mSegment until the end and no more data will be 567 // added. Note that mSegment might still contain data for the current 568 // iteration. 569 bool mEnded; 570 571 // True after track listeners have been notified that this track has ended. 572 bool mNotifiedEnded; 573 574 // Client-set volume of this track 575 nsTArray<RefPtr<MediaTrackListener>> mTrackListeners; 576 nsTArray<MainThreadMediaTrackListener*> mMainThreadListeners; 577 // This track's associated disabled mode. It can either by disabled by frames 578 // being replaced by black, or by retaining the previous frame. 579 DisabledTrackMode mDisabledMode; 580 581 // GraphTime at which this track starts blocking. 582 // This is only valid up to mStateComputedTime. The track is considered to 583 // have not been blocked before mCurrentTime (its mStartTime is 584 // increased as necessary to account for that time instead). 585 GraphTime mStartBlocking; 586 587 // MediaInputPorts to which this is connected 588 nsTArray<MediaInputPort*> mConsumers; 589 590 /** 591 * Number of outstanding suspend operations on this track. Track is 592 * suspended when this is > 0. 593 */ 594 int32_t mSuspendedCount; 595 596 // Main-thread views of state 597 TrackTime mMainThreadCurrentTime; 598 bool mMainThreadEnded; 599 bool mEndedNotificationSent; 600 bool mMainThreadDestroyed; 601 602 // Our media track graph. null if destroyed on the graph thread. 603 MediaTrackGraphImpl* mGraph; 604 }; 605 606 /** 607 * This is a track into which a decoder can write audio or video. 608 * 609 * Audio or video can be written on any thread, but you probably want to 610 * always write from the same thread to avoid unexpected interleavings. 611 * 612 * For audio the sample rate of the written data can differ from the sample rate 613 * of the graph itself. Use SetAppendDataSourceRate to inform the track what 614 * rate written audio data will be sampled in. 615 */ 616 class SourceMediaTrack : public MediaTrack { 617 public: 618 SourceMediaTrack(MediaSegment::Type aType, TrackRate aSampleRate); 619 AsSourceTrack()620 SourceMediaTrack* AsSourceTrack() override { return this; } 621 622 // Main thread only 623 624 /** 625 * Enable or disable pulling. 626 * When pulling is enabled, NotifyPull gets called on the 627 * MediaTrackListeners for this track during the MediaTrackGraph 628 * control loop. Pulling is initially disabled. Due to unavoidable race 629 * conditions, after a call to SetPullingEnabled(false) it is still possible 630 * for a NotifyPull to occur. 631 */ 632 void SetPullingEnabled(bool aEnabled); 633 634 // MediaTrackGraph thread only 635 void DestroyImpl() override; 636 637 // Call these on any thread. 638 /** 639 * Call all MediaTrackListeners to request new data via the NotifyPull 640 * API (if enabled). 641 * aDesiredUpToTime (in): end time of new data requested. 642 * 643 * Returns true if new data is about to be added. 644 */ 645 bool PullNewData(GraphTime aDesiredUpToTime); 646 647 /** 648 * Extract any state updates pending in the track, and apply them. 649 */ 650 void ExtractPendingInput(GraphTime aCurrentTime, GraphTime aDesiredUpToTime); 651 652 /** 653 * All data appended with AppendData() from this point on will be resampled 654 * from aRate to the graph rate. 655 * 656 * Resampling for video does not make sense and is forbidden. 657 */ 658 void SetAppendDataSourceRate(TrackRate aRate); 659 660 /** 661 * Append media data to this track. Ownership of aSegment remains with the 662 * caller, but aSegment is emptied. Returns 0 if the data was not appended 663 * because the stream has ended. Returns the duration of the appended data in 664 * the graph's track rate otherwise. 665 */ 666 TrackTime AppendData(MediaSegment* aSegment, 667 MediaSegment* aRawSegment = nullptr); 668 669 /** 670 * Clear any data appended with AppendData() that hasn't entered the graph 671 * yet. Returns the duration of the cleared data in the graph's track rate. 672 */ 673 TrackTime ClearFutureData(); 674 675 /** 676 * Indicate that this track has ended. Do not do any more API calls affecting 677 * this track. 678 */ 679 void End(); 680 681 // Overriding allows us to hold the mMutex lock while changing the track 682 // enable status 683 void SetDisabledTrackModeImpl(DisabledTrackMode aMode) override; 684 685 // Overriding allows us to ensure mMutex is locked while changing the track 686 // enable status 687 void ApplyTrackDisabling(MediaSegment* aSegment, 688 MediaSegment* aRawSegment = nullptr) override { 689 mMutex.AssertCurrentThreadOwns(); 690 MediaTrack::ApplyTrackDisabling(aSegment, aRawSegment); 691 } 692 693 uint32_t NumberOfChannels() const override; 694 695 void RemoveAllDirectListenersImpl() override; 696 697 // The value set here is applied in MoveToSegment so we can avoid the 698 // buffering delay in applying the change. See Bug 1443511. 699 void SetVolume(float aVolume); 700 float GetVolumeLocked(); 701 702 friend class MediaTrackGraphImpl; 703 704 protected: 705 enum TrackCommands : uint32_t; 706 707 virtual ~SourceMediaTrack(); 708 709 /** 710 * Data to cater for appending media data to this track. 711 */ 712 struct TrackData { 713 // Sample rate of the input data. 714 TrackRate mInputRate; 715 // Resampler if the rate of the input track does not match the 716 // MediaTrackGraph's. 717 nsAutoRef<SpeexResamplerState> mResampler; 718 uint32_t mResamplerChannelCount; 719 // Each time the track updates are flushed to the media graph thread, 720 // the segment buffer is emptied. 721 UniquePtr<MediaSegment> mData; 722 // True once the producer has signaled that no more data is coming. 723 bool mEnded; 724 // True if the producer of this track is having data pulled by the graph. 725 bool mPullingEnabled; 726 // True if the graph has notified this track that it will not be used 727 // again on the graph thread. 728 bool mGraphThreadDone; 729 }; 730 731 bool NeedsMixing(); 732 733 void ResampleAudioToGraphSampleRate(MediaSegment* aSegment); 734 735 void AddDirectListenerImpl( 736 already_AddRefed<DirectMediaTrackListener> aListener) override; 737 void RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) override; 738 739 /** 740 * Notify direct consumers of new data to this track. 741 * The data doesn't have to be resampled (though it may be). This is called 742 * from AppendData on the thread providing the data, and will call 743 * the Listeners on this thread. 744 */ 745 void NotifyDirectConsumers(MediaSegment* aSegment); 746 OnGraphThreadDone()747 void OnGraphThreadDone() override { 748 MutexAutoLock lock(mMutex); 749 if (!mUpdateTrack) { 750 return; 751 } 752 mUpdateTrack->mGraphThreadDone = true; 753 if (!mUpdateTrack->mData) { 754 return; 755 } 756 mUpdateTrack->mData->Clear(); 757 } 758 759 virtual void AdvanceTimeVaryingValuesToCurrentTime( 760 GraphTime aCurrentTime, GraphTime aBlockedTime) override; 761 762 // This must be acquired *before* MediaTrackGraphImpl's lock, if they are 763 // held together. 764 Mutex mMutex; 765 // protected by mMutex 766 float mVolume = 1.0; 767 UniquePtr<TrackData> mUpdateTrack; 768 nsTArray<RefPtr<DirectMediaTrackListener>> mDirectTrackListeners; 769 }; 770 771 /** 772 * A ref-counted wrapper of a MediaTrack that allows multiple users to share a 773 * reference to the same MediaTrack with the purpose of being guaranteed that 774 * the graph it is in is kept alive. 775 * 776 * Automatically suspended on creation and destroyed on destruction. Main thread 777 * only. 778 */ 779 struct SharedDummyTrack { NS_INLINE_DECL_REFCOUNTINGSharedDummyTrack780 NS_INLINE_DECL_REFCOUNTING(SharedDummyTrack) 781 explicit SharedDummyTrack(MediaTrack* aTrack) : mTrack(aTrack) { 782 mTrack->Suspend(); 783 } 784 const RefPtr<MediaTrack> mTrack; 785 786 private: ~SharedDummyTrackSharedDummyTrack787 ~SharedDummyTrack() { mTrack->Destroy(); } 788 }; 789 790 /** 791 * Represents a connection between a ProcessedMediaTrack and one of its 792 * input tracks. 793 * We make these refcounted so that track-related messages with MediaInputPort* 794 * pointers can be sent to the main thread safely. 795 * 796 * When a port's source or destination track dies, the track's DestroyImpl 797 * calls MediaInputPort::Disconnect to disconnect the port from 798 * the source and destination tracks. 799 * 800 * The lifetimes of MediaInputPort are controlled from the main thread. 801 * The media graph adds a reference to the port. When a MediaInputPort is no 802 * longer needed, main-thread code sends a Destroy message for the port and 803 * clears its reference (the last main-thread reference to the object). When 804 * the Destroy message is processed on the graph manager thread we disconnect 805 * the port and drop the graph's reference, destroying the object. 806 */ 807 class MediaInputPort final { 808 private: 809 // Do not call this constructor directly. Instead call 810 // aDest->AllocateInputPort. MediaInputPort(MediaTrackGraphImpl * aGraph,MediaTrack * aSource,ProcessedMediaTrack * aDest,uint16_t aInputNumber,uint16_t aOutputNumber)811 MediaInputPort(MediaTrackGraphImpl* aGraph, MediaTrack* aSource, 812 ProcessedMediaTrack* aDest, uint16_t aInputNumber, 813 uint16_t aOutputNumber) 814 : mSource(aSource), 815 mDest(aDest), 816 mInputNumber(aInputNumber), 817 mOutputNumber(aOutputNumber), 818 mGraph(aGraph) { 819 MOZ_COUNT_CTOR(MediaInputPort); 820 } 821 822 // Private destructor, to discourage deletion outside of Release(): 823 MOZ_COUNTED_DTOR(MediaInputPort) 824 825 public: 826 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort) 827 828 /** 829 * Disconnects and destroys the port. The caller must not reference this 830 * object again. Main thread. 831 */ 832 void Destroy(); 833 834 // The remaining methods and members must always be called on the graph thread 835 // from within MediaTrackGraph.cpp. 836 837 void Init(); 838 // Called during message processing to trigger removal of this port's source 839 // and destination tracks. 840 void Disconnect(); 841 GetSource()842 MediaTrack* GetSource() const { return mSource; } GetDestination()843 ProcessedMediaTrack* GetDestination() const { return mDest; } 844 InputNumber()845 uint16_t InputNumber() const { return mInputNumber; } OutputNumber()846 uint16_t OutputNumber() const { return mOutputNumber; } 847 848 struct InputInterval { 849 GraphTime mStart; 850 GraphTime mEnd; 851 bool mInputIsBlocked; 852 }; 853 // Find the next time interval starting at or after aTime during which 854 // aPort->mDest is not blocked and aPort->mSource's blocking status does not 855 // change. A null aPort returns a blocked interval starting at aTime. 856 static InputInterval GetNextInputInterval(MediaInputPort const* aPort, 857 GraphTime aTime); 858 859 /** 860 * Returns the graph that owns this port. 861 */ 862 MediaTrackGraphImpl* GraphImpl() const; 863 MediaTrackGraph* Graph() const; 864 865 /** 866 * Sets the graph that owns this track. Should only be called once. 867 */ 868 void SetGraphImpl(MediaTrackGraphImpl* aGraph); 869 870 /** 871 * Notify the port that the source MediaTrack has been suspended. 872 */ 873 void Suspended(); 874 875 /** 876 * Notify the port that the source MediaTrack has been resumed. 877 */ 878 void Resumed(); 879 SizeOfExcludingThis(MallocSizeOf aMallocSizeOf)880 size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { 881 size_t amount = 0; 882 883 // Not owned: 884 // - mSource 885 // - mDest 886 // - mGraph 887 return amount; 888 } 889 SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)890 size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { 891 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); 892 } 893 894 private: 895 friend class ProcessedMediaTrack; 896 // Never modified after Init() 897 MediaTrack* mSource; 898 ProcessedMediaTrack* mDest; 899 // The input and output numbers are optional, and are currently only used by 900 // Web Audio. 901 const uint16_t mInputNumber; 902 const uint16_t mOutputNumber; 903 904 // Our media track graph 905 MediaTrackGraphImpl* mGraph; 906 }; 907 908 /** 909 * This track processes zero or more input tracks in parallel to produce 910 * its output. The details of how the output is produced are handled by 911 * subclasses overriding the ProcessInput method. 912 */ 913 class ProcessedMediaTrack : public MediaTrack { 914 public: ProcessedMediaTrack(TrackRate aSampleRate,MediaSegment::Type aType,MediaSegment * aSegment)915 ProcessedMediaTrack(TrackRate aSampleRate, MediaSegment::Type aType, 916 MediaSegment* aSegment) 917 : MediaTrack(aSampleRate, aType, aSegment), 918 mAutoend(true), 919 mCycleMarker(0) {} 920 921 // Control API. 922 /** 923 * Allocates a new input port attached to source aTrack. 924 * This port can be removed by calling MediaInputPort::Destroy(). 925 */ 926 already_AddRefed<MediaInputPort> AllocateInputPort( 927 MediaTrack* aTrack, uint16_t aInputNumber = 0, 928 uint16_t aOutputNumber = 0); 929 /** 930 * Queue a message to set the autoend flag on this track (defaults to 931 * true). When this flag is set, and the input track has ended playout 932 * (including if there is no input track), this track automatically 933 * enters the ended state. 934 */ 935 virtual void QueueSetAutoend(bool aAutoend); 936 AsProcessedTrack()937 ProcessedMediaTrack* AsProcessedTrack() override { return this; } 938 939 friend class MediaTrackGraphImpl; 940 941 // Do not call these from outside MediaTrackGraph.cpp! 942 virtual void AddInput(MediaInputPort* aPort); RemoveInput(MediaInputPort * aPort)943 virtual void RemoveInput(MediaInputPort* aPort) { 944 mInputs.RemoveElement(aPort) || mSuspendedInputs.RemoveElement(aPort); 945 } HasInputPort(MediaInputPort * aPort)946 bool HasInputPort(MediaInputPort* aPort) const { 947 return mInputs.Contains(aPort) || mSuspendedInputs.Contains(aPort); 948 } InputPortCount()949 uint32_t InputPortCount() const { 950 return mInputs.Length() + mSuspendedInputs.Length(); 951 } 952 void InputSuspended(MediaInputPort* aPort); 953 void InputResumed(MediaInputPort* aPort); 954 void DestroyImpl() override; 955 void DecrementSuspendCount() override; 956 /** 957 * This gets called after we've computed the blocking states for all 958 * tracks (mBlocked is up to date up to mStateComputedTime). 959 * Also, we've produced output for all tracks up to this one. If this track 960 * is not in a cycle, then all its source tracks have produced data. 961 * Generate output from aFrom to aTo. 962 * This will be called on tracks that have ended. Most track types should 963 * just return immediately if they're ended, but some may wish to update 964 * internal state (see AudioNodeTrack). 965 * ProcessInput is allowed to set mEnded only if ALLOW_END is in aFlags. (This 966 * flag will be set when aTo >= mStateComputedTime, i.e. when we've produced 967 * the last block of data we need to produce.) Otherwise we can get into a 968 * situation where we've determined the track should not block before 969 * mStateComputedTime, but the track ends before mStateComputedTime, violating 970 * the invariant that ended tracks are blocked. 971 */ 972 enum { ALLOW_END = 0x01 }; 973 virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, 974 uint32_t aFlags) = 0; SetAutoendImpl(bool aAutoend)975 void SetAutoendImpl(bool aAutoend) { mAutoend = aAutoend; } 976 977 // Only valid after MediaTrackGraphImpl::UpdateTrackOrder() has run. 978 // A DelayNode is considered to break a cycle and so this will not return 979 // true for echo loops, only for muted cycles. InMutedCycle()980 bool InMutedCycle() const { return mCycleMarker; } 981 982 // Used by ForwardedInputTrack to propagate the disabled mode along the graph. OnInputDisabledModeChanged(DisabledTrackMode aMode)983 virtual void OnInputDisabledModeChanged(DisabledTrackMode aMode) {} 984 SizeOfExcludingThis(MallocSizeOf aMallocSizeOf)985 size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { 986 size_t amount = MediaTrack::SizeOfExcludingThis(aMallocSizeOf); 987 // Not owned: 988 // - mInputs elements 989 // - mSuspendedInputs elements 990 amount += mInputs.ShallowSizeOfExcludingThis(aMallocSizeOf); 991 amount += mSuspendedInputs.ShallowSizeOfExcludingThis(aMallocSizeOf); 992 return amount; 993 } 994 SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)995 size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override { 996 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); 997 } 998 999 protected: 1000 // This state is all accessed only on the media graph thread. 1001 1002 // The list of all inputs that are not currently suspended. 1003 nsTArray<MediaInputPort*> mInputs; 1004 // The list of all inputs that are currently suspended. 1005 nsTArray<MediaInputPort*> mSuspendedInputs; 1006 bool mAutoend; 1007 // After UpdateTrackOrder(), mCycleMarker is either 0 or 1 to indicate 1008 // whether this track is in a muted cycle. During ordering it can contain 1009 // other marker values - see MediaTrackGraphImpl::UpdateTrackOrder(). 1010 uint32_t mCycleMarker; 1011 }; 1012 1013 /** 1014 * There is a single MediaTrackGraph per window. 1015 * Additionaly, each OfflineAudioContext object creates its own MediaTrackGraph 1016 * object too. 1017 */ 1018 class MediaTrackGraph { 1019 public: 1020 // We ensure that the graph current time advances in multiples of 1021 // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A track that 1022 // never blocks and has the ideal audio rate will produce audio in multiples 1023 // of the block size. 1024 1025 // Initializing a graph that outputs audio can take quite long on some 1026 // platforms. Code that want to output audio at some point can express the 1027 // fact that they will need an audio track at some point by passing 1028 // AUDIO_THREAD_DRIVER when getting an instance of MediaTrackGraph, so that 1029 // the graph starts with the right driver. 1030 enum GraphDriverType { 1031 AUDIO_THREAD_DRIVER, 1032 SYSTEM_THREAD_DRIVER, 1033 OFFLINE_THREAD_DRIVER 1034 }; 1035 // A MediaTrackGraph running an AudioWorklet must always be run from the 1036 // same thread, in order to run js. To acheive this, create the graph with 1037 // a SINGLE_THREAD RunType. DIRECT_DRIVER will run the graph directly off 1038 // the GraphDriver's thread. 1039 enum GraphRunType { 1040 DIRECT_DRIVER, 1041 SINGLE_THREAD, 1042 }; 1043 static const uint32_t AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT = 20 * 1000; 1044 static const TrackRate REQUEST_DEFAULT_SAMPLE_RATE = 0; 1045 constexpr static const CubebUtils::AudioDeviceID DEFAULT_OUTPUT_DEVICE = 1046 nullptr; 1047 1048 // Main thread only 1049 static MediaTrackGraph* GetInstanceIfExists( 1050 nsPIDOMWindowInner* aWindow, TrackRate aSampleRate, 1051 CubebUtils::AudioDeviceID aOutputDeviceID); 1052 static MediaTrackGraph* GetInstance( 1053 GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow, 1054 TrackRate aSampleRate, CubebUtils::AudioDeviceID aOutputDeviceID); 1055 static MediaTrackGraph* CreateNonRealtimeInstance( 1056 TrackRate aSampleRate, nsPIDOMWindowInner* aWindowId); 1057 1058 // Idempotent 1059 void ForceShutDown(); 1060 1061 virtual nsresult OpenAudioInput(CubebUtils::AudioDeviceID aID, 1062 AudioDataListener* aListener) = 0; 1063 virtual void CloseAudioInput(CubebUtils::AudioDeviceID aID, 1064 AudioDataListener* aListener) = 0; 1065 1066 // Control API. 1067 /** 1068 * Create a track that a media decoder (or some other source of 1069 * media data, such as a camera) can write to. 1070 */ 1071 SourceMediaTrack* CreateSourceTrack(MediaSegment::Type aType); 1072 /** 1073 * Create a track that will forward data from its input track. 1074 * 1075 * A TrackUnionStream can have 0 or 1 input streams. Adding more than that is 1076 * an error. 1077 * 1078 * A TrackUnionStream will end when autoending is enabled (default) and there 1079 * is no input, or the input's source is ended. If there is no input and 1080 * autoending is disabled, TrackUnionStream will continue to produce silence 1081 * for audio or the last video frame for video. 1082 */ 1083 ProcessedMediaTrack* CreateForwardedInputTrack(MediaSegment::Type aType); 1084 /** 1085 * Create a track that will mix all its audio inputs. 1086 */ 1087 AudioCaptureTrack* CreateAudioCaptureTrack(); 1088 1089 CrossGraphTransmitter* CreateCrossGraphTransmitter( 1090 CrossGraphReceiver* aReceiver); 1091 CrossGraphReceiver* CreateCrossGraphReceiver(TrackRate aTransmitterRate); 1092 1093 /** 1094 * Add a new track to the graph. Main thread. 1095 */ 1096 void AddTrack(MediaTrack* aTrack); 1097 1098 /* From the main thread, ask the MTG to resolve the returned promise when 1099 * the device has started. 1100 * The promise is rejected with NS_ERROR_NOT_AVAILABLE if aTrack 1101 * is destroyed, or NS_ERROR_ILLEGAL_DURING_SHUTDOWN if the graph is shut 1102 * down, before the promise could be resolved. 1103 * (Audio is initially processed in the FallbackDriver's thread while the 1104 * device is starting up.) 1105 */ 1106 using GraphStartedPromise = GenericPromise; 1107 RefPtr<GraphStartedPromise> NotifyWhenDeviceStarted(MediaTrack* aTrack); 1108 1109 /* From the main thread, suspend, resume or close an AudioContext. Calls 1110 * are not counted. Even Resume calls can be more frequent than Suspend 1111 * calls. 1112 * 1113 * aTracks are the tracks of all the AudioNodes of the AudioContext that 1114 * need to be suspended or resumed. Suspend and Resume operations on these 1115 * tracks are counted. Resume operations must not outnumber Suspends and a 1116 * track will not resume until the number of Resume operations matches the 1117 * number of Suspends. This array may be empty if, for example, this is a 1118 * second consecutive suspend call and all the nodes are already suspended. 1119 * 1120 * This can possibly pause the graph thread, releasing system resources, if 1121 * all tracks have been suspended/closed. 1122 * 1123 * When the operation is complete, the returned promise is resolved. 1124 */ 1125 using AudioContextOperationPromise = 1126 MozPromise<dom::AudioContextState, bool, true>; 1127 RefPtr<AudioContextOperationPromise> ApplyAudioContextOperation( 1128 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks, 1129 dom::AudioContextOperation aOperation); 1130 1131 bool IsNonRealtime() const; 1132 /** 1133 * Start processing non-realtime for a specific number of ticks. 1134 */ 1135 void StartNonRealtimeProcessing(uint32_t aTicksToProcess); 1136 1137 /** 1138 * NotifyJSContext() is called on the graph thread before content script 1139 * runs. 1140 */ 1141 void NotifyJSContext(JSContext* aCx); 1142 1143 /** 1144 * Media graph thread only. 1145 * Dispatches a runnable that will run on the main thread after all 1146 * main-thread track state has been updated, i.e., during stable state. 1147 * 1148 * Should only be called during MediaTrackListener callbacks or during 1149 * ProcessedMediaTrack::ProcessInput(). 1150 * 1151 * Note that if called during shutdown the runnable will be ignored and 1152 * released on main thread. 1153 */ 1154 void DispatchToMainThreadStableState(already_AddRefed<nsIRunnable> aRunnable); 1155 1156 /** 1157 * Returns graph sample rate in Hz. 1158 */ GraphRate()1159 TrackRate GraphRate() const { return mSampleRate; } 1160 1161 double AudioOutputLatency(); 1162 1163 void RegisterCaptureTrackForWindow(uint64_t aWindowId, 1164 ProcessedMediaTrack* aCaptureTrack); 1165 void UnregisterCaptureTrackForWindow(uint64_t aWindowId); 1166 already_AddRefed<MediaInputPort> ConnectToCaptureTrack( 1167 uint64_t aWindowId, MediaTrack* aMediaTrack); 1168 AssertOnGraphThreadOrNotRunning()1169 void AssertOnGraphThreadOrNotRunning() const { 1170 MOZ_ASSERT(OnGraphThreadOrNotRunning()); 1171 } 1172 1173 /** 1174 * Returns a watchable of the graph's main-thread observable graph time. 1175 * Main thread only. 1176 */ 1177 virtual Watchable<GraphTime>& CurrentTime() = 0; 1178 1179 /** 1180 * Graph thread function to return the time at which all processing has been 1181 * completed. Some tracks may have performed processing beyond this time. 1182 */ 1183 GraphTime ProcessedTime() const; 1184 1185 protected: MediaTrackGraph(TrackRate aSampleRate)1186 explicit MediaTrackGraph(TrackRate aSampleRate) : mSampleRate(aSampleRate) { 1187 MOZ_COUNT_CTOR(MediaTrackGraph); 1188 } 1189 MOZ_COUNTED_DTOR_VIRTUAL(MediaTrackGraph) 1190 1191 // Intended only for assertions, either on graph thread or not running (in 1192 // which case we must be on the main thread). 1193 virtual bool OnGraphThreadOrNotRunning() const = 0; 1194 virtual bool OnGraphThread() const = 0; 1195 1196 // Intended only for internal assertions. Main thread only. 1197 virtual bool Destroyed() const = 0; 1198 1199 /** 1200 * Sample rate at which this graph runs. For real time graphs, this is 1201 * the rate of the audio mixer. For offline graphs, this is the rate specified 1202 * at construction. 1203 */ 1204 const TrackRate mSampleRate; 1205 }; 1206 1207 } // namespace mozilla 1208 1209 #endif /* MOZILLA_MEDIATRACKGRAPH_H_ */ 1210