1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 
6 #ifndef MOZILLA_AUDIONODETRACK_H_
7 #define MOZILLA_AUDIONODETRACK_H_
8 
9 #include "MediaTrackGraph.h"
10 #include "mozilla/dom/AudioNodeBinding.h"
11 #include "AlignedTArray.h"
12 #include "AudioBlock.h"
13 #include "AudioSegment.h"
14 
15 namespace WebCore {
16 class Reverb;
17 }  // namespace WebCore
18 
19 namespace mozilla {
20 
21 namespace dom {
22 struct ThreeDPoint;
23 struct AudioTimelineEvent;
24 class AudioContext;
25 }  // namespace dom
26 
27 class AbstractThread;
28 class ThreadSharedFloatArrayBufferList;
29 class AudioNodeEngine;
30 
31 typedef AlignedAutoTArray<float, GUESS_AUDIO_CHANNELS * WEBAUDIO_BLOCK_SIZE, 16>
32     DownmixBufferType;
33 
34 /**
35  * An AudioNodeTrack produces one audio track with ID AUDIO_TRACK.
36  * The start time of the AudioTrack is aligned to the start time of the
37  * AudioContext's destination node track, plus some multiple of BLOCK_SIZE
38  * samples.
39  *
40  * An AudioNodeTrack has an AudioNodeEngine plugged into it that does the
41  * actual audio processing. AudioNodeTrack contains the glue code that
42  * integrates audio processing with the MediaTrackGraph.
43  */
44 class AudioNodeTrack : public ProcessedMediaTrack {
45   typedef dom::ChannelCountMode ChannelCountMode;
46   typedef dom::ChannelInterpretation ChannelInterpretation;
47 
48  public:
49   typedef mozilla::dom::AudioContext AudioContext;
50 
51   enum { AUDIO_TRACK = 1 };
52 
53   typedef AutoTArray<AudioBlock, 1> OutputChunks;
54 
55   // Flags re main thread updates and track output.
56   typedef unsigned Flags;
57   enum : Flags {
58     NO_TRACK_FLAGS = 0U,
59     NEED_MAIN_THREAD_ENDED = 1U << 0,
60     NEED_MAIN_THREAD_CURRENT_TIME = 1U << 1,
61     // Internal AudioNodeTracks can only pass their output to another
62     // AudioNode, whereas external AudioNodeTracks can pass their output
63     // to other ProcessedMediaTracks or hardware audio output.
64     EXTERNAL_OUTPUT = 1U << 2,
65   };
66   /**
67    * Create a track that will process audio for an AudioNode.
68    * Takes ownership of aEngine.
69    * aGraph is required and equals the graph of aCtx in most cases. An exception
70    * is AudioDestinationNode where the context's graph hasn't been set up yet.
71    */
72   static already_AddRefed<AudioNodeTrack> Create(AudioContext* aCtx,
73                                                  AudioNodeEngine* aEngine,
74                                                  Flags aKind,
75                                                  MediaTrackGraph* aGraph);
76 
77  protected:
78   /**
79    * Transfers ownership of aEngine to the new AudioNodeTrack.
80    */
81   AudioNodeTrack(AudioNodeEngine* aEngine, Flags aFlags, TrackRate aSampleRate);
82 
83   ~AudioNodeTrack();
84 
85  public:
86   // Control API
87   /**
88    * Sets a parameter that's a time relative to some track's played time.
89    * This time is converted to a time relative to this track when it's set.
90    */
91   void SetTrackTimeParameter(uint32_t aIndex, AudioContext* aContext,
92                              double aTrackTime);
93   void SetDoubleParameter(uint32_t aIndex, double aValue);
94   void SetInt32Parameter(uint32_t aIndex, int32_t aValue);
95   void SetThreeDPointParameter(uint32_t aIndex, const dom::ThreeDPoint& aValue);
96   void SetBuffer(AudioChunk&& aBuffer);
97   void SetReverb(WebCore::Reverb* aReverb, uint32_t aImpulseChannelCount);
98   // This sends a single event to the timeline on the MTG thread side.
99   void SendTimelineEvent(uint32_t aIndex,
100                          const dom::AudioTimelineEvent& aEvent);
101   // This consumes the contents of aData.  aData will be emptied after this
102   // returns.
103   void SetRawArrayData(nsTArray<float>&& aData);
104   void SetChannelMixingParameters(uint32_t aNumberOfChannels,
105                                   ChannelCountMode aChannelCountMoe,
106                                   ChannelInterpretation aChannelInterpretation);
107   void SetPassThrough(bool aPassThrough);
108   void SendRunnable(already_AddRefed<nsIRunnable> aRunnable);
GetChannelInterpretation()109   ChannelInterpretation GetChannelInterpretation() {
110     return mChannelInterpretation;
111   }
112 
SetAudioParamHelperTrack()113   void SetAudioParamHelperTrack() {
114     MOZ_ASSERT(!mAudioParamTrack, "Can only do this once");
115     mAudioParamTrack = true;
116   }
117   // The value for channelCount on an AudioNode, but on the audio thread side.
118   uint32_t NumberOfChannels() const override;
119 
120   /*
121    * Resume track after updating its concept of current time by aAdvance.
122    * Main thread.  Used only from AudioDestinationNode when resuming a track
123    * suspended to save running the MediaTrackGraph when there are no other
124    * nodes in the AudioContext.
125    */
126   void AdvanceAndResume(TrackTime aAdvance);
127 
AsAudioNodeTrack()128   AudioNodeTrack* AsAudioNodeTrack() override { return this; }
129   void AddInput(MediaInputPort* aPort) override;
130   void RemoveInput(MediaInputPort* aPort) override;
131 
132   // Graph thread only
133   void SetTrackTimeParameterImpl(uint32_t aIndex, MediaTrack* aRelativeToTrack,
134                                  double aTrackTime);
135   void SetChannelMixingParametersImpl(
136       uint32_t aNumberOfChannels, ChannelCountMode aChannelCountMoe,
137       ChannelInterpretation aChannelInterpretation);
138   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
139   /**
140    * Produce the next block of output, before input is provided.
141    * ProcessInput() will be called later, and it then should not change
142    * the output.  This is used only for DelayNodeEngine in a feedback loop.
143    */
144   void ProduceOutputBeforeInput(GraphTime aFrom);
IsAudioParamTrack()145   bool IsAudioParamTrack() const { return mAudioParamTrack; }
146 
LastChunks()147   const OutputChunks& LastChunks() const { return mLastChunks; }
MainThreadNeedsUpdates()148   bool MainThreadNeedsUpdates() const override {
149     return ((mFlags & NEED_MAIN_THREAD_ENDED) && mEnded) ||
150            (mFlags & NEED_MAIN_THREAD_CURRENT_TIME);
151   }
152 
153   // Any thread
Engine()154   AudioNodeEngine* Engine() { return mEngine.get(); }
155 
156   size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
157   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override;
158 
159   void SizeOfAudioNodesIncludingThis(MallocSizeOf aMallocSizeOf,
160                                      AudioNodeSizes& aUsage) const;
161 
162   /*
163    * SetActive() is called when either an active input is added or the engine
164    * for a source node transitions from inactive to active.  This is not
165    * called from engines for processing nodes because they only become active
166    * when there are active input tracks, in which case this track is already
167    * active.
168    */
169   void SetActive();
170   /*
171    * ScheduleCheckForInactive() is called during track processing when the
172    * engine transitions from active to inactive, or the track finishes.  It
173    * schedules a call to CheckForInactive() after track processing.
174    */
175   void ScheduleCheckForInactive();
176 
177  protected:
178   class AdvanceAndResumeMessage;
179   class CheckForInactiveMessage;
180 
181   void OnGraphThreadDone() override;
182   void DestroyImpl() override;
183 
184   /*
185    * CheckForInactive() is called when the engine transitions from active to
186    * inactive, or an active input is removed, or the track finishes.  If the
187    * track is now inactive, then mInputChunks will be cleared and mLastChunks
188    * will be set to null.  ProcessBlock() will not be called on the engine
189    * again until SetActive() is called.
190    */
191   void CheckForInactive();
192 
193   void AdvanceOutputSegment();
194   void FinishOutput();
195   void AccumulateInputChunk(uint32_t aInputIndex, const AudioBlock& aChunk,
196                             AudioBlock* aBlock,
197                             DownmixBufferType* aDownmixBuffer);
198   void UpMixDownMixChunk(const AudioBlock* aChunk, uint32_t aOutputChannelCount,
199                          nsTArray<const float*>& aOutputChannels,
200                          DownmixBufferType& aDownmixBuffer);
201 
202   uint32_t ComputedNumberOfChannels(uint32_t aInputChannelCount);
203   void ObtainInputBlock(AudioBlock& aTmpChunk, uint32_t aPortIndex);
204   void IncrementActiveInputCount();
205   void DecrementActiveInputCount();
206 
207   // The engine that will generate output for this node.
208   const UniquePtr<AudioNodeEngine> mEngine;
209   // The mixed input blocks are kept from iteration to iteration to avoid
210   // reallocating channel data arrays and any buffers for mixing.
211   OutputChunks mInputChunks;
212   // The last block produced by this node.
213   OutputChunks mLastChunks;
214   // Whether this is an internal or external track
215   const Flags mFlags;
216   // The number of input tracks that may provide non-silent input.
217   uint32_t mActiveInputCount = 0;
218   // The number of input channels that this track requires. 0 means don't care.
219   uint32_t mNumberOfInputChannels;
220   // The mixing modes
221   ChannelCountMode mChannelCountMode;
222   ChannelInterpretation mChannelInterpretation;
223   // Tracks are considered active if the track has not finished and either
224   // the engine is active or there are active input tracks.
225   bool mIsActive;
226   // Whether the track should be marked as ended as soon
227   // as the current time range has been computed block by block.
228   bool mMarkAsEndedAfterThisBlock;
229   // Whether the track is an AudioParamHelper track.
230   bool mAudioParamTrack;
231   // Whether the track just passes its input through.
232   bool mPassThrough;
233 };
234 
235 }  // namespace mozilla
236 
237 #endif /* MOZILLA_AUDIONODETRACK_H_ */
238