1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 #ifndef MOZILLA_AUDIONODEENGINE_H_ 7 #define MOZILLA_AUDIONODEENGINE_H_ 8 9 #include "AudioSegment.h" 10 #include "mozilla/dom/AudioNode.h" 11 #include "mozilla/MemoryReporting.h" 12 #include "mozilla/Mutex.h" 13 14 namespace WebCore { 15 class Reverb; 16 } // namespace WebCore 17 18 namespace mozilla { 19 20 namespace dom { 21 struct ThreeDPoint; 22 class AudioParamTimeline; 23 class DelayNodeEngine; 24 struct AudioTimelineEvent; 25 } // namespace dom 26 27 class AbstractThread; 28 class AudioBlock; 29 class AudioNodeTrack; 30 31 /** 32 * This class holds onto a set of immutable channel buffers. The storage 33 * for the buffers must be malloced, but the buffer pointers and the malloc 34 * pointers can be different (e.g. if the buffers are contained inside 35 * some malloced object). 36 */ 37 class ThreadSharedFloatArrayBufferList final : public ThreadSharedObject { 38 public: 39 /** 40 * Construct with null channel data pointers. 41 */ ThreadSharedFloatArrayBufferList(uint32_t aCount)42 explicit ThreadSharedFloatArrayBufferList(uint32_t aCount) { 43 mContents.SetLength(aCount); 44 } 45 /** 46 * Create with buffers suitable for transfer to 47 * JS::NewArrayBufferWithContents(). The buffer contents are uninitialized 48 * and so should be set using GetDataForWrite(). 49 */ 50 static already_AddRefed<ThreadSharedFloatArrayBufferList> Create( 51 uint32_t aChannelCount, size_t aLength, const mozilla::fallible_t&); 52 AsThreadSharedFloatArrayBufferList()53 ThreadSharedFloatArrayBufferList* AsThreadSharedFloatArrayBufferList() 54 override { 55 return this; 56 }; 57 58 struct Storage final { Storagefinal59 Storage() : mDataToFree(nullptr), mFree(nullptr), mSampleData(nullptr) {} ~Storagefinal60 ~Storage() { 61 if (mFree) { 62 mFree(mDataToFree); 63 } else { 64 MOZ_ASSERT(!mDataToFree); 65 } 66 } SizeOfExcludingThisfinal67 size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { 68 // NB: mSampleData might not be owned, if it is it just points to 69 // mDataToFree. 70 return aMallocSizeOf(mDataToFree); 71 } 72 void* mDataToFree; 73 void (*mFree)(void*); 74 float* mSampleData; 75 }; 76 77 /** 78 * This can be called on any thread. 79 */ GetChannels()80 uint32_t GetChannels() const { return mContents.Length(); } 81 /** 82 * This can be called on any thread. 83 */ GetData(uint32_t aIndex)84 const float* GetData(uint32_t aIndex) const { 85 return mContents[aIndex].mSampleData; 86 } 87 /** 88 * This can be called on any thread, but only when the calling thread is the 89 * only owner. 90 */ GetDataForWrite(uint32_t aIndex)91 float* GetDataForWrite(uint32_t aIndex) { 92 MOZ_ASSERT(!IsShared()); 93 return mContents[aIndex].mSampleData; 94 } 95 96 /** 97 * Call this only during initialization, before the object is handed to 98 * any other thread. 99 */ SetData(uint32_t aIndex,void * aDataToFree,void (* aFreeFunc)(void *),float * aData)100 void SetData(uint32_t aIndex, void* aDataToFree, void (*aFreeFunc)(void*), 101 float* aData) { 102 Storage* s = &mContents[aIndex]; 103 if (s->mFree) { 104 s->mFree(s->mDataToFree); 105 } else { 106 MOZ_ASSERT(!s->mDataToFree); 107 } 108 109 s->mDataToFree = aDataToFree; 110 s->mFree = aFreeFunc; 111 s->mSampleData = aData; 112 } 113 114 /** 115 * Put this object into an error state where there are no channels. 116 */ Clear()117 void Clear() { mContents.Clear(); } 118 SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)119 size_t SizeOfExcludingThis( 120 mozilla::MallocSizeOf aMallocSizeOf) const override { 121 size_t amount = ThreadSharedObject::SizeOfExcludingThis(aMallocSizeOf); 122 amount += mContents.ShallowSizeOfExcludingThis(aMallocSizeOf); 123 for (size_t i = 0; i < mContents.Length(); i++) { 124 amount += mContents[i].SizeOfExcludingThis(aMallocSizeOf); 125 } 126 127 return amount; 128 } 129 SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)130 size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override { 131 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); 132 } 133 134 private: 135 AutoTArray<Storage, 2> mContents; 136 }; 137 138 /** 139 * aChunk must have been allocated by AllocateAudioBlock. 140 */ 141 void WriteZeroesToAudioBlock(AudioBlock* aChunk, uint32_t aStart, 142 uint32_t aLength); 143 144 /** 145 * Copy with scale. aScale == 1.0f should be optimized. 146 */ 147 void AudioBufferCopyWithScale(const float* aInput, float aScale, float* aOutput, 148 uint32_t aSize); 149 150 /** 151 * Pointwise multiply-add operation. aScale == 1.0f should be optimized. 152 */ 153 void AudioBufferAddWithScale(const float* aInput, float aScale, float* aOutput, 154 uint32_t aSize); 155 156 /** 157 * Pointwise multiply-add operation. aScale == 1.0f should be optimized. 158 */ 159 void AudioBlockAddChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE], 160 float aScale, 161 float aOutput[WEBAUDIO_BLOCK_SIZE]); 162 163 /** 164 * Pointwise copy-scaled operation. aScale == 1.0f should be optimized. 165 * 166 * Buffer size is implicitly assumed to be WEBAUDIO_BLOCK_SIZE. 167 */ 168 void AudioBlockCopyChannelWithScale(const float* aInput, float aScale, 169 float* aOutput); 170 171 /** 172 * Vector copy-scaled operation. 173 */ 174 void AudioBlockCopyChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE], 175 const float aScale[WEBAUDIO_BLOCK_SIZE], 176 float aOutput[WEBAUDIO_BLOCK_SIZE]); 177 178 /** 179 * Vector complex multiplication on arbitrary sized buffers. 180 */ 181 void BufferComplexMultiply(const float* aInput, const float* aScale, 182 float* aOutput, uint32_t aSize); 183 184 /** 185 * Vector maximum element magnitude ( max(abs(aInput)) ). 186 */ 187 float AudioBufferPeakValue(const float* aInput, uint32_t aSize); 188 189 /** 190 * In place gain. aScale == 1.0f should be optimized. 191 */ 192 void AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE], float aScale); 193 194 /** 195 * In place gain. aScale == 1.0f should be optimized. 196 */ 197 void AudioBufferInPlaceScale(float* aBlock, float aScale, uint32_t aSize); 198 199 /** 200 * a-rate in place gain. 201 */ 202 void AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE], 203 float aScale[WEBAUDIO_BLOCK_SIZE]); 204 /** 205 * a-rate in place gain. 206 */ 207 void AudioBufferInPlaceScale(float* aBlock, float* aScale, uint32_t aSize); 208 209 /** 210 * Upmix a mono input to a stereo output, scaling the two output channels by two 211 * different gain value. 212 * This algorithm is specified in the WebAudio spec. 213 */ 214 void AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE], 215 float aGainL, float aGainR, 216 float aOutputL[WEBAUDIO_BLOCK_SIZE], 217 float aOutputR[WEBAUDIO_BLOCK_SIZE]); 218 219 void AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE], 220 float aGainL[WEBAUDIO_BLOCK_SIZE], 221 float aGainR[WEBAUDIO_BLOCK_SIZE], 222 float aOutputL[WEBAUDIO_BLOCK_SIZE], 223 float aOutputR[WEBAUDIO_BLOCK_SIZE]); 224 /** 225 * Pan a stereo source according to right and left gain, and the position 226 * (whether the listener is on the left of the source or not). 227 * This algorithm is specified in the WebAudio spec. 228 */ 229 void AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE], 230 const float aInputR[WEBAUDIO_BLOCK_SIZE], 231 float aGainL, float aGainR, bool aIsOnTheLeft, 232 float aOutputL[WEBAUDIO_BLOCK_SIZE], 233 float aOutputR[WEBAUDIO_BLOCK_SIZE]); 234 void AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE], 235 const float aInputR[WEBAUDIO_BLOCK_SIZE], 236 const float aGainL[WEBAUDIO_BLOCK_SIZE], 237 const float aGainR[WEBAUDIO_BLOCK_SIZE], 238 const bool aIsOnTheLeft[WEBAUDIO_BLOCK_SIZE], 239 float aOutputL[WEBAUDIO_BLOCK_SIZE], 240 float aOutputR[WEBAUDIO_BLOCK_SIZE]); 241 242 /** 243 * Replace NaN by zeros in aSamples. 244 */ 245 void NaNToZeroInPlace(float* aSamples, size_t aCount); 246 247 /** 248 * Return the sum of squares of all of the samples in the input. 249 */ 250 float AudioBufferSumOfSquares(const float* aInput, uint32_t aLength); 251 252 /** 253 * All methods of this class and its subclasses are called on the 254 * MediaTrackGraph thread. 255 */ 256 class AudioNodeEngine { 257 public: 258 // This should be compatible with AudioNodeTrack::OutputChunks. 259 typedef AutoTArray<AudioBlock, 1> OutputChunks; 260 261 explicit AudioNodeEngine(dom::AudioNode* aNode); 262 ~AudioNodeEngine()263 virtual ~AudioNodeEngine() { 264 MOZ_ASSERT(!mNode, "The node reference must be already cleared"); 265 MOZ_COUNT_DTOR(AudioNodeEngine); 266 } 267 AsDelayNodeEngine()268 virtual dom::DelayNodeEngine* AsDelayNodeEngine() { return nullptr; } 269 SetTrackTimeParameter(uint32_t aIndex,TrackTime aParam)270 virtual void SetTrackTimeParameter(uint32_t aIndex, TrackTime aParam) { 271 NS_ERROR("Invalid SetTrackTimeParameter index"); 272 } SetDoubleParameter(uint32_t aIndex,double aParam)273 virtual void SetDoubleParameter(uint32_t aIndex, double aParam) { 274 NS_ERROR("Invalid SetDoubleParameter index"); 275 } SetInt32Parameter(uint32_t aIndex,int32_t aParam)276 virtual void SetInt32Parameter(uint32_t aIndex, int32_t aParam) { 277 NS_ERROR("Invalid SetInt32Parameter index"); 278 } RecvTimelineEvent(uint32_t aIndex,dom::AudioTimelineEvent & aValue)279 virtual void RecvTimelineEvent(uint32_t aIndex, 280 dom::AudioTimelineEvent& aValue) { 281 NS_ERROR("Invalid RecvTimelineEvent index"); 282 } SetBuffer(AudioChunk && aBuffer)283 virtual void SetBuffer(AudioChunk&& aBuffer) { 284 NS_ERROR("SetBuffer called on engine that doesn't support it"); 285 } 286 // This consumes the contents of aData. aData will be emptied after this 287 // returns. SetRawArrayData(nsTArray<float> && aData)288 virtual void SetRawArrayData(nsTArray<float>&& aData) { 289 NS_ERROR("SetRawArrayData called on an engine that doesn't support it"); 290 } 291 SetReverb(WebCore::Reverb * aBuffer,uint32_t aImpulseChannelCount)292 virtual void SetReverb(WebCore::Reverb* aBuffer, 293 uint32_t aImpulseChannelCount) { 294 NS_ERROR("SetReverb called on engine that doesn't support it"); 295 } 296 297 /** 298 * Produce the next block of audio samples, given input samples aInput 299 * (the mixed data for input 0). 300 * aInput is guaranteed to have float sample format (if it has samples at all) 301 * and to have been resampled to the sampling rate for the track, and to have 302 * exactly WEBAUDIO_BLOCK_SIZE samples. 303 * *aFinished is set to false by the caller. The callee must not set this to 304 * true unless silent output is produced. If set to true, we'll finish the 305 * track, consider this input inactive on any downstream nodes, and not 306 * call this again. 307 */ 308 virtual void ProcessBlock(AudioNodeTrack* aTrack, GraphTime aFrom, 309 const AudioBlock& aInput, AudioBlock* aOutput, 310 bool* aFinished); 311 /** 312 * Produce the next block of audio samples, before input is provided. 313 * ProcessBlock() will be called later, and it then should not change 314 * aOutput. This is used only for DelayNodeEngine in a feedback loop. 315 */ ProduceBlockBeforeInput(AudioNodeTrack * aTrack,GraphTime aFrom,AudioBlock * aOutput)316 virtual void ProduceBlockBeforeInput(AudioNodeTrack* aTrack, GraphTime aFrom, 317 AudioBlock* aOutput) { 318 MOZ_ASSERT_UNREACHABLE("ProduceBlockBeforeInput called on wrong engine"); 319 } 320 321 /** 322 * Produce the next block of audio samples, given input samples in the aInput 323 * array. There is one input sample per port in aInput, in order. 324 * This is the multi-input/output version of ProcessBlock. Only one kind 325 * of ProcessBlock is called on each node. ProcessBlocksOnPorts() is called 326 * instead of ProcessBlock() if either the number of inputs or the number of 327 * outputs is greater than 1. 328 * 329 * The numbers of AudioBlocks in aInput and aOutput are always guaranteed to 330 * match the numbers of inputs and outputs for the node. 331 */ 332 virtual void ProcessBlocksOnPorts(AudioNodeTrack* aTrack, GraphTime aFrom, 333 Span<const AudioBlock> aInput, 334 Span<AudioBlock> aOutput, bool* aFinished); 335 336 // IsActive() returns true if the engine needs to continue processing an 337 // unfinished track even when it has silent or no input connections. This 338 // includes tail-times and when sources have been scheduled to start. If 339 // returning false, then the track can be suspended. IsActive()340 virtual bool IsActive() const { return false; } 341 342 // Called on graph thread when the engine will not be used again. OnGraphThreadDone()343 virtual void OnGraphThreadDone() {} 344 HasNode()345 bool HasNode() const { 346 MOZ_ASSERT(NS_IsMainThread()); 347 return !!mNode; 348 } 349 NodeMainThread()350 dom::AudioNode* NodeMainThread() const { 351 MOZ_ASSERT(NS_IsMainThread()); 352 return mNode; 353 } 354 ClearNode()355 void ClearNode() { 356 MOZ_ASSERT(NS_IsMainThread()); 357 MOZ_ASSERT(mNode != nullptr); 358 mNode = nullptr; 359 } 360 InputCount()361 uint16_t InputCount() const { return mInputCount; } OutputCount()362 uint16_t OutputCount() const { return mOutputCount; } 363 SizeOfExcludingThis(MallocSizeOf aMallocSizeOf)364 virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { 365 // NB: |mNode| is tracked separately so it is excluded here. 366 return 0; 367 } 368 SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)369 virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { 370 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); 371 } 372 SizeOfIncludingThis(MallocSizeOf aMallocSizeOf,AudioNodeSizes & aUsage)373 void SizeOfIncludingThis(MallocSizeOf aMallocSizeOf, 374 AudioNodeSizes& aUsage) const { 375 aUsage.mEngine = SizeOfIncludingThis(aMallocSizeOf); 376 aUsage.mNodeType = mNodeType; 377 } 378 379 private: 380 // This is cleared from AudioNode::DestroyMediaTrack() 381 dom::AudioNode* MOZ_NON_OWNING_REF mNode; // main thread only 382 const char* const mNodeType; 383 const uint16_t mInputCount; 384 const uint16_t mOutputCount; 385 386 protected: 387 const RefPtr<AbstractThread> mAbstractMainThread; 388 }; 389 390 } // namespace mozilla 391 392 #endif /* MOZILLA_AUDIONODEENGINE_H_ */ 393