1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "MediaEngineWebRTC.h"
7
8 #include <stdio.h>
9 #include <algorithm>
10
11 #include "AllocationHandle.h"
12 #include "AudioConverter.h"
13 #include "MediaManager.h"
14 #include "MediaStreamGraphImpl.h"
15 #include "MediaTrackConstraints.h"
16 #include "mozilla/Assertions.h"
17 #include "mtransport/runnable_utils.h"
18 #include "nsAutoPtr.h"
19
20 // scoped_ptr.h uses FF
21 #ifdef FF
22 #undef FF
23 #endif
24 #include "webrtc/modules/audio_device/opensl/single_rw_fifo.h"
25 #include "webrtc/voice_engine/voice_engine_defines.h"
26 #include "webrtc/modules/audio_processing/include/audio_processing.h"
27 #include "webrtc/common_audio/include/audio_util.h"
28
29 using namespace webrtc;
30
31 // These are restrictions from the webrtc.org code
32 #define MAX_CHANNELS 2
33 #define MAX_SAMPLING_FREQ 48000 // Hz - multiple of 100
34
35 #define MAX_AEC_FIFO_DEPTH 200 // ms - multiple of 10
36 static_assert(!(MAX_AEC_FIFO_DEPTH % 10), "Invalid MAX_AEC_FIFO_DEPTH");
37
38 namespace mozilla {
39
40 #ifdef LOG
41 #undef LOG
42 #endif
43
44 LogModule* GetMediaManagerLog();
45 #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg)
46 #define LOG_FRAMES(msg) \
47 MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg)
48
AudioLogModule()49 LogModule* AudioLogModule() {
50 static mozilla::LazyLogModule log("AudioLatency");
51 return static_cast<LogModule*>(log);
52 }
53
NotifyOutputData(MediaStreamGraph * aGraph,AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)54 void WebRTCAudioDataListener::NotifyOutputData(MediaStreamGraph* aGraph,
55 AudioDataValue* aBuffer,
56 size_t aFrames, TrackRate aRate,
57 uint32_t aChannels) {
58 MutexAutoLock lock(mMutex);
59 if (mAudioSource) {
60 mAudioSource->NotifyOutputData(aGraph, aBuffer, aFrames, aRate, aChannels);
61 }
62 }
63
NotifyInputData(MediaStreamGraph * aGraph,const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)64 void WebRTCAudioDataListener::NotifyInputData(MediaStreamGraph* aGraph,
65 const AudioDataValue* aBuffer,
66 size_t aFrames, TrackRate aRate,
67 uint32_t aChannels) {
68 MutexAutoLock lock(mMutex);
69 if (mAudioSource) {
70 mAudioSource->NotifyInputData(aGraph, aBuffer, aFrames, aRate, aChannels);
71 }
72 }
73
DeviceChanged()74 void WebRTCAudioDataListener::DeviceChanged() {
75 MutexAutoLock lock(mMutex);
76 if (mAudioSource) {
77 mAudioSource->DeviceChanged();
78 }
79 }
80
Shutdown()81 void WebRTCAudioDataListener::Shutdown() {
82 MutexAutoLock lock(mMutex);
83 mAudioSource = nullptr;
84 }
85
86 /**
87 * WebRTC Microphone MediaEngineSource.
88 */
89 int MediaEngineWebRTCMicrophoneSource::sChannelsOpen = 0;
90
Allocation(const RefPtr<AllocationHandle> & aHandle)91 MediaEngineWebRTCMicrophoneSource::Allocation::Allocation(
92 const RefPtr<AllocationHandle>& aHandle)
93 : mHandle(aHandle) {}
94
95 MediaEngineWebRTCMicrophoneSource::Allocation::~Allocation() = default;
96
MediaEngineWebRTCMicrophoneSource(mozilla::AudioInput * aAudioInput,int aIndex,const char * aDeviceName,const char * aDeviceUUID,bool aDelayAgnostic,bool aExtendedFilter)97 MediaEngineWebRTCMicrophoneSource::MediaEngineWebRTCMicrophoneSource(
98 mozilla::AudioInput* aAudioInput, int aIndex, const char* aDeviceName,
99 const char* aDeviceUUID, bool aDelayAgnostic, bool aExtendedFilter)
100 : mAudioInput(aAudioInput),
101 mAudioProcessing(AudioProcessing::Create()),
102 mMutex("WebRTCMic::Mutex"),
103 mCapIndex(aIndex),
104 mDelayAgnostic(aDelayAgnostic),
105 mExtendedFilter(aExtendedFilter),
106 mStarted(false),
107 mDeviceName(NS_ConvertUTF8toUTF16(aDeviceName)),
108 mDeviceUUID(aDeviceUUID),
109 mSettings(new nsMainThreadPtrHolder<
110 media::Refcountable<dom::MediaTrackSettings>>(
111 "MediaEngineWebRTCMicrophoneSource::mSettings",
112 new media::Refcountable<dom::MediaTrackSettings>(),
113 // Non-strict means it won't assert main thread for us.
114 // It would be great if it did but we're already on the media thread.
115 /* aStrict = */ false)),
116 mTotalFrames(0),
117 mLastLogFrames(0),
118 mSkipProcessing(false),
119 mInputDownmixBuffer(MAX_SAMPLING_FREQ * MAX_CHANNELS / 100) {
120 MOZ_ASSERT(aAudioInput);
121 mSettings->mEchoCancellation.Construct(0);
122 mSettings->mAutoGainControl.Construct(0);
123 mSettings->mNoiseSuppression.Construct(0);
124 mSettings->mChannelCount.Construct(0);
125 // We'll init lazily as needed
126 }
127
GetName() const128 nsString MediaEngineWebRTCMicrophoneSource::GetName() const {
129 return mDeviceName;
130 }
131
GetUUID() const132 nsCString MediaEngineWebRTCMicrophoneSource::GetUUID() const {
133 return mDeviceUUID;
134 }
135
136 // GetBestFitnessDistance returns the best distance the capture device can offer
137 // as a whole, given an accumulated number of ConstraintSets.
138 // Ideal values are considered in the first ConstraintSet only.
139 // Plain values are treated as Ideal in the first ConstraintSet.
140 // Plain values are treated as Exact in subsequent ConstraintSets.
141 // Infinity = UINT32_MAX e.g. device cannot satisfy accumulated ConstraintSets.
142 // A finite result may be used to calculate this device's ranking as a choice.
143
GetBestFitnessDistance(const nsTArray<const NormalizedConstraintSet * > & aConstraintSets,const nsString & aDeviceId) const144 uint32_t MediaEngineWebRTCMicrophoneSource::GetBestFitnessDistance(
145 const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
146 const nsString& aDeviceId) const {
147 uint32_t distance = 0;
148
149 for (const auto* cs : aConstraintSets) {
150 distance =
151 MediaConstraintsHelper::GetMinimumFitnessDistance(*cs, aDeviceId);
152 break; // distance is read from first entry only
153 }
154 return distance;
155 }
156
ReevaluateAllocation(const RefPtr<AllocationHandle> & aHandle,const NormalizedConstraints * aConstraintsUpdate,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const char ** aOutBadConstraint)157 nsresult MediaEngineWebRTCMicrophoneSource::ReevaluateAllocation(
158 const RefPtr<AllocationHandle>& aHandle,
159 const NormalizedConstraints* aConstraintsUpdate,
160 const MediaEnginePrefs& aPrefs, const nsString& aDeviceId,
161 const char** aOutBadConstraint) {
162 AssertIsOnOwningThread();
163
164 // aHandle and/or aConstraintsUpdate may be nullptr (see below)
165
166 AutoTArray<const NormalizedConstraints*, 10> allConstraints;
167 for (const Allocation& registered : mAllocations) {
168 if (aConstraintsUpdate && registered.mHandle == aHandle) {
169 continue; // Don't count old constraints
170 }
171 allConstraints.AppendElement(®istered.mHandle->mConstraints);
172 }
173 if (aConstraintsUpdate) {
174 allConstraints.AppendElement(aConstraintsUpdate);
175 } else if (aHandle) {
176 // In the case of AddShareOfSingleSource, the handle isn't registered yet.
177 allConstraints.AppendElement(&aHandle->mConstraints);
178 }
179
180 NormalizedConstraints netConstraints(allConstraints);
181 if (netConstraints.mBadConstraint) {
182 *aOutBadConstraint = netConstraints.mBadConstraint;
183 return NS_ERROR_FAILURE;
184 }
185
186 nsresult rv = UpdateSingleSource(aHandle, netConstraints, aPrefs, aDeviceId,
187 aOutBadConstraint);
188 if (NS_FAILED(rv)) {
189 return rv;
190 }
191 if (aHandle && aConstraintsUpdate) {
192 aHandle->mConstraints = *aConstraintsUpdate;
193 }
194 return NS_OK;
195 }
196
Reconfigure(const RefPtr<AllocationHandle> & aHandle,const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const char ** aOutBadConstraint)197 nsresult MediaEngineWebRTCMicrophoneSource::Reconfigure(
198 const RefPtr<AllocationHandle>& aHandle,
199 const dom::MediaTrackConstraints& aConstraints,
200 const MediaEnginePrefs& aPrefs, const nsString& aDeviceId,
201 const char** aOutBadConstraint) {
202 AssertIsOnOwningThread();
203 MOZ_ASSERT(aHandle);
204
205 LOG(("Mic source %p allocation %p Reconfigure()", this, aHandle.get()));
206
207 NormalizedConstraints constraints(aConstraints);
208 nsresult rv = ReevaluateAllocation(aHandle, &constraints, aPrefs, aDeviceId,
209 aOutBadConstraint);
210
211 size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator());
212 MOZ_DIAGNOSTIC_ASSERT(i != mAllocations.NoIndex);
213 ApplySettings(mNetPrefs, mAllocations[i].mStream->GraphImpl());
214
215 return rv;
216 }
217
operator ==(const MediaEnginePrefs & a,const MediaEnginePrefs & b)218 bool operator==(const MediaEnginePrefs& a, const MediaEnginePrefs& b) {
219 return !memcmp(&a, &b, sizeof(MediaEnginePrefs));
220 };
221
222 // This does an early return in case of error.
223 #define HANDLE_APM_ERROR(fn) \
224 do { \
225 int rv = fn; \
226 if (rv != AudioProcessing::kNoError) { \
227 MOZ_ASSERT_UNREACHABLE("APM error in " #fn); \
228 return; \
229 } \
230 } while (0);
231
UpdateAECSettingsIfNeeded(bool aEnable,EcModes aMode)232 void MediaEngineWebRTCMicrophoneSource::UpdateAECSettingsIfNeeded(
233 bool aEnable, EcModes aMode) {
234 AssertIsOnOwningThread();
235
236 using webrtc::EcModes;
237
238 EchoCancellation::SuppressionLevel level;
239
240 switch (aMode) {
241 case EcModes::kEcUnchanged:
242 level = mAudioProcessing->echo_cancellation()->suppression_level();
243 break;
244 case EcModes::kEcConference:
245 level = EchoCancellation::kHighSuppression;
246 break;
247 case EcModes::kEcDefault:
248 level = EchoCancellation::kModerateSuppression;
249 break;
250 case EcModes::kEcAec:
251 level = EchoCancellation::kModerateSuppression;
252 break;
253 case EcModes::kEcAecm:
254 // No suppression level to set for the mobile echo canceller
255 break;
256 default:
257 MOZ_LOG(GetMediaManagerLog(), LogLevel::Error, ("Bad EcMode value"));
258 MOZ_ASSERT_UNREACHABLE(
259 "Bad pref set in all.js or in about:config"
260 " for the echo cancelation mode.");
261 // fall back to something sensible in release
262 level = EchoCancellation::kModerateSuppression;
263 break;
264 }
265
266 // AECm and AEC are mutually exclusive.
267 if (aMode == EcModes::kEcAecm) {
268 HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(false));
269 HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(aEnable));
270 } else {
271 HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(false));
272 HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(aEnable));
273 HANDLE_APM_ERROR(
274 mAudioProcessing->echo_cancellation()->set_suppression_level(level));
275 }
276 }
277
UpdateAGCSettingsIfNeeded(bool aEnable,AgcModes aMode)278 void MediaEngineWebRTCMicrophoneSource::UpdateAGCSettingsIfNeeded(
279 bool aEnable, AgcModes aMode) {
280 AssertIsOnOwningThread();
281
282 #if defined(WEBRTC_IOS) || defined(ATA) || defined(WEBRTC_ANDROID)
283 if (aMode == kAgcAdaptiveAnalog) {
284 MOZ_LOG(GetMediaManagerLog(), LogLevel::Error,
285 ("Invalid AGC mode kAgcAdaptiveAnalog on mobile"));
286 MOZ_ASSERT_UNREACHABLE(
287 "Bad pref set in all.js or in about:config"
288 " for the auto gain, on mobile.");
289 aMode = kAgcDefault;
290 }
291 #endif
292 GainControl::Mode mode = kDefaultAgcMode;
293
294 switch (aMode) {
295 case AgcModes::kAgcDefault:
296 mode = kDefaultAgcMode;
297 break;
298 case AgcModes::kAgcUnchanged:
299 mode = mAudioProcessing->gain_control()->mode();
300 break;
301 case AgcModes::kAgcFixedDigital:
302 mode = GainControl::Mode::kFixedDigital;
303 break;
304 case AgcModes::kAgcAdaptiveAnalog:
305 mode = GainControl::Mode::kAdaptiveAnalog;
306 break;
307 case AgcModes::kAgcAdaptiveDigital:
308 mode = GainControl::Mode::kAdaptiveDigital;
309 break;
310 default:
311 MOZ_ASSERT_UNREACHABLE(
312 "Bad pref set in all.js or in about:config"
313 " for the auto gain.");
314 // This is a good fallback, it works regardless of the platform.
315 mode = GainControl::Mode::kAdaptiveDigital;
316 break;
317 }
318
319 HANDLE_APM_ERROR(mAudioProcessing->gain_control()->set_mode(mode));
320 HANDLE_APM_ERROR(mAudioProcessing->gain_control()->Enable(aEnable));
321 }
322
UpdateNSSettingsIfNeeded(bool aEnable,NsModes aMode)323 void MediaEngineWebRTCMicrophoneSource::UpdateNSSettingsIfNeeded(
324 bool aEnable, NsModes aMode) {
325 AssertIsOnOwningThread();
326
327 NoiseSuppression::Level nsLevel;
328
329 switch (aMode) {
330 case NsModes::kNsDefault:
331 nsLevel = kDefaultNsMode;
332 break;
333 case NsModes::kNsUnchanged:
334 nsLevel = mAudioProcessing->noise_suppression()->level();
335 break;
336 case NsModes::kNsConference:
337 nsLevel = NoiseSuppression::kHigh;
338 break;
339 case NsModes::kNsLowSuppression:
340 nsLevel = NoiseSuppression::kLow;
341 break;
342 case NsModes::kNsModerateSuppression:
343 nsLevel = NoiseSuppression::kModerate;
344 break;
345 case NsModes::kNsHighSuppression:
346 nsLevel = NoiseSuppression::kHigh;
347 break;
348 case NsModes::kNsVeryHighSuppression:
349 nsLevel = NoiseSuppression::kVeryHigh;
350 break;
351 default:
352 MOZ_ASSERT_UNREACHABLE(
353 "Bad pref set in all.js or in about:config"
354 " for the noise suppression.");
355 // Pick something sensible as a faillback in release.
356 nsLevel = NoiseSuppression::kModerate;
357 }
358 HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->set_level(nsLevel));
359 HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->Enable(aEnable));
360 }
361
362 #undef HANDLE_APM_ERROR
363
UpdateSingleSource(const RefPtr<const AllocationHandle> & aHandle,const NormalizedConstraints & aNetConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const char ** aOutBadConstraint)364 nsresult MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
365 const RefPtr<const AllocationHandle>& aHandle,
366 const NormalizedConstraints& aNetConstraints,
367 const MediaEnginePrefs& aPrefs, const nsString& aDeviceId,
368 const char** aOutBadConstraint) {
369 AssertIsOnOwningThread();
370
371 FlattenedConstraints c(aNetConstraints);
372
373 MediaEnginePrefs prefs = aPrefs;
374 prefs.mAecOn = c.mEchoCancellation.Get(prefs.mAecOn);
375 prefs.mAgcOn = c.mAutoGainControl.Get(prefs.mAgcOn);
376 prefs.mNoiseOn = c.mNoiseSuppression.Get(prefs.mNoiseOn);
377 uint32_t maxChannels = 1;
378 if (mAudioInput->GetMaxAvailableChannels(maxChannels) != 0) {
379 return NS_ERROR_FAILURE;
380 }
381 // Check channelCount violation
382 if (static_cast<int32_t>(maxChannels) < c.mChannelCount.mMin ||
383 static_cast<int32_t>(maxChannels) > c.mChannelCount.mMax) {
384 *aOutBadConstraint = "channelCount";
385 return NS_ERROR_FAILURE;
386 }
387 // Clamp channelCount to a valid value
388 if (prefs.mChannels <= 0) {
389 prefs.mChannels = static_cast<int32_t>(maxChannels);
390 }
391 prefs.mChannels = c.mChannelCount.Get(
392 std::min(prefs.mChannels, static_cast<int32_t>(maxChannels)));
393 // Clamp channelCount to a valid value
394 prefs.mChannels =
395 std::max(1, std::min(prefs.mChannels, static_cast<int32_t>(maxChannels)));
396
397 LOG(("Audio config: aec: %d, agc: %d, noise: %d, channels: %d",
398 prefs.mAecOn ? prefs.mAec : -1, prefs.mAgcOn ? prefs.mAgc : -1,
399 prefs.mNoiseOn ? prefs.mNoise : -1, prefs.mChannels));
400
401 switch (mState) {
402 case kReleased:
403 MOZ_ASSERT(aHandle);
404 if (sChannelsOpen != 0) {
405 // Until we fix (or wallpaper) support for multiple mic input
406 // (Bug 1238038) fail allocation for a second device
407 return NS_ERROR_FAILURE;
408 }
409 if (mAudioInput->SetRecordingDevice(mCapIndex)) {
410 return NS_ERROR_FAILURE;
411 }
412 mAudioInput->SetUserChannelCount(prefs.mChannels);
413 {
414 MutexAutoLock lock(mMutex);
415 mState = kAllocated;
416 }
417 sChannelsOpen++;
418 LOG(("Audio device %d allocated", mCapIndex));
419 {
420 // Update with the actual applied channelCount in order
421 // to store it in settings.
422 uint32_t channelCount = 0;
423 mAudioInput->GetChannelCount(channelCount);
424 MOZ_ASSERT(channelCount > 0);
425 prefs.mChannels = channelCount;
426 }
427 break;
428
429 case kStarted:
430 case kStopped:
431 if (prefs == mNetPrefs) {
432 return NS_OK;
433 }
434
435 if (prefs.mChannels != mNetPrefs.mChannels) {
436 // If the channel count changed, tell the MSG to open a new driver with
437 // the correct channel count.
438 MOZ_ASSERT(!mAllocations.IsEmpty());
439 RefPtr<SourceMediaStream> stream;
440 for (const Allocation& allocation : mAllocations) {
441 if (allocation.mStream) {
442 stream = allocation.mStream;
443 break;
444 }
445 }
446 MOZ_ASSERT(stream);
447
448 mAudioInput->SetUserChannelCount(prefs.mChannels);
449 // Get validated number of channel
450 uint32_t channelCount = 0;
451 mAudioInput->GetChannelCount(channelCount);
452 MOZ_ASSERT(channelCount > 0 && mNetPrefs.mChannels > 0);
453 if (mNetPrefs.mChannels != prefs.mChannels &&
454 !stream->OpenNewAudioCallbackDriver(mListener)) {
455 MOZ_LOG(GetMediaManagerLog(), LogLevel::Error,
456 ("Could not open a new AudioCallbackDriver for input"));
457 return NS_ERROR_FAILURE;
458 }
459 }
460
461 if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) {
462 if (mAllocations.IsEmpty()) {
463 LOG(("Audio device %d reallocated", mCapIndex));
464 } else {
465 LOG(("Audio device %d allocated shared", mCapIndex));
466 }
467 }
468 break;
469
470 default:
471 LOG(("Audio device %d in ignored state %d", mCapIndex, mState));
472 break;
473 }
474
475 if (sChannelsOpen > 0) {
476 UpdateAGCSettingsIfNeeded(prefs.mAgcOn, static_cast<AgcModes>(prefs.mAgc));
477 UpdateNSSettingsIfNeeded(prefs.mNoiseOn,
478 static_cast<NsModes>(prefs.mNoise));
479 UpdateAECSettingsIfNeeded(prefs.mAecOn, static_cast<EcModes>(prefs.mAec));
480
481 webrtc::Config config;
482 config.Set<webrtc::ExtendedFilter>(
483 new webrtc::ExtendedFilter(mExtendedFilter));
484 config.Set<webrtc::DelayAgnostic>(
485 new webrtc::DelayAgnostic(mDelayAgnostic));
486 mAudioProcessing->SetExtraOptions(config);
487 }
488 mNetPrefs = prefs;
489 return NS_OK;
490 }
491
492 #undef HANDLE_APM_ERROR
493
ApplySettings(const MediaEnginePrefs & aPrefs,RefPtr<MediaStreamGraphImpl> aGraph)494 void MediaEngineWebRTCMicrophoneSource::ApplySettings(
495 const MediaEnginePrefs& aPrefs, RefPtr<MediaStreamGraphImpl> aGraph) {
496 AssertIsOnOwningThread();
497 MOZ_DIAGNOSTIC_ASSERT(aGraph);
498
499 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
500 NS_DispatchToMainThread(
501 media::NewRunnableFrom([ that, graph = Move(aGraph), aPrefs ]() mutable {
502 that->mSettings->mEchoCancellation.Value() = aPrefs.mAecOn;
503 that->mSettings->mAutoGainControl.Value() = aPrefs.mAgcOn;
504 that->mSettings->mNoiseSuppression.Value() = aPrefs.mNoiseOn;
505 that->mSettings->mChannelCount.Value() = aPrefs.mChannels;
506
507 class Message : public ControlMessage {
508 public:
509 Message(MediaEngineWebRTCMicrophoneSource* aSource, bool aPassThrough)
510 : ControlMessage(nullptr),
511 mMicrophoneSource(aSource),
512 mPassThrough(aPassThrough) {}
513
514 void Run() override {
515 mMicrophoneSource->SetPassThrough(mPassThrough);
516 }
517
518 protected:
519 RefPtr<MediaEngineWebRTCMicrophoneSource> mMicrophoneSource;
520 bool mPassThrough;
521 };
522
523 bool passThrough = !(aPrefs.mAecOn || aPrefs.mAgcOn || aPrefs.mNoiseOn);
524 if (graph) {
525 graph->AppendMessage(MakeUnique<Message>(that, passThrough));
526 }
527
528 return NS_OK;
529 }));
530 }
531
Allocate(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const ipc::PrincipalInfo & aPrincipalInfo,AllocationHandle ** aOutHandle,const char ** aOutBadConstraint)532 nsresult MediaEngineWebRTCMicrophoneSource::Allocate(
533 const dom::MediaTrackConstraints& aConstraints,
534 const MediaEnginePrefs& aPrefs, const nsString& aDeviceId,
535 const ipc::PrincipalInfo& aPrincipalInfo, AllocationHandle** aOutHandle,
536 const char** aOutBadConstraint) {
537 AssertIsOnOwningThread();
538 MOZ_ASSERT(aOutHandle);
539 auto handle = MakeRefPtr<AllocationHandle>(aConstraints, aPrincipalInfo,
540 aPrefs, aDeviceId);
541
542 LOG(("Mic source %p allocation %p Allocate()", this, handle.get()));
543
544 nsresult rv = ReevaluateAllocation(handle, nullptr, aPrefs, aDeviceId,
545 aOutBadConstraint);
546 if (NS_FAILED(rv)) {
547 return rv;
548 }
549
550 {
551 MutexAutoLock lock(mMutex);
552 mAllocations.AppendElement(Allocation(handle));
553 }
554
555 handle.forget(aOutHandle);
556 return NS_OK;
557 }
558
Deallocate(const RefPtr<const AllocationHandle> & aHandle)559 nsresult MediaEngineWebRTCMicrophoneSource::Deallocate(
560 const RefPtr<const AllocationHandle>& aHandle) {
561 AssertIsOnOwningThread();
562
563 size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator());
564 MOZ_DIAGNOSTIC_ASSERT(i != mAllocations.NoIndex);
565 MOZ_DIAGNOSTIC_ASSERT(
566 !mAllocations[i].mEnabled,
567 "Source should be stopped for the track before removing");
568
569 LOG(("Mic source %p allocation %p Deallocate()", this, aHandle.get()));
570
571 if (mAllocations[i].mStream && IsTrackIDExplicit(mAllocations[i].mTrackID)) {
572 mAllocations[i].mStream->EndTrack(mAllocations[i].mTrackID);
573 }
574
575 {
576 MutexAutoLock lock(mMutex);
577 mAllocations.RemoveElementAt(i);
578 }
579
580 if (mAllocations.IsEmpty()) {
581 // If empty, no callbacks to deliver data should be occuring
582 MOZ_ASSERT(mState != kReleased, "Source not allocated");
583 MOZ_ASSERT(mState != kStarted, "Source not stopped");
584 MOZ_ASSERT(sChannelsOpen > 0);
585 --sChannelsOpen;
586
587 MutexAutoLock lock(mMutex);
588 mState = kReleased;
589 LOG(("Audio device %d deallocated", mCapIndex));
590 } else {
591 LOG(("Audio device %d deallocated but still in use", mCapIndex));
592 }
593 return NS_OK;
594 }
595
SetTrack(const RefPtr<const AllocationHandle> & aHandle,const RefPtr<SourceMediaStream> & aStream,TrackID aTrackID,const PrincipalHandle & aPrincipal)596 nsresult MediaEngineWebRTCMicrophoneSource::SetTrack(
597 const RefPtr<const AllocationHandle>& aHandle,
598 const RefPtr<SourceMediaStream>& aStream, TrackID aTrackID,
599 const PrincipalHandle& aPrincipal) {
600 AssertIsOnOwningThread();
601 MOZ_ASSERT(aStream);
602 MOZ_ASSERT(IsTrackIDExplicit(aTrackID));
603
604 LOG(("Mic source %p allocation %p SetTrack() stream=%p, track=%" PRId32, this,
605 aHandle.get(), aStream.get(), aTrackID));
606
607 // Until we fix bug 1400488 we need to block a second tab (OuterWindow)
608 // from opening an already-open device. If it's the same tab, they
609 // will share a Graph(), and we can allow it.
610 if (!mAllocations.IsEmpty() && mAllocations[0].mStream &&
611 mAllocations[0].mStream->Graph() != aStream->Graph()) {
612 return NS_ERROR_NOT_AVAILABLE;
613 }
614
615 size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator());
616 MOZ_DIAGNOSTIC_ASSERT(i != mAllocations.NoIndex);
617 MOZ_ASSERT(!mAllocations[i].mStream);
618 MOZ_ASSERT(mAllocations[i].mTrackID == TRACK_NONE);
619 MOZ_ASSERT(mAllocations[i].mPrincipal == PRINCIPAL_HANDLE_NONE);
620 {
621 MutexAutoLock lock(mMutex);
622 mAllocations[i].mStream = aStream;
623 mAllocations[i].mTrackID = aTrackID;
624 mAllocations[i].mPrincipal = aPrincipal;
625 }
626
627 AudioSegment* segment = new AudioSegment();
628
629 aStream->AddAudioTrack(aTrackID, aStream->GraphRate(), 0, segment,
630 SourceMediaStream::ADDTRACK_QUEUED);
631
632 // XXX Make this based on the pref.
633 aStream->RegisterForAudioMixing();
634
635 LOG(("Stream %p registered for microphone capture", aStream.get()));
636 return NS_OK;
637 }
638
Start(const RefPtr<const AllocationHandle> & aHandle)639 nsresult MediaEngineWebRTCMicrophoneSource::Start(
640 const RefPtr<const AllocationHandle>& aHandle) {
641 AssertIsOnOwningThread();
642
643 if (sChannelsOpen == 0) {
644 return NS_ERROR_FAILURE;
645 }
646
647 LOG(("Mic source %p allocation %p Start()", this, aHandle.get()));
648
649 size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator());
650 MOZ_DIAGNOSTIC_ASSERT(i != mAllocations.NoIndex,
651 "Can't start track that hasn't been added");
652 Allocation& allocation = mAllocations[i];
653
654 MOZ_ASSERT(!allocation.mEnabled, "Source already started");
655 {
656 // This spans setting both the enabled state and mState.
657 MutexAutoLock lock(mMutex);
658 allocation.mEnabled = true;
659
660 #ifdef DEBUG
661 // Ensure that callback-tracking state is reset when callbacks start coming.
662 allocation.mLastCallbackAppendTime = 0;
663 #endif
664 allocation.mLiveFramesAppended = false;
665 allocation.mLiveSilenceAppended = false;
666
667 if (!mListener) {
668 mListener = new WebRTCAudioDataListener(this);
669 }
670
671 // Make sure logger starts before capture
672 AsyncLatencyLogger::Get(true);
673
674 // Must be *before* StartSend() so it will notice we selected external input
675 // (full_duplex)
676 mAudioInput->StartRecording(allocation.mStream, mListener);
677
678 MOZ_ASSERT(mState != kReleased);
679 mState = kStarted;
680 }
681
682 ApplySettings(mNetPrefs, allocation.mStream->GraphImpl());
683
684 return NS_OK;
685 }
686
Stop(const RefPtr<const AllocationHandle> & aHandle)687 nsresult MediaEngineWebRTCMicrophoneSource::Stop(
688 const RefPtr<const AllocationHandle>& aHandle) {
689 AssertIsOnOwningThread();
690
691 LOG(("Mic source %p allocation %p Stop()", this, aHandle.get()));
692
693 size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator());
694 MOZ_DIAGNOSTIC_ASSERT(i != mAllocations.NoIndex,
695 "Cannot stop track that we don't know about");
696 Allocation& allocation = mAllocations[i];
697
698 if (!allocation.mEnabled) {
699 // Already stopped - this is allowed
700 return NS_OK;
701 }
702
703 {
704 // This spans setting both the enabled state and mState.
705 MutexAutoLock lock(mMutex);
706 allocation.mEnabled = false;
707
708 mAudioInput->StopRecording(allocation.mStream);
709
710 if (HasEnabledTrack()) {
711 // Another track is keeping us from stopping
712 return NS_OK;
713 }
714
715 MOZ_ASSERT(mState == kStarted, "Should be started when stopping");
716 mState = kStopped;
717 }
718
719 if (mListener) {
720 // breaks a cycle, since the WebRTCAudioDataListener has a RefPtr to us
721 mListener->Shutdown();
722 mListener = nullptr;
723 }
724
725 return NS_OK;
726 }
727
GetSettings(dom::MediaTrackSettings & aOutSettings) const728 void MediaEngineWebRTCMicrophoneSource::GetSettings(
729 dom::MediaTrackSettings& aOutSettings) const {
730 MOZ_ASSERT(NS_IsMainThread());
731 aOutSettings = *mSettings;
732 }
733
Pull(const RefPtr<const AllocationHandle> & aHandle,const RefPtr<SourceMediaStream> & aStream,TrackID aTrackID,StreamTime aDesiredTime,const PrincipalHandle & aPrincipalHandle)734 void MediaEngineWebRTCMicrophoneSource::Pull(
735 const RefPtr<const AllocationHandle>& aHandle,
736 const RefPtr<SourceMediaStream>& aStream, TrackID aTrackID,
737 StreamTime aDesiredTime, const PrincipalHandle& aPrincipalHandle) {
738 StreamTime delta;
739
740 {
741 MutexAutoLock lock(mMutex);
742 size_t i = mAllocations.IndexOf(aHandle, 0, AllocationHandleComparator());
743 if (i == mAllocations.NoIndex) {
744 // This handle must have been deallocated. That's fine, and its track
745 // will already be ended. No need to do anything.
746 return;
747 }
748
749 // We don't want to GetEndOfAppendedData() above at the declaration if the
750 // allocation was removed and the track non-existant. An assert will fail.
751 delta = aDesiredTime - aStream->GetEndOfAppendedData(aTrackID);
752
753 if (!mAllocations[i].mLiveFramesAppended ||
754 !mAllocations[i].mLiveSilenceAppended) {
755 // These are the iterations after starting or resuming audio capture.
756 // Make sure there's at least one extra block buffered until audio
757 // callbacks come in. We also allow appending silence one time after
758 // audio callbacks have started, to cover the case where audio callbacks
759 // start appending data immediately and there is no extra data buffered.
760 delta += WEBAUDIO_BLOCK_SIZE;
761 }
762
763 if (delta < 0) {
764 LOG_FRAMES(("Not appending silence for allocation %p; %" PRId64
765 " frames already buffered",
766 mAllocations[i].mHandle.get(), -delta));
767 return;
768 }
769
770 LOG_FRAMES(("Pulling %" PRId64 " frames of silence for allocation %p",
771 delta, mAllocations[i].mHandle.get()));
772
773 // This assertion fails when we append silence here in the same iteration
774 // as there were real audio samples already appended by the audio callback.
775 // Note that this is exempted until live samples and a subsequent chunk of
776 // silence have been appended to the track. This will cover cases like:
777 // - After Start(), there is silence (maybe multiple times) appended before
778 // the first audio callback.
779 // - After Start(), there is real data (maybe multiple times) appended
780 // before the first graph iteration.
781 // And other combinations of order of audio sample sources.
782 MOZ_ASSERT_IF(mAllocations[i].mEnabled &&
783 mAllocations[i].mLiveFramesAppended &&
784 mAllocations[i].mLiveSilenceAppended,
785 aStream->GraphImpl()->IterationEnd() >
786 mAllocations[i].mLastCallbackAppendTime);
787
788 if (mAllocations[i].mLiveFramesAppended) {
789 mAllocations[i].mLiveSilenceAppended = true;
790 }
791 }
792
793 AudioSegment audio;
794 audio.AppendNullData(delta);
795 aStream->AppendToTrack(aTrackID, &audio);
796 }
797
NotifyOutputData(MediaStreamGraph * aGraph,AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)798 void MediaEngineWebRTCMicrophoneSource::NotifyOutputData(
799 MediaStreamGraph* aGraph, AudioDataValue* aBuffer, size_t aFrames,
800 TrackRate aRate, uint32_t aChannels) {
801 if (!mPacketizerOutput || mPacketizerOutput->PacketSize() != aRate / 100u ||
802 mPacketizerOutput->Channels() != aChannels) {
803 // It's ok to drop the audio still in the packetizer here: if this changes,
804 // we changed devices or something.
805 mPacketizerOutput =
806 new AudioPacketizer<AudioDataValue, float>(aRate / 100, aChannels);
807 }
808
809 mPacketizerOutput->Input(aBuffer, aFrames);
810
811 while (mPacketizerOutput->PacketsAvailable()) {
812 uint32_t samplesPerPacket =
813 mPacketizerOutput->PacketSize() * mPacketizerOutput->Channels();
814 if (mOutputBuffer.Length() < samplesPerPacket) {
815 mOutputBuffer.SetLength(samplesPerPacket);
816 }
817 if (mDeinterleavedBuffer.Length() < samplesPerPacket) {
818 mDeinterleavedBuffer.SetLength(samplesPerPacket);
819 }
820 float* packet = mOutputBuffer.Data();
821 mPacketizerOutput->Output(packet);
822
823 AutoTArray<float*, MAX_CHANNELS> deinterleavedPacketDataChannelPointers;
824 float* interleavedFarend = nullptr;
825 uint32_t channelCountFarend = 0;
826 uint32_t framesPerPacketFarend = 0;
827
828 // Downmix from aChannels to MAX_CHANNELS if needed. We always have floats
829 // here, the packetized performed the conversion.
830 if (aChannels > MAX_CHANNELS) {
831 AudioConverter converter(
832 AudioConfig(aChannels, 0, AudioConfig::FORMAT_FLT),
833 AudioConfig(MAX_CHANNELS, 0, AudioConfig::FORMAT_FLT));
834 framesPerPacketFarend = mPacketizerOutput->PacketSize();
835 framesPerPacketFarend =
836 converter.Process(mInputDownmixBuffer, packet, framesPerPacketFarend);
837 interleavedFarend = mInputDownmixBuffer.Data();
838 channelCountFarend = MAX_CHANNELS;
839 deinterleavedPacketDataChannelPointers.SetLength(MAX_CHANNELS);
840 } else {
841 interleavedFarend = packet;
842 channelCountFarend = aChannels;
843 framesPerPacketFarend = mPacketizerOutput->PacketSize();
844 deinterleavedPacketDataChannelPointers.SetLength(aChannels);
845 }
846
847 MOZ_ASSERT(interleavedFarend &&
848 (channelCountFarend == 1 || channelCountFarend == 2) &&
849 framesPerPacketFarend);
850
851 if (mInputBuffer.Length() < framesPerPacketFarend * channelCountFarend) {
852 mInputBuffer.SetLength(framesPerPacketFarend * channelCountFarend);
853 }
854
855 size_t offset = 0;
856 for (size_t i = 0; i < deinterleavedPacketDataChannelPointers.Length();
857 ++i) {
858 deinterleavedPacketDataChannelPointers[i] = mInputBuffer.Data() + offset;
859 offset += framesPerPacketFarend;
860 }
861
862 // Deinterleave, prepare a channel pointers array, with enough storage for
863 // the frames.
864 DeinterleaveAndConvertBuffer(
865 interleavedFarend, framesPerPacketFarend, channelCountFarend,
866 deinterleavedPacketDataChannelPointers.Elements());
867
868 // Having the same config for input and output means we potentially save
869 // some CPU.
870 StreamConfig inputConfig(aRate, channelCountFarend, false);
871 StreamConfig outputConfig = inputConfig;
872
873 // Passing the same pointers here saves a copy inside this function.
874 DebugOnly<int> err = mAudioProcessing->ProcessReverseStream(
875 deinterleavedPacketDataChannelPointers.Elements(), inputConfig,
876 outputConfig, deinterleavedPacketDataChannelPointers.Elements());
877
878 MOZ_ASSERT(!err, "Could not process the reverse stream.");
879 }
880 }
881
882 // Only called if we're not in passthrough mode
PacketizeAndProcess(MediaStreamGraph * aGraph,const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)883 void MediaEngineWebRTCMicrophoneSource::PacketizeAndProcess(
884 MediaStreamGraph* aGraph, const AudioDataValue* aBuffer, size_t aFrames,
885 TrackRate aRate, uint32_t aChannels) {
886 MOZ_ASSERT(!PassThrough(),
887 "This should be bypassed when in PassThrough mode.");
888 size_t offset = 0;
889
890 if (!mPacketizerInput || mPacketizerInput->PacketSize() != aRate / 100u ||
891 mPacketizerInput->Channels() != aChannels) {
892 // It's ok to drop the audio still in the packetizer here.
893 mPacketizerInput =
894 new AudioPacketizer<AudioDataValue, float>(aRate / 100, aChannels);
895 }
896
897 // On initial capture, throw away all far-end data except the most recent
898 // sample since it's already irrelevant and we want to avoid confusing the AEC
899 // far-end input code with "old" audio.
900 if (!mStarted) {
901 mStarted = true;
902 }
903
904 // Packetize our input data into 10ms chunks, deinterleave into planar channel
905 // buffers, process, and append to the right MediaStreamTrack.
906 mPacketizerInput->Input(aBuffer, static_cast<uint32_t>(aFrames));
907
908 while (mPacketizerInput->PacketsAvailable()) {
909 uint32_t samplesPerPacket =
910 mPacketizerInput->PacketSize() * mPacketizerInput->Channels();
911 if (mInputBuffer.Length() < samplesPerPacket) {
912 mInputBuffer.SetLength(samplesPerPacket);
913 }
914 if (mDeinterleavedBuffer.Length() < samplesPerPacket) {
915 mDeinterleavedBuffer.SetLength(samplesPerPacket);
916 }
917 float* packet = mInputBuffer.Data();
918 mPacketizerInput->Output(packet);
919
920 // Deinterleave the input data
921 // Prepare an array pointing to deinterleaved channels.
922 AutoTArray<float*, 8> deinterleavedPacketizedInputDataChannelPointers;
923 deinterleavedPacketizedInputDataChannelPointers.SetLength(aChannels);
924 offset = 0;
925 for (size_t i = 0;
926 i < deinterleavedPacketizedInputDataChannelPointers.Length(); ++i) {
927 deinterleavedPacketizedInputDataChannelPointers[i] =
928 mDeinterleavedBuffer.Data() + offset;
929 offset += mPacketizerInput->PacketSize();
930 }
931
932 // Deinterleave to mInputBuffer, pointed to by inputBufferChannelPointers.
933 Deinterleave(packet, mPacketizerInput->PacketSize(), aChannels,
934 deinterleavedPacketizedInputDataChannelPointers.Elements());
935
936 StreamConfig inputConfig(aRate, aChannels,
937 false /* we don't use typing detection*/);
938 StreamConfig outputConfig = inputConfig;
939
940 // Bug 1404965: Get the right delay here, it saves some work down the line.
941 mAudioProcessing->set_stream_delay_ms(0);
942
943 // Bug 1414837: find a way to not allocate here.
944 RefPtr<SharedBuffer> buffer = SharedBuffer::Create(
945 mPacketizerInput->PacketSize() * aChannels * sizeof(float));
946
947 // Prepare channel pointers to the SharedBuffer created above.
948 AutoTArray<float*, 8> processedOutputChannelPointers;
949 AutoTArray<const float*, 8> processedOutputChannelPointersConst;
950 processedOutputChannelPointers.SetLength(aChannels);
951 processedOutputChannelPointersConst.SetLength(aChannels);
952
953 offset = 0;
954 for (size_t i = 0; i < processedOutputChannelPointers.Length(); ++i) {
955 processedOutputChannelPointers[i] =
956 static_cast<float*>(buffer->Data()) + offset;
957 processedOutputChannelPointersConst[i] =
958 static_cast<float*>(buffer->Data()) + offset;
959 offset += mPacketizerInput->PacketSize();
960 }
961
962 mAudioProcessing->ProcessStream(
963 deinterleavedPacketizedInputDataChannelPointers.Elements(), inputConfig,
964 outputConfig, processedOutputChannelPointers.Elements());
965 MutexAutoLock lock(mMutex);
966 if (mState != kStarted) {
967 return;
968 }
969
970 AudioSegment segment;
971 for (Allocation& allocation : mAllocations) {
972 if (!allocation.mStream) {
973 continue;
974 }
975
976 if (!allocation.mStream->GraphImpl()) {
977 // The DOMMediaStream that owns allocation.mStream has been cleaned up
978 // and MediaStream::DestroyImpl() has run in the MSG. This is fine and
979 // can happen before the MediaManager thread gets to stop capture for
980 // this allocation.
981 continue;
982 }
983
984 if (!allocation.mEnabled) {
985 continue;
986 }
987
988 LOG_FRAMES(("Appending %" PRIu32
989 " frames of packetized audio for allocation %p",
990 mPacketizerInput->PacketSize(), allocation.mHandle.get()));
991
992 #ifdef DEBUG
993 allocation.mLastCallbackAppendTime =
994 allocation.mStream->GraphImpl()->IterationEnd();
995 #endif
996 allocation.mLiveFramesAppended = true;
997
998 // We already have planar audio data of the right format. Insert into the
999 // MSG.
1000 MOZ_ASSERT(processedOutputChannelPointers.Length() == aChannels);
1001 RefPtr<SharedBuffer> other = buffer;
1002 segment.AppendFrames(other.forget(), processedOutputChannelPointersConst,
1003 mPacketizerInput->PacketSize(),
1004 allocation.mPrincipal);
1005 allocation.mStream->AppendToTrack(allocation.mTrackID, &segment);
1006 }
1007 }
1008 }
1009
PassThrough() const1010 bool MediaEngineWebRTCMicrophoneSource::PassThrough() const {
1011 return mSkipProcessing;
1012 }
1013
SetPassThrough(bool aPassThrough)1014 void MediaEngineWebRTCMicrophoneSource::SetPassThrough(bool aPassThrough) {
1015 mSkipProcessing = aPassThrough;
1016 }
1017
1018 template <typename T>
InsertInGraph(const T * aBuffer,size_t aFrames,uint32_t aChannels)1019 void MediaEngineWebRTCMicrophoneSource::InsertInGraph(const T* aBuffer,
1020 size_t aFrames,
1021 uint32_t aChannels) {
1022 MutexAutoLock lock(mMutex);
1023
1024 if (mState != kStarted) {
1025 return;
1026 }
1027
1028 if (MOZ_LOG_TEST(AudioLogModule(), LogLevel::Debug)) {
1029 mTotalFrames += aFrames;
1030 if (!mAllocations.IsEmpty() && mAllocations[0].mStream &&
1031 mTotalFrames >
1032 mLastLogFrames +
1033 mAllocations[0].mStream->GraphRate()) { // ~ 1 second
1034 MOZ_LOG(AudioLogModule(), LogLevel::Debug,
1035 ("%p: Inserting %zu samples into graph, total frames = %" PRIu64,
1036 (void*)this, aFrames, mTotalFrames));
1037 mLastLogFrames = mTotalFrames;
1038 }
1039 }
1040
1041 for (Allocation& allocation : mAllocations) {
1042 if (!allocation.mStream) {
1043 continue;
1044 }
1045
1046 if (!allocation.mStream->GraphImpl()) {
1047 // The DOMMediaStream that owns allocation.mStream has been cleaned up
1048 // and MediaStream::DestroyImpl() has run in the MSG. This is fine and
1049 // can happen before the MediaManager thread gets to stop capture for
1050 // this allocation.
1051 continue;
1052 }
1053
1054 if (!allocation.mEnabled) {
1055 continue;
1056 }
1057
1058 #ifdef DEBUG
1059 allocation.mLastCallbackAppendTime =
1060 allocation.mStream->GraphImpl()->IterationEnd();
1061 #endif
1062 allocation.mLiveFramesAppended = true;
1063
1064 TimeStamp insertTime;
1065 // Make sure we include the stream and the track.
1066 // The 0:1 is a flag to note when we've done the final insert for a given
1067 // input block.
1068 LogTime(AsyncLatencyLogger::AudioTrackInsertion,
1069 LATENCY_STREAM_ID(allocation.mStream.get(), allocation.mTrackID),
1070 (&allocation != &mAllocations.LastElement()) ? 0 : 1, insertTime);
1071
1072 // Bug 971528 - Support stereo capture in gUM
1073 MOZ_ASSERT(aChannels >= 1 && aChannels <= 8, "Support up to 8 channels");
1074
1075 AudioSegment segment;
1076 RefPtr<SharedBuffer> buffer =
1077 SharedBuffer::Create(aFrames * aChannels * sizeof(T));
1078 AutoTArray<const T*, 8> channels;
1079 if (aChannels == 1) {
1080 PodCopy(static_cast<T*>(buffer->Data()), aBuffer, aFrames);
1081 channels.AppendElement(static_cast<T*>(buffer->Data()));
1082 } else {
1083 channels.SetLength(aChannels);
1084 AutoTArray<T*, 8> write_channels;
1085 write_channels.SetLength(aChannels);
1086 T* samples = static_cast<T*>(buffer->Data());
1087
1088 size_t offset = 0;
1089 for (uint32_t i = 0; i < aChannels; ++i) {
1090 channels[i] = write_channels[i] = samples + offset;
1091 offset += aFrames;
1092 }
1093
1094 DeinterleaveAndConvertBuffer(aBuffer, aFrames, aChannels,
1095 write_channels.Elements());
1096 }
1097
1098 LOG_FRAMES(("Appending %zu frames of raw audio for allocation %p", aFrames,
1099 allocation.mHandle.get()));
1100
1101 MOZ_ASSERT(aChannels == channels.Length());
1102 segment.AppendFrames(buffer.forget(), channels, aFrames,
1103 allocation.mPrincipal);
1104 segment.GetStartTime(insertTime);
1105
1106 allocation.mStream->AppendToTrack(allocation.mTrackID, &segment);
1107 }
1108 }
1109
1110 // Called back on GraphDriver thread!
1111 // Note this can be called back after ::Shutdown()
NotifyInputData(MediaStreamGraph * aGraph,const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)1112 void MediaEngineWebRTCMicrophoneSource::NotifyInputData(
1113 MediaStreamGraph* aGraph, const AudioDataValue* aBuffer, size_t aFrames,
1114 TrackRate aRate, uint32_t aChannels) {
1115 // If some processing is necessary, packetize and insert in the WebRTC.org
1116 // code. Otherwise, directly insert the mic data in the MSG, bypassing all
1117 // processing.
1118 if (PassThrough()) {
1119 InsertInGraph<AudioDataValue>(aBuffer, aFrames, aChannels);
1120 } else {
1121 PacketizeAndProcess(aGraph, aBuffer, aFrames, aRate, aChannels);
1122 }
1123 }
1124
1125 #define ResetProcessingIfNeeded(_processing) \
1126 do { \
1127 bool enabled = mAudioProcessing->_processing()->is_enabled(); \
1128 \
1129 if (enabled) { \
1130 int rv = mAudioProcessing->_processing()->Enable(!enabled); \
1131 if (rv) { \
1132 NS_WARNING("Could not reset the status of the " #_processing \
1133 " on device change."); \
1134 return; \
1135 } \
1136 rv = mAudioProcessing->_processing()->Enable(enabled); \
1137 if (rv) { \
1138 NS_WARNING("Could not reset the status of the " #_processing \
1139 " on device change."); \
1140 return; \
1141 } \
1142 } \
1143 } while (0)
1144
DeviceChanged()1145 void MediaEngineWebRTCMicrophoneSource::DeviceChanged() {
1146 // Reset some processing
1147 ResetProcessingIfNeeded(gain_control);
1148 ResetProcessingIfNeeded(echo_cancellation);
1149 ResetProcessingIfNeeded(noise_suppression);
1150 }
1151
Shutdown()1152 void MediaEngineWebRTCMicrophoneSource::Shutdown() {
1153 AssertIsOnOwningThread();
1154
1155 if (mListener) {
1156 // breaks a cycle, since the WebRTCAudioDataListener has a RefPtr to us
1157 mListener->Shutdown();
1158 // Don't release the webrtc.org pointers yet until the Listener is (async)
1159 // shutdown
1160 mListener = nullptr;
1161 }
1162
1163 if (mState == kStarted) {
1164 for (const Allocation& allocation : mAllocations) {
1165 if (allocation.mEnabled) {
1166 Stop(allocation.mHandle);
1167 }
1168 }
1169 MOZ_ASSERT(mState == kStopped);
1170 }
1171
1172 while (!mAllocations.IsEmpty()) {
1173 MOZ_ASSERT(mState == kAllocated || mState == kStopped);
1174 Deallocate(mAllocations[0].mHandle);
1175 }
1176 MOZ_ASSERT(mState == kReleased);
1177 }
1178
GetName() const1179 nsString MediaEngineWebRTCAudioCaptureSource::GetName() const {
1180 return NS_LITERAL_STRING(u"AudioCapture");
1181 }
1182
GetUUID() const1183 nsCString MediaEngineWebRTCAudioCaptureSource::GetUUID() const {
1184 nsID uuid;
1185 char uuidBuffer[NSID_LENGTH];
1186 nsCString asciiString;
1187 ErrorResult rv;
1188
1189 rv = nsContentUtils::GenerateUUIDInPlace(uuid);
1190 if (rv.Failed()) {
1191 return NS_LITERAL_CSTRING("");
1192 }
1193
1194 uuid.ToProvidedString(uuidBuffer);
1195 asciiString.AssignASCII(uuidBuffer);
1196
1197 // Remove {} and the null terminator
1198 return nsCString(Substring(asciiString, 1, NSID_LENGTH - 3));
1199 }
1200
HasEnabledTrack() const1201 bool MediaEngineWebRTCMicrophoneSource::HasEnabledTrack() const {
1202 AssertIsOnOwningThread();
1203 for (const Allocation& allocation : mAllocations) {
1204 if (allocation.mEnabled) {
1205 return true;
1206 }
1207 }
1208 return false;
1209 }
1210
SetTrack(const RefPtr<const AllocationHandle> & aHandle,const RefPtr<SourceMediaStream> & aStream,TrackID aTrackID,const PrincipalHandle & aPrincipalHandle)1211 nsresult MediaEngineWebRTCAudioCaptureSource::SetTrack(
1212 const RefPtr<const AllocationHandle>& aHandle,
1213 const RefPtr<SourceMediaStream>& aStream, TrackID aTrackID,
1214 const PrincipalHandle& aPrincipalHandle) {
1215 AssertIsOnOwningThread();
1216 // Nothing to do here. aStream is a placeholder dummy and not exposed.
1217 return NS_OK;
1218 }
1219
Start(const RefPtr<const AllocationHandle> & aHandle)1220 nsresult MediaEngineWebRTCAudioCaptureSource::Start(
1221 const RefPtr<const AllocationHandle>& aHandle) {
1222 AssertIsOnOwningThread();
1223 return NS_OK;
1224 }
1225
Stop(const RefPtr<const AllocationHandle> & aHandle)1226 nsresult MediaEngineWebRTCAudioCaptureSource::Stop(
1227 const RefPtr<const AllocationHandle>& aHandle) {
1228 AssertIsOnOwningThread();
1229 return NS_OK;
1230 }
1231
Reconfigure(const RefPtr<AllocationHandle> & aHandle,const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const char ** aOutBadConstraint)1232 nsresult MediaEngineWebRTCAudioCaptureSource::Reconfigure(
1233 const RefPtr<AllocationHandle>& aHandle,
1234 const dom::MediaTrackConstraints& aConstraints,
1235 const MediaEnginePrefs& aPrefs, const nsString& aDeviceId,
1236 const char** aOutBadConstraint) {
1237 MOZ_ASSERT(!aHandle);
1238 return NS_OK;
1239 }
1240
GetBestFitnessDistance(const nsTArray<const NormalizedConstraintSet * > & aConstraintSets,const nsString & aDeviceId) const1241 uint32_t MediaEngineWebRTCAudioCaptureSource::GetBestFitnessDistance(
1242 const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
1243 const nsString& aDeviceId) const {
1244 // There is only one way of capturing audio for now, and it's always adequate.
1245 return 0;
1246 }
1247 }
1248