1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "MediaEngineWebRTCAudio.h"
7
8 #include <stdio.h>
9 #include <algorithm>
10
11 #include "AudioConverter.h"
12 #include "MediaManager.h"
13 #include "MediaTrackGraphImpl.h"
14 #include "MediaTrackConstraints.h"
15 #include "mozilla/Assertions.h"
16 #include "mozilla/ErrorNames.h"
17 #include "nsContentUtils.h"
18 #include "transport/runnable_utils.h"
19 #include "Tracing.h"
20
21 // scoped_ptr.h uses FF
22 #ifdef FF
23 # undef FF
24 #endif
25 #include "webrtc/voice_engine/voice_engine_defines.h"
26 #include "webrtc/modules/audio_processing/include/audio_processing.h"
27 #include "webrtc/common_audio/include/audio_util.h"
28
29 using namespace webrtc;
30
31 // These are restrictions from the webrtc.org code
32 #define MAX_CHANNELS 2
33 #define MONO 1
34 #define MAX_SAMPLING_FREQ 48000 // Hz - multiple of 100
35
36 namespace mozilla {
37
38 extern LazyLogModule gMediaManagerLog;
39 #define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
40 #define LOG_FRAME(...) \
41 MOZ_LOG(gMediaManagerLog, LogLevel::Verbose, (__VA_ARGS__))
42 #define LOG_ERROR(...) MOZ_LOG(gMediaManagerLog, LogLevel::Error, (__VA_ARGS__))
43
44 /**
45 * WebRTC Microphone MediaEngineSource.
46 */
47
MediaEngineWebRTCMicrophoneSource(RefPtr<AudioDeviceInfo> aInfo,const nsString & aDeviceName,const nsCString & aDeviceUUID,const nsString & aDeviceGroup,uint32_t aMaxChannelCount,bool aDelayAgnostic,bool aExtendedFilter)48 MediaEngineWebRTCMicrophoneSource::MediaEngineWebRTCMicrophoneSource(
49 RefPtr<AudioDeviceInfo> aInfo, const nsString& aDeviceName,
50 const nsCString& aDeviceUUID, const nsString& aDeviceGroup,
51 uint32_t aMaxChannelCount, bool aDelayAgnostic, bool aExtendedFilter)
52 : mPrincipal(PRINCIPAL_HANDLE_NONE),
53 mDeviceInfo(std::move(aInfo)),
54 mDelayAgnostic(aDelayAgnostic),
55 mExtendedFilter(aExtendedFilter),
56 mDeviceName(aDeviceName),
57 mDeviceUUID(aDeviceUUID),
58 mDeviceGroup(aDeviceGroup),
59 mDeviceMaxChannelCount(aMaxChannelCount),
60 mSettings(new nsMainThreadPtrHolder<
61 media::Refcountable<dom::MediaTrackSettings>>(
62 "MediaEngineWebRTCMicrophoneSource::mSettings",
63 new media::Refcountable<dom::MediaTrackSettings>(),
64 // Non-strict means it won't assert main thread for us.
65 // It would be great if it did but we're already on the media thread.
66 /* aStrict = */ false)) {
67 #ifndef ANDROID
68 MOZ_ASSERT(mDeviceInfo->DeviceID());
69 #endif
70
71 // We'll init lazily as needed
72 mSettings->mEchoCancellation.Construct(0);
73 mSettings->mAutoGainControl.Construct(0);
74 mSettings->mNoiseSuppression.Construct(0);
75 mSettings->mChannelCount.Construct(0);
76
77 mState = kReleased;
78 }
79
GetName() const80 nsString MediaEngineWebRTCMicrophoneSource::GetName() const {
81 return mDeviceName;
82 }
83
GetUUID() const84 nsCString MediaEngineWebRTCMicrophoneSource::GetUUID() const {
85 return mDeviceUUID;
86 }
87
GetGroupId() const88 nsString MediaEngineWebRTCMicrophoneSource::GetGroupId() const {
89 return mDeviceGroup;
90 }
91
EvaluateSettings(const NormalizedConstraints & aConstraintsUpdate,const MediaEnginePrefs & aInPrefs,MediaEnginePrefs * aOutPrefs,const char ** aOutBadConstraint)92 nsresult MediaEngineWebRTCMicrophoneSource::EvaluateSettings(
93 const NormalizedConstraints& aConstraintsUpdate,
94 const MediaEnginePrefs& aInPrefs, MediaEnginePrefs* aOutPrefs,
95 const char** aOutBadConstraint) {
96 AssertIsOnOwningThread();
97
98 FlattenedConstraints c(aConstraintsUpdate);
99 MediaEnginePrefs prefs = aInPrefs;
100
101 prefs.mAecOn = c.mEchoCancellation.Get(aInPrefs.mAecOn);
102 prefs.mAgcOn = c.mAutoGainControl.Get(aInPrefs.mAgcOn && prefs.mAecOn);
103 prefs.mNoiseOn = c.mNoiseSuppression.Get(aInPrefs.mNoiseOn && prefs.mAecOn);
104
105 // Determine an actual channel count to use for this source. Three factors at
106 // play here: the device capabilities, the constraints passed in by content,
107 // and a pref that can force things (for testing)
108 int32_t maxChannels = mDeviceInfo->MaxChannels();
109
110 // First, check channelCount violation wrt constraints. This fails in case of
111 // error.
112 if (c.mChannelCount.mMin > maxChannels) {
113 *aOutBadConstraint = "channelCount";
114 return NS_ERROR_FAILURE;
115 }
116 // A pref can force the channel count to use. If the pref has a value of zero
117 // or lower, it has no effect.
118 if (aInPrefs.mChannels <= 0) {
119 prefs.mChannels = maxChannels;
120 }
121
122 // Get the number of channels asked for by content, and clamp it between the
123 // pref and the maximum number of channels that the device supports.
124 prefs.mChannels = c.mChannelCount.Get(std::min(prefs.mChannels, maxChannels));
125 prefs.mChannels = std::max(1, std::min(prefs.mChannels, maxChannels));
126
127 LOG("Mic source %p Audio config: aec: %d, agc: %d, noise: %d, channels: %d",
128 this, prefs.mAecOn ? prefs.mAec : -1, prefs.mAgcOn ? prefs.mAgc : -1,
129 prefs.mNoiseOn ? prefs.mNoise : -1, prefs.mChannels);
130
131 *aOutPrefs = prefs;
132
133 return NS_OK;
134 }
135
Reconfigure(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const char ** aOutBadConstraint)136 nsresult MediaEngineWebRTCMicrophoneSource::Reconfigure(
137 const dom::MediaTrackConstraints& aConstraints,
138 const MediaEnginePrefs& aPrefs, const char** aOutBadConstraint) {
139 AssertIsOnOwningThread();
140 MOZ_ASSERT(mTrack);
141
142 LOG("Mic source %p Reconfigure ", this);
143
144 NormalizedConstraints constraints(aConstraints);
145 MediaEnginePrefs outputPrefs;
146 nsresult rv =
147 EvaluateSettings(constraints, aPrefs, &outputPrefs, aOutBadConstraint);
148 if (NS_FAILED(rv)) {
149 if (aOutBadConstraint) {
150 return NS_ERROR_INVALID_ARG;
151 }
152
153 nsAutoCString name;
154 GetErrorName(rv, name);
155 LOG("Mic source %p Reconfigure() failed unexpectedly. rv=%s", this,
156 name.Data());
157 Stop();
158 return NS_ERROR_UNEXPECTED;
159 }
160
161 ApplySettings(outputPrefs);
162
163 mCurrentPrefs = outputPrefs;
164
165 return NS_OK;
166 }
167
UpdateAECSettings(bool aEnable,bool aUseAecMobile,EchoCancellation::SuppressionLevel aLevel,EchoControlMobile::RoutingMode aRoutingMode)168 void MediaEngineWebRTCMicrophoneSource::UpdateAECSettings(
169 bool aEnable, bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
170 EchoControlMobile::RoutingMode aRoutingMode) {
171 AssertIsOnOwningThread();
172
173 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
174 NS_DispatchToMainThread(NS_NewRunnableFunction(
175 __func__,
176 [that, track = mTrack, aEnable, aUseAecMobile, aLevel, aRoutingMode] {
177 class Message : public ControlMessage {
178 public:
179 Message(AudioInputProcessing* aInputProcessing, bool aEnable,
180 bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
181 EchoControlMobile::RoutingMode aRoutingMode)
182 : ControlMessage(nullptr),
183 mInputProcessing(aInputProcessing),
184 mEnable(aEnable),
185 mUseAecMobile(aUseAecMobile),
186 mLevel(aLevel),
187 mRoutingMode(aRoutingMode) {}
188
189 void Run() override {
190 mInputProcessing->UpdateAECSettings(mEnable, mUseAecMobile, mLevel,
191 mRoutingMode);
192 }
193
194 protected:
195 RefPtr<AudioInputProcessing> mInputProcessing;
196 bool mEnable;
197 bool mUseAecMobile;
198 EchoCancellation::SuppressionLevel mLevel;
199 EchoControlMobile::RoutingMode mRoutingMode;
200 };
201
202 if (track->IsDestroyed()) {
203 return;
204 }
205
206 track->GraphImpl()->AppendMessage(
207 MakeUnique<Message>(that->mInputProcessing, aEnable, aUseAecMobile,
208 aLevel, aRoutingMode));
209 }));
210 }
211
UpdateAGCSettings(bool aEnable,GainControl::Mode aMode)212 void MediaEngineWebRTCMicrophoneSource::UpdateAGCSettings(
213 bool aEnable, GainControl::Mode aMode) {
214 AssertIsOnOwningThread();
215
216 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
217 NS_DispatchToMainThread(
218 NS_NewRunnableFunction(__func__, [that, track = mTrack, aEnable, aMode] {
219 class Message : public ControlMessage {
220 public:
221 Message(AudioInputProcessing* aInputProcessing, bool aEnable,
222 GainControl::Mode aMode)
223 : ControlMessage(nullptr),
224 mInputProcessing(aInputProcessing),
225 mEnable(aEnable),
226 mMode(aMode) {}
227
228 void Run() override {
229 mInputProcessing->UpdateAGCSettings(mEnable, mMode);
230 }
231
232 protected:
233 RefPtr<AudioInputProcessing> mInputProcessing;
234 bool mEnable;
235 GainControl::Mode mMode;
236 };
237
238 if (track->IsDestroyed()) {
239 return;
240 }
241
242 track->GraphImpl()->AppendMessage(
243 MakeUnique<Message>(that->mInputProcessing, aEnable, aMode));
244 }));
245 }
246
UpdateHPFSettings(bool aEnable)247 void MediaEngineWebRTCMicrophoneSource::UpdateHPFSettings(bool aEnable) {
248 AssertIsOnOwningThread();
249
250 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
251 NS_DispatchToMainThread(
252 NS_NewRunnableFunction(__func__, [that, track = mTrack, aEnable] {
253 class Message : public ControlMessage {
254 public:
255 Message(AudioInputProcessing* aInputProcessing, bool aEnable)
256 : ControlMessage(nullptr),
257 mInputProcessing(aInputProcessing),
258 mEnable(aEnable) {}
259
260 void Run() override { mInputProcessing->UpdateHPFSettings(mEnable); }
261
262 protected:
263 RefPtr<AudioInputProcessing> mInputProcessing;
264 bool mEnable;
265 };
266
267 if (track->IsDestroyed()) {
268 return;
269 }
270
271 track->GraphImpl()->AppendMessage(
272 MakeUnique<Message>(that->mInputProcessing, aEnable));
273 }));
274 }
275
UpdateNSSettings(bool aEnable,webrtc::NoiseSuppression::Level aLevel)276 void MediaEngineWebRTCMicrophoneSource::UpdateNSSettings(
277 bool aEnable, webrtc::NoiseSuppression::Level aLevel) {
278 AssertIsOnOwningThread();
279
280 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
281 NS_DispatchToMainThread(
282 NS_NewRunnableFunction(__func__, [that, track = mTrack, aEnable, aLevel] {
283 class Message : public ControlMessage {
284 public:
285 Message(AudioInputProcessing* aInputProcessing, bool aEnable,
286 webrtc::NoiseSuppression::Level aLevel)
287 : ControlMessage(nullptr),
288 mInputProcessing(aInputProcessing),
289 mEnable(aEnable),
290 mLevel(aLevel) {}
291
292 void Run() override {
293 mInputProcessing->UpdateNSSettings(mEnable, mLevel);
294 }
295
296 protected:
297 RefPtr<AudioInputProcessing> mInputProcessing;
298 bool mEnable;
299 webrtc::NoiseSuppression::Level mLevel;
300 };
301
302 if (track->IsDestroyed()) {
303 return;
304 }
305
306 track->GraphImpl()->AppendMessage(
307 MakeUnique<Message>(that->mInputProcessing, aEnable, aLevel));
308 }));
309 }
310
UpdateAPMExtraOptions(bool aExtendedFilter,bool aDelayAgnostic)311 void MediaEngineWebRTCMicrophoneSource::UpdateAPMExtraOptions(
312 bool aExtendedFilter, bool aDelayAgnostic) {
313 AssertIsOnOwningThread();
314
315 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
316 NS_DispatchToMainThread(NS_NewRunnableFunction(
317 __func__, [that, track = mTrack, aExtendedFilter, aDelayAgnostic] {
318 class Message : public ControlMessage {
319 public:
320 Message(AudioInputProcessing* aInputProcessing, bool aExtendedFilter,
321 bool aDelayAgnostic)
322 : ControlMessage(nullptr),
323 mInputProcessing(aInputProcessing),
324 mExtendedFilter(aExtendedFilter),
325 mDelayAgnostic(aDelayAgnostic) {}
326
327 void Run() override {
328 mInputProcessing->UpdateAPMExtraOptions(mExtendedFilter,
329 mDelayAgnostic);
330 }
331
332 protected:
333 RefPtr<AudioInputProcessing> mInputProcessing;
334 bool mExtendedFilter;
335 bool mDelayAgnostic;
336 };
337
338 if (track->IsDestroyed()) {
339 return;
340 }
341
342 track->GraphImpl()->AppendMessage(MakeUnique<Message>(
343 that->mInputProcessing, aExtendedFilter, aDelayAgnostic));
344 }));
345 }
346
ApplySettings(const MediaEnginePrefs & aPrefs)347 void MediaEngineWebRTCMicrophoneSource::ApplySettings(
348 const MediaEnginePrefs& aPrefs) {
349 AssertIsOnOwningThread();
350
351 MOZ_ASSERT(
352 mTrack,
353 "ApplySetting is to be called only after SetTrack has been called");
354
355 if (mTrack) {
356 UpdateAGCSettings(aPrefs.mAgcOn,
357 static_cast<webrtc::GainControl::Mode>(aPrefs.mAgc));
358 UpdateNSSettings(
359 aPrefs.mNoiseOn,
360 static_cast<webrtc::NoiseSuppression::Level>(aPrefs.mNoise));
361 UpdateAECSettings(
362 aPrefs.mAecOn, aPrefs.mUseAecMobile,
363 static_cast<webrtc::EchoCancellation::SuppressionLevel>(aPrefs.mAec),
364 static_cast<webrtc::EchoControlMobile::RoutingMode>(
365 aPrefs.mRoutingMode));
366 UpdateHPFSettings(aPrefs.mHPFOn);
367
368 UpdateAPMExtraOptions(mExtendedFilter, mDelayAgnostic);
369 }
370
371 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
372 NS_DispatchToMainThread(
373 NS_NewRunnableFunction(__func__, [that, track = mTrack, prefs = aPrefs] {
374 that->mSettings->mEchoCancellation.Value() = prefs.mAecOn;
375 that->mSettings->mAutoGainControl.Value() = prefs.mAgcOn;
376 that->mSettings->mNoiseSuppression.Value() = prefs.mNoiseOn;
377 that->mSettings->mChannelCount.Value() = prefs.mChannels;
378
379 class Message : public ControlMessage {
380 public:
381 Message(MediaTrack* aTrack, AudioInputProcessing* aInputProcessing,
382 bool aPassThrough, uint32_t aRequestedInputChannelCount)
383 : ControlMessage(aTrack),
384 mInputProcessing(aInputProcessing),
385 mPassThrough(aPassThrough),
386 mRequestedInputChannelCount(aRequestedInputChannelCount) {}
387
388 void Run() override {
389 mInputProcessing->SetPassThrough(mTrack->GraphImpl(), mPassThrough);
390 mInputProcessing->SetRequestedInputChannelCount(
391 mTrack->GraphImpl(), mRequestedInputChannelCount);
392 }
393
394 protected:
395 RefPtr<AudioInputProcessing> mInputProcessing;
396 bool mPassThrough;
397 uint32_t mRequestedInputChannelCount;
398 };
399
400 // The high-pass filter is not taken into account when activating the
401 // pass through, since it's not controllable from content.
402 bool passThrough = !(prefs.mAecOn || prefs.mAgcOn || prefs.mNoiseOn);
403 if (track->IsDestroyed()) {
404 return;
405 }
406
407 track->GraphImpl()->AppendMessage(MakeUnique<Message>(
408 track, that->mInputProcessing, passThrough, prefs.mChannels));
409 }));
410 }
411
Allocate(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,uint64_t aWindowID,const char ** aOutBadConstraint)412 nsresult MediaEngineWebRTCMicrophoneSource::Allocate(
413 const dom::MediaTrackConstraints& aConstraints,
414 const MediaEnginePrefs& aPrefs, uint64_t aWindowID,
415 const char** aOutBadConstraint) {
416 AssertIsOnOwningThread();
417
418 mState = kAllocated;
419
420 NormalizedConstraints normalized(aConstraints);
421 MediaEnginePrefs outputPrefs;
422 nsresult rv =
423 EvaluateSettings(normalized, aPrefs, &outputPrefs, aOutBadConstraint);
424 if (NS_FAILED(rv)) {
425 return rv;
426 }
427
428 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
429 NS_DispatchToMainThread(
430 NS_NewRunnableFunction(__func__, [that, prefs = outputPrefs] {
431 that->mSettings->mEchoCancellation.Value() = prefs.mAecOn;
432 that->mSettings->mAutoGainControl.Value() = prefs.mAgcOn;
433 that->mSettings->mNoiseSuppression.Value() = prefs.mNoiseOn;
434 that->mSettings->mChannelCount.Value() = prefs.mChannels;
435 }));
436
437 mCurrentPrefs = outputPrefs;
438
439 return rv;
440 }
441
Deallocate()442 nsresult MediaEngineWebRTCMicrophoneSource::Deallocate() {
443 AssertIsOnOwningThread();
444
445 MOZ_ASSERT(mState == kStopped || mState == kAllocated);
446
447 class EndTrackMessage : public ControlMessage {
448 public:
449 EndTrackMessage(AudioInputTrack* aTrack,
450 AudioInputProcessing* aAudioInputProcessing)
451 : ControlMessage(aTrack),
452 mInputProcessing(aAudioInputProcessing),
453 mInputTrack(aTrack) {}
454
455 void Run() override { mInputProcessing->End(); }
456
457 protected:
458 const RefPtr<AudioInputProcessing> mInputProcessing;
459 AudioInputTrack const* mInputTrack;
460 };
461
462 if (mTrack) {
463 RefPtr<AudioInputProcessing> inputProcessing = mInputProcessing;
464 NS_DispatchToMainThread(NS_NewRunnableFunction(
465 __func__, [track = std::move(mTrack),
466 audioInputProcessing = std::move(inputProcessing)] {
467 if (track->IsDestroyed()) {
468 // This track has already been destroyed on main thread by its
469 // DOMMediaStream. No cleanup left to do.
470 return;
471 }
472 track->GraphImpl()->AppendMessage(
473 MakeUnique<EndTrackMessage>(track, audioInputProcessing));
474 }));
475 }
476
477 // Reset all state. This is not strictly necessary, this instance will get
478 // destroyed soon.
479 mTrack = nullptr;
480 mPrincipal = PRINCIPAL_HANDLE_NONE;
481
482 // If empty, no callbacks to deliver data should be occuring
483 MOZ_ASSERT(mState != kReleased, "Source not allocated");
484 MOZ_ASSERT(mState != kStarted, "Source not stopped");
485
486 mState = kReleased;
487 LOG("Mic source %p Audio device %s deallocated", this,
488 NS_ConvertUTF16toUTF8(mDeviceName).get());
489 return NS_OK;
490 }
491
SetTrack(const RefPtr<MediaTrack> & aTrack,const PrincipalHandle & aPrincipal)492 void MediaEngineWebRTCMicrophoneSource::SetTrack(
493 const RefPtr<MediaTrack>& aTrack, const PrincipalHandle& aPrincipal) {
494 AssertIsOnOwningThread();
495 MOZ_ASSERT(aTrack);
496 MOZ_ASSERT(aTrack->AsAudioInputTrack());
497
498 MOZ_ASSERT(!mTrack);
499 MOZ_ASSERT(mPrincipal == PRINCIPAL_HANDLE_NONE);
500 mTrack = aTrack->AsAudioInputTrack();
501 mPrincipal = aPrincipal;
502
503 mInputProcessing =
504 MakeAndAddRef<AudioInputProcessing>(mDeviceMaxChannelCount, mPrincipal);
505
506 NS_DispatchToMainThread(NS_NewRunnableFunction(
507 __func__, [track = mTrack, processing = mInputProcessing]() mutable {
508 track->SetInputProcessing(std::move(processing));
509 track->Resume(); // Suspended by MediaManager
510 }));
511
512 LOG("Mic source %p Track %p registered for microphone capture", this,
513 aTrack.get());
514 }
515
516 class StartStopMessage : public ControlMessage {
517 public:
518 enum StartStop { Start, Stop };
519
StartStopMessage(AudioInputProcessing * aInputProcessing,StartStop aAction)520 StartStopMessage(AudioInputProcessing* aInputProcessing, StartStop aAction)
521 : ControlMessage(nullptr),
522 mInputProcessing(aInputProcessing),
523 mAction(aAction) {}
524
Run()525 void Run() override {
526 if (mAction == StartStopMessage::Start) {
527 mInputProcessing->Start();
528 } else if (mAction == StartStopMessage::Stop) {
529 mInputProcessing->Stop();
530 } else {
531 MOZ_CRASH("Invalid enum value");
532 }
533 }
534
535 protected:
536 RefPtr<AudioInputProcessing> mInputProcessing;
537 StartStop mAction;
538 };
539
Start()540 nsresult MediaEngineWebRTCMicrophoneSource::Start() {
541 AssertIsOnOwningThread();
542
543 // This spans setting both the enabled state and mState.
544 if (mState == kStarted) {
545 return NS_OK;
546 }
547
548 MOZ_ASSERT(mState == kAllocated || mState == kStopped);
549
550 // This check is unreliable due to potential in-flight device updates.
551 // Multiple input devices are reliably excluded in OpenAudioInputImpl(),
552 // but the check here provides some error reporting most of the
553 // time.
554 CubebUtils::AudioDeviceID deviceID = mDeviceInfo->DeviceID();
555 if (mTrack->GraphImpl()->InputDeviceID() &&
556 mTrack->GraphImpl()->InputDeviceID() != deviceID) {
557 // For now, we only allow opening a single audio input device per document,
558 // because we can only have one MTG per document.
559 return NS_ERROR_NOT_AVAILABLE;
560 }
561
562 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
563 NS_DispatchToMainThread(
564 NS_NewRunnableFunction(__func__, [that, deviceID, track = mTrack] {
565 if (track->IsDestroyed()) {
566 return;
567 }
568
569 track->GraphImpl()->AppendMessage(MakeUnique<StartStopMessage>(
570 that->mInputProcessing, StartStopMessage::Start));
571 track->OpenAudioInput(deviceID, that->mInputProcessing);
572 }));
573
574 ApplySettings(mCurrentPrefs);
575
576 MOZ_ASSERT(mState != kReleased);
577 mState = kStarted;
578
579 return NS_OK;
580 }
581
Stop()582 nsresult MediaEngineWebRTCMicrophoneSource::Stop() {
583 AssertIsOnOwningThread();
584
585 LOG("Mic source %p Stop()", this);
586 MOZ_ASSERT(mTrack, "SetTrack must have been called before ::Stop");
587
588 if (mState == kStopped) {
589 // Already stopped - this is allowed
590 return NS_OK;
591 }
592
593 RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
594 NS_DispatchToMainThread(
595 NS_NewRunnableFunction(__func__, [that, track = mTrack] {
596 if (track->IsDestroyed()) {
597 return;
598 }
599
600 track->GraphImpl()->AppendMessage(MakeUnique<StartStopMessage>(
601 that->mInputProcessing, StartStopMessage::Stop));
602 MOZ_ASSERT(track->DeviceId().value() == that->mDeviceInfo->DeviceID());
603 track->CloseAudioInput();
604 }));
605
606 MOZ_ASSERT(mState == kStarted, "Should be started when stopping");
607 mState = kStopped;
608
609 return NS_OK;
610 }
611
GetSettings(dom::MediaTrackSettings & aOutSettings) const612 void MediaEngineWebRTCMicrophoneSource::GetSettings(
613 dom::MediaTrackSettings& aOutSettings) const {
614 MOZ_ASSERT(NS_IsMainThread());
615 aOutSettings = *mSettings;
616 }
617
AudioInputProcessing(uint32_t aMaxChannelCount,const PrincipalHandle & aPrincipalHandle)618 AudioInputProcessing::AudioInputProcessing(
619 uint32_t aMaxChannelCount, const PrincipalHandle& aPrincipalHandle)
620 : mAudioProcessing(AudioProcessing::Create()),
621 mRequestedInputChannelCount(aMaxChannelCount),
622 mSkipProcessing(false),
623 mInputDownmixBuffer(MAX_SAMPLING_FREQ * MAX_CHANNELS / 100),
624 mLiveFramesAppended(false),
625 mLiveBufferingAppended(0),
626 mPrincipal(aPrincipalHandle),
627 mEnabled(false),
628 mEnded(false) {}
629
Disconnect(MediaTrackGraphImpl * aGraph)630 void AudioInputProcessing::Disconnect(MediaTrackGraphImpl* aGraph) {
631 // This method is just for asserts.
632 MOZ_ASSERT(aGraph->OnGraphThread());
633 }
634
PassThrough(MediaTrackGraphImpl * aGraph) const635 bool AudioInputProcessing::PassThrough(MediaTrackGraphImpl* aGraph) const {
636 MOZ_ASSERT(aGraph->OnGraphThread());
637 return mSkipProcessing;
638 }
639
SetPassThrough(MediaTrackGraphImpl * aGraph,bool aPassThrough)640 void AudioInputProcessing::SetPassThrough(MediaTrackGraphImpl* aGraph,
641 bool aPassThrough) {
642 MOZ_ASSERT(aGraph->OnGraphThread());
643 if (!mSkipProcessing && aPassThrough && mPacketizerInput) {
644 MOZ_ASSERT(mPacketizerInput->PacketsAvailable() == 0);
645 LOG_FRAME(
646 "AudioInputProcessing %p Appending %u frames of null data for data "
647 "discarded in the packetizer",
648 this, mPacketizerInput->FramesAvailable());
649 mSegment.AppendNullData(mPacketizerInput->FramesAvailable());
650 mPacketizerInput->Clear();
651 }
652 mSkipProcessing = aPassThrough;
653 }
654
GetRequestedInputChannelCount()655 uint32_t AudioInputProcessing::GetRequestedInputChannelCount() {
656 return mRequestedInputChannelCount;
657 }
658
SetRequestedInputChannelCount(MediaTrackGraphImpl * aGraph,uint32_t aRequestedInputChannelCount)659 void AudioInputProcessing::SetRequestedInputChannelCount(
660 MediaTrackGraphImpl* aGraph, uint32_t aRequestedInputChannelCount) {
661 mRequestedInputChannelCount = aRequestedInputChannelCount;
662
663 aGraph->ReevaluateInputDevice();
664 }
665
666 // This does an early return in case of error.
667 #define HANDLE_APM_ERROR(fn) \
668 do { \
669 int rv = fn; \
670 if (rv != AudioProcessing::kNoError) { \
671 MOZ_ASSERT_UNREACHABLE("APM error in " #fn); \
672 return; \
673 } \
674 } while (0);
675
UpdateAECSettings(bool aEnable,bool aUseAecMobile,EchoCancellation::SuppressionLevel aLevel,EchoControlMobile::RoutingMode aRoutingMode)676 void AudioInputProcessing::UpdateAECSettings(
677 bool aEnable, bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
678 EchoControlMobile::RoutingMode aRoutingMode) {
679 if (aUseAecMobile) {
680 HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(aEnable));
681 HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->set_routing_mode(
682 aRoutingMode));
683 HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(false));
684 } else {
685 if (aLevel != EchoCancellation::SuppressionLevel::kLowSuppression &&
686 aLevel != EchoCancellation::SuppressionLevel::kModerateSuppression &&
687 aLevel != EchoCancellation::SuppressionLevel::kHighSuppression) {
688 LOG_ERROR(
689 "AudioInputProcessing %p Attempt to set invalid AEC suppression "
690 "level %d",
691 this, static_cast<int>(aLevel));
692
693 aLevel = EchoCancellation::SuppressionLevel::kModerateSuppression;
694 }
695
696 HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(false));
697 HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(aEnable));
698 HANDLE_APM_ERROR(
699 mAudioProcessing->echo_cancellation()->set_suppression_level(aLevel));
700 }
701 }
702
UpdateAGCSettings(bool aEnable,GainControl::Mode aMode)703 void AudioInputProcessing::UpdateAGCSettings(bool aEnable,
704 GainControl::Mode aMode) {
705 if (aMode != GainControl::Mode::kAdaptiveAnalog &&
706 aMode != GainControl::Mode::kAdaptiveDigital &&
707 aMode != GainControl::Mode::kFixedDigital) {
708 LOG_ERROR("AudioInputProcessing %p Attempt to set invalid AGC mode %d",
709 this, static_cast<int>(aMode));
710
711 aMode = GainControl::Mode::kAdaptiveDigital;
712 }
713
714 #if defined(WEBRTC_IOS) || defined(ATA) || defined(WEBRTC_ANDROID)
715 if (aMode == GainControl::Mode::kAdaptiveAnalog) {
716 LOG_ERROR(
717 "AudioInputProcessing %p Invalid AGC mode kAgcAdaptiveAnalog on "
718 "mobile",
719 this);
720 MOZ_ASSERT_UNREACHABLE(
721 "Bad pref set in all.js or in about:config"
722 " for the auto gain, on mobile.");
723 aMode = GainControl::Mode::kFixedDigital;
724 }
725 #endif
726 HANDLE_APM_ERROR(mAudioProcessing->gain_control()->set_mode(aMode));
727 HANDLE_APM_ERROR(mAudioProcessing->gain_control()->Enable(aEnable));
728 }
729
UpdateHPFSettings(bool aEnable)730 void AudioInputProcessing::UpdateHPFSettings(bool aEnable) {
731 HANDLE_APM_ERROR(mAudioProcessing->high_pass_filter()->Enable(aEnable));
732 }
733
UpdateNSSettings(bool aEnable,webrtc::NoiseSuppression::Level aLevel)734 void AudioInputProcessing::UpdateNSSettings(
735 bool aEnable, webrtc::NoiseSuppression::Level aLevel) {
736 if (aLevel != NoiseSuppression::Level::kLow &&
737 aLevel != NoiseSuppression::Level::kModerate &&
738 aLevel != NoiseSuppression::Level::kHigh &&
739 aLevel != NoiseSuppression::Level::kVeryHigh) {
740 LOG_ERROR(
741 "AudioInputProcessing %p Attempt to set invalid noise suppression "
742 "level %d",
743 this, static_cast<int>(aLevel));
744
745 aLevel = NoiseSuppression::Level::kModerate;
746 }
747
748 HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->set_level(aLevel));
749 HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->Enable(aEnable));
750 }
751
752 #undef HANDLE_APM_ERROR
753
UpdateAPMExtraOptions(bool aExtendedFilter,bool aDelayAgnostic)754 void AudioInputProcessing::UpdateAPMExtraOptions(bool aExtendedFilter,
755 bool aDelayAgnostic) {
756 webrtc::Config config;
757 config.Set<webrtc::ExtendedFilter>(
758 new webrtc::ExtendedFilter(aExtendedFilter));
759 config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(aDelayAgnostic));
760
761 mAudioProcessing->SetExtraOptions(config);
762 }
763
Start()764 void AudioInputProcessing::Start() {
765 mEnabled = true;
766 mLiveFramesAppended = false;
767 }
768
Stop()769 void AudioInputProcessing::Stop() { mEnabled = false; }
770
Pull(MediaTrackGraphImpl * aGraph,GraphTime aFrom,GraphTime aTo,GraphTime aTrackEnd,AudioSegment * aSegment,bool aLastPullThisIteration,bool * aEnded)771 void AudioInputProcessing::Pull(MediaTrackGraphImpl* aGraph, GraphTime aFrom,
772 GraphTime aTo, GraphTime aTrackEnd,
773 AudioSegment* aSegment,
774 bool aLastPullThisIteration, bool* aEnded) {
775 MOZ_ASSERT(aGraph->OnGraphThread());
776
777 if (mEnded) {
778 *aEnded = true;
779 return;
780 }
781
782 TrackTime delta = aTo - aTrackEnd;
783 MOZ_ASSERT(delta >= 0, "We shouldn't append more than requested");
784 TrackTime buffering = 0;
785
786 // Add the amount of buffering required to not underrun and glitch.
787
788 // Make sure there's at least one extra block buffered until audio callbacks
789 // come in, since we round graph iteration durations up to the nearest block.
790 buffering += WEBAUDIO_BLOCK_SIZE;
791
792 // If we're supposed to be packetizing but there's no packetizer yet,
793 // there must not have been any live frames appended yet.
794 MOZ_ASSERT_IF(!PassThrough(aGraph) && !mPacketizerInput,
795 mSegment.GetDuration() == 0);
796
797 if (!PassThrough(aGraph) && mPacketizerInput) {
798 // Processing is active and is processed in chunks of 10ms through the
799 // input packetizer. We allow for 10ms of silence on the track to
800 // accomodate the buffering worst-case.
801 buffering += mPacketizerInput->mPacketSize;
802 }
803
804 if (delta <= 0) {
805 return;
806 }
807
808 if (MOZ_LIKELY(mLiveFramesAppended)) {
809 if (MOZ_UNLIKELY(buffering > mLiveBufferingAppended)) {
810 // We need to buffer more data. This could happen the first time we pull
811 // input data, or the first iteration after starting to use the
812 // packetizer.
813 LOG_FRAME("AudioInputProcessing %p Inserting %" PRId64
814 " frames of silence due to buffer increase",
815 this, buffering - mLiveBufferingAppended);
816 mSegment.InsertNullDataAtStart(buffering - mLiveBufferingAppended);
817 mLiveBufferingAppended = buffering;
818 } else if (MOZ_UNLIKELY(buffering < mLiveBufferingAppended)) {
819 // We need to clear some buffered data to reduce latency now that the
820 // packetizer is no longer used.
821 MOZ_ASSERT(PassThrough(aGraph), "Must have turned on passthrough");
822 MOZ_ASSERT(mSegment.GetDuration() >=
823 (mLiveBufferingAppended - buffering));
824 TrackTime frames =
825 std::min(mSegment.GetDuration(), mLiveBufferingAppended - buffering);
826 LOG_FRAME("AudioInputProcessing %p Removing %" PRId64
827 " frames of silence due to buffer decrease",
828 this, frames);
829 mLiveBufferingAppended -= frames;
830 mSegment.RemoveLeading(frames);
831 }
832 }
833
834 if (mSegment.GetDuration() > 0) {
835 MOZ_ASSERT(buffering == mLiveBufferingAppended);
836 TrackTime frames = std::min(mSegment.GetDuration(), delta);
837 LOG_FRAME("AudioInputProcessing %p Appending %" PRId64
838 " frames of real data for %u channels.",
839 this, frames, mRequestedInputChannelCount);
840 aSegment->AppendSlice(mSegment, 0, frames);
841 mSegment.RemoveLeading(frames);
842 delta -= frames;
843
844 // Assert that the amount of data buffered doesn't grow unboundedly.
845 MOZ_ASSERT_IF(aLastPullThisIteration, mSegment.GetDuration() <= buffering);
846 }
847
848 if (delta <= 0) {
849 if (mSegment.GetDuration() == 0) {
850 mLiveBufferingAppended = -delta;
851 }
852 return;
853 }
854
855 LOG_FRAME("AudioInputProcessing %p Pulling %" PRId64
856 " frames of silence for %u channels.",
857 this, delta, mRequestedInputChannelCount);
858
859 // This assertion fails if we append silence here after having appended live
860 // frames. Before appending live frames we should add sufficient buffering to
861 // not have to glitch (aka append silence). Failing this meant the buffering
862 // was not sufficient.
863 MOZ_ASSERT_IF(mEnabled, !mLiveFramesAppended);
864 mLiveBufferingAppended = 0;
865
866 aSegment->AppendNullData(delta);
867 }
868
NotifyOutputData(MediaTrackGraphImpl * aGraph,BufferInfo aInfo)869 void AudioInputProcessing::NotifyOutputData(MediaTrackGraphImpl* aGraph,
870 BufferInfo aInfo) {
871 MOZ_ASSERT(aGraph->OnGraphThread());
872 MOZ_ASSERT(mEnabled);
873
874 if (!mPacketizerOutput ||
875 mPacketizerOutput->mPacketSize != aInfo.mRate / 100u ||
876 mPacketizerOutput->mChannels != aInfo.mChannels) {
877 // It's ok to drop the audio still in the packetizer here: if this changes,
878 // we changed devices or something.
879 mPacketizerOutput = MakeUnique<AudioPacketizer<AudioDataValue, float>>(
880 aInfo.mRate / 100, aInfo.mChannels);
881 }
882
883 mPacketizerOutput->Input(aInfo.mBuffer, aInfo.mFrames);
884
885 while (mPacketizerOutput->PacketsAvailable()) {
886 uint32_t samplesPerPacket =
887 mPacketizerOutput->mPacketSize * mPacketizerOutput->mChannels;
888 if (mOutputBuffer.Length() < samplesPerPacket) {
889 mOutputBuffer.SetLength(samplesPerPacket);
890 }
891 if (mDeinterleavedBuffer.Length() < samplesPerPacket) {
892 mDeinterleavedBuffer.SetLength(samplesPerPacket);
893 }
894 float* packet = mOutputBuffer.Data();
895 mPacketizerOutput->Output(packet);
896
897 AutoTArray<float*, MAX_CHANNELS> deinterleavedPacketDataChannelPointers;
898 float* interleavedFarend = nullptr;
899 uint32_t channelCountFarend = 0;
900 uint32_t framesPerPacketFarend = 0;
901
902 // Downmix from aInfo.mChannels to MAX_CHANNELS if needed. We always have
903 // floats here, the packetized performed the conversion.
904 if (aInfo.mChannels > MAX_CHANNELS) {
905 AudioConverter converter(
906 AudioConfig(aInfo.mChannels, 0, AudioConfig::FORMAT_FLT),
907 AudioConfig(MAX_CHANNELS, 0, AudioConfig::FORMAT_FLT));
908 framesPerPacketFarend = mPacketizerOutput->mPacketSize;
909 framesPerPacketFarend =
910 converter.Process(mInputDownmixBuffer, packet, framesPerPacketFarend);
911 interleavedFarend = mInputDownmixBuffer.Data();
912 channelCountFarend = MAX_CHANNELS;
913 deinterleavedPacketDataChannelPointers.SetLength(MAX_CHANNELS);
914 } else {
915 interleavedFarend = packet;
916 channelCountFarend = aInfo.mChannels;
917 framesPerPacketFarend = mPacketizerOutput->mPacketSize;
918 deinterleavedPacketDataChannelPointers.SetLength(aInfo.mChannels);
919 }
920
921 MOZ_ASSERT(interleavedFarend &&
922 (channelCountFarend == 1 || channelCountFarend == 2) &&
923 framesPerPacketFarend);
924
925 if (mInputBuffer.Length() < framesPerPacketFarend * channelCountFarend) {
926 mInputBuffer.SetLength(framesPerPacketFarend * channelCountFarend);
927 }
928
929 size_t offset = 0;
930 for (size_t i = 0; i < deinterleavedPacketDataChannelPointers.Length();
931 ++i) {
932 deinterleavedPacketDataChannelPointers[i] = mInputBuffer.Data() + offset;
933 offset += framesPerPacketFarend;
934 }
935
936 // Deinterleave, prepare a channel pointers array, with enough storage for
937 // the frames.
938 DeinterleaveAndConvertBuffer(
939 interleavedFarend, framesPerPacketFarend, channelCountFarend,
940 deinterleavedPacketDataChannelPointers.Elements());
941
942 // Having the same config for input and output means we potentially save
943 // some CPU.
944 StreamConfig inputConfig(aInfo.mRate, channelCountFarend, false);
945 StreamConfig outputConfig = inputConfig;
946
947 // Passing the same pointers here saves a copy inside this function.
948 DebugOnly<int> err = mAudioProcessing->ProcessReverseStream(
949 deinterleavedPacketDataChannelPointers.Elements(), inputConfig,
950 outputConfig, deinterleavedPacketDataChannelPointers.Elements());
951
952 MOZ_ASSERT(!err, "Could not process the reverse stream.");
953 }
954 }
955
956 // Only called if we're not in passthrough mode
PacketizeAndProcess(MediaTrackGraphImpl * aGraph,const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)957 void AudioInputProcessing::PacketizeAndProcess(MediaTrackGraphImpl* aGraph,
958 const AudioDataValue* aBuffer,
959 size_t aFrames, TrackRate aRate,
960 uint32_t aChannels) {
961 MOZ_ASSERT(!PassThrough(aGraph),
962 "This should be bypassed when in PassThrough mode.");
963 MOZ_ASSERT(mEnabled);
964 size_t offset = 0;
965
966 if (!mPacketizerInput || mPacketizerInput->mPacketSize != aRate / 100u ||
967 mPacketizerInput->mChannels != aChannels) {
968 // It's ok to drop the audio still in the packetizer here.
969 mPacketizerInput = MakeUnique<AudioPacketizer<AudioDataValue, float>>(
970 aRate / 100, aChannels);
971 }
972
973 LOG_FRAME("AudioInputProcessing %p Appending %zu frames to packetizer", this,
974 aFrames);
975
976 // Packetize our input data into 10ms chunks, deinterleave into planar channel
977 // buffers, process, and append to the right MediaStreamTrack.
978 mPacketizerInput->Input(aBuffer, static_cast<uint32_t>(aFrames));
979
980 while (mPacketizerInput->PacketsAvailable()) {
981 uint32_t samplesPerPacket =
982 mPacketizerInput->mPacketSize * mPacketizerInput->mChannels;
983 if (mInputBuffer.Length() < samplesPerPacket) {
984 mInputBuffer.SetLength(samplesPerPacket);
985 }
986 if (mDeinterleavedBuffer.Length() < samplesPerPacket) {
987 mDeinterleavedBuffer.SetLength(samplesPerPacket);
988 }
989 float* packet = mInputBuffer.Data();
990 mPacketizerInput->Output(packet);
991
992 // Downmix from aChannels to mono if needed. We always have floats
993 // here, the packetizer performed the conversion. This handles sound cards
994 // with multiple physical jacks exposed as a single device with _n_
995 // discrete channels, where only a single mic is plugged in. Those channels
996 // are not correlated temporaly since they are discrete channels, mixing is
997 // just a sum.
998 AutoTArray<float*, 8> deinterleavedPacketizedInputDataChannelPointers;
999 uint32_t channelCountInput = 0;
1000 if (aChannels > MAX_CHANNELS) {
1001 channelCountInput = MONO;
1002 deinterleavedPacketizedInputDataChannelPointers.SetLength(
1003 channelCountInput);
1004 deinterleavedPacketizedInputDataChannelPointers[0] =
1005 mDeinterleavedBuffer.Data();
1006 // Downmix to mono (and effectively have a planar buffer) by summing all
1007 // channels in the first channel.
1008 size_t readIndex = 0;
1009 for (size_t i = 0; i < mPacketizerInput->mPacketSize; i++) {
1010 mDeinterleavedBuffer.Data()[i] = 0.;
1011 for (size_t j = 0; j < aChannels; j++) {
1012 mDeinterleavedBuffer.Data()[i] += packet[readIndex++];
1013 }
1014 }
1015 } else {
1016 channelCountInput = aChannels;
1017 // Deinterleave the input data
1018 // Prepare an array pointing to deinterleaved channels.
1019 deinterleavedPacketizedInputDataChannelPointers.SetLength(
1020 channelCountInput);
1021 offset = 0;
1022 for (size_t i = 0;
1023 i < deinterleavedPacketizedInputDataChannelPointers.Length(); ++i) {
1024 deinterleavedPacketizedInputDataChannelPointers[i] =
1025 mDeinterleavedBuffer.Data() + offset;
1026 offset += mPacketizerInput->mPacketSize;
1027 }
1028 // Deinterleave to mInputBuffer, pointed to by inputBufferChannelPointers.
1029 Deinterleave(packet, mPacketizerInput->mPacketSize, channelCountInput,
1030 deinterleavedPacketizedInputDataChannelPointers.Elements());
1031 }
1032
1033 StreamConfig inputConfig(aRate, channelCountInput,
1034 false /* we don't use typing detection*/);
1035 StreamConfig outputConfig = inputConfig;
1036
1037 // Bug 1404965: Get the right delay here, it saves some work down the line.
1038 mAudioProcessing->set_stream_delay_ms(0);
1039
1040 // Bug 1414837: find a way to not allocate here.
1041 CheckedInt<size_t> bufferSize(sizeof(float));
1042 bufferSize *= mPacketizerInput->mPacketSize;
1043 bufferSize *= channelCountInput;
1044 RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize);
1045
1046 // Prepare channel pointers to the SharedBuffer created above.
1047 AutoTArray<float*, 8> processedOutputChannelPointers;
1048 AutoTArray<const float*, 8> processedOutputChannelPointersConst;
1049 processedOutputChannelPointers.SetLength(channelCountInput);
1050 processedOutputChannelPointersConst.SetLength(channelCountInput);
1051
1052 offset = 0;
1053 for (size_t i = 0; i < processedOutputChannelPointers.Length(); ++i) {
1054 processedOutputChannelPointers[i] =
1055 static_cast<float*>(buffer->Data()) + offset;
1056 processedOutputChannelPointersConst[i] =
1057 static_cast<float*>(buffer->Data()) + offset;
1058 offset += mPacketizerInput->mPacketSize;
1059 }
1060
1061 mAudioProcessing->ProcessStream(
1062 deinterleavedPacketizedInputDataChannelPointers.Elements(), inputConfig,
1063 outputConfig, processedOutputChannelPointers.Elements());
1064
1065 if (mEnded) {
1066 continue;
1067 }
1068
1069 LOG_FRAME("AudioInputProcessing %p Appending %u frames of packetized audio",
1070 this, mPacketizerInput->mPacketSize);
1071
1072 // We already have planar audio data of the right format. Insert into the
1073 // MTG.
1074 MOZ_ASSERT(processedOutputChannelPointers.Length() == channelCountInput);
1075 RefPtr<SharedBuffer> other = buffer;
1076 mSegment.AppendFrames(other.forget(), processedOutputChannelPointersConst,
1077 mPacketizerInput->mPacketSize, mPrincipal);
1078 }
1079 }
1080
ProcessInput(MediaTrackGraphImpl * aGraph,const AudioSegment * aSegment)1081 void AudioInputProcessing::ProcessInput(MediaTrackGraphImpl* aGraph,
1082 const AudioSegment* aSegment) {
1083 MOZ_ASSERT(aGraph);
1084 MOZ_ASSERT(aGraph->OnGraphThread());
1085
1086 if (mEnded || !mEnabled || !mLiveFramesAppended || !mInputData) {
1087 return;
1088 }
1089
1090 // One NotifyInputData might have multiple following ProcessInput calls, but
1091 // we only process one input per NotifyInputData call.
1092 BufferInfo inputInfo = mInputData.extract();
1093
1094 // If some processing is necessary, packetize and insert in the WebRTC.org
1095 // code. Otherwise, directly insert the mic data in the MTG, bypassing all
1096 // processing.
1097 if (PassThrough(aGraph)) {
1098 if (aSegment) {
1099 mSegment.AppendSegment(aSegment, mPrincipal);
1100 } else {
1101 mSegment.AppendFromInterleavedBuffer(inputInfo.mBuffer, inputInfo.mFrames,
1102 inputInfo.mChannels, mPrincipal);
1103 }
1104 } else {
1105 MOZ_ASSERT(aGraph->GraphRate() == inputInfo.mRate);
1106 PacketizeAndProcess(aGraph, inputInfo.mBuffer, inputInfo.mFrames,
1107 inputInfo.mRate, inputInfo.mChannels);
1108 }
1109 }
1110
NotifyInputStopped(MediaTrackGraphImpl * aGraph)1111 void AudioInputProcessing::NotifyInputStopped(MediaTrackGraphImpl* aGraph) {
1112 MOZ_ASSERT(aGraph->OnGraphThread());
1113 // This is called when an AudioCallbackDriver switch has happened for any
1114 // reason, including other reasons than starting this audio input stream. We
1115 // reset state when this happens, as a fallback driver may have fiddled with
1116 // the amount of buffered silence during the switch.
1117 mLiveFramesAppended = false;
1118 mSegment.Clear();
1119 if (mPacketizerInput) {
1120 mPacketizerInput->Clear();
1121 }
1122 mInputData.take();
1123 }
1124
1125 // Called back on GraphDriver thread!
1126 // Note this can be called back after ::Stop()
NotifyInputData(MediaTrackGraphImpl * aGraph,const BufferInfo aInfo,uint32_t aAlreadyBuffered)1127 void AudioInputProcessing::NotifyInputData(MediaTrackGraphImpl* aGraph,
1128 const BufferInfo aInfo,
1129 uint32_t aAlreadyBuffered) {
1130 MOZ_ASSERT(aGraph->OnGraphThread());
1131 TRACE();
1132
1133 MOZ_ASSERT(mEnabled);
1134
1135 if (!mLiveFramesAppended) {
1136 // First time we see live frames getting added. Use what's already buffered
1137 // in the driver's scratch buffer as a starting point.
1138 mLiveFramesAppended = true;
1139 mLiveBufferingAppended = aAlreadyBuffered;
1140 }
1141
1142 mInputData = Some(aInfo);
1143 }
1144
1145 #define ResetProcessingIfNeeded(_processing) \
1146 do { \
1147 bool enabled = mAudioProcessing->_processing()->is_enabled(); \
1148 \
1149 if (enabled) { \
1150 int rv = mAudioProcessing->_processing()->Enable(!enabled); \
1151 if (rv) { \
1152 NS_WARNING("Could not reset the status of the " #_processing \
1153 " on device change."); \
1154 return; \
1155 } \
1156 rv = mAudioProcessing->_processing()->Enable(enabled); \
1157 if (rv) { \
1158 NS_WARNING("Could not reset the status of the " #_processing \
1159 " on device change."); \
1160 return; \
1161 } \
1162 } \
1163 } while (0)
1164
DeviceChanged(MediaTrackGraphImpl * aGraph)1165 void AudioInputProcessing::DeviceChanged(MediaTrackGraphImpl* aGraph) {
1166 MOZ_ASSERT(aGraph->OnGraphThread());
1167 // Reset some processing
1168 ResetProcessingIfNeeded(gain_control);
1169 ResetProcessingIfNeeded(echo_cancellation);
1170 ResetProcessingIfNeeded(noise_suppression);
1171 }
1172
End()1173 void AudioInputProcessing::End() {
1174 mEnded = true;
1175 mSegment.Clear();
1176 mInputData.take();
1177 }
1178
NumBufferedFrames(MediaTrackGraphImpl * aGraph) const1179 TrackTime AudioInputProcessing::NumBufferedFrames(
1180 MediaTrackGraphImpl* aGraph) const {
1181 MOZ_ASSERT(aGraph->OnGraphThread());
1182 return mSegment.GetDuration();
1183 }
1184
Destroy()1185 void AudioInputTrack::Destroy() {
1186 MOZ_ASSERT(NS_IsMainThread());
1187 CloseAudioInput();
1188
1189 MediaTrack::Destroy();
1190 }
1191
SetInputProcessing(RefPtr<AudioInputProcessing> aInputProcessing)1192 void AudioInputTrack::SetInputProcessing(
1193 RefPtr<AudioInputProcessing> aInputProcessing) {
1194 class Message : public ControlMessage {
1195 RefPtr<AudioInputTrack> mTrack;
1196 RefPtr<AudioInputProcessing> mProcessing;
1197
1198 public:
1199 Message(RefPtr<AudioInputTrack> aTrack,
1200 RefPtr<AudioInputProcessing> aProcessing)
1201 : ControlMessage(aTrack),
1202 mTrack(std::move(aTrack)),
1203 mProcessing(std::move(aProcessing)) {}
1204 void Run() override {
1205 mTrack->SetInputProcessingImpl(std::move(mProcessing));
1206 }
1207 };
1208
1209 if (IsDestroyed()) {
1210 return;
1211 }
1212 GraphImpl()->AppendMessage(
1213 MakeUnique<Message>(std::move(this), std::move(aInputProcessing)));
1214 }
1215
Create(MediaTrackGraph * aGraph)1216 AudioInputTrack* AudioInputTrack::Create(MediaTrackGraph* aGraph) {
1217 MOZ_ASSERT(NS_IsMainThread());
1218 AudioInputTrack* track = new AudioInputTrack(aGraph->GraphRate());
1219 aGraph->AddTrack(track);
1220 return track;
1221 }
1222
DestroyImpl()1223 void AudioInputTrack::DestroyImpl() {
1224 ProcessedMediaTrack::DestroyImpl();
1225 if (mInputProcessing) {
1226 mInputProcessing->End();
1227 }
1228 }
1229
ProcessInput(GraphTime aFrom,GraphTime aTo,uint32_t aFlags)1230 void AudioInputTrack::ProcessInput(GraphTime aFrom, GraphTime aTo,
1231 uint32_t aFlags) {
1232 TRACE_COMMENT("AudioInputTrack %p", this);
1233
1234 // Check if there is a connected NativeInputTrack
1235 NativeInputTrack* source = nullptr;
1236 if (!mInputs.IsEmpty()) {
1237 for (const MediaInputPort* input : mInputs) {
1238 MOZ_ASSERT(input->GetSource());
1239 if (input->GetSource()->AsNativeInputTrack()) {
1240 source = input->GetSource()->AsNativeInputTrack();
1241 break;
1242 }
1243 }
1244 }
1245
1246 // Push the input data from the connected NativeInputTrack to mInputProcessing
1247 if (source) {
1248 MOZ_ASSERT(source->GraphImpl() == GraphImpl());
1249 MOZ_ASSERT(source->mSampleRate == mSampleRate);
1250 MOZ_ASSERT(GraphImpl()->GraphRate() == mSampleRate);
1251 mInputProcessing->ProcessInput(GraphImpl(),
1252 source->GetData<AudioSegment>());
1253 }
1254
1255 bool ended = false;
1256 mInputProcessing->Pull(
1257 GraphImpl(), aFrom, aTo, TrackTimeToGraphTime(GetEnd()),
1258 GetData<AudioSegment>(), aTo == GraphImpl()->mStateComputedTime, &ended);
1259 ApplyTrackDisabling(mSegment.get());
1260 if (ended && (aFlags & ALLOW_END)) {
1261 mEnded = true;
1262 }
1263 }
1264
SetInputProcessingImpl(RefPtr<AudioInputProcessing> aInputProcessing)1265 void AudioInputTrack::SetInputProcessingImpl(
1266 RefPtr<AudioInputProcessing> aInputProcessing) {
1267 MOZ_ASSERT(GraphImpl()->OnGraphThread());
1268 mInputProcessing = std::move(aInputProcessing);
1269 }
1270
OpenAudioInput(CubebUtils::AudioDeviceID aId,AudioDataListener * aListener)1271 nsresult AudioInputTrack::OpenAudioInput(CubebUtils::AudioDeviceID aId,
1272 AudioDataListener* aListener) {
1273 MOZ_ASSERT(NS_IsMainThread());
1274 MOZ_ASSERT(GraphImpl());
1275 MOZ_ASSERT(!mInputListener);
1276 MOZ_ASSERT(mDeviceId.isNothing());
1277 mInputListener = aListener;
1278 ProcessedMediaTrack* input = GraphImpl()->GetDeviceTrack(aId);
1279 MOZ_ASSERT(input);
1280 LOG("Open device %p (InputTrack=%p) for Mic source %p", aId, input, this);
1281 mPort = AllocateInputPort(input);
1282 mDeviceId.emplace(aId);
1283 return GraphImpl()->OpenAudioInput(aId, aListener);
1284 }
1285
CloseAudioInput()1286 void AudioInputTrack::CloseAudioInput() {
1287 MOZ_ASSERT(NS_IsMainThread());
1288 MOZ_ASSERT(GraphImpl());
1289 if (!mInputListener) {
1290 return;
1291 }
1292 MOZ_ASSERT(mPort);
1293 MOZ_ASSERT(mDeviceId.isSome());
1294 LOG("Close device %p (InputTrack=%p) for Mic source %p ", mDeviceId.value(),
1295 mPort->GetSource(), this);
1296 mPort->Destroy();
1297 GraphImpl()->CloseAudioInput(mDeviceId.extract(), mInputListener);
1298 mInputListener = nullptr;
1299 }
1300
DeviceId() const1301 Maybe<CubebUtils::AudioDeviceID> AudioInputTrack::DeviceId() const {
1302 MOZ_ASSERT(NS_IsMainThread());
1303 return mDeviceId;
1304 }
1305
GetName() const1306 nsString MediaEngineWebRTCAudioCaptureSource::GetName() const {
1307 return u"AudioCapture"_ns;
1308 }
1309
GetUUID() const1310 nsCString MediaEngineWebRTCAudioCaptureSource::GetUUID() const {
1311 nsID uuid;
1312 char uuidBuffer[NSID_LENGTH];
1313 nsCString asciiString;
1314 ErrorResult rv;
1315
1316 rv = nsContentUtils::GenerateUUIDInPlace(uuid);
1317 if (rv.Failed()) {
1318 return ""_ns;
1319 }
1320
1321 uuid.ToProvidedString(uuidBuffer);
1322 asciiString.AssignASCII(uuidBuffer);
1323
1324 // Remove {} and the null terminator
1325 return nsCString(Substring(asciiString, 1, NSID_LENGTH - 3));
1326 }
1327
GetGroupId() const1328 nsString MediaEngineWebRTCAudioCaptureSource::GetGroupId() const {
1329 return u"AudioCaptureGroup"_ns;
1330 }
1331
SetTrack(const RefPtr<MediaTrack> & aTrack,const PrincipalHandle & aPrincipalHandle)1332 void MediaEngineWebRTCAudioCaptureSource::SetTrack(
1333 const RefPtr<MediaTrack>& aTrack, const PrincipalHandle& aPrincipalHandle) {
1334 AssertIsOnOwningThread();
1335 // Nothing to do here. aTrack is a placeholder dummy and not exposed.
1336 }
1337
Start()1338 nsresult MediaEngineWebRTCAudioCaptureSource::Start() {
1339 AssertIsOnOwningThread();
1340 return NS_OK;
1341 }
1342
Stop()1343 nsresult MediaEngineWebRTCAudioCaptureSource::Stop() {
1344 AssertIsOnOwningThread();
1345 return NS_OK;
1346 }
1347
Reconfigure(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const char ** aOutBadConstraint)1348 nsresult MediaEngineWebRTCAudioCaptureSource::Reconfigure(
1349 const dom::MediaTrackConstraints& aConstraints,
1350 const MediaEnginePrefs& aPrefs, const char** aOutBadConstraint) {
1351 return NS_OK;
1352 }
1353
GetSettings(dom::MediaTrackSettings & aOutSettings) const1354 void MediaEngineWebRTCAudioCaptureSource::GetSettings(
1355 dom::MediaTrackSettings& aOutSettings) const {
1356 aOutSettings.mAutoGainControl.Construct(false);
1357 aOutSettings.mEchoCancellation.Construct(false);
1358 aOutSettings.mNoiseSuppression.Construct(false);
1359 aOutSettings.mChannelCount.Construct(1);
1360 }
1361
1362 } // namespace mozilla
1363