1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 
6 #include "MediaEngineWebRTCAudio.h"
7 
8 #include <stdio.h>
9 #include <algorithm>
10 
11 #include "AudioConverter.h"
12 #include "MediaManager.h"
13 #include "MediaTrackGraphImpl.h"
14 #include "MediaTrackConstraints.h"
15 #include "mozilla/Assertions.h"
16 #include "mozilla/ErrorNames.h"
17 #include "nsContentUtils.h"
18 #include "transport/runnable_utils.h"
19 #include "Tracing.h"
20 
21 // scoped_ptr.h uses FF
22 #ifdef FF
23 #  undef FF
24 #endif
25 #include "webrtc/voice_engine/voice_engine_defines.h"
26 #include "webrtc/modules/audio_processing/include/audio_processing.h"
27 #include "webrtc/common_audio/include/audio_util.h"
28 
29 using namespace webrtc;
30 
31 // These are restrictions from the webrtc.org code
32 #define MAX_CHANNELS 2
33 #define MONO 1
34 #define MAX_SAMPLING_FREQ 48000  // Hz - multiple of 100
35 
36 namespace mozilla {
37 
38 extern LazyLogModule gMediaManagerLog;
39 #define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
40 #define LOG_FRAME(...) \
41   MOZ_LOG(gMediaManagerLog, LogLevel::Verbose, (__VA_ARGS__))
42 #define LOG_ERROR(...) MOZ_LOG(gMediaManagerLog, LogLevel::Error, (__VA_ARGS__))
43 
44 /**
45  * WebRTC Microphone MediaEngineSource.
46  */
47 
MediaEngineWebRTCMicrophoneSource(RefPtr<AudioDeviceInfo> aInfo,const nsString & aDeviceName,const nsCString & aDeviceUUID,const nsString & aDeviceGroup,uint32_t aMaxChannelCount,bool aDelayAgnostic,bool aExtendedFilter)48 MediaEngineWebRTCMicrophoneSource::MediaEngineWebRTCMicrophoneSource(
49     RefPtr<AudioDeviceInfo> aInfo, const nsString& aDeviceName,
50     const nsCString& aDeviceUUID, const nsString& aDeviceGroup,
51     uint32_t aMaxChannelCount, bool aDelayAgnostic, bool aExtendedFilter)
52     : mPrincipal(PRINCIPAL_HANDLE_NONE),
53       mDeviceInfo(std::move(aInfo)),
54       mDelayAgnostic(aDelayAgnostic),
55       mExtendedFilter(aExtendedFilter),
56       mDeviceName(aDeviceName),
57       mDeviceUUID(aDeviceUUID),
58       mDeviceGroup(aDeviceGroup),
59       mDeviceMaxChannelCount(aMaxChannelCount),
60       mSettings(new nsMainThreadPtrHolder<
61                 media::Refcountable<dom::MediaTrackSettings>>(
62           "MediaEngineWebRTCMicrophoneSource::mSettings",
63           new media::Refcountable<dom::MediaTrackSettings>(),
64           // Non-strict means it won't assert main thread for us.
65           // It would be great if it did but we're already on the media thread.
66           /* aStrict = */ false)) {
67 #ifndef ANDROID
68   MOZ_ASSERT(mDeviceInfo->DeviceID());
69 #endif
70 
71   // We'll init lazily as needed
72   mSettings->mEchoCancellation.Construct(0);
73   mSettings->mAutoGainControl.Construct(0);
74   mSettings->mNoiseSuppression.Construct(0);
75   mSettings->mChannelCount.Construct(0);
76 
77   mState = kReleased;
78 }
79 
GetName() const80 nsString MediaEngineWebRTCMicrophoneSource::GetName() const {
81   return mDeviceName;
82 }
83 
GetUUID() const84 nsCString MediaEngineWebRTCMicrophoneSource::GetUUID() const {
85   return mDeviceUUID;
86 }
87 
GetGroupId() const88 nsString MediaEngineWebRTCMicrophoneSource::GetGroupId() const {
89   return mDeviceGroup;
90 }
91 
EvaluateSettings(const NormalizedConstraints & aConstraintsUpdate,const MediaEnginePrefs & aInPrefs,MediaEnginePrefs * aOutPrefs,const char ** aOutBadConstraint)92 nsresult MediaEngineWebRTCMicrophoneSource::EvaluateSettings(
93     const NormalizedConstraints& aConstraintsUpdate,
94     const MediaEnginePrefs& aInPrefs, MediaEnginePrefs* aOutPrefs,
95     const char** aOutBadConstraint) {
96   AssertIsOnOwningThread();
97 
98   FlattenedConstraints c(aConstraintsUpdate);
99   MediaEnginePrefs prefs = aInPrefs;
100 
101   prefs.mAecOn = c.mEchoCancellation.Get(aInPrefs.mAecOn);
102   prefs.mAgcOn = c.mAutoGainControl.Get(aInPrefs.mAgcOn && prefs.mAecOn);
103   prefs.mNoiseOn = c.mNoiseSuppression.Get(aInPrefs.mNoiseOn && prefs.mAecOn);
104 
105   // Determine an actual channel count to use for this source. Three factors at
106   // play here: the device capabilities, the constraints passed in by content,
107   // and a pref that can force things (for testing)
108   int32_t maxChannels = mDeviceInfo->MaxChannels();
109 
110   // First, check channelCount violation wrt constraints. This fails in case of
111   // error.
112   if (c.mChannelCount.mMin > maxChannels) {
113     *aOutBadConstraint = "channelCount";
114     return NS_ERROR_FAILURE;
115   }
116   // A pref can force the channel count to use. If the pref has a value of zero
117   // or lower, it has no effect.
118   if (aInPrefs.mChannels <= 0) {
119     prefs.mChannels = maxChannels;
120   }
121 
122   // Get the number of channels asked for by content, and clamp it between the
123   // pref and the maximum number of channels that the device supports.
124   prefs.mChannels = c.mChannelCount.Get(std::min(prefs.mChannels, maxChannels));
125   prefs.mChannels = std::max(1, std::min(prefs.mChannels, maxChannels));
126 
127   LOG("Mic source %p Audio config: aec: %d, agc: %d, noise: %d, channels: %d",
128       this, prefs.mAecOn ? prefs.mAec : -1, prefs.mAgcOn ? prefs.mAgc : -1,
129       prefs.mNoiseOn ? prefs.mNoise : -1, prefs.mChannels);
130 
131   *aOutPrefs = prefs;
132 
133   return NS_OK;
134 }
135 
Reconfigure(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const char ** aOutBadConstraint)136 nsresult MediaEngineWebRTCMicrophoneSource::Reconfigure(
137     const dom::MediaTrackConstraints& aConstraints,
138     const MediaEnginePrefs& aPrefs, const char** aOutBadConstraint) {
139   AssertIsOnOwningThread();
140   MOZ_ASSERT(mTrack);
141 
142   LOG("Mic source %p Reconfigure ", this);
143 
144   NormalizedConstraints constraints(aConstraints);
145   MediaEnginePrefs outputPrefs;
146   nsresult rv =
147       EvaluateSettings(constraints, aPrefs, &outputPrefs, aOutBadConstraint);
148   if (NS_FAILED(rv)) {
149     if (aOutBadConstraint) {
150       return NS_ERROR_INVALID_ARG;
151     }
152 
153     nsAutoCString name;
154     GetErrorName(rv, name);
155     LOG("Mic source %p Reconfigure() failed unexpectedly. rv=%s", this,
156         name.Data());
157     Stop();
158     return NS_ERROR_UNEXPECTED;
159   }
160 
161   ApplySettings(outputPrefs);
162 
163   mCurrentPrefs = outputPrefs;
164 
165   return NS_OK;
166 }
167 
UpdateAECSettings(bool aEnable,bool aUseAecMobile,EchoCancellation::SuppressionLevel aLevel,EchoControlMobile::RoutingMode aRoutingMode)168 void MediaEngineWebRTCMicrophoneSource::UpdateAECSettings(
169     bool aEnable, bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
170     EchoControlMobile::RoutingMode aRoutingMode) {
171   AssertIsOnOwningThread();
172 
173   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
174   NS_DispatchToMainThread(NS_NewRunnableFunction(
175       __func__,
176       [that, track = mTrack, aEnable, aUseAecMobile, aLevel, aRoutingMode] {
177         class Message : public ControlMessage {
178          public:
179           Message(AudioInputProcessing* aInputProcessing, bool aEnable,
180                   bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
181                   EchoControlMobile::RoutingMode aRoutingMode)
182               : ControlMessage(nullptr),
183                 mInputProcessing(aInputProcessing),
184                 mEnable(aEnable),
185                 mUseAecMobile(aUseAecMobile),
186                 mLevel(aLevel),
187                 mRoutingMode(aRoutingMode) {}
188 
189           void Run() override {
190             mInputProcessing->UpdateAECSettings(mEnable, mUseAecMobile, mLevel,
191                                                 mRoutingMode);
192           }
193 
194          protected:
195           RefPtr<AudioInputProcessing> mInputProcessing;
196           bool mEnable;
197           bool mUseAecMobile;
198           EchoCancellation::SuppressionLevel mLevel;
199           EchoControlMobile::RoutingMode mRoutingMode;
200         };
201 
202         if (track->IsDestroyed()) {
203           return;
204         }
205 
206         track->GraphImpl()->AppendMessage(
207             MakeUnique<Message>(that->mInputProcessing, aEnable, aUseAecMobile,
208                                 aLevel, aRoutingMode));
209       }));
210 }
211 
UpdateAGCSettings(bool aEnable,GainControl::Mode aMode)212 void MediaEngineWebRTCMicrophoneSource::UpdateAGCSettings(
213     bool aEnable, GainControl::Mode aMode) {
214   AssertIsOnOwningThread();
215 
216   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
217   NS_DispatchToMainThread(
218       NS_NewRunnableFunction(__func__, [that, track = mTrack, aEnable, aMode] {
219         class Message : public ControlMessage {
220          public:
221           Message(AudioInputProcessing* aInputProcessing, bool aEnable,
222                   GainControl::Mode aMode)
223               : ControlMessage(nullptr),
224                 mInputProcessing(aInputProcessing),
225                 mEnable(aEnable),
226                 mMode(aMode) {}
227 
228           void Run() override {
229             mInputProcessing->UpdateAGCSettings(mEnable, mMode);
230           }
231 
232          protected:
233           RefPtr<AudioInputProcessing> mInputProcessing;
234           bool mEnable;
235           GainControl::Mode mMode;
236         };
237 
238         if (track->IsDestroyed()) {
239           return;
240         }
241 
242         track->GraphImpl()->AppendMessage(
243             MakeUnique<Message>(that->mInputProcessing, aEnable, aMode));
244       }));
245 }
246 
UpdateHPFSettings(bool aEnable)247 void MediaEngineWebRTCMicrophoneSource::UpdateHPFSettings(bool aEnable) {
248   AssertIsOnOwningThread();
249 
250   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
251   NS_DispatchToMainThread(
252       NS_NewRunnableFunction(__func__, [that, track = mTrack, aEnable] {
253         class Message : public ControlMessage {
254          public:
255           Message(AudioInputProcessing* aInputProcessing, bool aEnable)
256               : ControlMessage(nullptr),
257                 mInputProcessing(aInputProcessing),
258                 mEnable(aEnable) {}
259 
260           void Run() override { mInputProcessing->UpdateHPFSettings(mEnable); }
261 
262          protected:
263           RefPtr<AudioInputProcessing> mInputProcessing;
264           bool mEnable;
265         };
266 
267         if (track->IsDestroyed()) {
268           return;
269         }
270 
271         track->GraphImpl()->AppendMessage(
272             MakeUnique<Message>(that->mInputProcessing, aEnable));
273       }));
274 }
275 
UpdateNSSettings(bool aEnable,webrtc::NoiseSuppression::Level aLevel)276 void MediaEngineWebRTCMicrophoneSource::UpdateNSSettings(
277     bool aEnable, webrtc::NoiseSuppression::Level aLevel) {
278   AssertIsOnOwningThread();
279 
280   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
281   NS_DispatchToMainThread(
282       NS_NewRunnableFunction(__func__, [that, track = mTrack, aEnable, aLevel] {
283         class Message : public ControlMessage {
284          public:
285           Message(AudioInputProcessing* aInputProcessing, bool aEnable,
286                   webrtc::NoiseSuppression::Level aLevel)
287               : ControlMessage(nullptr),
288                 mInputProcessing(aInputProcessing),
289                 mEnable(aEnable),
290                 mLevel(aLevel) {}
291 
292           void Run() override {
293             mInputProcessing->UpdateNSSettings(mEnable, mLevel);
294           }
295 
296          protected:
297           RefPtr<AudioInputProcessing> mInputProcessing;
298           bool mEnable;
299           webrtc::NoiseSuppression::Level mLevel;
300         };
301 
302         if (track->IsDestroyed()) {
303           return;
304         }
305 
306         track->GraphImpl()->AppendMessage(
307             MakeUnique<Message>(that->mInputProcessing, aEnable, aLevel));
308       }));
309 }
310 
UpdateAPMExtraOptions(bool aExtendedFilter,bool aDelayAgnostic)311 void MediaEngineWebRTCMicrophoneSource::UpdateAPMExtraOptions(
312     bool aExtendedFilter, bool aDelayAgnostic) {
313   AssertIsOnOwningThread();
314 
315   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
316   NS_DispatchToMainThread(NS_NewRunnableFunction(
317       __func__, [that, track = mTrack, aExtendedFilter, aDelayAgnostic] {
318         class Message : public ControlMessage {
319          public:
320           Message(AudioInputProcessing* aInputProcessing, bool aExtendedFilter,
321                   bool aDelayAgnostic)
322               : ControlMessage(nullptr),
323                 mInputProcessing(aInputProcessing),
324                 mExtendedFilter(aExtendedFilter),
325                 mDelayAgnostic(aDelayAgnostic) {}
326 
327           void Run() override {
328             mInputProcessing->UpdateAPMExtraOptions(mExtendedFilter,
329                                                     mDelayAgnostic);
330           }
331 
332          protected:
333           RefPtr<AudioInputProcessing> mInputProcessing;
334           bool mExtendedFilter;
335           bool mDelayAgnostic;
336         };
337 
338         if (track->IsDestroyed()) {
339           return;
340         }
341 
342         track->GraphImpl()->AppendMessage(MakeUnique<Message>(
343             that->mInputProcessing, aExtendedFilter, aDelayAgnostic));
344       }));
345 }
346 
ApplySettings(const MediaEnginePrefs & aPrefs)347 void MediaEngineWebRTCMicrophoneSource::ApplySettings(
348     const MediaEnginePrefs& aPrefs) {
349   AssertIsOnOwningThread();
350 
351   MOZ_ASSERT(
352       mTrack,
353       "ApplySetting is to be called only after SetTrack has been called");
354 
355   if (mTrack) {
356     UpdateAGCSettings(aPrefs.mAgcOn,
357                       static_cast<webrtc::GainControl::Mode>(aPrefs.mAgc));
358     UpdateNSSettings(
359         aPrefs.mNoiseOn,
360         static_cast<webrtc::NoiseSuppression::Level>(aPrefs.mNoise));
361     UpdateAECSettings(
362         aPrefs.mAecOn, aPrefs.mUseAecMobile,
363         static_cast<webrtc::EchoCancellation::SuppressionLevel>(aPrefs.mAec),
364         static_cast<webrtc::EchoControlMobile::RoutingMode>(
365             aPrefs.mRoutingMode));
366     UpdateHPFSettings(aPrefs.mHPFOn);
367 
368     UpdateAPMExtraOptions(mExtendedFilter, mDelayAgnostic);
369   }
370 
371   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
372   NS_DispatchToMainThread(
373       NS_NewRunnableFunction(__func__, [that, track = mTrack, prefs = aPrefs] {
374         that->mSettings->mEchoCancellation.Value() = prefs.mAecOn;
375         that->mSettings->mAutoGainControl.Value() = prefs.mAgcOn;
376         that->mSettings->mNoiseSuppression.Value() = prefs.mNoiseOn;
377         that->mSettings->mChannelCount.Value() = prefs.mChannels;
378 
379         class Message : public ControlMessage {
380          public:
381           Message(MediaTrack* aTrack, AudioInputProcessing* aInputProcessing,
382                   bool aPassThrough, uint32_t aRequestedInputChannelCount)
383               : ControlMessage(aTrack),
384                 mInputProcessing(aInputProcessing),
385                 mPassThrough(aPassThrough),
386                 mRequestedInputChannelCount(aRequestedInputChannelCount) {}
387 
388           void Run() override {
389             mInputProcessing->SetPassThrough(mTrack->GraphImpl(), mPassThrough);
390             mInputProcessing->SetRequestedInputChannelCount(
391                 mTrack->GraphImpl(), mRequestedInputChannelCount);
392           }
393 
394          protected:
395           RefPtr<AudioInputProcessing> mInputProcessing;
396           bool mPassThrough;
397           uint32_t mRequestedInputChannelCount;
398         };
399 
400         // The high-pass filter is not taken into account when activating the
401         // pass through, since it's not controllable from content.
402         bool passThrough = !(prefs.mAecOn || prefs.mAgcOn || prefs.mNoiseOn);
403         if (track->IsDestroyed()) {
404           return;
405         }
406 
407         track->GraphImpl()->AppendMessage(MakeUnique<Message>(
408             track, that->mInputProcessing, passThrough, prefs.mChannels));
409       }));
410 }
411 
Allocate(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,uint64_t aWindowID,const char ** aOutBadConstraint)412 nsresult MediaEngineWebRTCMicrophoneSource::Allocate(
413     const dom::MediaTrackConstraints& aConstraints,
414     const MediaEnginePrefs& aPrefs, uint64_t aWindowID,
415     const char** aOutBadConstraint) {
416   AssertIsOnOwningThread();
417 
418   mState = kAllocated;
419 
420   NormalizedConstraints normalized(aConstraints);
421   MediaEnginePrefs outputPrefs;
422   nsresult rv =
423       EvaluateSettings(normalized, aPrefs, &outputPrefs, aOutBadConstraint);
424   if (NS_FAILED(rv)) {
425     return rv;
426   }
427 
428   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
429   NS_DispatchToMainThread(
430       NS_NewRunnableFunction(__func__, [that, prefs = outputPrefs] {
431         that->mSettings->mEchoCancellation.Value() = prefs.mAecOn;
432         that->mSettings->mAutoGainControl.Value() = prefs.mAgcOn;
433         that->mSettings->mNoiseSuppression.Value() = prefs.mNoiseOn;
434         that->mSettings->mChannelCount.Value() = prefs.mChannels;
435       }));
436 
437   mCurrentPrefs = outputPrefs;
438 
439   return rv;
440 }
441 
Deallocate()442 nsresult MediaEngineWebRTCMicrophoneSource::Deallocate() {
443   AssertIsOnOwningThread();
444 
445   MOZ_ASSERT(mState == kStopped || mState == kAllocated);
446 
447   class EndTrackMessage : public ControlMessage {
448    public:
449     EndTrackMessage(AudioInputTrack* aTrack,
450                     AudioInputProcessing* aAudioInputProcessing)
451         : ControlMessage(aTrack),
452           mInputProcessing(aAudioInputProcessing),
453           mInputTrack(aTrack) {}
454 
455     void Run() override { mInputProcessing->End(); }
456 
457    protected:
458     const RefPtr<AudioInputProcessing> mInputProcessing;
459     AudioInputTrack const* mInputTrack;
460   };
461 
462   if (mTrack) {
463     RefPtr<AudioInputProcessing> inputProcessing = mInputProcessing;
464     NS_DispatchToMainThread(NS_NewRunnableFunction(
465         __func__, [track = std::move(mTrack),
466                    audioInputProcessing = std::move(inputProcessing)] {
467           if (track->IsDestroyed()) {
468             // This track has already been destroyed on main thread by its
469             // DOMMediaStream. No cleanup left to do.
470             return;
471           }
472           track->GraphImpl()->AppendMessage(
473               MakeUnique<EndTrackMessage>(track, audioInputProcessing));
474         }));
475   }
476 
477   // Reset all state. This is not strictly necessary, this instance will get
478   // destroyed soon.
479   mTrack = nullptr;
480   mPrincipal = PRINCIPAL_HANDLE_NONE;
481 
482   // If empty, no callbacks to deliver data should be occuring
483   MOZ_ASSERT(mState != kReleased, "Source not allocated");
484   MOZ_ASSERT(mState != kStarted, "Source not stopped");
485 
486   mState = kReleased;
487   LOG("Mic source %p Audio device %s deallocated", this,
488       NS_ConvertUTF16toUTF8(mDeviceName).get());
489   return NS_OK;
490 }
491 
SetTrack(const RefPtr<MediaTrack> & aTrack,const PrincipalHandle & aPrincipal)492 void MediaEngineWebRTCMicrophoneSource::SetTrack(
493     const RefPtr<MediaTrack>& aTrack, const PrincipalHandle& aPrincipal) {
494   AssertIsOnOwningThread();
495   MOZ_ASSERT(aTrack);
496   MOZ_ASSERT(aTrack->AsAudioInputTrack());
497 
498   MOZ_ASSERT(!mTrack);
499   MOZ_ASSERT(mPrincipal == PRINCIPAL_HANDLE_NONE);
500   mTrack = aTrack->AsAudioInputTrack();
501   mPrincipal = aPrincipal;
502 
503   mInputProcessing =
504       MakeAndAddRef<AudioInputProcessing>(mDeviceMaxChannelCount, mPrincipal);
505 
506   NS_DispatchToMainThread(NS_NewRunnableFunction(
507       __func__, [track = mTrack, processing = mInputProcessing]() mutable {
508         track->SetInputProcessing(std::move(processing));
509         track->Resume();  // Suspended by MediaManager
510       }));
511 
512   LOG("Mic source %p Track %p registered for microphone capture", this,
513       aTrack.get());
514 }
515 
516 class StartStopMessage : public ControlMessage {
517  public:
518   enum StartStop { Start, Stop };
519 
StartStopMessage(AudioInputProcessing * aInputProcessing,StartStop aAction)520   StartStopMessage(AudioInputProcessing* aInputProcessing, StartStop aAction)
521       : ControlMessage(nullptr),
522         mInputProcessing(aInputProcessing),
523         mAction(aAction) {}
524 
Run()525   void Run() override {
526     if (mAction == StartStopMessage::Start) {
527       mInputProcessing->Start();
528     } else if (mAction == StartStopMessage::Stop) {
529       mInputProcessing->Stop();
530     } else {
531       MOZ_CRASH("Invalid enum value");
532     }
533   }
534 
535  protected:
536   RefPtr<AudioInputProcessing> mInputProcessing;
537   StartStop mAction;
538 };
539 
Start()540 nsresult MediaEngineWebRTCMicrophoneSource::Start() {
541   AssertIsOnOwningThread();
542 
543   // This spans setting both the enabled state and mState.
544   if (mState == kStarted) {
545     return NS_OK;
546   }
547 
548   MOZ_ASSERT(mState == kAllocated || mState == kStopped);
549 
550   // This check is unreliable due to potential in-flight device updates.
551   // Multiple input devices are reliably excluded in OpenAudioInputImpl(),
552   // but the check here provides some error reporting most of the
553   // time.
554   CubebUtils::AudioDeviceID deviceID = mDeviceInfo->DeviceID();
555   if (mTrack->GraphImpl()->InputDeviceID() &&
556       mTrack->GraphImpl()->InputDeviceID() != deviceID) {
557     // For now, we only allow opening a single audio input device per document,
558     // because we can only have one MTG per document.
559     return NS_ERROR_NOT_AVAILABLE;
560   }
561 
562   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
563   NS_DispatchToMainThread(
564       NS_NewRunnableFunction(__func__, [that, deviceID, track = mTrack] {
565         if (track->IsDestroyed()) {
566           return;
567         }
568 
569         track->GraphImpl()->AppendMessage(MakeUnique<StartStopMessage>(
570             that->mInputProcessing, StartStopMessage::Start));
571         track->OpenAudioInput(deviceID, that->mInputProcessing);
572       }));
573 
574   ApplySettings(mCurrentPrefs);
575 
576   MOZ_ASSERT(mState != kReleased);
577   mState = kStarted;
578 
579   return NS_OK;
580 }
581 
Stop()582 nsresult MediaEngineWebRTCMicrophoneSource::Stop() {
583   AssertIsOnOwningThread();
584 
585   LOG("Mic source %p Stop()", this);
586   MOZ_ASSERT(mTrack, "SetTrack must have been called before ::Stop");
587 
588   if (mState == kStopped) {
589     // Already stopped - this is allowed
590     return NS_OK;
591   }
592 
593   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
594   NS_DispatchToMainThread(
595       NS_NewRunnableFunction(__func__, [that, track = mTrack] {
596         if (track->IsDestroyed()) {
597           return;
598         }
599 
600         track->GraphImpl()->AppendMessage(MakeUnique<StartStopMessage>(
601             that->mInputProcessing, StartStopMessage::Stop));
602         MOZ_ASSERT(track->DeviceId().value() == that->mDeviceInfo->DeviceID());
603         track->CloseAudioInput();
604       }));
605 
606   MOZ_ASSERT(mState == kStarted, "Should be started when stopping");
607   mState = kStopped;
608 
609   return NS_OK;
610 }
611 
GetSettings(dom::MediaTrackSettings & aOutSettings) const612 void MediaEngineWebRTCMicrophoneSource::GetSettings(
613     dom::MediaTrackSettings& aOutSettings) const {
614   MOZ_ASSERT(NS_IsMainThread());
615   aOutSettings = *mSettings;
616 }
617 
AudioInputProcessing(uint32_t aMaxChannelCount,const PrincipalHandle & aPrincipalHandle)618 AudioInputProcessing::AudioInputProcessing(
619     uint32_t aMaxChannelCount, const PrincipalHandle& aPrincipalHandle)
620     : mAudioProcessing(AudioProcessing::Create()),
621       mRequestedInputChannelCount(aMaxChannelCount),
622       mSkipProcessing(false),
623       mInputDownmixBuffer(MAX_SAMPLING_FREQ * MAX_CHANNELS / 100),
624       mLiveFramesAppended(false),
625       mLiveBufferingAppended(0),
626       mPrincipal(aPrincipalHandle),
627       mEnabled(false),
628       mEnded(false) {}
629 
Disconnect(MediaTrackGraphImpl * aGraph)630 void AudioInputProcessing::Disconnect(MediaTrackGraphImpl* aGraph) {
631   // This method is just for asserts.
632   MOZ_ASSERT(aGraph->OnGraphThread());
633 }
634 
PassThrough(MediaTrackGraphImpl * aGraph) const635 bool AudioInputProcessing::PassThrough(MediaTrackGraphImpl* aGraph) const {
636   MOZ_ASSERT(aGraph->OnGraphThread());
637   return mSkipProcessing;
638 }
639 
SetPassThrough(MediaTrackGraphImpl * aGraph,bool aPassThrough)640 void AudioInputProcessing::SetPassThrough(MediaTrackGraphImpl* aGraph,
641                                           bool aPassThrough) {
642   MOZ_ASSERT(aGraph->OnGraphThread());
643   if (!mSkipProcessing && aPassThrough && mPacketizerInput) {
644     MOZ_ASSERT(mPacketizerInput->PacketsAvailable() == 0);
645     LOG_FRAME(
646         "AudioInputProcessing %p Appending %u frames of null data for data "
647         "discarded in the packetizer",
648         this, mPacketizerInput->FramesAvailable());
649     mSegment.AppendNullData(mPacketizerInput->FramesAvailable());
650     mPacketizerInput->Clear();
651   }
652   mSkipProcessing = aPassThrough;
653 }
654 
GetRequestedInputChannelCount()655 uint32_t AudioInputProcessing::GetRequestedInputChannelCount() {
656   return mRequestedInputChannelCount;
657 }
658 
SetRequestedInputChannelCount(MediaTrackGraphImpl * aGraph,uint32_t aRequestedInputChannelCount)659 void AudioInputProcessing::SetRequestedInputChannelCount(
660     MediaTrackGraphImpl* aGraph, uint32_t aRequestedInputChannelCount) {
661   mRequestedInputChannelCount = aRequestedInputChannelCount;
662 
663   aGraph->ReevaluateInputDevice();
664 }
665 
666 // This does an early return in case of error.
667 #define HANDLE_APM_ERROR(fn)                       \
668   do {                                             \
669     int rv = fn;                                   \
670     if (rv != AudioProcessing::kNoError) {         \
671       MOZ_ASSERT_UNREACHABLE("APM error in " #fn); \
672       return;                                      \
673     }                                              \
674   } while (0);
675 
UpdateAECSettings(bool aEnable,bool aUseAecMobile,EchoCancellation::SuppressionLevel aLevel,EchoControlMobile::RoutingMode aRoutingMode)676 void AudioInputProcessing::UpdateAECSettings(
677     bool aEnable, bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
678     EchoControlMobile::RoutingMode aRoutingMode) {
679   if (aUseAecMobile) {
680     HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(aEnable));
681     HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->set_routing_mode(
682         aRoutingMode));
683     HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(false));
684   } else {
685     if (aLevel != EchoCancellation::SuppressionLevel::kLowSuppression &&
686         aLevel != EchoCancellation::SuppressionLevel::kModerateSuppression &&
687         aLevel != EchoCancellation::SuppressionLevel::kHighSuppression) {
688       LOG_ERROR(
689           "AudioInputProcessing %p Attempt to set invalid AEC suppression "
690           "level %d",
691           this, static_cast<int>(aLevel));
692 
693       aLevel = EchoCancellation::SuppressionLevel::kModerateSuppression;
694     }
695 
696     HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(false));
697     HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(aEnable));
698     HANDLE_APM_ERROR(
699         mAudioProcessing->echo_cancellation()->set_suppression_level(aLevel));
700   }
701 }
702 
UpdateAGCSettings(bool aEnable,GainControl::Mode aMode)703 void AudioInputProcessing::UpdateAGCSettings(bool aEnable,
704                                              GainControl::Mode aMode) {
705   if (aMode != GainControl::Mode::kAdaptiveAnalog &&
706       aMode != GainControl::Mode::kAdaptiveDigital &&
707       aMode != GainControl::Mode::kFixedDigital) {
708     LOG_ERROR("AudioInputProcessing %p Attempt to set invalid AGC mode %d",
709               this, static_cast<int>(aMode));
710 
711     aMode = GainControl::Mode::kAdaptiveDigital;
712   }
713 
714 #if defined(WEBRTC_IOS) || defined(ATA) || defined(WEBRTC_ANDROID)
715   if (aMode == GainControl::Mode::kAdaptiveAnalog) {
716     LOG_ERROR(
717         "AudioInputProcessing %p Invalid AGC mode kAgcAdaptiveAnalog on "
718         "mobile",
719         this);
720     MOZ_ASSERT_UNREACHABLE(
721         "Bad pref set in all.js or in about:config"
722         " for the auto gain, on mobile.");
723     aMode = GainControl::Mode::kFixedDigital;
724   }
725 #endif
726   HANDLE_APM_ERROR(mAudioProcessing->gain_control()->set_mode(aMode));
727   HANDLE_APM_ERROR(mAudioProcessing->gain_control()->Enable(aEnable));
728 }
729 
UpdateHPFSettings(bool aEnable)730 void AudioInputProcessing::UpdateHPFSettings(bool aEnable) {
731   HANDLE_APM_ERROR(mAudioProcessing->high_pass_filter()->Enable(aEnable));
732 }
733 
UpdateNSSettings(bool aEnable,webrtc::NoiseSuppression::Level aLevel)734 void AudioInputProcessing::UpdateNSSettings(
735     bool aEnable, webrtc::NoiseSuppression::Level aLevel) {
736   if (aLevel != NoiseSuppression::Level::kLow &&
737       aLevel != NoiseSuppression::Level::kModerate &&
738       aLevel != NoiseSuppression::Level::kHigh &&
739       aLevel != NoiseSuppression::Level::kVeryHigh) {
740     LOG_ERROR(
741         "AudioInputProcessing %p Attempt to set invalid noise suppression "
742         "level %d",
743         this, static_cast<int>(aLevel));
744 
745     aLevel = NoiseSuppression::Level::kModerate;
746   }
747 
748   HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->set_level(aLevel));
749   HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->Enable(aEnable));
750 }
751 
752 #undef HANDLE_APM_ERROR
753 
UpdateAPMExtraOptions(bool aExtendedFilter,bool aDelayAgnostic)754 void AudioInputProcessing::UpdateAPMExtraOptions(bool aExtendedFilter,
755                                                  bool aDelayAgnostic) {
756   webrtc::Config config;
757   config.Set<webrtc::ExtendedFilter>(
758       new webrtc::ExtendedFilter(aExtendedFilter));
759   config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(aDelayAgnostic));
760 
761   mAudioProcessing->SetExtraOptions(config);
762 }
763 
Start()764 void AudioInputProcessing::Start() {
765   mEnabled = true;
766   mLiveFramesAppended = false;
767 }
768 
Stop()769 void AudioInputProcessing::Stop() { mEnabled = false; }
770 
Pull(MediaTrackGraphImpl * aGraph,GraphTime aFrom,GraphTime aTo,GraphTime aTrackEnd,AudioSegment * aSegment,bool aLastPullThisIteration,bool * aEnded)771 void AudioInputProcessing::Pull(MediaTrackGraphImpl* aGraph, GraphTime aFrom,
772                                 GraphTime aTo, GraphTime aTrackEnd,
773                                 AudioSegment* aSegment,
774                                 bool aLastPullThisIteration, bool* aEnded) {
775   MOZ_ASSERT(aGraph->OnGraphThread());
776 
777   if (mEnded) {
778     *aEnded = true;
779     return;
780   }
781 
782   TrackTime delta = aTo - aTrackEnd;
783   MOZ_ASSERT(delta >= 0, "We shouldn't append more than requested");
784   TrackTime buffering = 0;
785 
786   // Add the amount of buffering required to not underrun and glitch.
787 
788   // Make sure there's at least one extra block buffered until audio callbacks
789   // come in, since we round graph iteration durations up to the nearest block.
790   buffering += WEBAUDIO_BLOCK_SIZE;
791 
792   // If we're supposed to be packetizing but there's no packetizer yet,
793   // there must not have been any live frames appended yet.
794   MOZ_ASSERT_IF(!PassThrough(aGraph) && !mPacketizerInput,
795                 mSegment.GetDuration() == 0);
796 
797   if (!PassThrough(aGraph) && mPacketizerInput) {
798     // Processing is active and is processed in chunks of 10ms through the
799     // input packetizer. We allow for 10ms of silence on the track to
800     // accomodate the buffering worst-case.
801     buffering += mPacketizerInput->mPacketSize;
802   }
803 
804   if (delta <= 0) {
805     return;
806   }
807 
808   if (MOZ_LIKELY(mLiveFramesAppended)) {
809     if (MOZ_UNLIKELY(buffering > mLiveBufferingAppended)) {
810       // We need to buffer more data. This could happen the first time we pull
811       // input data, or the first iteration after starting to use the
812       // packetizer.
813       LOG_FRAME("AudioInputProcessing %p Inserting %" PRId64
814                 " frames of silence due to buffer increase",
815                 this, buffering - mLiveBufferingAppended);
816       mSegment.InsertNullDataAtStart(buffering - mLiveBufferingAppended);
817       mLiveBufferingAppended = buffering;
818     } else if (MOZ_UNLIKELY(buffering < mLiveBufferingAppended)) {
819       // We need to clear some buffered data to reduce latency now that the
820       // packetizer is no longer used.
821       MOZ_ASSERT(PassThrough(aGraph), "Must have turned on passthrough");
822       MOZ_ASSERT(mSegment.GetDuration() >=
823                  (mLiveBufferingAppended - buffering));
824       TrackTime frames =
825           std::min(mSegment.GetDuration(), mLiveBufferingAppended - buffering);
826       LOG_FRAME("AudioInputProcessing %p Removing %" PRId64
827                 " frames of silence due to buffer decrease",
828                 this, frames);
829       mLiveBufferingAppended -= frames;
830       mSegment.RemoveLeading(frames);
831     }
832   }
833 
834   if (mSegment.GetDuration() > 0) {
835     MOZ_ASSERT(buffering == mLiveBufferingAppended);
836     TrackTime frames = std::min(mSegment.GetDuration(), delta);
837     LOG_FRAME("AudioInputProcessing %p Appending %" PRId64
838               " frames of real data for %u channels.",
839               this, frames, mRequestedInputChannelCount);
840     aSegment->AppendSlice(mSegment, 0, frames);
841     mSegment.RemoveLeading(frames);
842     delta -= frames;
843 
844     // Assert that the amount of data buffered doesn't grow unboundedly.
845     MOZ_ASSERT_IF(aLastPullThisIteration, mSegment.GetDuration() <= buffering);
846   }
847 
848   if (delta <= 0) {
849     if (mSegment.GetDuration() == 0) {
850       mLiveBufferingAppended = -delta;
851     }
852     return;
853   }
854 
855   LOG_FRAME("AudioInputProcessing %p Pulling %" PRId64
856             " frames of silence for %u channels.",
857             this, delta, mRequestedInputChannelCount);
858 
859   // This assertion fails if we append silence here after having appended live
860   // frames. Before appending live frames we should add sufficient buffering to
861   // not have to glitch (aka append silence). Failing this meant the buffering
862   // was not sufficient.
863   MOZ_ASSERT_IF(mEnabled, !mLiveFramesAppended);
864   mLiveBufferingAppended = 0;
865 
866   aSegment->AppendNullData(delta);
867 }
868 
NotifyOutputData(MediaTrackGraphImpl * aGraph,BufferInfo aInfo)869 void AudioInputProcessing::NotifyOutputData(MediaTrackGraphImpl* aGraph,
870                                             BufferInfo aInfo) {
871   MOZ_ASSERT(aGraph->OnGraphThread());
872   MOZ_ASSERT(mEnabled);
873 
874   if (!mPacketizerOutput ||
875       mPacketizerOutput->mPacketSize != aInfo.mRate / 100u ||
876       mPacketizerOutput->mChannels != aInfo.mChannels) {
877     // It's ok to drop the audio still in the packetizer here: if this changes,
878     // we changed devices or something.
879     mPacketizerOutput = MakeUnique<AudioPacketizer<AudioDataValue, float>>(
880         aInfo.mRate / 100, aInfo.mChannels);
881   }
882 
883   mPacketizerOutput->Input(aInfo.mBuffer, aInfo.mFrames);
884 
885   while (mPacketizerOutput->PacketsAvailable()) {
886     uint32_t samplesPerPacket =
887         mPacketizerOutput->mPacketSize * mPacketizerOutput->mChannels;
888     if (mOutputBuffer.Length() < samplesPerPacket) {
889       mOutputBuffer.SetLength(samplesPerPacket);
890     }
891     if (mDeinterleavedBuffer.Length() < samplesPerPacket) {
892       mDeinterleavedBuffer.SetLength(samplesPerPacket);
893     }
894     float* packet = mOutputBuffer.Data();
895     mPacketizerOutput->Output(packet);
896 
897     AutoTArray<float*, MAX_CHANNELS> deinterleavedPacketDataChannelPointers;
898     float* interleavedFarend = nullptr;
899     uint32_t channelCountFarend = 0;
900     uint32_t framesPerPacketFarend = 0;
901 
902     // Downmix from aInfo.mChannels to MAX_CHANNELS if needed. We always have
903     // floats here, the packetized performed the conversion.
904     if (aInfo.mChannels > MAX_CHANNELS) {
905       AudioConverter converter(
906           AudioConfig(aInfo.mChannels, 0, AudioConfig::FORMAT_FLT),
907           AudioConfig(MAX_CHANNELS, 0, AudioConfig::FORMAT_FLT));
908       framesPerPacketFarend = mPacketizerOutput->mPacketSize;
909       framesPerPacketFarend =
910           converter.Process(mInputDownmixBuffer, packet, framesPerPacketFarend);
911       interleavedFarend = mInputDownmixBuffer.Data();
912       channelCountFarend = MAX_CHANNELS;
913       deinterleavedPacketDataChannelPointers.SetLength(MAX_CHANNELS);
914     } else {
915       interleavedFarend = packet;
916       channelCountFarend = aInfo.mChannels;
917       framesPerPacketFarend = mPacketizerOutput->mPacketSize;
918       deinterleavedPacketDataChannelPointers.SetLength(aInfo.mChannels);
919     }
920 
921     MOZ_ASSERT(interleavedFarend &&
922                (channelCountFarend == 1 || channelCountFarend == 2) &&
923                framesPerPacketFarend);
924 
925     if (mInputBuffer.Length() < framesPerPacketFarend * channelCountFarend) {
926       mInputBuffer.SetLength(framesPerPacketFarend * channelCountFarend);
927     }
928 
929     size_t offset = 0;
930     for (size_t i = 0; i < deinterleavedPacketDataChannelPointers.Length();
931          ++i) {
932       deinterleavedPacketDataChannelPointers[i] = mInputBuffer.Data() + offset;
933       offset += framesPerPacketFarend;
934     }
935 
936     // Deinterleave, prepare a channel pointers array, with enough storage for
937     // the frames.
938     DeinterleaveAndConvertBuffer(
939         interleavedFarend, framesPerPacketFarend, channelCountFarend,
940         deinterleavedPacketDataChannelPointers.Elements());
941 
942     // Having the same config for input and output means we potentially save
943     // some CPU.
944     StreamConfig inputConfig(aInfo.mRate, channelCountFarend, false);
945     StreamConfig outputConfig = inputConfig;
946 
947     // Passing the same pointers here saves a copy inside this function.
948     DebugOnly<int> err = mAudioProcessing->ProcessReverseStream(
949         deinterleavedPacketDataChannelPointers.Elements(), inputConfig,
950         outputConfig, deinterleavedPacketDataChannelPointers.Elements());
951 
952     MOZ_ASSERT(!err, "Could not process the reverse stream.");
953   }
954 }
955 
956 // Only called if we're not in passthrough mode
PacketizeAndProcess(MediaTrackGraphImpl * aGraph,const AudioDataValue * aBuffer,size_t aFrames,TrackRate aRate,uint32_t aChannels)957 void AudioInputProcessing::PacketizeAndProcess(MediaTrackGraphImpl* aGraph,
958                                                const AudioDataValue* aBuffer,
959                                                size_t aFrames, TrackRate aRate,
960                                                uint32_t aChannels) {
961   MOZ_ASSERT(!PassThrough(aGraph),
962              "This should be bypassed when in PassThrough mode.");
963   MOZ_ASSERT(mEnabled);
964   size_t offset = 0;
965 
966   if (!mPacketizerInput || mPacketizerInput->mPacketSize != aRate / 100u ||
967       mPacketizerInput->mChannels != aChannels) {
968     // It's ok to drop the audio still in the packetizer here.
969     mPacketizerInput = MakeUnique<AudioPacketizer<AudioDataValue, float>>(
970         aRate / 100, aChannels);
971   }
972 
973   LOG_FRAME("AudioInputProcessing %p Appending %zu frames to packetizer", this,
974             aFrames);
975 
976   // Packetize our input data into 10ms chunks, deinterleave into planar channel
977   // buffers, process, and append to the right MediaStreamTrack.
978   mPacketizerInput->Input(aBuffer, static_cast<uint32_t>(aFrames));
979 
980   while (mPacketizerInput->PacketsAvailable()) {
981     uint32_t samplesPerPacket =
982         mPacketizerInput->mPacketSize * mPacketizerInput->mChannels;
983     if (mInputBuffer.Length() < samplesPerPacket) {
984       mInputBuffer.SetLength(samplesPerPacket);
985     }
986     if (mDeinterleavedBuffer.Length() < samplesPerPacket) {
987       mDeinterleavedBuffer.SetLength(samplesPerPacket);
988     }
989     float* packet = mInputBuffer.Data();
990     mPacketizerInput->Output(packet);
991 
992     // Downmix from aChannels to mono if needed. We always have floats
993     // here, the packetizer performed the conversion. This handles sound cards
994     // with multiple physical jacks exposed as a single device with _n_
995     // discrete channels, where only a single mic is plugged in. Those channels
996     // are not correlated temporaly since they are discrete channels, mixing is
997     // just a sum.
998     AutoTArray<float*, 8> deinterleavedPacketizedInputDataChannelPointers;
999     uint32_t channelCountInput = 0;
1000     if (aChannels > MAX_CHANNELS) {
1001       channelCountInput = MONO;
1002       deinterleavedPacketizedInputDataChannelPointers.SetLength(
1003           channelCountInput);
1004       deinterleavedPacketizedInputDataChannelPointers[0] =
1005           mDeinterleavedBuffer.Data();
1006       // Downmix to mono (and effectively have a planar buffer) by summing all
1007       // channels in the first channel.
1008       size_t readIndex = 0;
1009       for (size_t i = 0; i < mPacketizerInput->mPacketSize; i++) {
1010         mDeinterleavedBuffer.Data()[i] = 0.;
1011         for (size_t j = 0; j < aChannels; j++) {
1012           mDeinterleavedBuffer.Data()[i] += packet[readIndex++];
1013         }
1014       }
1015     } else {
1016       channelCountInput = aChannels;
1017       // Deinterleave the input data
1018       // Prepare an array pointing to deinterleaved channels.
1019       deinterleavedPacketizedInputDataChannelPointers.SetLength(
1020           channelCountInput);
1021       offset = 0;
1022       for (size_t i = 0;
1023            i < deinterleavedPacketizedInputDataChannelPointers.Length(); ++i) {
1024         deinterleavedPacketizedInputDataChannelPointers[i] =
1025             mDeinterleavedBuffer.Data() + offset;
1026         offset += mPacketizerInput->mPacketSize;
1027       }
1028       // Deinterleave to mInputBuffer, pointed to by inputBufferChannelPointers.
1029       Deinterleave(packet, mPacketizerInput->mPacketSize, channelCountInput,
1030                    deinterleavedPacketizedInputDataChannelPointers.Elements());
1031     }
1032 
1033     StreamConfig inputConfig(aRate, channelCountInput,
1034                              false /* we don't use typing detection*/);
1035     StreamConfig outputConfig = inputConfig;
1036 
1037     // Bug 1404965: Get the right delay here, it saves some work down the line.
1038     mAudioProcessing->set_stream_delay_ms(0);
1039 
1040     // Bug 1414837: find a way to not allocate here.
1041     CheckedInt<size_t> bufferSize(sizeof(float));
1042     bufferSize *= mPacketizerInput->mPacketSize;
1043     bufferSize *= channelCountInput;
1044     RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize);
1045 
1046     // Prepare channel pointers to the SharedBuffer created above.
1047     AutoTArray<float*, 8> processedOutputChannelPointers;
1048     AutoTArray<const float*, 8> processedOutputChannelPointersConst;
1049     processedOutputChannelPointers.SetLength(channelCountInput);
1050     processedOutputChannelPointersConst.SetLength(channelCountInput);
1051 
1052     offset = 0;
1053     for (size_t i = 0; i < processedOutputChannelPointers.Length(); ++i) {
1054       processedOutputChannelPointers[i] =
1055           static_cast<float*>(buffer->Data()) + offset;
1056       processedOutputChannelPointersConst[i] =
1057           static_cast<float*>(buffer->Data()) + offset;
1058       offset += mPacketizerInput->mPacketSize;
1059     }
1060 
1061     mAudioProcessing->ProcessStream(
1062         deinterleavedPacketizedInputDataChannelPointers.Elements(), inputConfig,
1063         outputConfig, processedOutputChannelPointers.Elements());
1064 
1065     if (mEnded) {
1066       continue;
1067     }
1068 
1069     LOG_FRAME("AudioInputProcessing %p Appending %u frames of packetized audio",
1070               this, mPacketizerInput->mPacketSize);
1071 
1072     // We already have planar audio data of the right format. Insert into the
1073     // MTG.
1074     MOZ_ASSERT(processedOutputChannelPointers.Length() == channelCountInput);
1075     RefPtr<SharedBuffer> other = buffer;
1076     mSegment.AppendFrames(other.forget(), processedOutputChannelPointersConst,
1077                           mPacketizerInput->mPacketSize, mPrincipal);
1078   }
1079 }
1080 
ProcessInput(MediaTrackGraphImpl * aGraph,const AudioSegment * aSegment)1081 void AudioInputProcessing::ProcessInput(MediaTrackGraphImpl* aGraph,
1082                                         const AudioSegment* aSegment) {
1083   MOZ_ASSERT(aGraph);
1084   MOZ_ASSERT(aGraph->OnGraphThread());
1085 
1086   if (mEnded || !mEnabled || !mLiveFramesAppended || !mInputData) {
1087     return;
1088   }
1089 
1090   // One NotifyInputData might have multiple following ProcessInput calls, but
1091   // we only process one input per NotifyInputData call.
1092   BufferInfo inputInfo = mInputData.extract();
1093 
1094   // If some processing is necessary, packetize and insert in the WebRTC.org
1095   // code. Otherwise, directly insert the mic data in the MTG, bypassing all
1096   // processing.
1097   if (PassThrough(aGraph)) {
1098     if (aSegment) {
1099       mSegment.AppendSegment(aSegment, mPrincipal);
1100     } else {
1101       mSegment.AppendFromInterleavedBuffer(inputInfo.mBuffer, inputInfo.mFrames,
1102                                            inputInfo.mChannels, mPrincipal);
1103     }
1104   } else {
1105     MOZ_ASSERT(aGraph->GraphRate() == inputInfo.mRate);
1106     PacketizeAndProcess(aGraph, inputInfo.mBuffer, inputInfo.mFrames,
1107                         inputInfo.mRate, inputInfo.mChannels);
1108   }
1109 }
1110 
NotifyInputStopped(MediaTrackGraphImpl * aGraph)1111 void AudioInputProcessing::NotifyInputStopped(MediaTrackGraphImpl* aGraph) {
1112   MOZ_ASSERT(aGraph->OnGraphThread());
1113   // This is called when an AudioCallbackDriver switch has happened for any
1114   // reason, including other reasons than starting this audio input stream. We
1115   // reset state when this happens, as a fallback driver may have fiddled with
1116   // the amount of buffered silence during the switch.
1117   mLiveFramesAppended = false;
1118   mSegment.Clear();
1119   if (mPacketizerInput) {
1120     mPacketizerInput->Clear();
1121   }
1122   mInputData.take();
1123 }
1124 
1125 // Called back on GraphDriver thread!
1126 // Note this can be called back after ::Stop()
NotifyInputData(MediaTrackGraphImpl * aGraph,const BufferInfo aInfo,uint32_t aAlreadyBuffered)1127 void AudioInputProcessing::NotifyInputData(MediaTrackGraphImpl* aGraph,
1128                                            const BufferInfo aInfo,
1129                                            uint32_t aAlreadyBuffered) {
1130   MOZ_ASSERT(aGraph->OnGraphThread());
1131   TRACE();
1132 
1133   MOZ_ASSERT(mEnabled);
1134 
1135   if (!mLiveFramesAppended) {
1136     // First time we see live frames getting added. Use what's already buffered
1137     // in the driver's scratch buffer as a starting point.
1138     mLiveFramesAppended = true;
1139     mLiveBufferingAppended = aAlreadyBuffered;
1140   }
1141 
1142   mInputData = Some(aInfo);
1143 }
1144 
1145 #define ResetProcessingIfNeeded(_processing)                         \
1146   do {                                                               \
1147     bool enabled = mAudioProcessing->_processing()->is_enabled();    \
1148                                                                      \
1149     if (enabled) {                                                   \
1150       int rv = mAudioProcessing->_processing()->Enable(!enabled);    \
1151       if (rv) {                                                      \
1152         NS_WARNING("Could not reset the status of the " #_processing \
1153                    " on device change.");                            \
1154         return;                                                      \
1155       }                                                              \
1156       rv = mAudioProcessing->_processing()->Enable(enabled);         \
1157       if (rv) {                                                      \
1158         NS_WARNING("Could not reset the status of the " #_processing \
1159                    " on device change.");                            \
1160         return;                                                      \
1161       }                                                              \
1162     }                                                                \
1163   } while (0)
1164 
DeviceChanged(MediaTrackGraphImpl * aGraph)1165 void AudioInputProcessing::DeviceChanged(MediaTrackGraphImpl* aGraph) {
1166   MOZ_ASSERT(aGraph->OnGraphThread());
1167   // Reset some processing
1168   ResetProcessingIfNeeded(gain_control);
1169   ResetProcessingIfNeeded(echo_cancellation);
1170   ResetProcessingIfNeeded(noise_suppression);
1171 }
1172 
End()1173 void AudioInputProcessing::End() {
1174   mEnded = true;
1175   mSegment.Clear();
1176   mInputData.take();
1177 }
1178 
NumBufferedFrames(MediaTrackGraphImpl * aGraph) const1179 TrackTime AudioInputProcessing::NumBufferedFrames(
1180     MediaTrackGraphImpl* aGraph) const {
1181   MOZ_ASSERT(aGraph->OnGraphThread());
1182   return mSegment.GetDuration();
1183 }
1184 
Destroy()1185 void AudioInputTrack::Destroy() {
1186   MOZ_ASSERT(NS_IsMainThread());
1187   CloseAudioInput();
1188 
1189   MediaTrack::Destroy();
1190 }
1191 
SetInputProcessing(RefPtr<AudioInputProcessing> aInputProcessing)1192 void AudioInputTrack::SetInputProcessing(
1193     RefPtr<AudioInputProcessing> aInputProcessing) {
1194   class Message : public ControlMessage {
1195     RefPtr<AudioInputTrack> mTrack;
1196     RefPtr<AudioInputProcessing> mProcessing;
1197 
1198    public:
1199     Message(RefPtr<AudioInputTrack> aTrack,
1200             RefPtr<AudioInputProcessing> aProcessing)
1201         : ControlMessage(aTrack),
1202           mTrack(std::move(aTrack)),
1203           mProcessing(std::move(aProcessing)) {}
1204     void Run() override {
1205       mTrack->SetInputProcessingImpl(std::move(mProcessing));
1206     }
1207   };
1208 
1209   if (IsDestroyed()) {
1210     return;
1211   }
1212   GraphImpl()->AppendMessage(
1213       MakeUnique<Message>(std::move(this), std::move(aInputProcessing)));
1214 }
1215 
Create(MediaTrackGraph * aGraph)1216 AudioInputTrack* AudioInputTrack::Create(MediaTrackGraph* aGraph) {
1217   MOZ_ASSERT(NS_IsMainThread());
1218   AudioInputTrack* track = new AudioInputTrack(aGraph->GraphRate());
1219   aGraph->AddTrack(track);
1220   return track;
1221 }
1222 
DestroyImpl()1223 void AudioInputTrack::DestroyImpl() {
1224   ProcessedMediaTrack::DestroyImpl();
1225   if (mInputProcessing) {
1226     mInputProcessing->End();
1227   }
1228 }
1229 
ProcessInput(GraphTime aFrom,GraphTime aTo,uint32_t aFlags)1230 void AudioInputTrack::ProcessInput(GraphTime aFrom, GraphTime aTo,
1231                                    uint32_t aFlags) {
1232   TRACE_COMMENT("AudioInputTrack %p", this);
1233 
1234   // Check if there is a connected NativeInputTrack
1235   NativeInputTrack* source = nullptr;
1236   if (!mInputs.IsEmpty()) {
1237     for (const MediaInputPort* input : mInputs) {
1238       MOZ_ASSERT(input->GetSource());
1239       if (input->GetSource()->AsNativeInputTrack()) {
1240         source = input->GetSource()->AsNativeInputTrack();
1241         break;
1242       }
1243     }
1244   }
1245 
1246   // Push the input data from the connected NativeInputTrack to mInputProcessing
1247   if (source) {
1248     MOZ_ASSERT(source->GraphImpl() == GraphImpl());
1249     MOZ_ASSERT(source->mSampleRate == mSampleRate);
1250     MOZ_ASSERT(GraphImpl()->GraphRate() == mSampleRate);
1251     mInputProcessing->ProcessInput(GraphImpl(),
1252                                    source->GetData<AudioSegment>());
1253   }
1254 
1255   bool ended = false;
1256   mInputProcessing->Pull(
1257       GraphImpl(), aFrom, aTo, TrackTimeToGraphTime(GetEnd()),
1258       GetData<AudioSegment>(), aTo == GraphImpl()->mStateComputedTime, &ended);
1259   ApplyTrackDisabling(mSegment.get());
1260   if (ended && (aFlags & ALLOW_END)) {
1261     mEnded = true;
1262   }
1263 }
1264 
SetInputProcessingImpl(RefPtr<AudioInputProcessing> aInputProcessing)1265 void AudioInputTrack::SetInputProcessingImpl(
1266     RefPtr<AudioInputProcessing> aInputProcessing) {
1267   MOZ_ASSERT(GraphImpl()->OnGraphThread());
1268   mInputProcessing = std::move(aInputProcessing);
1269 }
1270 
OpenAudioInput(CubebUtils::AudioDeviceID aId,AudioDataListener * aListener)1271 nsresult AudioInputTrack::OpenAudioInput(CubebUtils::AudioDeviceID aId,
1272                                          AudioDataListener* aListener) {
1273   MOZ_ASSERT(NS_IsMainThread());
1274   MOZ_ASSERT(GraphImpl());
1275   MOZ_ASSERT(!mInputListener);
1276   MOZ_ASSERT(mDeviceId.isNothing());
1277   mInputListener = aListener;
1278   ProcessedMediaTrack* input = GraphImpl()->GetDeviceTrack(aId);
1279   MOZ_ASSERT(input);
1280   LOG("Open device %p (InputTrack=%p) for Mic source %p", aId, input, this);
1281   mPort = AllocateInputPort(input);
1282   mDeviceId.emplace(aId);
1283   return GraphImpl()->OpenAudioInput(aId, aListener);
1284 }
1285 
CloseAudioInput()1286 void AudioInputTrack::CloseAudioInput() {
1287   MOZ_ASSERT(NS_IsMainThread());
1288   MOZ_ASSERT(GraphImpl());
1289   if (!mInputListener) {
1290     return;
1291   }
1292   MOZ_ASSERT(mPort);
1293   MOZ_ASSERT(mDeviceId.isSome());
1294   LOG("Close device %p (InputTrack=%p) for Mic source %p ", mDeviceId.value(),
1295       mPort->GetSource(), this);
1296   mPort->Destroy();
1297   GraphImpl()->CloseAudioInput(mDeviceId.extract(), mInputListener);
1298   mInputListener = nullptr;
1299 }
1300 
DeviceId() const1301 Maybe<CubebUtils::AudioDeviceID> AudioInputTrack::DeviceId() const {
1302   MOZ_ASSERT(NS_IsMainThread());
1303   return mDeviceId;
1304 }
1305 
GetName() const1306 nsString MediaEngineWebRTCAudioCaptureSource::GetName() const {
1307   return u"AudioCapture"_ns;
1308 }
1309 
GetUUID() const1310 nsCString MediaEngineWebRTCAudioCaptureSource::GetUUID() const {
1311   nsID uuid;
1312   char uuidBuffer[NSID_LENGTH];
1313   nsCString asciiString;
1314   ErrorResult rv;
1315 
1316   rv = nsContentUtils::GenerateUUIDInPlace(uuid);
1317   if (rv.Failed()) {
1318     return ""_ns;
1319   }
1320 
1321   uuid.ToProvidedString(uuidBuffer);
1322   asciiString.AssignASCII(uuidBuffer);
1323 
1324   // Remove {} and the null terminator
1325   return nsCString(Substring(asciiString, 1, NSID_LENGTH - 3));
1326 }
1327 
GetGroupId() const1328 nsString MediaEngineWebRTCAudioCaptureSource::GetGroupId() const {
1329   return u"AudioCaptureGroup"_ns;
1330 }
1331 
SetTrack(const RefPtr<MediaTrack> & aTrack,const PrincipalHandle & aPrincipalHandle)1332 void MediaEngineWebRTCAudioCaptureSource::SetTrack(
1333     const RefPtr<MediaTrack>& aTrack, const PrincipalHandle& aPrincipalHandle) {
1334   AssertIsOnOwningThread();
1335   // Nothing to do here. aTrack is a placeholder dummy and not exposed.
1336 }
1337 
Start()1338 nsresult MediaEngineWebRTCAudioCaptureSource::Start() {
1339   AssertIsOnOwningThread();
1340   return NS_OK;
1341 }
1342 
Stop()1343 nsresult MediaEngineWebRTCAudioCaptureSource::Stop() {
1344   AssertIsOnOwningThread();
1345   return NS_OK;
1346 }
1347 
Reconfigure(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const char ** aOutBadConstraint)1348 nsresult MediaEngineWebRTCAudioCaptureSource::Reconfigure(
1349     const dom::MediaTrackConstraints& aConstraints,
1350     const MediaEnginePrefs& aPrefs, const char** aOutBadConstraint) {
1351   return NS_OK;
1352 }
1353 
GetSettings(dom::MediaTrackSettings & aOutSettings) const1354 void MediaEngineWebRTCAudioCaptureSource::GetSettings(
1355     dom::MediaTrackSettings& aOutSettings) const {
1356   aOutSettings.mAutoGainControl.Construct(false);
1357   aOutSettings.mEchoCancellation.Construct(false);
1358   aOutSettings.mNoiseSuppression.Construct(false);
1359   aOutSettings.mChannelCount.Construct(1);
1360 }
1361 
1362 }  // namespace mozilla
1363