1 /*
2  *  Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifdef HAVE_WEBRTC_VOICE
12 
13 #include "media/engine/webrtcvoiceengine.h"
14 
15 #include <algorithm>
16 #include <cstdio>
17 #include <functional>
18 #include <string>
19 #include <utility>
20 #include <vector>
21 
22 #include "api/call/audio_sink.h"
23 #include "media/base/audiosource.h"
24 #include "media/base/mediaconstants.h"
25 #include "media/base/streamparams.h"
26 #include "media/engine/adm_helpers.h"
27 #include "media/engine/apm_helpers.h"
28 #include "media/engine/payload_type_mapper.h"
29 #include "media/engine/webrtcmediaengine.h"
30 #include "media/engine/webrtcvoe.h"
31 #include "modules/audio_device/audio_device_impl.h"
32 #include "modules/audio_mixer/audio_mixer_impl.h"
33 #include "modules/audio_processing/aec_dump/aec_dump_factory.h"
34 #include "modules/audio_processing/include/audio_processing.h"
35 #include "rtc_base/arraysize.h"
36 #include "rtc_base/base64.h"
37 #include "rtc_base/byteorder.h"
38 #include "rtc_base/constructormagic.h"
39 #include "rtc_base/helpers.h"
40 #include "rtc_base/logging.h"
41 #include "rtc_base/race_checker.h"
42 #include "rtc_base/stringencode.h"
43 #include "rtc_base/stringutils.h"
44 #include "rtc_base/trace_event.h"
45 #include "system_wrappers/include/field_trial.h"
46 #include "system_wrappers/include/metrics.h"
47 #include "voice_engine/transmit_mixer.h"
48 
49 namespace cricket {
50 namespace {
51 
52 constexpr size_t kMaxUnsignaledRecvStreams = 4;
53 
54 constexpr int kNackRtpHistoryMs = 5000;
55 
56 // Check to verify that the define for the intelligibility enhancer is properly
57 // set.
58 #if !defined(WEBRTC_INTELLIGIBILITY_ENHANCER) || \
59     (WEBRTC_INTELLIGIBILITY_ENHANCER != 0 &&     \
60      WEBRTC_INTELLIGIBILITY_ENHANCER != 1)
61 #error "Set WEBRTC_INTELLIGIBILITY_ENHANCER to either 0 or 1"
62 #endif
63 
64 // For SendSideBwe, Opus bitrate should be in the range between 6000 and 32000.
65 const int kOpusMinBitrateBps = 6000;
66 const int kOpusBitrateFbBps = 32000;
67 
68 // Default audio dscp value.
69 // See http://tools.ietf.org/html/rfc2474 for details.
70 // See also http://tools.ietf.org/html/draft-jennings-rtcweb-qos-00
71 const rtc::DiffServCodePoint kAudioDscpValue = rtc::DSCP_EF;
72 
73 const int kMinTelephoneEventCode = 0;           // RFC4733 (Section 2.3.1)
74 const int kMaxTelephoneEventCode = 255;
75 
76 const int kMinPayloadType = 0;
77 const int kMaxPayloadType = 127;
78 
79 class ProxySink : public webrtc::AudioSinkInterface {
80  public:
ProxySink(AudioSinkInterface * sink)81   explicit ProxySink(AudioSinkInterface* sink) : sink_(sink) {
82     RTC_DCHECK(sink);
83   }
84 
OnData(const Data & audio)85   void OnData(const Data& audio) override { sink_->OnData(audio); }
86 
87  private:
88   webrtc::AudioSinkInterface* sink_;
89 };
90 
ValidateStreamParams(const StreamParams & sp)91 bool ValidateStreamParams(const StreamParams& sp) {
92   if (sp.ssrcs.empty()) {
93     RTC_LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString();
94     return false;
95   }
96   if (sp.ssrcs.size() > 1) {
97     RTC_LOG(LS_ERROR) << "Multiple SSRCs in stream parameters: "
98                       << sp.ToString();
99     return false;
100   }
101   return true;
102 }
103 
104 // Dumps an AudioCodec in RFC 2327-ish format.
ToString(const AudioCodec & codec)105 std::string ToString(const AudioCodec& codec) {
106   std::stringstream ss;
107   ss << codec.name << "/" << codec.clockrate << "/" << codec.channels;
108   if (!codec.params.empty()) {
109     ss << " {";
110     for (const auto& param : codec.params) {
111       ss << " " << param.first << "=" << param.second;
112     }
113     ss << " }";
114   }
115   ss << " (" << codec.id << ")";
116   return ss.str();
117 }
118 
IsCodec(const AudioCodec & codec,const char * ref_name)119 bool IsCodec(const AudioCodec& codec, const char* ref_name) {
120   return (_stricmp(codec.name.c_str(), ref_name) == 0);
121 }
122 
FindCodec(const std::vector<AudioCodec> & codecs,const AudioCodec & codec,AudioCodec * found_codec)123 bool FindCodec(const std::vector<AudioCodec>& codecs,
124                const AudioCodec& codec,
125                AudioCodec* found_codec) {
126   for (const AudioCodec& c : codecs) {
127     if (c.Matches(codec)) {
128       if (found_codec != NULL) {
129         *found_codec = c;
130       }
131       return true;
132     }
133   }
134   return false;
135 }
136 
VerifyUniquePayloadTypes(const std::vector<AudioCodec> & codecs)137 bool VerifyUniquePayloadTypes(const std::vector<AudioCodec>& codecs) {
138   if (codecs.empty()) {
139     return true;
140   }
141   std::vector<int> payload_types;
142   for (const AudioCodec& codec : codecs) {
143     payload_types.push_back(codec.id);
144   }
145   std::sort(payload_types.begin(), payload_types.end());
146   auto it = std::unique(payload_types.begin(), payload_types.end());
147   return it == payload_types.end();
148 }
149 
GetAudioNetworkAdaptorConfig(const AudioOptions & options)150 rtc::Optional<std::string> GetAudioNetworkAdaptorConfig(
151     const AudioOptions& options) {
152   if (options.audio_network_adaptor && *options.audio_network_adaptor &&
153       options.audio_network_adaptor_config) {
154     // Turn on audio network adaptor only when |options_.audio_network_adaptor|
155     // equals true and |options_.audio_network_adaptor_config| has a value.
156     return options.audio_network_adaptor_config;
157   }
158   return rtc::nullopt;
159 }
160 
MakeAudioStateConfig(VoEWrapper * voe_wrapper,rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing)161 webrtc::AudioState::Config MakeAudioStateConfig(
162     VoEWrapper* voe_wrapper,
163     rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,
164     rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing) {
165   webrtc::AudioState::Config config;
166   config.voice_engine = voe_wrapper->engine();
167   if (audio_mixer) {
168     config.audio_mixer = audio_mixer;
169   } else {
170     config.audio_mixer = webrtc::AudioMixerImpl::Create();
171   }
172   config.audio_processing = audio_processing;
173   return config;
174 }
175 
176 // |max_send_bitrate_bps| is the bitrate from "b=" in SDP.
177 // |rtp_max_bitrate_bps| is the bitrate from RtpSender::SetParameters.
ComputeSendBitrate(int max_send_bitrate_bps,rtc::Optional<int> rtp_max_bitrate_bps,const webrtc::AudioCodecSpec & spec)178 rtc::Optional<int> ComputeSendBitrate(int max_send_bitrate_bps,
179                                       rtc::Optional<int> rtp_max_bitrate_bps,
180                                       const webrtc::AudioCodecSpec& spec) {
181   // If application-configured bitrate is set, take minimum of that and SDP
182   // bitrate.
183   const int bps =
184       rtp_max_bitrate_bps
185           ? webrtc::MinPositive(max_send_bitrate_bps, *rtp_max_bitrate_bps)
186           : max_send_bitrate_bps;
187   if (bps <= 0) {
188     return spec.info.default_bitrate_bps;
189   }
190 
191   if (bps < spec.info.min_bitrate_bps) {
192     // If codec is not multi-rate and |bps| is less than the fixed bitrate then
193     // fail. If codec is not multi-rate and |bps| exceeds or equal the fixed
194     // bitrate then ignore.
195     RTC_LOG(LS_ERROR) << "Failed to set codec " << spec.format.name
196                       << " to bitrate " << bps << " bps"
197                       << ", requires at least " << spec.info.min_bitrate_bps
198                       << " bps.";
199     return rtc::nullopt;
200   }
201 
202   if (spec.info.HasFixedBitrate()) {
203     return spec.info.default_bitrate_bps;
204   } else {
205     // If codec is multi-rate then just set the bitrate.
206     return std::min(bps, spec.info.max_bitrate_bps);
207   }
208 }
209 
210 }  // namespace
211 
WebRtcVoiceEngine(webrtc::AudioDeviceModule * adm,const rtc::scoped_refptr<webrtc::AudioEncoderFactory> & encoder_factory,const rtc::scoped_refptr<webrtc::AudioDecoderFactory> & decoder_factory,rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing)212 WebRtcVoiceEngine::WebRtcVoiceEngine(
213     webrtc::AudioDeviceModule* adm,
214     const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory,
215     const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
216     rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,
217     rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing)
218     : WebRtcVoiceEngine(adm,
219                         encoder_factory,
220                         decoder_factory,
221                         audio_mixer,
222                         audio_processing,
223                         nullptr) {}
224 
WebRtcVoiceEngine(webrtc::AudioDeviceModule * adm,const rtc::scoped_refptr<webrtc::AudioEncoderFactory> & encoder_factory,const rtc::scoped_refptr<webrtc::AudioDecoderFactory> & decoder_factory,rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing,VoEWrapper * voe_wrapper)225 WebRtcVoiceEngine::WebRtcVoiceEngine(
226     webrtc::AudioDeviceModule* adm,
227     const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory,
228     const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
229     rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,
230     rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing,
231     VoEWrapper* voe_wrapper)
232     : adm_(adm),
233       encoder_factory_(encoder_factory),
234       decoder_factory_(decoder_factory),
235       audio_mixer_(audio_mixer),
236       apm_(audio_processing),
237       voe_wrapper_(voe_wrapper) {
238   // This may be called from any thread, so detach thread checkers.
239   worker_thread_checker_.DetachFromThread();
240   signal_thread_checker_.DetachFromThread();
241   RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::WebRtcVoiceEngine";
242   RTC_DCHECK(decoder_factory);
243   RTC_DCHECK(encoder_factory);
244   RTC_DCHECK(audio_processing);
245   // The rest of our initialization will happen in Init.
246 }
247 
~WebRtcVoiceEngine()248 WebRtcVoiceEngine::~WebRtcVoiceEngine() {
249   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
250   RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::~WebRtcVoiceEngine";
251   if (initialized_) {
252     StopAecDump();
253     voe_wrapper_->base()->Terminate();
254 
255     // Stop AudioDevice.
256     adm()->StopPlayout();
257     adm()->StopRecording();
258     adm()->RegisterAudioCallback(nullptr);
259     adm()->Terminate();
260   }
261 }
262 
Init()263 void WebRtcVoiceEngine::Init() {
264   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
265   RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::Init";
266 
267   // TaskQueue expects to be created/destroyed on the same thread.
268   low_priority_worker_queue_.reset(
269       new rtc::TaskQueue("rtc-low-prio", rtc::TaskQueue::Priority::LOW));
270 
271   // VoEWrapper needs to be created on the worker thread. It's expected to be
272   // null here unless it's being injected for testing.
273   if (!voe_wrapper_) {
274     voe_wrapper_.reset(new VoEWrapper());
275   }
276 
277   // Load our audio codec lists.
278   RTC_LOG(LS_INFO) << "Supported send codecs in order of preference:";
279   send_codecs_ = CollectCodecs(encoder_factory_->GetSupportedEncoders());
280   for (const AudioCodec& codec : send_codecs_) {
281     RTC_LOG(LS_INFO) << ToString(codec);
282   }
283 
284   RTC_LOG(LS_INFO) << "Supported recv codecs in order of preference:";
285   recv_codecs_ = CollectCodecs(decoder_factory_->GetSupportedDecoders());
286   for (const AudioCodec& codec : recv_codecs_) {
287     RTC_LOG(LS_INFO) << ToString(codec);
288   }
289 
290   channel_config_.enable_voice_pacing = true;
291 
292 #if defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE)
293   // No ADM supplied? Create a default one.
294   if (!adm_) {
295     adm_ = webrtc::AudioDeviceModule::Create(
296         webrtc::AudioDeviceModule::kPlatformDefaultAudio);
297   }
298 #endif  // WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE
299   RTC_CHECK(adm());
300   webrtc::adm_helpers::Init(adm());
301   webrtc::apm_helpers::Init(apm());
302   RTC_CHECK_EQ(0, voe_wrapper_->base()->Init(adm(), apm(), decoder_factory_));
303   transmit_mixer_ = voe_wrapper_->base()->transmit_mixer();
304   RTC_DCHECK(transmit_mixer_);
305 
306   // Save the default AGC configuration settings. This must happen before
307   // calling ApplyOptions or the default will be overwritten.
308   default_agc_config_ = webrtc::apm_helpers::GetAgcConfig(apm());
309 
310   // Set default engine options.
311   {
312     AudioOptions options;
313     options.echo_cancellation = true;
314     options.auto_gain_control = true;
315     options.noise_suppression = true;
316     options.highpass_filter = true;
317     options.stereo_swapping = false;
318     options.audio_jitter_buffer_max_packets = 50;
319     options.audio_jitter_buffer_fast_accelerate = false;
320     options.typing_detection = true;
321     options.adjust_agc_delta = 0;
322     options.experimental_agc = false;
323     options.extended_filter_aec = false;
324     options.delay_agnostic_aec = false;
325     options.experimental_ns = false;
326     options.intelligibility_enhancer = false;
327     options.level_control = false;
328     options.residual_echo_detector = true;
329     bool error = ApplyOptions(options);
330     RTC_DCHECK(error);
331   }
332 
333   // May be null for VoE injected for testing.
334   if (voe()->engine()) {
335     audio_state_ = webrtc::AudioState::Create(
336         MakeAudioStateConfig(voe(), audio_mixer_, apm_));
337 
338     // Connect the ADM to our audio path.
339     adm()->RegisterAudioCallback(audio_state_->audio_transport());
340   }
341 
342   initialized_ = true;
343 }
344 
345 rtc::scoped_refptr<webrtc::AudioState>
GetAudioState() const346     WebRtcVoiceEngine::GetAudioState() const {
347   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
348   return audio_state_;
349 }
350 
CreateChannel(webrtc::Call * call,const MediaConfig & config,const AudioOptions & options)351 VoiceMediaChannel* WebRtcVoiceEngine::CreateChannel(
352     webrtc::Call* call,
353     const MediaConfig& config,
354     const AudioOptions& options) {
355   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
356   return new WebRtcVoiceMediaChannel(this, config, options, call);
357 }
358 
ApplyOptions(const AudioOptions & options_in)359 bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
360   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
361   RTC_LOG(LS_INFO) << "WebRtcVoiceEngine::ApplyOptions: "
362                    << options_in.ToString();
363   AudioOptions options = options_in;  // The options are modified below.
364 
365   // Set and adjust echo canceller options.
366   // kEcConference is AEC with high suppression.
367   webrtc::EcModes ec_mode = webrtc::kEcConference;
368   if (options.aecm_generate_comfort_noise) {
369     RTC_LOG(LS_VERBOSE) << "Comfort noise explicitly set to "
370                         << *options.aecm_generate_comfort_noise
371                         << " (default is false).";
372   }
373 
374 #if defined(WEBRTC_IOS)
375   // On iOS, VPIO provides built-in EC.
376   options.echo_cancellation = false;
377   options.extended_filter_aec = false;
378   RTC_LOG(LS_INFO) << "Always disable AEC on iOS. Use built-in instead.";
379 #elif defined(WEBRTC_ANDROID)
380   ec_mode = webrtc::kEcAecm;
381   options.extended_filter_aec = false;
382 #endif
383 
384   // Delay Agnostic AEC automatically turns on EC if not set except on iOS
385   // where the feature is not supported.
386   bool use_delay_agnostic_aec = false;
387 #if !defined(WEBRTC_IOS)
388   if (options.delay_agnostic_aec) {
389     use_delay_agnostic_aec = *options.delay_agnostic_aec;
390     if (use_delay_agnostic_aec) {
391       options.echo_cancellation = true;
392       options.extended_filter_aec = true;
393       ec_mode = webrtc::kEcConference;
394     }
395   }
396 #endif
397 
398 // Set and adjust noise suppressor options.
399 #if defined(WEBRTC_IOS)
400   // On iOS, VPIO provides built-in NS.
401   options.noise_suppression = false;
402   options.typing_detection = false;
403   options.experimental_ns = false;
404   RTC_LOG(LS_INFO) << "Always disable NS on iOS. Use built-in instead.";
405 #elif defined(WEBRTC_ANDROID)
406   options.typing_detection = false;
407   options.experimental_ns = false;
408 #endif
409 
410 // Set and adjust gain control options.
411 #if defined(WEBRTC_IOS)
412   // On iOS, VPIO provides built-in AGC.
413   options.auto_gain_control = false;
414   options.experimental_agc = false;
415   RTC_LOG(LS_INFO) << "Always disable AGC on iOS. Use built-in instead.";
416 #elif defined(WEBRTC_ANDROID)
417   options.experimental_agc = false;
418 #endif
419 
420 #if defined(WEBRTC_IOS) || defined(WEBRTC_ANDROID)
421   // Turn off the gain control if specified by the field trial.
422   // The purpose of the field trial is to reduce the amount of resampling
423   // performed inside the audio processing module on mobile platforms by
424   // whenever possible turning off the fixed AGC mode and the high-pass filter.
425   // (https://bugs.chromium.org/p/webrtc/issues/detail?id=6181).
426   if (webrtc::field_trial::IsEnabled(
427           "WebRTC-Audio-MinimizeResamplingOnMobile")) {
428     options.auto_gain_control = false;
429     RTC_LOG(LS_INFO) << "Disable AGC according to field trial.";
430     if (!(options.noise_suppression.value_or(false) ||
431           options.echo_cancellation.value_or(false))) {
432       // If possible, turn off the high-pass filter.
433       RTC_LOG(LS_INFO)
434           << "Disable high-pass filter in response to field trial.";
435       options.highpass_filter = false;
436     }
437   }
438 #endif
439 
440 #if (WEBRTC_INTELLIGIBILITY_ENHANCER == 0)
441   // Hardcode the intelligibility enhancer to be off.
442   options.intelligibility_enhancer = false;
443 #endif
444 
445   if (options.echo_cancellation) {
446     // Check if platform supports built-in EC. Currently only supported on
447     // Android and in combination with Java based audio layer.
448     // TODO(henrika): investigate possibility to support built-in EC also
449     // in combination with Open SL ES audio.
450     const bool built_in_aec = adm()->BuiltInAECIsAvailable();
451     if (built_in_aec) {
452       // Built-in EC exists on this device and use_delay_agnostic_aec is not
453       // overriding it. Enable/Disable it according to the echo_cancellation
454       // audio option.
455       const bool enable_built_in_aec =
456           *options.echo_cancellation && !use_delay_agnostic_aec;
457       if (adm()->EnableBuiltInAEC(enable_built_in_aec) == 0 &&
458           enable_built_in_aec) {
459         // Disable internal software EC if built-in EC is enabled,
460         // i.e., replace the software EC with the built-in EC.
461         options.echo_cancellation = false;
462         RTC_LOG(LS_INFO)
463             << "Disabling EC since built-in EC will be used instead";
464       }
465     }
466     webrtc::apm_helpers::SetEcStatus(
467         apm(), *options.echo_cancellation, ec_mode);
468 #if !defined(WEBRTC_ANDROID)
469     webrtc::apm_helpers::SetEcMetricsStatus(apm(), *options.echo_cancellation);
470 #endif
471     if (ec_mode == webrtc::kEcAecm) {
472       bool cn = options.aecm_generate_comfort_noise.value_or(false);
473       webrtc::apm_helpers::SetAecmMode(apm(), cn);
474     }
475   }
476 
477   if (options.auto_gain_control) {
478     bool built_in_agc_avaliable = adm()->BuiltInAGCIsAvailable();
479     if (built_in_agc_avaliable) {
480       if (adm()->EnableBuiltInAGC(*options.auto_gain_control) == 0 &&
481           *options.auto_gain_control) {
482         // Disable internal software AGC if built-in AGC is enabled,
483         // i.e., replace the software AGC with the built-in AGC.
484         options.auto_gain_control = false;
485         RTC_LOG(LS_INFO)
486             << "Disabling AGC since built-in AGC will be used instead";
487       }
488     }
489     webrtc::apm_helpers::SetAgcStatus(apm(), adm(), *options.auto_gain_control);
490   }
491 
492   if (options.tx_agc_target_dbov || options.tx_agc_digital_compression_gain ||
493       options.tx_agc_limiter || options.adjust_agc_delta) {
494     // Override default_agc_config_. Generally, an unset option means "leave
495     // the VoE bits alone" in this function, so we want whatever is set to be
496     // stored as the new "default". If we didn't, then setting e.g.
497     // tx_agc_target_dbov would reset digital compression gain and limiter
498     // settings.
499     // Also, if we don't update default_agc_config_, then adjust_agc_delta
500     // would be an offset from the original values, and not whatever was set
501     // explicitly.
502     default_agc_config_.targetLeveldBOv = options.tx_agc_target_dbov.value_or(
503         default_agc_config_.targetLeveldBOv);
504     default_agc_config_.digitalCompressionGaindB =
505         options.tx_agc_digital_compression_gain.value_or(
506             default_agc_config_.digitalCompressionGaindB);
507     default_agc_config_.limiterEnable =
508         options.tx_agc_limiter.value_or(default_agc_config_.limiterEnable);
509 
510     webrtc::AgcConfig config = default_agc_config_;
511     if (options.adjust_agc_delta) {
512       config.targetLeveldBOv -= *options.adjust_agc_delta;
513       RTC_LOG(LS_INFO) << "Adjusting AGC level from default -"
514                        << default_agc_config_.targetLeveldBOv << "dB to -"
515                        << config.targetLeveldBOv << "dB";
516     }
517     webrtc::apm_helpers::SetAgcConfig(apm(), config);
518   }
519 
520   if (options.intelligibility_enhancer) {
521     intelligibility_enhancer_ = options.intelligibility_enhancer;
522   }
523   if (intelligibility_enhancer_ && *intelligibility_enhancer_) {
524     RTC_LOG(LS_INFO) << "Enabling NS when Intelligibility Enhancer is active.";
525     options.noise_suppression = intelligibility_enhancer_;
526   }
527 
528   if (options.noise_suppression) {
529     if (adm()->BuiltInNSIsAvailable()) {
530       bool builtin_ns =
531           *options.noise_suppression &&
532           !(intelligibility_enhancer_ && *intelligibility_enhancer_);
533       if (adm()->EnableBuiltInNS(builtin_ns) == 0 && builtin_ns) {
534         // Disable internal software NS if built-in NS is enabled,
535         // i.e., replace the software NS with the built-in NS.
536         options.noise_suppression = false;
537         RTC_LOG(LS_INFO)
538             << "Disabling NS since built-in NS will be used instead";
539       }
540     }
541     webrtc::apm_helpers::SetNsStatus(apm(), *options.noise_suppression);
542   }
543 
544   if (options.stereo_swapping) {
545     RTC_LOG(LS_INFO) << "Stereo swapping enabled? " << *options.stereo_swapping;
546     transmit_mixer()->EnableStereoChannelSwapping(*options.stereo_swapping);
547   }
548 
549   if (options.audio_jitter_buffer_max_packets) {
550     RTC_LOG(LS_INFO) << "NetEq capacity is "
551                      << *options.audio_jitter_buffer_max_packets;
552     channel_config_.acm_config.neteq_config.max_packets_in_buffer =
553         std::max(20, *options.audio_jitter_buffer_max_packets);
554   }
555   if (options.audio_jitter_buffer_fast_accelerate) {
556     RTC_LOG(LS_INFO) << "NetEq fast mode? "
557                      << *options.audio_jitter_buffer_fast_accelerate;
558     channel_config_.acm_config.neteq_config.enable_fast_accelerate =
559         *options.audio_jitter_buffer_fast_accelerate;
560   }
561 
562   if (options.typing_detection) {
563     RTC_LOG(LS_INFO) << "Typing detection is enabled? "
564                      << *options.typing_detection;
565     webrtc::apm_helpers::SetTypingDetectionStatus(
566         apm(), *options.typing_detection);
567   }
568 
569   webrtc::Config config;
570 
571   if (options.delay_agnostic_aec)
572     delay_agnostic_aec_ = options.delay_agnostic_aec;
573   if (delay_agnostic_aec_) {
574     RTC_LOG(LS_INFO) << "Delay agnostic aec is enabled? "
575                      << *delay_agnostic_aec_;
576     config.Set<webrtc::DelayAgnostic>(
577         new webrtc::DelayAgnostic(*delay_agnostic_aec_));
578   }
579 
580   if (options.extended_filter_aec) {
581     extended_filter_aec_ = options.extended_filter_aec;
582   }
583   if (extended_filter_aec_) {
584     RTC_LOG(LS_INFO) << "Extended filter aec is enabled? "
585                      << *extended_filter_aec_;
586     config.Set<webrtc::ExtendedFilter>(
587         new webrtc::ExtendedFilter(*extended_filter_aec_));
588   }
589 
590   if (options.experimental_ns) {
591     experimental_ns_ = options.experimental_ns;
592   }
593   if (experimental_ns_) {
594     RTC_LOG(LS_INFO) << "Experimental ns is enabled? " << *experimental_ns_;
595     config.Set<webrtc::ExperimentalNs>(
596         new webrtc::ExperimentalNs(*experimental_ns_));
597   }
598 
599   if (intelligibility_enhancer_) {
600     RTC_LOG(LS_INFO) << "Intelligibility Enhancer is enabled? "
601                      << *intelligibility_enhancer_;
602     config.Set<webrtc::Intelligibility>(
603         new webrtc::Intelligibility(*intelligibility_enhancer_));
604   }
605 
606   if (options.level_control) {
607     level_control_ = options.level_control;
608   }
609 
610   webrtc::AudioProcessing::Config apm_config = apm()->GetConfig();
611 
612   RTC_LOG(LS_INFO) << "Level control: "
613                    << (!!level_control_ ? *level_control_ : -1);
614   if (level_control_) {
615     apm_config.level_controller.enabled = *level_control_;
616     if (options.level_control_initial_peak_level_dbfs) {
617       apm_config.level_controller.initial_peak_level_dbfs =
618           *options.level_control_initial_peak_level_dbfs;
619     }
620   }
621 
622   if (options.highpass_filter) {
623     apm_config.high_pass_filter.enabled = *options.highpass_filter;
624   }
625 
626   if (options.residual_echo_detector) {
627     apm_config.residual_echo_detector.enabled = *options.residual_echo_detector;
628   }
629 
630   apm()->SetExtraOptions(config);
631   apm()->ApplyConfig(apm_config);
632   return true;
633 }
634 
635 // TODO(solenberg): Remove, once AudioMonitor is gone.
GetInputLevel()636 int WebRtcVoiceEngine::GetInputLevel() {
637   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
638   int8_t level = transmit_mixer()->AudioLevel();
639   RTC_DCHECK_LE(0, level);
640   return level;
641 }
642 
send_codecs() const643 const std::vector<AudioCodec>& WebRtcVoiceEngine::send_codecs() const {
644   RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
645   return send_codecs_;
646 }
647 
recv_codecs() const648 const std::vector<AudioCodec>& WebRtcVoiceEngine::recv_codecs() const {
649   RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
650   return recv_codecs_;
651 }
652 
GetCapabilities() const653 RtpCapabilities WebRtcVoiceEngine::GetCapabilities() const {
654   RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
655   RtpCapabilities capabilities;
656   capabilities.header_extensions.push_back(
657       webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri,
658                            webrtc::RtpExtension::kAudioLevelDefaultId));
659   if (webrtc::field_trial::IsEnabled("WebRTC-Audio-SendSideBwe")) {
660     capabilities.header_extensions.push_back(webrtc::RtpExtension(
661         webrtc::RtpExtension::kTransportSequenceNumberUri,
662         webrtc::RtpExtension::kTransportSequenceNumberDefaultId));
663   }
664   return capabilities;
665 }
666 
RegisterChannel(WebRtcVoiceMediaChannel * channel)667 void WebRtcVoiceEngine::RegisterChannel(WebRtcVoiceMediaChannel* channel) {
668   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
669   RTC_DCHECK(channel);
670   channels_.push_back(channel);
671 }
672 
UnregisterChannel(WebRtcVoiceMediaChannel * channel)673 void WebRtcVoiceEngine::UnregisterChannel(WebRtcVoiceMediaChannel* channel) {
674   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
675   auto it = std::find(channels_.begin(), channels_.end(), channel);
676   RTC_DCHECK(it != channels_.end());
677   channels_.erase(it);
678 }
679 
StartAecDump(rtc::PlatformFile file,int64_t max_size_bytes)680 bool WebRtcVoiceEngine::StartAecDump(rtc::PlatformFile file,
681                                      int64_t max_size_bytes) {
682   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
683   auto aec_dump = webrtc::AecDumpFactory::Create(
684       file, max_size_bytes, low_priority_worker_queue_.get());
685   if (!aec_dump) {
686     return false;
687   }
688   apm()->AttachAecDump(std::move(aec_dump));
689   return true;
690 }
691 
StartAecDump(const std::string & filename)692 void WebRtcVoiceEngine::StartAecDump(const std::string& filename) {
693   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
694 
695   auto aec_dump = webrtc::AecDumpFactory::Create(
696       filename, -1, low_priority_worker_queue_.get());
697   if (aec_dump) {
698     apm()->AttachAecDump(std::move(aec_dump));
699   }
700 }
701 
StopAecDump()702 void WebRtcVoiceEngine::StopAecDump() {
703   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
704   apm()->DetachAecDump();
705 }
706 
CreateVoEChannel()707 int WebRtcVoiceEngine::CreateVoEChannel() {
708   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
709   return voe_wrapper_->base()->CreateChannel(channel_config_);
710 }
711 
adm()712 webrtc::AudioDeviceModule* WebRtcVoiceEngine::adm() {
713   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
714   RTC_DCHECK(adm_);
715   return adm_.get();
716 }
717 
apm() const718 webrtc::AudioProcessing* WebRtcVoiceEngine::apm() const {
719   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
720   RTC_DCHECK(apm_);
721   return apm_.get();
722 }
723 
transmit_mixer()724 webrtc::voe::TransmitMixer* WebRtcVoiceEngine::transmit_mixer() {
725   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
726   RTC_DCHECK(transmit_mixer_);
727   return transmit_mixer_;
728 }
729 
CollectCodecs(const std::vector<webrtc::AudioCodecSpec> & specs) const730 AudioCodecs WebRtcVoiceEngine::CollectCodecs(
731     const std::vector<webrtc::AudioCodecSpec>& specs) const {
732   PayloadTypeMapper mapper;
733   AudioCodecs out;
734 
735   // Only generate CN payload types for these clockrates:
736   std::map<int, bool, std::greater<int>> generate_cn = {{ 8000,  false },
737                                                         { 16000, false },
738                                                         { 32000, false }};
739   // Only generate telephone-event payload types for these clockrates:
740   std::map<int, bool, std::greater<int>> generate_dtmf = {{ 8000,  false },
741                                                           { 16000, false },
742                                                           { 32000, false },
743                                                           { 48000, false }};
744 
745   auto map_format = [&mapper](const webrtc::SdpAudioFormat& format,
746                               AudioCodecs* out) {
747     rtc::Optional<AudioCodec> opt_codec = mapper.ToAudioCodec(format);
748     if (opt_codec) {
749       if (out) {
750         out->push_back(*opt_codec);
751       }
752     } else {
753       RTC_LOG(LS_ERROR) << "Unable to assign payload type to format: "
754                         << format;
755     }
756 
757     return opt_codec;
758   };
759 
760   for (const auto& spec : specs) {
761     // We need to do some extra stuff before adding the main codecs to out.
762     rtc::Optional<AudioCodec> opt_codec = map_format(spec.format, nullptr);
763     if (opt_codec) {
764       AudioCodec& codec = *opt_codec;
765       if (spec.info.supports_network_adaption) {
766         codec.AddFeedbackParam(
767             FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty));
768       }
769 
770       if (spec.info.allow_comfort_noise) {
771         // Generate a CN entry if the decoder allows it and we support the
772         // clockrate.
773         auto cn = generate_cn.find(spec.format.clockrate_hz);
774         if (cn != generate_cn.end()) {
775           cn->second = true;
776         }
777       }
778 
779       // Generate a telephone-event entry if we support the clockrate.
780       auto dtmf = generate_dtmf.find(spec.format.clockrate_hz);
781       if (dtmf != generate_dtmf.end()) {
782         dtmf->second = true;
783       }
784 
785       out.push_back(codec);
786     }
787   }
788 
789   // Add CN codecs after "proper" audio codecs.
790   for (const auto& cn : generate_cn) {
791     if (cn.second) {
792       map_format({kCnCodecName, cn.first, 1}, &out);
793     }
794   }
795 
796   // Add telephone-event codecs last.
797   for (const auto& dtmf : generate_dtmf) {
798     if (dtmf.second) {
799       map_format({kDtmfCodecName, dtmf.first, 1}, &out);
800     }
801   }
802 
803   return out;
804 }
805 
806 class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
807     : public AudioSource::Sink {
808  public:
WebRtcAudioSendStream(int ch,webrtc::AudioTransport * voe_audio_transport,uint32_t ssrc,const std::string & c_name,const std::string track_id,const rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec> & send_codec_spec,const std::vector<webrtc::RtpExtension> & extensions,int max_send_bitrate_bps,const rtc::Optional<std::string> & audio_network_adaptor_config,webrtc::Call * call,webrtc::Transport * send_transport,const rtc::scoped_refptr<webrtc::AudioEncoderFactory> & encoder_factory)809   WebRtcAudioSendStream(
810       int ch,
811       webrtc::AudioTransport* voe_audio_transport,
812       uint32_t ssrc,
813       const std::string& c_name,
814       const std::string track_id,
815       const rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec>&
816           send_codec_spec,
817       const std::vector<webrtc::RtpExtension>& extensions,
818       int max_send_bitrate_bps,
819       const rtc::Optional<std::string>& audio_network_adaptor_config,
820       webrtc::Call* call,
821       webrtc::Transport* send_transport,
822       const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory)
823       : voe_audio_transport_(voe_audio_transport),
824         call_(call),
825         config_(send_transport),
826         send_side_bwe_with_overhead_(
827             webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")),
828         max_send_bitrate_bps_(max_send_bitrate_bps),
829         rtp_parameters_(CreateRtpParametersWithOneEncoding()) {
830     RTC_DCHECK_GE(ch, 0);
831     // TODO(solenberg): Once we're not using FakeWebRtcVoiceEngine anymore:
832     // RTC_DCHECK(voe_audio_transport);
833     RTC_DCHECK(call);
834     RTC_DCHECK(encoder_factory);
835     config_.rtp.ssrc = ssrc;
836     config_.rtp.c_name = c_name;
837     config_.voe_channel_id = ch;
838     config_.rtp.extensions = extensions;
839     config_.audio_network_adaptor_config = audio_network_adaptor_config;
840     config_.encoder_factory = encoder_factory;
841     config_.track_id = track_id;
842     rtp_parameters_.encodings[0].ssrc = ssrc;
843 
844     if (send_codec_spec) {
845       UpdateSendCodecSpec(*send_codec_spec);
846     }
847 
848     stream_ = call_->CreateAudioSendStream(config_);
849   }
850 
~WebRtcAudioSendStream()851   ~WebRtcAudioSendStream() override {
852     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
853     ClearSource();
854     call_->DestroyAudioSendStream(stream_);
855   }
856 
SetSendCodecSpec(const webrtc::AudioSendStream::Config::SendCodecSpec & send_codec_spec)857   void SetSendCodecSpec(
858       const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) {
859     UpdateSendCodecSpec(send_codec_spec);
860     ReconfigureAudioSendStream();
861   }
862 
SetRtpExtensions(const std::vector<webrtc::RtpExtension> & extensions)863   void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions) {
864     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
865     config_.rtp.extensions = extensions;
866     ReconfigureAudioSendStream();
867   }
868 
SetAudioNetworkAdaptorConfig(const rtc::Optional<std::string> & audio_network_adaptor_config)869   void SetAudioNetworkAdaptorConfig(
870       const rtc::Optional<std::string>& audio_network_adaptor_config) {
871     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
872     if (config_.audio_network_adaptor_config == audio_network_adaptor_config) {
873       return;
874     }
875     config_.audio_network_adaptor_config = audio_network_adaptor_config;
876     UpdateAllowedBitrateRange();
877     ReconfigureAudioSendStream();
878   }
879 
SetMaxSendBitrate(int bps)880   bool SetMaxSendBitrate(int bps) {
881     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
882     RTC_DCHECK(config_.send_codec_spec);
883     RTC_DCHECK(audio_codec_spec_);
884     auto send_rate = ComputeSendBitrate(
885         bps, rtp_parameters_.encodings[0].max_bitrate_bps, *audio_codec_spec_);
886 
887     if (!send_rate) {
888       return false;
889     }
890 
891     max_send_bitrate_bps_ = bps;
892 
893     if (send_rate != config_.send_codec_spec->target_bitrate_bps) {
894       config_.send_codec_spec->target_bitrate_bps = send_rate;
895       ReconfigureAudioSendStream();
896     }
897     return true;
898   }
899 
SendTelephoneEvent(int payload_type,int payload_freq,int event,int duration_ms)900   bool SendTelephoneEvent(int payload_type, int payload_freq, int event,
901                           int duration_ms) {
902     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
903     RTC_DCHECK(stream_);
904     return stream_->SendTelephoneEvent(payload_type, payload_freq, event,
905                                        duration_ms);
906   }
907 
SetSend(bool send)908   void SetSend(bool send) {
909     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
910     send_ = send;
911     UpdateSendState();
912   }
913 
SetMuted(bool muted)914   void SetMuted(bool muted) {
915     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
916     RTC_DCHECK(stream_);
917     stream_->SetMuted(muted);
918     muted_ = muted;
919   }
920 
muted() const921   bool muted() const {
922     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
923     return muted_;
924   }
925 
GetStats(bool has_remote_tracks) const926   webrtc::AudioSendStream::Stats GetStats(bool has_remote_tracks) const {
927     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
928     RTC_DCHECK(stream_);
929     return stream_->GetStats(has_remote_tracks);
930   }
931 
932   // Starts the sending by setting ourselves as a sink to the AudioSource to
933   // get data callbacks.
934   // This method is called on the libjingle worker thread.
935   // TODO(xians): Make sure Start() is called only once.
SetSource(AudioSource * source)936   void SetSource(AudioSource* source) {
937     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
938     RTC_DCHECK(source);
939     if (source_) {
940       RTC_DCHECK(source_ == source);
941       return;
942     }
943     source->SetSink(this);
944     source_ = source;
945     UpdateSendState();
946   }
947 
948   // Stops sending by setting the sink of the AudioSource to nullptr. No data
949   // callback will be received after this method.
950   // This method is called on the libjingle worker thread.
ClearSource()951   void ClearSource() {
952     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
953     if (source_) {
954       source_->SetSink(nullptr);
955       source_ = nullptr;
956     }
957     UpdateSendState();
958   }
959 
960   // AudioSource::Sink implementation.
961   // This method is called on the audio thread.
OnData(const void * audio_data,int bits_per_sample,int sample_rate,size_t number_of_channels,size_t number_of_frames)962   void OnData(const void* audio_data,
963               int bits_per_sample,
964               int sample_rate,
965               size_t number_of_channels,
966               size_t number_of_frames) override {
967     RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_);
968     RTC_DCHECK(voe_audio_transport_);
969     voe_audio_transport_->PushCaptureData(config_.voe_channel_id, audio_data,
970                                           bits_per_sample, sample_rate,
971                                           number_of_channels, number_of_frames);
972   }
973 
974   // Callback from the |source_| when it is going away. In case Start() has
975   // never been called, this callback won't be triggered.
OnClose()976   void OnClose() override {
977     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
978     // Set |source_| to nullptr to make sure no more callback will get into
979     // the source.
980     source_ = nullptr;
981     UpdateSendState();
982   }
983 
984   // Accessor to the VoE channel ID.
channel() const985   int channel() const {
986     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
987     return config_.voe_channel_id;
988   }
989 
rtp_parameters() const990   const webrtc::RtpParameters& rtp_parameters() const {
991     return rtp_parameters_;
992   }
993 
ValidateRtpParameters(const webrtc::RtpParameters & rtp_parameters)994   bool ValidateRtpParameters(const webrtc::RtpParameters& rtp_parameters) {
995     if (rtp_parameters.encodings.size() != 1) {
996       RTC_LOG(LS_ERROR)
997           << "Attempted to set RtpParameters without exactly one encoding";
998       return false;
999     }
1000     if (rtp_parameters.encodings[0].ssrc != rtp_parameters_.encodings[0].ssrc) {
1001       RTC_LOG(LS_ERROR) << "Attempted to set RtpParameters with modified SSRC";
1002       return false;
1003     }
1004     return true;
1005   }
1006 
SetRtpParameters(const webrtc::RtpParameters & parameters)1007   bool SetRtpParameters(const webrtc::RtpParameters& parameters) {
1008     if (!ValidateRtpParameters(parameters)) {
1009       return false;
1010     }
1011 
1012     rtc::Optional<int> send_rate;
1013     if (audio_codec_spec_) {
1014       send_rate = ComputeSendBitrate(max_send_bitrate_bps_,
1015                                      parameters.encodings[0].max_bitrate_bps,
1016                                      *audio_codec_spec_);
1017       if (!send_rate) {
1018         return false;
1019       }
1020     }
1021 
1022     const rtc::Optional<int> old_rtp_max_bitrate =
1023         rtp_parameters_.encodings[0].max_bitrate_bps;
1024 
1025     rtp_parameters_ = parameters;
1026 
1027     if (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) {
1028       // Reconfigure AudioSendStream with new bit rate.
1029       if (send_rate) {
1030         config_.send_codec_spec->target_bitrate_bps = send_rate;
1031       }
1032       UpdateAllowedBitrateRange();
1033       ReconfigureAudioSendStream();
1034     } else {
1035       // parameters.encodings[0].active could have changed.
1036       UpdateSendState();
1037     }
1038     return true;
1039   }
1040 
1041  private:
UpdateSendState()1042   void UpdateSendState() {
1043     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1044     RTC_DCHECK(stream_);
1045     RTC_DCHECK_EQ(1UL, rtp_parameters_.encodings.size());
1046     if (send_ && source_ != nullptr && rtp_parameters_.encodings[0].active) {
1047       stream_->Start();
1048     } else {  // !send || source_ = nullptr
1049       stream_->Stop();
1050     }
1051   }
1052 
UpdateAllowedBitrateRange()1053   void UpdateAllowedBitrateRange() {
1054     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1055     const bool is_opus =
1056         config_.send_codec_spec &&
1057         !STR_CASE_CMP(config_.send_codec_spec->format.name.c_str(),
1058                       kOpusCodecName);
1059     if (is_opus && webrtc::field_trial::IsEnabled("WebRTC-Audio-SendSideBwe")) {
1060       config_.min_bitrate_bps = kOpusMinBitrateBps;
1061 
1062       // This means that when RtpParameters is reset, we may change the
1063       // encoder's bit rate immediately (through ReconfigureAudioSendStream()),
1064       // meanwhile change the cap to the output of BWE.
1065       config_.max_bitrate_bps =
1066           rtp_parameters_.encodings[0].max_bitrate_bps
1067               ? *rtp_parameters_.encodings[0].max_bitrate_bps
1068               : kOpusBitrateFbBps;
1069 
1070       // TODO(mflodman): Keep testing this and set proper values.
1071       // Note: This is an early experiment currently only supported by Opus.
1072       if (send_side_bwe_with_overhead_) {
1073         const int max_packet_size_ms =
1074             WEBRTC_OPUS_SUPPORT_120MS_PTIME ? 120 : 60;
1075 
1076         // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
1077         constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12;
1078 
1079         int min_overhead_bps =
1080             kOverheadPerPacket * 8 * 1000 / max_packet_size_ms;
1081 
1082         // We assume that |config_.max_bitrate_bps| before the next line is
1083         // a hard limit on the payload bitrate, so we add min_overhead_bps to
1084         // it to ensure that, when overhead is deducted, the payload rate
1085         // never goes beyond the limit.
1086         // Note: this also means that if a higher overhead is forced, we
1087         // cannot reach the limit.
1088         // TODO(minyue): Reconsider this when the signaling to BWE is done
1089         // through a dedicated API.
1090         config_.max_bitrate_bps += min_overhead_bps;
1091 
1092         // In contrast to max_bitrate_bps, we let min_bitrate_bps always be
1093         // reachable.
1094         config_.min_bitrate_bps += min_overhead_bps;
1095       }
1096     }
1097   }
1098 
UpdateSendCodecSpec(const webrtc::AudioSendStream::Config::SendCodecSpec & send_codec_spec)1099   void UpdateSendCodecSpec(
1100       const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) {
1101     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1102     config_.rtp.nack.rtp_history_ms =
1103         send_codec_spec.nack_enabled ? kNackRtpHistoryMs : 0;
1104     config_.send_codec_spec = send_codec_spec;
1105     auto info =
1106         config_.encoder_factory->QueryAudioEncoder(send_codec_spec.format);
1107     RTC_DCHECK(info);
1108     // If a specific target bitrate has been set for the stream, use that as
1109     // the new default bitrate when computing send bitrate.
1110     if (send_codec_spec.target_bitrate_bps) {
1111       info->default_bitrate_bps = std::max(
1112           info->min_bitrate_bps,
1113           std::min(info->max_bitrate_bps, *send_codec_spec.target_bitrate_bps));
1114     }
1115 
1116     audio_codec_spec_.emplace(
1117         webrtc::AudioCodecSpec{send_codec_spec.format, *info});
1118 
1119     config_.send_codec_spec->target_bitrate_bps = ComputeSendBitrate(
1120         max_send_bitrate_bps_, rtp_parameters_.encodings[0].max_bitrate_bps,
1121         *audio_codec_spec_);
1122 
1123     UpdateAllowedBitrateRange();
1124   }
1125 
ReconfigureAudioSendStream()1126   void ReconfigureAudioSendStream() {
1127     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1128     RTC_DCHECK(stream_);
1129     stream_->Reconfigure(config_);
1130   }
1131 
1132   rtc::ThreadChecker worker_thread_checker_;
1133   rtc::RaceChecker audio_capture_race_checker_;
1134   webrtc::AudioTransport* const voe_audio_transport_ = nullptr;
1135   webrtc::Call* call_ = nullptr;
1136   webrtc::AudioSendStream::Config config_;
1137   const bool send_side_bwe_with_overhead_;
1138   // The stream is owned by WebRtcAudioSendStream and may be reallocated if
1139   // configuration changes.
1140   webrtc::AudioSendStream* stream_ = nullptr;
1141 
1142   // Raw pointer to AudioSource owned by LocalAudioTrackHandler.
1143   // PeerConnection will make sure invalidating the pointer before the object
1144   // goes away.
1145   AudioSource* source_ = nullptr;
1146   bool send_ = false;
1147   bool muted_ = false;
1148   int max_send_bitrate_bps_;
1149   webrtc::RtpParameters rtp_parameters_;
1150   rtc::Optional<webrtc::AudioCodecSpec> audio_codec_spec_;
1151 
1152   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcAudioSendStream);
1153 };
1154 
1155 class WebRtcVoiceMediaChannel::WebRtcAudioReceiveStream {
1156  public:
WebRtcAudioReceiveStream(int ch,uint32_t remote_ssrc,uint32_t local_ssrc,bool use_transport_cc,bool use_nack,const std::string & sync_group,const std::vector<webrtc::RtpExtension> & extensions,webrtc::Call * call,webrtc::Transport * rtcp_send_transport,const rtc::scoped_refptr<webrtc::AudioDecoderFactory> & decoder_factory,const std::map<int,webrtc::SdpAudioFormat> & decoder_map)1157   WebRtcAudioReceiveStream(
1158       int ch,
1159       uint32_t remote_ssrc,
1160       uint32_t local_ssrc,
1161       bool use_transport_cc,
1162       bool use_nack,
1163       const std::string& sync_group,
1164       const std::vector<webrtc::RtpExtension>& extensions,
1165       webrtc::Call* call,
1166       webrtc::Transport* rtcp_send_transport,
1167       const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
1168       const std::map<int, webrtc::SdpAudioFormat>& decoder_map)
1169       : call_(call), config_() {
1170     RTC_DCHECK_GE(ch, 0);
1171     RTC_DCHECK(call);
1172     config_.rtp.remote_ssrc = remote_ssrc;
1173     config_.rtp.local_ssrc = local_ssrc;
1174     config_.rtp.transport_cc = use_transport_cc;
1175     config_.rtp.nack.rtp_history_ms = use_nack ? kNackRtpHistoryMs : 0;
1176     config_.rtp.extensions = extensions;
1177     config_.rtcp_send_transport = rtcp_send_transport;
1178     config_.voe_channel_id = ch;
1179     config_.sync_group = sync_group;
1180     config_.decoder_factory = decoder_factory;
1181     config_.decoder_map = decoder_map;
1182     RecreateAudioReceiveStream();
1183   }
1184 
~WebRtcAudioReceiveStream()1185   ~WebRtcAudioReceiveStream() {
1186     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1187     call_->DestroyAudioReceiveStream(stream_);
1188   }
1189 
RecreateAudioReceiveStream(uint32_t local_ssrc)1190   void RecreateAudioReceiveStream(uint32_t local_ssrc) {
1191     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1192     config_.rtp.local_ssrc = local_ssrc;
1193     RecreateAudioReceiveStream();
1194   }
1195 
RecreateAudioReceiveStream(bool use_transport_cc,bool use_nack)1196   void RecreateAudioReceiveStream(bool use_transport_cc, bool use_nack) {
1197     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1198     config_.rtp.transport_cc = use_transport_cc;
1199     config_.rtp.nack.rtp_history_ms = use_nack ? kNackRtpHistoryMs : 0;
1200     RecreateAudioReceiveStream();
1201   }
1202 
RecreateAudioReceiveStream(const std::vector<webrtc::RtpExtension> & extensions)1203   void RecreateAudioReceiveStream(
1204       const std::vector<webrtc::RtpExtension>& extensions) {
1205     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1206     config_.rtp.extensions = extensions;
1207     RecreateAudioReceiveStream();
1208   }
1209 
1210   // Set a new payload type -> decoder map.
RecreateAudioReceiveStream(const std::map<int,webrtc::SdpAudioFormat> & decoder_map)1211   void RecreateAudioReceiveStream(
1212       const std::map<int, webrtc::SdpAudioFormat>& decoder_map) {
1213     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1214     config_.decoder_map = decoder_map;
1215     RecreateAudioReceiveStream();
1216   }
1217 
MaybeRecreateAudioReceiveStream(const std::string & sync_group)1218   void MaybeRecreateAudioReceiveStream(const std::string& sync_group) {
1219     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1220     if (config_.sync_group != sync_group) {
1221       config_.sync_group = sync_group;
1222       RecreateAudioReceiveStream();
1223     }
1224   }
1225 
GetStats() const1226   webrtc::AudioReceiveStream::Stats GetStats() const {
1227     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1228     RTC_DCHECK(stream_);
1229     return stream_->GetStats();
1230   }
1231 
GetOutputLevel() const1232   int GetOutputLevel() const {
1233     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1234     RTC_DCHECK(stream_);
1235     return stream_->GetOutputLevel();
1236   }
1237 
channel() const1238   int channel() const {
1239     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1240     return config_.voe_channel_id;
1241   }
1242 
SetRawAudioSink(std::unique_ptr<webrtc::AudioSinkInterface> sink)1243   void SetRawAudioSink(std::unique_ptr<webrtc::AudioSinkInterface> sink) {
1244     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1245     stream_->SetSink(std::move(sink));
1246   }
1247 
SetOutputVolume(double volume)1248   void SetOutputVolume(double volume) {
1249     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1250     stream_->SetGain(volume);
1251   }
1252 
SetPlayout(bool playout)1253   void SetPlayout(bool playout) {
1254     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1255     RTC_DCHECK(stream_);
1256     if (playout) {
1257       RTC_LOG(LS_INFO) << "Starting playout for channel #" << channel();
1258       stream_->Start();
1259     } else {
1260       RTC_LOG(LS_INFO) << "Stopping playout for channel #" << channel();
1261       stream_->Stop();
1262     }
1263     playout_ = playout;
1264   }
1265 
GetSources()1266   std::vector<webrtc::RtpSource> GetSources() {
1267     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1268     RTC_DCHECK(stream_);
1269     return stream_->GetSources();
1270   }
1271 
1272  private:
RecreateAudioReceiveStream()1273   void RecreateAudioReceiveStream() {
1274     RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1275     if (stream_) {
1276       call_->DestroyAudioReceiveStream(stream_);
1277     }
1278     stream_ = call_->CreateAudioReceiveStream(config_);
1279     RTC_CHECK(stream_);
1280     SetPlayout(playout_);
1281   }
1282 
1283   rtc::ThreadChecker worker_thread_checker_;
1284   webrtc::Call* call_ = nullptr;
1285   webrtc::AudioReceiveStream::Config config_;
1286   // The stream is owned by WebRtcAudioReceiveStream and may be reallocated if
1287   // configuration changes.
1288   webrtc::AudioReceiveStream* stream_ = nullptr;
1289   bool playout_ = false;
1290 
1291   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcAudioReceiveStream);
1292 };
1293 
WebRtcVoiceMediaChannel(WebRtcVoiceEngine * engine,const MediaConfig & config,const AudioOptions & options,webrtc::Call * call)1294 WebRtcVoiceMediaChannel::WebRtcVoiceMediaChannel(WebRtcVoiceEngine* engine,
1295                                                  const MediaConfig& config,
1296                                                  const AudioOptions& options,
1297                                                  webrtc::Call* call)
1298     : VoiceMediaChannel(config), engine_(engine), call_(call) {
1299   RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::WebRtcVoiceMediaChannel";
1300   RTC_DCHECK(call);
1301   engine->RegisterChannel(this);
1302   SetOptions(options);
1303 }
1304 
~WebRtcVoiceMediaChannel()1305 WebRtcVoiceMediaChannel::~WebRtcVoiceMediaChannel() {
1306   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1307   RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::~WebRtcVoiceMediaChannel";
1308   // TODO(solenberg): Should be able to delete the streams directly, without
1309   //                  going through RemoveNnStream(), once stream objects handle
1310   //                  all (de)configuration.
1311   while (!send_streams_.empty()) {
1312     RemoveSendStream(send_streams_.begin()->first);
1313   }
1314   while (!recv_streams_.empty()) {
1315     RemoveRecvStream(recv_streams_.begin()->first);
1316   }
1317   engine()->UnregisterChannel(this);
1318 }
1319 
PreferredDscp() const1320 rtc::DiffServCodePoint WebRtcVoiceMediaChannel::PreferredDscp() const {
1321   return kAudioDscpValue;
1322 }
1323 
SetSendParameters(const AudioSendParameters & params)1324 bool WebRtcVoiceMediaChannel::SetSendParameters(
1325     const AudioSendParameters& params) {
1326   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetSendParameters");
1327   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1328   RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetSendParameters: "
1329                    << params.ToString();
1330   // TODO(pthatcher): Refactor this to be more clean now that we have
1331   // all the information at once.
1332 
1333   if (!SetSendCodecs(params.codecs)) {
1334     return false;
1335   }
1336 
1337   if (!ValidateRtpExtensions(params.extensions)) {
1338     return false;
1339   }
1340   std::vector<webrtc::RtpExtension> filtered_extensions =
1341       FilterRtpExtensions(params.extensions,
1342                           webrtc::RtpExtension::IsSupportedForAudio, true);
1343   if (send_rtp_extensions_ != filtered_extensions) {
1344     send_rtp_extensions_.swap(filtered_extensions);
1345     for (auto& it : send_streams_) {
1346       it.second->SetRtpExtensions(send_rtp_extensions_);
1347     }
1348   }
1349 
1350   if (!SetMaxSendBitrate(params.max_bandwidth_bps)) {
1351     return false;
1352   }
1353   return SetOptions(params.options);
1354 }
1355 
SetRecvParameters(const AudioRecvParameters & params)1356 bool WebRtcVoiceMediaChannel::SetRecvParameters(
1357     const AudioRecvParameters& params) {
1358   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetRecvParameters");
1359   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1360   RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetRecvParameters: "
1361                    << params.ToString();
1362   // TODO(pthatcher): Refactor this to be more clean now that we have
1363   // all the information at once.
1364 
1365   if (!SetRecvCodecs(params.codecs)) {
1366     return false;
1367   }
1368 
1369   if (!ValidateRtpExtensions(params.extensions)) {
1370     return false;
1371   }
1372   std::vector<webrtc::RtpExtension> filtered_extensions =
1373       FilterRtpExtensions(params.extensions,
1374                           webrtc::RtpExtension::IsSupportedForAudio, false);
1375   if (recv_rtp_extensions_ != filtered_extensions) {
1376     recv_rtp_extensions_.swap(filtered_extensions);
1377     for (auto& it : recv_streams_) {
1378       it.second->RecreateAudioReceiveStream(recv_rtp_extensions_);
1379     }
1380   }
1381   return true;
1382 }
1383 
GetRtpSendParameters(uint32_t ssrc) const1384 webrtc::RtpParameters WebRtcVoiceMediaChannel::GetRtpSendParameters(
1385     uint32_t ssrc) const {
1386   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1387   auto it = send_streams_.find(ssrc);
1388   if (it == send_streams_.end()) {
1389     RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream "
1390                         << "with ssrc " << ssrc << " which doesn't exist.";
1391     return webrtc::RtpParameters();
1392   }
1393 
1394   webrtc::RtpParameters rtp_params = it->second->rtp_parameters();
1395   // Need to add the common list of codecs to the send stream-specific
1396   // RTP parameters.
1397   for (const AudioCodec& codec : send_codecs_) {
1398     rtp_params.codecs.push_back(codec.ToCodecParameters());
1399   }
1400   return rtp_params;
1401 }
1402 
SetRtpSendParameters(uint32_t ssrc,const webrtc::RtpParameters & parameters)1403 bool WebRtcVoiceMediaChannel::SetRtpSendParameters(
1404     uint32_t ssrc,
1405     const webrtc::RtpParameters& parameters) {
1406   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1407   auto it = send_streams_.find(ssrc);
1408   if (it == send_streams_.end()) {
1409     RTC_LOG(LS_WARNING) << "Attempting to set RTP send parameters for stream "
1410                         << "with ssrc " << ssrc << " which doesn't exist.";
1411     return false;
1412   }
1413 
1414   // TODO(deadbeef): Handle setting parameters with a list of codecs in a
1415   // different order (which should change the send codec).
1416   webrtc::RtpParameters current_parameters = GetRtpSendParameters(ssrc);
1417   if (current_parameters.codecs != parameters.codecs) {
1418     RTC_LOG(LS_ERROR) << "Using SetParameters to change the set of codecs "
1419                       << "is not currently supported.";
1420     return false;
1421   }
1422 
1423   // TODO(minyue): The following legacy actions go into
1424   // |WebRtcAudioSendStream::SetRtpParameters()| which is called at the end,
1425   // though there are two difference:
1426   // 1. |WebRtcVoiceMediaChannel::SetChannelSendParameters()| only calls
1427   // |SetSendCodec| while |WebRtcAudioSendStream::SetRtpParameters()| calls
1428   // |SetSendCodecs|. The outcome should be the same.
1429   // 2. AudioSendStream can be recreated.
1430 
1431   // Codecs are handled at the WebRtcVoiceMediaChannel level.
1432   webrtc::RtpParameters reduced_params = parameters;
1433   reduced_params.codecs.clear();
1434   return it->second->SetRtpParameters(reduced_params);
1435 }
1436 
GetRtpReceiveParameters(uint32_t ssrc) const1437 webrtc::RtpParameters WebRtcVoiceMediaChannel::GetRtpReceiveParameters(
1438     uint32_t ssrc) const {
1439   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1440   webrtc::RtpParameters rtp_params;
1441   // SSRC of 0 represents the default receive stream.
1442   if (ssrc == 0) {
1443     if (!default_sink_) {
1444       RTC_LOG(LS_WARNING)
1445           << "Attempting to get RTP parameters for the default, "
1446              "unsignaled audio receive stream, but not yet "
1447              "configured to receive such a stream.";
1448       return rtp_params;
1449     }
1450     rtp_params.encodings.emplace_back();
1451   } else {
1452     auto it = recv_streams_.find(ssrc);
1453     if (it == recv_streams_.end()) {
1454       RTC_LOG(LS_WARNING)
1455           << "Attempting to get RTP receive parameters for stream "
1456           << "with ssrc " << ssrc << " which doesn't exist.";
1457       return webrtc::RtpParameters();
1458     }
1459     rtp_params.encodings.emplace_back();
1460     // TODO(deadbeef): Return stream-specific parameters.
1461     rtp_params.encodings[0].ssrc = ssrc;
1462   }
1463 
1464   for (const AudioCodec& codec : recv_codecs_) {
1465     rtp_params.codecs.push_back(codec.ToCodecParameters());
1466   }
1467   return rtp_params;
1468 }
1469 
SetRtpReceiveParameters(uint32_t ssrc,const webrtc::RtpParameters & parameters)1470 bool WebRtcVoiceMediaChannel::SetRtpReceiveParameters(
1471     uint32_t ssrc,
1472     const webrtc::RtpParameters& parameters) {
1473   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1474   // SSRC of 0 represents the default receive stream.
1475   if (ssrc == 0) {
1476     if (!default_sink_) {
1477       RTC_LOG(LS_WARNING)
1478           << "Attempting to set RTP parameters for the default, "
1479              "unsignaled audio receive stream, but not yet "
1480              "configured to receive such a stream.";
1481       return false;
1482     }
1483   } else {
1484     auto it = recv_streams_.find(ssrc);
1485     if (it == recv_streams_.end()) {
1486       RTC_LOG(LS_WARNING)
1487           << "Attempting to set RTP receive parameters for stream "
1488           << "with ssrc " << ssrc << " which doesn't exist.";
1489       return false;
1490     }
1491   }
1492 
1493   webrtc::RtpParameters current_parameters = GetRtpReceiveParameters(ssrc);
1494   if (current_parameters != parameters) {
1495     RTC_LOG(LS_ERROR) << "Changing the RTP receive parameters is currently "
1496                       << "unsupported.";
1497     return false;
1498   }
1499   return true;
1500 }
1501 
SetOptions(const AudioOptions & options)1502 bool WebRtcVoiceMediaChannel::SetOptions(const AudioOptions& options) {
1503   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1504   RTC_LOG(LS_INFO) << "Setting voice channel options: " << options.ToString();
1505 
1506   // We retain all of the existing options, and apply the given ones
1507   // on top.  This means there is no way to "clear" options such that
1508   // they go back to the engine default.
1509   options_.SetAll(options);
1510   if (!engine()->ApplyOptions(options_)) {
1511     RTC_LOG(LS_WARNING)
1512         << "Failed to apply engine options during channel SetOptions.";
1513     return false;
1514   }
1515 
1516   rtc::Optional<std::string> audio_network_adaptor_config =
1517       GetAudioNetworkAdaptorConfig(options_);
1518   for (auto& it : send_streams_) {
1519     it.second->SetAudioNetworkAdaptorConfig(audio_network_adaptor_config);
1520   }
1521 
1522   RTC_LOG(LS_INFO) << "Set voice channel options. Current options: "
1523                    << options_.ToString();
1524   return true;
1525 }
1526 
SetRecvCodecs(const std::vector<AudioCodec> & codecs)1527 bool WebRtcVoiceMediaChannel::SetRecvCodecs(
1528     const std::vector<AudioCodec>& codecs) {
1529   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1530 
1531   // Set the payload types to be used for incoming media.
1532   RTC_LOG(LS_INFO) << "Setting receive voice codecs.";
1533 
1534   if (!VerifyUniquePayloadTypes(codecs)) {
1535     RTC_LOG(LS_ERROR) << "Codec payload types overlap.";
1536     return false;
1537   }
1538 
1539   // Create a payload type -> SdpAudioFormat map with all the decoders. Fail
1540   // unless the factory claims to support all decoders.
1541   std::map<int, webrtc::SdpAudioFormat> decoder_map;
1542   for (const AudioCodec& codec : codecs) {
1543     // Log a warning if a codec's payload type is changing. This used to be
1544     // treated as an error. It's abnormal, but not really illegal.
1545     AudioCodec old_codec;
1546     if (FindCodec(recv_codecs_, codec, &old_codec) &&
1547         old_codec.id != codec.id) {
1548       RTC_LOG(LS_WARNING) << codec.name << " mapped to a second payload type ("
1549                           << codec.id << ", was already mapped to "
1550                           << old_codec.id << ")";
1551     }
1552     auto format = AudioCodecToSdpAudioFormat(codec);
1553     if (!IsCodec(codec, "cn") && !IsCodec(codec, "telephone-event") &&
1554         !engine()->decoder_factory_->IsSupportedDecoder(format)) {
1555       RTC_LOG(LS_ERROR) << "Unsupported codec: " << format;
1556       return false;
1557     }
1558     // We allow adding new codecs but don't allow changing the payload type of
1559     // codecs that are already configured since we might already be receiving
1560     // packets with that payload type. See RFC3264, Section 8.3.2.
1561     // TODO(deadbeef): Also need to check for clashes with previously mapped
1562     // payload types, and not just currently mapped ones. For example, this
1563     // should be illegal:
1564     // 1. {100: opus/48000/2, 101: ISAC/16000}
1565     // 2. {100: opus/48000/2}
1566     // 3. {100: opus/48000/2, 101: ISAC/32000}
1567     // Though this check really should happen at a higher level, since this
1568     // conflict could happen between audio and video codecs.
1569     auto existing = decoder_map_.find(codec.id);
1570     if (existing != decoder_map_.end() && !existing->second.Matches(format)) {
1571       RTC_LOG(LS_ERROR) << "Attempting to use payload type " << codec.id
1572                         << " for " << codec.name
1573                         << ", but it is already used for "
1574                         << existing->second.name;
1575       return false;
1576     }
1577     decoder_map.insert({codec.id, std::move(format)});
1578   }
1579 
1580   if (decoder_map == decoder_map_) {
1581     // There's nothing new to configure.
1582     return true;
1583   }
1584 
1585   if (playout_) {
1586     // Receive codecs can not be changed while playing. So we temporarily
1587     // pause playout.
1588     ChangePlayout(false);
1589   }
1590 
1591   decoder_map_ = std::move(decoder_map);
1592   for (auto& kv : recv_streams_) {
1593     kv.second->RecreateAudioReceiveStream(decoder_map_);
1594   }
1595   recv_codecs_ = codecs;
1596 
1597   if (desired_playout_ && !playout_) {
1598     ChangePlayout(desired_playout_);
1599   }
1600   return true;
1601 }
1602 
1603 // Utility function called from SetSendParameters() to extract current send
1604 // codec settings from the given list of codecs (originally from SDP). Both send
1605 // and receive streams may be reconfigured based on the new settings.
SetSendCodecs(const std::vector<AudioCodec> & codecs)1606 bool WebRtcVoiceMediaChannel::SetSendCodecs(
1607     const std::vector<AudioCodec>& codecs) {
1608   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1609   dtmf_payload_type_ = rtc::nullopt;
1610   dtmf_payload_freq_ = -1;
1611 
1612   // Validate supplied codecs list.
1613   for (const AudioCodec& codec : codecs) {
1614     // TODO(solenberg): Validate more aspects of input - that payload types
1615     //                  don't overlap, remove redundant/unsupported codecs etc -
1616     //                  the same way it is done for RtpHeaderExtensions.
1617     if (codec.id < kMinPayloadType || codec.id > kMaxPayloadType) {
1618       RTC_LOG(LS_WARNING) << "Codec payload type out of range: "
1619                           << ToString(codec);
1620       return false;
1621     }
1622   }
1623 
1624   // Find PT of telephone-event codec with lowest clockrate, as a fallback, in
1625   // case we don't have a DTMF codec with a rate matching the send codec's, or
1626   // if this function returns early.
1627   std::vector<AudioCodec> dtmf_codecs;
1628   for (const AudioCodec& codec : codecs) {
1629     if (IsCodec(codec, kDtmfCodecName)) {
1630       dtmf_codecs.push_back(codec);
1631       if (!dtmf_payload_type_ || codec.clockrate < dtmf_payload_freq_) {
1632         dtmf_payload_type_ = codec.id;
1633         dtmf_payload_freq_ = codec.clockrate;
1634       }
1635     }
1636   }
1637 
1638   // Scan through the list to figure out the codec to use for sending.
1639   rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec> send_codec_spec;
1640   webrtc::Call::Config::BitrateConfig bitrate_config;
1641   rtc::Optional<webrtc::AudioCodecInfo> voice_codec_info;
1642   for (const AudioCodec& voice_codec : codecs) {
1643     if (!(IsCodec(voice_codec, kCnCodecName) ||
1644           IsCodec(voice_codec, kDtmfCodecName) ||
1645           IsCodec(voice_codec, kRedCodecName))) {
1646       webrtc::SdpAudioFormat format(voice_codec.name, voice_codec.clockrate,
1647                                     voice_codec.channels, voice_codec.params);
1648 
1649       voice_codec_info = engine()->encoder_factory_->QueryAudioEncoder(format);
1650       if (!voice_codec_info) {
1651         RTC_LOG(LS_WARNING) << "Unknown codec " << ToString(voice_codec);
1652         continue;
1653       }
1654 
1655       send_codec_spec = webrtc::AudioSendStream::Config::SendCodecSpec(
1656           voice_codec.id, format);
1657       if (voice_codec.bitrate > 0) {
1658         send_codec_spec->target_bitrate_bps = voice_codec.bitrate;
1659       }
1660       send_codec_spec->transport_cc_enabled = HasTransportCc(voice_codec);
1661       send_codec_spec->nack_enabled = HasNack(voice_codec);
1662       bitrate_config = GetBitrateConfigForCodec(voice_codec);
1663       break;
1664     }
1665   }
1666 
1667   if (!send_codec_spec) {
1668     return false;
1669   }
1670 
1671   RTC_DCHECK(voice_codec_info);
1672   if (voice_codec_info->allow_comfort_noise) {
1673     // Loop through the codecs list again to find the CN codec.
1674     // TODO(solenberg): Break out into a separate function?
1675     for (const AudioCodec& cn_codec : codecs) {
1676       if (IsCodec(cn_codec, kCnCodecName) &&
1677           cn_codec.clockrate == send_codec_spec->format.clockrate_hz) {
1678         switch (cn_codec.clockrate) {
1679           case 8000:
1680           case 16000:
1681           case 32000:
1682             send_codec_spec->cng_payload_type = cn_codec.id;
1683             break;
1684           default:
1685             RTC_LOG(LS_WARNING)
1686                 << "CN frequency " << cn_codec.clockrate << " not supported.";
1687             break;
1688         }
1689         break;
1690       }
1691     }
1692 
1693     // Find the telephone-event PT exactly matching the preferred send codec.
1694     for (const AudioCodec& dtmf_codec : dtmf_codecs) {
1695       if (dtmf_codec.clockrate == send_codec_spec->format.clockrate_hz) {
1696         dtmf_payload_type_ = dtmf_codec.id;
1697         dtmf_payload_freq_ = dtmf_codec.clockrate;
1698         break;
1699       }
1700     }
1701   }
1702 
1703   if (send_codec_spec_ != send_codec_spec) {
1704     send_codec_spec_ = std::move(send_codec_spec);
1705     // Apply new settings to all streams.
1706     for (const auto& kv : send_streams_) {
1707       kv.second->SetSendCodecSpec(*send_codec_spec_);
1708     }
1709   } else {
1710     // If the codec isn't changing, set the start bitrate to -1 which means
1711     // "unchanged" so that BWE isn't affected.
1712     bitrate_config.start_bitrate_bps = -1;
1713   }
1714   call_->SetBitrateConfig(bitrate_config);
1715 
1716   // Check if the transport cc feedback or NACK status has changed on the
1717   // preferred send codec, and in that case reconfigure all receive streams.
1718   if (recv_transport_cc_enabled_ != send_codec_spec_->transport_cc_enabled ||
1719       recv_nack_enabled_ != send_codec_spec_->nack_enabled) {
1720     RTC_LOG(LS_INFO) << "Recreate all the receive streams because the send "
1721                         "codec has changed.";
1722     recv_transport_cc_enabled_ = send_codec_spec_->transport_cc_enabled;
1723     recv_nack_enabled_ = send_codec_spec_->nack_enabled;
1724     for (auto& kv : recv_streams_) {
1725       kv.second->RecreateAudioReceiveStream(recv_transport_cc_enabled_,
1726                                             recv_nack_enabled_);
1727     }
1728   }
1729 
1730   send_codecs_ = codecs;
1731   return true;
1732 }
1733 
SetPlayout(bool playout)1734 void WebRtcVoiceMediaChannel::SetPlayout(bool playout) {
1735   desired_playout_ = playout;
1736   return ChangePlayout(desired_playout_);
1737 }
1738 
ChangePlayout(bool playout)1739 void WebRtcVoiceMediaChannel::ChangePlayout(bool playout) {
1740   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::ChangePlayout");
1741   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1742   if (playout_ == playout) {
1743     return;
1744   }
1745 
1746   for (const auto& kv : recv_streams_) {
1747     kv.second->SetPlayout(playout);
1748   }
1749   playout_ = playout;
1750 }
1751 
SetSend(bool send)1752 void WebRtcVoiceMediaChannel::SetSend(bool send) {
1753   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetSend");
1754   if (send_ == send) {
1755     return;
1756   }
1757 
1758   // Apply channel specific options, and initialize the ADM for recording (this
1759   // may take time on some platforms, e.g. Android).
1760   if (send) {
1761     engine()->ApplyOptions(options_);
1762 
1763     // InitRecording() may return an error if the ADM is already recording.
1764     if (!engine()->adm()->RecordingIsInitialized() &&
1765         !engine()->adm()->Recording()) {
1766       if (engine()->adm()->InitRecording() != 0) {
1767         RTC_LOG(LS_WARNING) << "Failed to initialize recording";
1768       }
1769     }
1770   }
1771 
1772   // Change the settings on each send channel.
1773   for (auto& kv : send_streams_) {
1774     kv.second->SetSend(send);
1775   }
1776 
1777   send_ = send;
1778 }
1779 
SetAudioSend(uint32_t ssrc,bool enable,const AudioOptions * options,AudioSource * source)1780 bool WebRtcVoiceMediaChannel::SetAudioSend(uint32_t ssrc,
1781                                            bool enable,
1782                                            const AudioOptions* options,
1783                                            AudioSource* source) {
1784   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1785   // TODO(solenberg): The state change should be fully rolled back if any one of
1786   //                  these calls fail.
1787   if (!SetLocalSource(ssrc, source)) {
1788     return false;
1789   }
1790   if (!MuteStream(ssrc, !enable)) {
1791     return false;
1792   }
1793   if (enable && options) {
1794     return SetOptions(*options);
1795   }
1796   return true;
1797 }
1798 
CreateVoEChannel()1799 int WebRtcVoiceMediaChannel::CreateVoEChannel() {
1800   int id = engine()->CreateVoEChannel();
1801   if (id == -1) {
1802     RTC_LOG(LS_WARNING) << "CreateVoEChannel() failed.";
1803     return -1;
1804   }
1805 
1806   return id;
1807 }
1808 
DeleteVoEChannel(int channel)1809 bool WebRtcVoiceMediaChannel::DeleteVoEChannel(int channel) {
1810   if (engine()->voe()->base()->DeleteChannel(channel) == -1) {
1811     RTC_LOG(LS_WARNING) << "DeleteChannel(" << channel << ") failed.";
1812     return false;
1813   }
1814   return true;
1815 }
1816 
AddSendStream(const StreamParams & sp)1817 bool WebRtcVoiceMediaChannel::AddSendStream(const StreamParams& sp) {
1818   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::AddSendStream");
1819   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1820   RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString();
1821 
1822   uint32_t ssrc = sp.first_ssrc();
1823   RTC_DCHECK(0 != ssrc);
1824 
1825   if (GetSendChannelId(ssrc) != -1) {
1826     RTC_LOG(LS_ERROR) << "Stream already exists with ssrc " << ssrc;
1827     return false;
1828   }
1829 
1830   // Create a new channel for sending audio data.
1831   int channel = CreateVoEChannel();
1832   if (channel == -1) {
1833     return false;
1834   }
1835 
1836   // Save the channel to send_streams_, so that RemoveSendStream() can still
1837   // delete the channel in case failure happens below.
1838   webrtc::AudioTransport* audio_transport =
1839       engine()->voe()->base()->audio_transport();
1840 
1841   rtc::Optional<std::string> audio_network_adaptor_config =
1842       GetAudioNetworkAdaptorConfig(options_);
1843   WebRtcAudioSendStream* stream = new WebRtcAudioSendStream(
1844       channel, audio_transport, ssrc, sp.cname, sp.id, send_codec_spec_,
1845       send_rtp_extensions_, max_send_bitrate_bps_, audio_network_adaptor_config,
1846       call_, this, engine()->encoder_factory_);
1847   send_streams_.insert(std::make_pair(ssrc, stream));
1848 
1849   // At this point the stream's local SSRC has been updated. If it is the first
1850   // send stream, make sure that all the receive streams are updated with the
1851   // same SSRC in order to send receiver reports.
1852   if (send_streams_.size() == 1) {
1853     receiver_reports_ssrc_ = ssrc;
1854     for (const auto& kv : recv_streams_) {
1855       // TODO(solenberg): Allow applications to set the RTCP SSRC of receive
1856       // streams instead, so we can avoid recreating the streams here.
1857       kv.second->RecreateAudioReceiveStream(ssrc);
1858     }
1859   }
1860 
1861   send_streams_[ssrc]->SetSend(send_);
1862   return true;
1863 }
1864 
RemoveSendStream(uint32_t ssrc)1865 bool WebRtcVoiceMediaChannel::RemoveSendStream(uint32_t ssrc) {
1866   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::RemoveSendStream");
1867   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1868   RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc;
1869 
1870   auto it = send_streams_.find(ssrc);
1871   if (it == send_streams_.end()) {
1872     RTC_LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc
1873                         << " which doesn't exist.";
1874     return false;
1875   }
1876 
1877   it->second->SetSend(false);
1878 
1879   // TODO(solenberg): If we're removing the receiver_reports_ssrc_ stream, find
1880   // the first active send stream and use that instead, reassociating receive
1881   // streams.
1882 
1883   // Clean up and delete the send stream+channel.
1884   int channel = it->second->channel();
1885   RTC_LOG(LS_INFO) << "Removing audio send stream " << ssrc
1886                    << " with VoiceEngine channel #" << channel << ".";
1887   delete it->second;
1888   send_streams_.erase(it);
1889   if (!DeleteVoEChannel(channel)) {
1890     return false;
1891   }
1892   if (send_streams_.empty()) {
1893     SetSend(false);
1894   }
1895   return true;
1896 }
1897 
AddRecvStream(const StreamParams & sp)1898 bool WebRtcVoiceMediaChannel::AddRecvStream(const StreamParams& sp) {
1899   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::AddRecvStream");
1900   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1901   RTC_LOG(LS_INFO) << "AddRecvStream: " << sp.ToString();
1902 
1903   if (!ValidateStreamParams(sp)) {
1904     return false;
1905   }
1906 
1907   const uint32_t ssrc = sp.first_ssrc();
1908   if (ssrc == 0) {
1909     RTC_LOG(LS_WARNING) << "AddRecvStream with ssrc==0 is not supported.";
1910     return false;
1911   }
1912 
1913   // If this stream was previously received unsignaled, we promote it, possibly
1914   // recreating the AudioReceiveStream, if sync_label has changed.
1915   if (MaybeDeregisterUnsignaledRecvStream(ssrc)) {
1916     recv_streams_[ssrc]->MaybeRecreateAudioReceiveStream(sp.sync_label);
1917     return true;
1918   }
1919 
1920   if (GetReceiveChannelId(ssrc) != -1) {
1921     RTC_LOG(LS_ERROR) << "Stream already exists with ssrc " << ssrc;
1922     return false;
1923   }
1924 
1925   // Create a new channel for receiving audio data.
1926   const int channel = CreateVoEChannel();
1927   if (channel == -1) {
1928     return false;
1929   }
1930 
1931   recv_streams_.insert(std::make_pair(
1932       ssrc,
1933       new WebRtcAudioReceiveStream(
1934           channel, ssrc, receiver_reports_ssrc_, recv_transport_cc_enabled_,
1935           recv_nack_enabled_, sp.sync_label, recv_rtp_extensions_, call_, this,
1936           engine()->decoder_factory_, decoder_map_)));
1937   recv_streams_[ssrc]->SetPlayout(playout_);
1938 
1939   return true;
1940 }
1941 
RemoveRecvStream(uint32_t ssrc)1942 bool WebRtcVoiceMediaChannel::RemoveRecvStream(uint32_t ssrc) {
1943   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::RemoveRecvStream");
1944   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1945   RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc;
1946 
1947   const auto it = recv_streams_.find(ssrc);
1948   if (it == recv_streams_.end()) {
1949     RTC_LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc
1950                         << " which doesn't exist.";
1951     return false;
1952   }
1953 
1954   MaybeDeregisterUnsignaledRecvStream(ssrc);
1955 
1956   const int channel = it->second->channel();
1957 
1958   // Clean up and delete the receive stream+channel.
1959   RTC_LOG(LS_INFO) << "Removing audio receive stream " << ssrc
1960                    << " with VoiceEngine channel #" << channel << ".";
1961   it->second->SetRawAudioSink(nullptr);
1962   delete it->second;
1963   recv_streams_.erase(it);
1964   return DeleteVoEChannel(channel);
1965 }
1966 
SetLocalSource(uint32_t ssrc,AudioSource * source)1967 bool WebRtcVoiceMediaChannel::SetLocalSource(uint32_t ssrc,
1968                                              AudioSource* source) {
1969   auto it = send_streams_.find(ssrc);
1970   if (it == send_streams_.end()) {
1971     if (source) {
1972       // Return an error if trying to set a valid source with an invalid ssrc.
1973       RTC_LOG(LS_ERROR) << "SetLocalSource failed with ssrc " << ssrc;
1974       return false;
1975     }
1976 
1977     // The channel likely has gone away, do nothing.
1978     return true;
1979   }
1980 
1981   if (source) {
1982     it->second->SetSource(source);
1983   } else {
1984     it->second->ClearSource();
1985   }
1986 
1987   return true;
1988 }
1989 
1990 // TODO(solenberg): Remove, once AudioMonitor is gone.
GetActiveStreams(AudioInfo::StreamList * actives)1991 bool WebRtcVoiceMediaChannel::GetActiveStreams(
1992     AudioInfo::StreamList* actives) {
1993   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
1994   actives->clear();
1995   for (const auto& ch : recv_streams_) {
1996     int level = ch.second->GetOutputLevel();
1997     if (level > 0) {
1998       actives->push_back(std::make_pair(ch.first, level));
1999     }
2000   }
2001   return true;
2002 }
2003 
2004 // TODO(solenberg): Remove, once AudioMonitor is gone.
GetOutputLevel()2005 int WebRtcVoiceMediaChannel::GetOutputLevel() {
2006   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2007   int highest = 0;
2008   for (const auto& ch : recv_streams_) {
2009     highest = std::max(ch.second->GetOutputLevel(), highest);
2010   }
2011   return highest;
2012 }
2013 
SetOutputVolume(uint32_t ssrc,double volume)2014 bool WebRtcVoiceMediaChannel::SetOutputVolume(uint32_t ssrc, double volume) {
2015   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2016   std::vector<uint32_t> ssrcs(1, ssrc);
2017   // SSRC of 0 represents the default receive stream.
2018   if (ssrc == 0) {
2019     default_recv_volume_ = volume;
2020     ssrcs = unsignaled_recv_ssrcs_;
2021   }
2022   for (uint32_t ssrc : ssrcs) {
2023     const auto it = recv_streams_.find(ssrc);
2024     if (it == recv_streams_.end()) {
2025       RTC_LOG(LS_WARNING) << "SetOutputVolume: no recv stream " << ssrc;
2026       return false;
2027     }
2028     it->second->SetOutputVolume(volume);
2029     RTC_LOG(LS_INFO) << "SetOutputVolume() to " << volume
2030                      << " for recv stream with ssrc " << ssrc;
2031   }
2032   return true;
2033 }
2034 
CanInsertDtmf()2035 bool WebRtcVoiceMediaChannel::CanInsertDtmf() {
2036   return dtmf_payload_type_ ? true : false;
2037 }
2038 
InsertDtmf(uint32_t ssrc,int event,int duration)2039 bool WebRtcVoiceMediaChannel::InsertDtmf(uint32_t ssrc, int event,
2040                                          int duration) {
2041   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2042   RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::InsertDtmf";
2043   if (!dtmf_payload_type_) {
2044     return false;
2045   }
2046 
2047   // Figure out which WebRtcAudioSendStream to send the event on.
2048   auto it = ssrc != 0 ? send_streams_.find(ssrc) : send_streams_.begin();
2049   if (it == send_streams_.end()) {
2050     RTC_LOG(LS_WARNING) << "The specified ssrc " << ssrc << " is not in use.";
2051     return false;
2052   }
2053   if (event < kMinTelephoneEventCode ||
2054       event > kMaxTelephoneEventCode) {
2055     RTC_LOG(LS_WARNING) << "DTMF event code " << event << " out of range.";
2056     return false;
2057   }
2058   RTC_DCHECK_NE(-1, dtmf_payload_freq_);
2059   return it->second->SendTelephoneEvent(*dtmf_payload_type_, dtmf_payload_freq_,
2060                                         event, duration);
2061 }
2062 
OnPacketReceived(rtc::CopyOnWriteBuffer * packet,const rtc::PacketTime & packet_time)2063 void WebRtcVoiceMediaChannel::OnPacketReceived(
2064     rtc::CopyOnWriteBuffer* packet, const rtc::PacketTime& packet_time) {
2065   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2066 
2067   const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp,
2068                                               packet_time.not_before);
2069   webrtc::PacketReceiver::DeliveryStatus delivery_result =
2070       call_->Receiver()->DeliverPacket(webrtc::MediaType::AUDIO,
2071                                        packet->cdata(), packet->size(),
2072                                        webrtc_packet_time);
2073   if (delivery_result != webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC) {
2074     return;
2075   }
2076 
2077   // Create an unsignaled receive stream for this previously not received ssrc.
2078   // If there already is N unsignaled receive streams, delete the oldest.
2079   // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=5208
2080   uint32_t ssrc = 0;
2081   if (!GetRtpSsrc(packet->cdata(), packet->size(), &ssrc)) {
2082     return;
2083   }
2084   RTC_DCHECK(std::find(unsignaled_recv_ssrcs_.begin(),
2085       unsignaled_recv_ssrcs_.end(), ssrc) == unsignaled_recv_ssrcs_.end());
2086 
2087   // Add new stream.
2088   StreamParams sp;
2089   sp.ssrcs.push_back(ssrc);
2090   RTC_LOG(LS_INFO) << "Creating unsignaled receive stream for SSRC=" << ssrc;
2091   if (!AddRecvStream(sp)) {
2092     RTC_LOG(LS_WARNING) << "Could not create unsignaled receive stream.";
2093     return;
2094   }
2095   unsignaled_recv_ssrcs_.push_back(ssrc);
2096   RTC_HISTOGRAM_COUNTS_LINEAR(
2097       "WebRTC.Audio.NumOfUnsignaledStreams", unsignaled_recv_ssrcs_.size(), 1,
2098       100, 101);
2099 
2100   // Remove oldest unsignaled stream, if we have too many.
2101   if (unsignaled_recv_ssrcs_.size() > kMaxUnsignaledRecvStreams) {
2102     uint32_t remove_ssrc = unsignaled_recv_ssrcs_.front();
2103     RTC_LOG(LS_INFO) << "Removing unsignaled receive stream with SSRC="
2104                      << remove_ssrc;
2105     RemoveRecvStream(remove_ssrc);
2106   }
2107   RTC_DCHECK_GE(kMaxUnsignaledRecvStreams, unsignaled_recv_ssrcs_.size());
2108 
2109   SetOutputVolume(ssrc, default_recv_volume_);
2110 
2111   // The default sink can only be attached to one stream at a time, so we hook
2112   // it up to the *latest* unsignaled stream we've seen, in order to support the
2113   // case where the SSRC of one unsignaled stream changes.
2114   if (default_sink_) {
2115     for (uint32_t drop_ssrc : unsignaled_recv_ssrcs_) {
2116       auto it = recv_streams_.find(drop_ssrc);
2117       it->second->SetRawAudioSink(nullptr);
2118     }
2119     std::unique_ptr<webrtc::AudioSinkInterface> proxy_sink(
2120         new ProxySink(default_sink_.get()));
2121     SetRawAudioSink(ssrc, std::move(proxy_sink));
2122   }
2123 
2124   delivery_result = call_->Receiver()->DeliverPacket(webrtc::MediaType::AUDIO,
2125                                                      packet->cdata(),
2126                                                      packet->size(),
2127                                                      webrtc_packet_time);
2128   RTC_DCHECK_NE(webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC, delivery_result);
2129 }
2130 
OnRtcpReceived(rtc::CopyOnWriteBuffer * packet,const rtc::PacketTime & packet_time)2131 void WebRtcVoiceMediaChannel::OnRtcpReceived(
2132     rtc::CopyOnWriteBuffer* packet, const rtc::PacketTime& packet_time) {
2133   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2134 
2135   // Forward packet to Call as well.
2136   const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp,
2137                                               packet_time.not_before);
2138   call_->Receiver()->DeliverPacket(webrtc::MediaType::AUDIO,
2139       packet->cdata(), packet->size(), webrtc_packet_time);
2140 }
2141 
OnNetworkRouteChanged(const std::string & transport_name,const rtc::NetworkRoute & network_route)2142 void WebRtcVoiceMediaChannel::OnNetworkRouteChanged(
2143     const std::string& transport_name,
2144     const rtc::NetworkRoute& network_route) {
2145   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2146   // TODO(zhihaung): Merge these two callbacks.
2147   call_->OnNetworkRouteChanged(transport_name, network_route);
2148   call_->OnTransportOverheadChanged(webrtc::MediaType::AUDIO,
2149                                     network_route.packet_overhead);
2150 }
2151 
MuteStream(uint32_t ssrc,bool muted)2152 bool WebRtcVoiceMediaChannel::MuteStream(uint32_t ssrc, bool muted) {
2153   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2154   const auto it = send_streams_.find(ssrc);
2155   if (it == send_streams_.end()) {
2156     RTC_LOG(LS_WARNING) << "The specified ssrc " << ssrc << " is not in use.";
2157     return false;
2158   }
2159   it->second->SetMuted(muted);
2160 
2161   // TODO(solenberg):
2162   // We set the AGC to mute state only when all the channels are muted.
2163   // This implementation is not ideal, instead we should signal the AGC when
2164   // the mic channel is muted/unmuted. We can't do it today because there
2165   // is no good way to know which stream is mapping to the mic channel.
2166   bool all_muted = muted;
2167   for (const auto& kv : send_streams_) {
2168     all_muted = all_muted && kv.second->muted();
2169   }
2170   engine()->apm()->set_output_will_be_muted(all_muted);
2171 
2172   return true;
2173 }
2174 
SetMaxSendBitrate(int bps)2175 bool WebRtcVoiceMediaChannel::SetMaxSendBitrate(int bps) {
2176   RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetMaxSendBitrate.";
2177   max_send_bitrate_bps_ = bps;
2178   bool success = true;
2179   for (const auto& kv : send_streams_) {
2180     if (!kv.second->SetMaxSendBitrate(max_send_bitrate_bps_)) {
2181       success = false;
2182     }
2183   }
2184   return success;
2185 }
2186 
OnReadyToSend(bool ready)2187 void WebRtcVoiceMediaChannel::OnReadyToSend(bool ready) {
2188   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2189   RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready.");
2190   call_->SignalChannelNetworkState(
2191       webrtc::MediaType::AUDIO,
2192       ready ? webrtc::kNetworkUp : webrtc::kNetworkDown);
2193 }
2194 
GetStats(VoiceMediaInfo * info)2195 bool WebRtcVoiceMediaChannel::GetStats(VoiceMediaInfo* info) {
2196   TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::GetStats");
2197   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2198   RTC_DCHECK(info);
2199 
2200   // Get SSRC and stats for each sender.
2201   RTC_DCHECK_EQ(info->senders.size(), 0U);
2202   for (const auto& stream : send_streams_) {
2203     webrtc::AudioSendStream::Stats stats =
2204         stream.second->GetStats(recv_streams_.size() > 0);
2205     VoiceSenderInfo sinfo;
2206     sinfo.add_ssrc(stats.local_ssrc);
2207     sinfo.bytes_sent = stats.bytes_sent;
2208     sinfo.packets_sent = stats.packets_sent;
2209     sinfo.packets_lost = stats.packets_lost;
2210     sinfo.fraction_lost = stats.fraction_lost;
2211     sinfo.codec_name = stats.codec_name;
2212     sinfo.codec_payload_type = stats.codec_payload_type;
2213     sinfo.ext_seqnum = stats.ext_seqnum;
2214     sinfo.jitter_ms = stats.jitter_ms;
2215     sinfo.rtt_ms = stats.rtt_ms;
2216     sinfo.audio_level = stats.audio_level;
2217     sinfo.total_input_energy = stats.total_input_energy;
2218     sinfo.total_input_duration = stats.total_input_duration;
2219     sinfo.typing_noise_detected = (send_ ? stats.typing_noise_detected : false);
2220     sinfo.ana_statistics = stats.ana_statistics;
2221     sinfo.apm_statistics = stats.apm_statistics;
2222     info->senders.push_back(sinfo);
2223   }
2224 
2225   // Get SSRC and stats for each receiver.
2226   RTC_DCHECK_EQ(info->receivers.size(), 0U);
2227   for (const auto& stream : recv_streams_) {
2228     uint32_t ssrc = stream.first;
2229     // When SSRCs are unsignaled, there's only one audio MediaStreamTrack, but
2230     // multiple RTP streams can be received over time (if the SSRC changes for
2231     // whatever reason). We only want the RTCMediaStreamTrackStats to represent
2232     // the stats for the most recent stream (the one whose audio is actually
2233     // routed to the MediaStreamTrack), so here we ignore any unsignaled SSRCs
2234     // except for the most recent one (last in the vector). This is somewhat of
2235     // a hack, and means you don't get *any* stats for these inactive streams,
2236     // but it's slightly better than the previous behavior, which was "highest
2237     // SSRC wins".
2238     // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=8158
2239     if (!unsignaled_recv_ssrcs_.empty()) {
2240       auto end_it = --unsignaled_recv_ssrcs_.end();
2241       if (std::find(unsignaled_recv_ssrcs_.begin(), end_it, ssrc) != end_it) {
2242         continue;
2243       }
2244     }
2245     webrtc::AudioReceiveStream::Stats stats = stream.second->GetStats();
2246     VoiceReceiverInfo rinfo;
2247     rinfo.add_ssrc(stats.remote_ssrc);
2248     rinfo.bytes_rcvd = stats.bytes_rcvd;
2249     rinfo.packets_rcvd = stats.packets_rcvd;
2250     rinfo.packets_lost = stats.packets_lost;
2251     rinfo.fraction_lost = stats.fraction_lost;
2252     rinfo.codec_name = stats.codec_name;
2253     rinfo.codec_payload_type = stats.codec_payload_type;
2254     rinfo.ext_seqnum = stats.ext_seqnum;
2255     rinfo.jitter_ms = stats.jitter_ms;
2256     rinfo.jitter_buffer_ms = stats.jitter_buffer_ms;
2257     rinfo.jitter_buffer_preferred_ms = stats.jitter_buffer_preferred_ms;
2258     rinfo.delay_estimate_ms = stats.delay_estimate_ms;
2259     rinfo.audio_level = stats.audio_level;
2260     rinfo.total_output_energy = stats.total_output_energy;
2261     rinfo.total_samples_received = stats.total_samples_received;
2262     rinfo.total_output_duration = stats.total_output_duration;
2263     rinfo.concealed_samples = stats.concealed_samples;
2264     rinfo.concealment_events = stats.concealment_events;
2265     rinfo.jitter_buffer_delay_seconds = stats.jitter_buffer_delay_seconds;
2266     rinfo.expand_rate = stats.expand_rate;
2267     rinfo.speech_expand_rate = stats.speech_expand_rate;
2268     rinfo.secondary_decoded_rate = stats.secondary_decoded_rate;
2269     rinfo.secondary_discarded_rate = stats.secondary_discarded_rate;
2270     rinfo.accelerate_rate = stats.accelerate_rate;
2271     rinfo.preemptive_expand_rate = stats.preemptive_expand_rate;
2272     rinfo.decoding_calls_to_silence_generator =
2273         stats.decoding_calls_to_silence_generator;
2274     rinfo.decoding_calls_to_neteq = stats.decoding_calls_to_neteq;
2275     rinfo.decoding_normal = stats.decoding_normal;
2276     rinfo.decoding_plc = stats.decoding_plc;
2277     rinfo.decoding_cng = stats.decoding_cng;
2278     rinfo.decoding_plc_cng = stats.decoding_plc_cng;
2279     rinfo.decoding_muted_output = stats.decoding_muted_output;
2280     rinfo.capture_start_ntp_time_ms = stats.capture_start_ntp_time_ms;
2281     info->receivers.push_back(rinfo);
2282   }
2283 
2284   // Get codec info
2285   for (const AudioCodec& codec : send_codecs_) {
2286     webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
2287     info->send_codecs.insert(
2288         std::make_pair(codec_params.payload_type, std::move(codec_params)));
2289   }
2290   for (const AudioCodec& codec : recv_codecs_) {
2291     webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
2292     info->receive_codecs.insert(
2293         std::make_pair(codec_params.payload_type, std::move(codec_params)));
2294   }
2295 
2296   return true;
2297 }
2298 
SetRawAudioSink(uint32_t ssrc,std::unique_ptr<webrtc::AudioSinkInterface> sink)2299 void WebRtcVoiceMediaChannel::SetRawAudioSink(
2300     uint32_t ssrc,
2301     std::unique_ptr<webrtc::AudioSinkInterface> sink) {
2302   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2303   RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::SetRawAudioSink: ssrc:"
2304                       << ssrc << " " << (sink ? "(ptr)" : "NULL");
2305   if (ssrc == 0) {
2306     if (!unsignaled_recv_ssrcs_.empty()) {
2307       std::unique_ptr<webrtc::AudioSinkInterface> proxy_sink(
2308           sink ? new ProxySink(sink.get()) : nullptr);
2309       SetRawAudioSink(unsignaled_recv_ssrcs_.back(), std::move(proxy_sink));
2310     }
2311     default_sink_ = std::move(sink);
2312     return;
2313   }
2314   const auto it = recv_streams_.find(ssrc);
2315   if (it == recv_streams_.end()) {
2316     RTC_LOG(LS_WARNING) << "SetRawAudioSink: no recv stream " << ssrc;
2317     return;
2318   }
2319   it->second->SetRawAudioSink(std::move(sink));
2320 }
2321 
GetSources(uint32_t ssrc) const2322 std::vector<webrtc::RtpSource> WebRtcVoiceMediaChannel::GetSources(
2323     uint32_t ssrc) const {
2324   auto it = recv_streams_.find(ssrc);
2325   if (it == recv_streams_.end()) {
2326     RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:"
2327                       << ssrc << " which doesn't exist.";
2328     return std::vector<webrtc::RtpSource>();
2329   }
2330   return it->second->GetSources();
2331 }
2332 
GetReceiveChannelId(uint32_t ssrc) const2333 int WebRtcVoiceMediaChannel::GetReceiveChannelId(uint32_t ssrc) const {
2334   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2335   const auto it = recv_streams_.find(ssrc);
2336   if (it != recv_streams_.end()) {
2337     return it->second->channel();
2338   }
2339   return -1;
2340 }
2341 
GetSendChannelId(uint32_t ssrc) const2342 int WebRtcVoiceMediaChannel::GetSendChannelId(uint32_t ssrc) const {
2343   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2344   const auto it = send_streams_.find(ssrc);
2345   if (it != send_streams_.end()) {
2346     return it->second->channel();
2347   }
2348   return -1;
2349 }
2350 
2351 bool WebRtcVoiceMediaChannel::
MaybeDeregisterUnsignaledRecvStream(uint32_t ssrc)2352     MaybeDeregisterUnsignaledRecvStream(uint32_t ssrc) {
2353   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
2354   auto it = std::find(unsignaled_recv_ssrcs_.begin(),
2355                       unsignaled_recv_ssrcs_.end(),
2356                       ssrc);
2357   if (it != unsignaled_recv_ssrcs_.end()) {
2358     unsignaled_recv_ssrcs_.erase(it);
2359     return true;
2360   }
2361   return false;
2362 }
2363 }  // namespace cricket
2364 
2365 #endif  // HAVE_WEBRTC_VOICE
2366