1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "audio/audio_send_stream.h"
12
13 #include <memory>
14 #include <string>
15 #include <utility>
16 #include <vector>
17
18 #include "api/audio_codecs/audio_encoder.h"
19 #include "api/audio_codecs/audio_encoder_factory.h"
20 #include "api/audio_codecs/audio_format.h"
21 #include "api/call/transport.h"
22 #include "api/crypto/frame_encryptor_interface.h"
23 #include "api/function_view.h"
24 #include "api/rtc_event_log/rtc_event_log.h"
25 #include "audio/audio_state.h"
26 #include "audio/channel_send.h"
27 #include "audio/conversion.h"
28 #include "call/rtp_config.h"
29 #include "call/rtp_transport_controller_send_interface.h"
30 #include "common_audio/vad/include/vad.h"
31 #include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h"
32 #include "logging/rtc_event_log/rtc_stream_config.h"
33 #include "modules/audio_coding/codecs/cng/audio_encoder_cng.h"
34 #include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h"
35 #include "modules/audio_processing/include/audio_processing.h"
36 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
37 #include "rtc_base/checks.h"
38 #include "rtc_base/event.h"
39 #include "rtc_base/logging.h"
40 #include "rtc_base/strings/audio_format_to_string.h"
41 #include "rtc_base/task_queue.h"
42 #include "system_wrappers/include/field_trial.h"
43
44 namespace webrtc {
45 namespace {
46
UpdateEventLogStreamConfig(RtcEventLog * event_log,const AudioSendStream::Config & config,const AudioSendStream::Config * old_config)47 void UpdateEventLogStreamConfig(RtcEventLog* event_log,
48 const AudioSendStream::Config& config,
49 const AudioSendStream::Config* old_config) {
50 using SendCodecSpec = AudioSendStream::Config::SendCodecSpec;
51 // Only update if any of the things we log have changed.
52 auto payload_types_equal = [](const absl::optional<SendCodecSpec>& a,
53 const absl::optional<SendCodecSpec>& b) {
54 if (a.has_value() && b.has_value()) {
55 return a->format.name == b->format.name &&
56 a->payload_type == b->payload_type;
57 }
58 return !a.has_value() && !b.has_value();
59 };
60
61 if (old_config && config.rtp.ssrc == old_config->rtp.ssrc &&
62 config.rtp.extensions == old_config->rtp.extensions &&
63 payload_types_equal(config.send_codec_spec,
64 old_config->send_codec_spec)) {
65 return;
66 }
67
68 auto rtclog_config = std::make_unique<rtclog::StreamConfig>();
69 rtclog_config->local_ssrc = config.rtp.ssrc;
70 rtclog_config->rtp_extensions = config.rtp.extensions;
71 if (config.send_codec_spec) {
72 rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name,
73 config.send_codec_spec->payload_type, 0);
74 }
75 event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>(
76 std::move(rtclog_config)));
77 }
78 } // namespace
79
80 constexpr char AudioAllocationConfig::kKey[];
81
Parser()82 std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() {
83 return StructParametersParser::Create( //
84 "min", &min_bitrate, //
85 "max", &max_bitrate, //
86 "prio_rate", &priority_bitrate, //
87 "prio_rate_raw", &priority_bitrate_raw, //
88 "rate_prio", &bitrate_priority);
89 }
90
AudioAllocationConfig()91 AudioAllocationConfig::AudioAllocationConfig() {
92 Parser()->Parse(field_trial::FindFullName(kKey));
93 if (priority_bitrate_raw && !priority_bitrate.IsZero()) {
94 RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually "
95 "exclusive but both were configured.";
96 }
97 }
98
99 namespace internal {
AudioSendStream(Clock * clock,const webrtc::AudioSendStream::Config & config,const rtc::scoped_refptr<webrtc::AudioState> & audio_state,TaskQueueFactory * task_queue_factory,ProcessThread * module_process_thread,RtpTransportControllerSendInterface * rtp_transport,BitrateAllocatorInterface * bitrate_allocator,RtcEventLog * event_log,RtcpRttStats * rtcp_rtt_stats,const absl::optional<RtpState> & suspended_rtp_state)100 AudioSendStream::AudioSendStream(
101 Clock* clock,
102 const webrtc::AudioSendStream::Config& config,
103 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
104 TaskQueueFactory* task_queue_factory,
105 ProcessThread* module_process_thread,
106 RtpTransportControllerSendInterface* rtp_transport,
107 BitrateAllocatorInterface* bitrate_allocator,
108 RtcEventLog* event_log,
109 RtcpRttStats* rtcp_rtt_stats,
110 const absl::optional<RtpState>& suspended_rtp_state)
111 : AudioSendStream(clock,
112 config,
113 audio_state,
114 task_queue_factory,
115 rtp_transport,
116 bitrate_allocator,
117 event_log,
118 suspended_rtp_state,
119 voe::CreateChannelSend(
120 clock,
121 task_queue_factory,
122 module_process_thread,
123 config.send_transport,
124 rtcp_rtt_stats,
125 event_log,
126 config.frame_encryptor,
127 config.crypto_options,
128 config.rtp.extmap_allow_mixed,
129 config.rtcp_report_interval_ms,
130 config.rtp.ssrc,
131 config.frame_transformer,
132 rtp_transport->transport_feedback_observer())) {}
133
AudioSendStream(Clock * clock,const webrtc::AudioSendStream::Config & config,const rtc::scoped_refptr<webrtc::AudioState> & audio_state,TaskQueueFactory * task_queue_factory,RtpTransportControllerSendInterface * rtp_transport,BitrateAllocatorInterface * bitrate_allocator,RtcEventLog * event_log,const absl::optional<RtpState> & suspended_rtp_state,std::unique_ptr<voe::ChannelSendInterface> channel_send)134 AudioSendStream::AudioSendStream(
135 Clock* clock,
136 const webrtc::AudioSendStream::Config& config,
137 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
138 TaskQueueFactory* task_queue_factory,
139 RtpTransportControllerSendInterface* rtp_transport,
140 BitrateAllocatorInterface* bitrate_allocator,
141 RtcEventLog* event_log,
142 const absl::optional<RtpState>& suspended_rtp_state,
143 std::unique_ptr<voe::ChannelSendInterface> channel_send)
144 : clock_(clock),
145 worker_queue_(rtp_transport->GetWorkerQueue()),
146 audio_send_side_bwe_(field_trial::IsEnabled("WebRTC-Audio-SendSideBwe")),
147 allocate_audio_without_feedback_(
148 field_trial::IsEnabled("WebRTC-Audio-ABWENoTWCC")),
149 enable_audio_alr_probing_(
150 !field_trial::IsDisabled("WebRTC-Audio-AlrProbing")),
151 send_side_bwe_with_overhead_(
152 !field_trial::IsDisabled("WebRTC-SendSideBwe-WithOverhead")),
153 config_(Config(/*send_transport=*/nullptr)),
154 audio_state_(audio_state),
155 channel_send_(std::move(channel_send)),
156 event_log_(event_log),
157 use_legacy_overhead_calculation_(
158 field_trial::IsEnabled("WebRTC-Audio-LegacyOverhead")),
159 bitrate_allocator_(bitrate_allocator),
160 rtp_transport_(rtp_transport),
161 rtp_rtcp_module_(channel_send_->GetRtpRtcp()),
162 suspended_rtp_state_(suspended_rtp_state) {
163 RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc;
164 RTC_DCHECK(worker_queue_);
165 RTC_DCHECK(audio_state_);
166 RTC_DCHECK(channel_send_);
167 RTC_DCHECK(bitrate_allocator_);
168 RTC_DCHECK(rtp_transport);
169
170 RTC_DCHECK(rtp_rtcp_module_);
171
172 ConfigureStream(config, true);
173
174 pacer_thread_checker_.Detach();
175 }
176
~AudioSendStream()177 AudioSendStream::~AudioSendStream() {
178 RTC_DCHECK(worker_thread_checker_.IsCurrent());
179 RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc;
180 RTC_DCHECK(!sending_);
181 channel_send_->ResetSenderCongestionControlObjects();
182 // Blocking call to synchronize state with worker queue to ensure that there
183 // are no pending tasks left that keeps references to audio.
184 rtc::Event thread_sync_event;
185 worker_queue_->PostTask([&] { thread_sync_event.Set(); });
186 thread_sync_event.Wait(rtc::Event::kForever);
187 }
188
GetConfig() const189 const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const {
190 RTC_DCHECK(worker_thread_checker_.IsCurrent());
191 return config_;
192 }
193
Reconfigure(const webrtc::AudioSendStream::Config & new_config)194 void AudioSendStream::Reconfigure(
195 const webrtc::AudioSendStream::Config& new_config) {
196 RTC_DCHECK(worker_thread_checker_.IsCurrent());
197 ConfigureStream(new_config, false);
198 }
199
FindExtensionIds(const std::vector<RtpExtension> & extensions)200 AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds(
201 const std::vector<RtpExtension>& extensions) {
202 ExtensionIds ids;
203 for (const auto& extension : extensions) {
204 if (extension.uri == RtpExtension::kAudioLevelUri) {
205 ids.audio_level = extension.id;
206 } else if (extension.uri == RtpExtension::kAbsSendTimeUri) {
207 ids.abs_send_time = extension.id;
208 } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
209 ids.transport_sequence_number = extension.id;
210 } else if (extension.uri == RtpExtension::kMidUri) {
211 ids.mid = extension.id;
212 } else if (extension.uri == RtpExtension::kRidUri) {
213 ids.rid = extension.id;
214 } else if (extension.uri == RtpExtension::kRepairedRidUri) {
215 ids.repaired_rid = extension.id;
216 } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) {
217 ids.abs_capture_time = extension.id;
218 }
219 }
220 return ids;
221 }
222
TransportSeqNumId(const AudioSendStream::Config & config)223 int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) {
224 return FindExtensionIds(config.rtp.extensions).transport_sequence_number;
225 }
226
ConfigureStream(const webrtc::AudioSendStream::Config & new_config,bool first_time)227 void AudioSendStream::ConfigureStream(
228 const webrtc::AudioSendStream::Config& new_config,
229 bool first_time) {
230 RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: "
231 << new_config.ToString();
232 UpdateEventLogStreamConfig(event_log_, new_config,
233 first_time ? nullptr : &config_);
234
235 const auto& old_config = config_;
236
237 // Configuration parameters which cannot be changed.
238 RTC_DCHECK(first_time ||
239 old_config.send_transport == new_config.send_transport);
240 RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc);
241 if (suspended_rtp_state_ && first_time) {
242 rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_);
243 }
244 if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) {
245 channel_send_->SetRTCP_CNAME(new_config.rtp.c_name);
246 }
247
248 // Enable the frame encryptor if a new frame encryptor has been provided.
249 if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) {
250 channel_send_->SetFrameEncryptor(new_config.frame_encryptor);
251 }
252
253 if (first_time ||
254 new_config.frame_transformer != old_config.frame_transformer) {
255 channel_send_->SetEncoderToPacketizerFrameTransformer(
256 new_config.frame_transformer);
257 }
258
259 if (first_time ||
260 new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) {
261 rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed);
262 }
263
264 const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions);
265 const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions);
266
267 // Audio level indication
268 if (first_time || new_ids.audio_level != old_ids.audio_level) {
269 channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0,
270 new_ids.audio_level);
271 }
272
273 if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) {
274 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(
275 kRtpExtensionAbsoluteSendTime);
276 if (new_ids.abs_send_time) {
277 rtp_rtcp_module_->RegisterRtpHeaderExtension(AbsoluteSendTime::kUri,
278 new_ids.abs_send_time);
279 }
280 }
281
282 bool transport_seq_num_id_changed =
283 new_ids.transport_sequence_number != old_ids.transport_sequence_number;
284 if (first_time ||
285 (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) {
286 if (!first_time) {
287 channel_send_->ResetSenderCongestionControlObjects();
288 }
289
290 RtcpBandwidthObserver* bandwidth_observer = nullptr;
291
292 if (audio_send_side_bwe_ && !allocate_audio_without_feedback_ &&
293 new_ids.transport_sequence_number != 0) {
294 rtp_rtcp_module_->RegisterRtpHeaderExtension(
295 TransportSequenceNumber::kUri, new_ids.transport_sequence_number);
296 // Probing in application limited region is only used in combination with
297 // send side congestion control, wich depends on feedback packets which
298 // requires transport sequence numbers to be enabled.
299 // Optionally request ALR probing but do not override any existing
300 // request from other streams.
301 if (enable_audio_alr_probing_) {
302 rtp_transport_->EnablePeriodicAlrProbing(true);
303 }
304 bandwidth_observer = rtp_transport_->GetBandwidthObserver();
305 }
306 channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_,
307 bandwidth_observer);
308 }
309 // MID RTP header extension.
310 if ((first_time || new_ids.mid != old_ids.mid ||
311 new_config.rtp.mid != old_config.rtp.mid) &&
312 new_ids.mid != 0 && !new_config.rtp.mid.empty()) {
313 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::kUri, new_ids.mid);
314 rtp_rtcp_module_->SetMid(new_config.rtp.mid);
315 }
316
317 // RID RTP header extension
318 if ((first_time || new_ids.rid != old_ids.rid ||
319 new_ids.repaired_rid != old_ids.repaired_rid ||
320 new_config.rtp.rid != old_config.rtp.rid)) {
321 if (new_ids.rid != 0 || new_ids.repaired_rid != 0) {
322 if (new_config.rtp.rid.empty()) {
323 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(RtpStreamId::kUri);
324 } else if (new_ids.repaired_rid != 0) {
325 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri,
326 new_ids.repaired_rid);
327 } else {
328 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri,
329 new_ids.rid);
330 }
331 }
332 rtp_rtcp_module_->SetRid(new_config.rtp.rid);
333 }
334
335 if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) {
336 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(
337 kRtpExtensionAbsoluteCaptureTime);
338 if (new_ids.abs_capture_time) {
339 rtp_rtcp_module_->RegisterRtpHeaderExtension(
340 AbsoluteCaptureTimeExtension::kUri, new_ids.abs_capture_time);
341 }
342 }
343
344 if (!ReconfigureSendCodec(new_config)) {
345 RTC_LOG(LS_ERROR) << "Failed to set up send codec state.";
346 }
347
348 // Set currently known overhead (used in ANA, opus only).
349 {
350 MutexLock lock(&overhead_per_packet_lock_);
351 UpdateOverheadForEncoder();
352 }
353
354 channel_send_->CallEncoder([this](AudioEncoder* encoder) {
355 if (!encoder) {
356 return;
357 }
358 worker_queue_->PostTask(
359 [this, length_range = encoder->GetFrameLengthRange()] {
360 RTC_DCHECK_RUN_ON(worker_queue_);
361 frame_length_range_ = length_range;
362 });
363 });
364
365 if (sending_) {
366 ReconfigureBitrateObserver(new_config);
367 }
368 config_ = new_config;
369 }
370
Start()371 void AudioSendStream::Start() {
372 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
373 if (sending_) {
374 return;
375 }
376 if (!config_.has_dscp && config_.min_bitrate_bps != -1 &&
377 config_.max_bitrate_bps != -1 &&
378 (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) {
379 rtp_transport_->AccountForAudioPacketsInPacedSender(true);
380 if (send_side_bwe_with_overhead_)
381 rtp_transport_->IncludeOverheadInPacedSender();
382 rtp_rtcp_module_->SetAsPartOfAllocation(true);
383 rtc::Event thread_sync_event;
384 worker_queue_->PostTask([&] {
385 RTC_DCHECK_RUN_ON(worker_queue_);
386 ConfigureBitrateObserver();
387 thread_sync_event.Set();
388 });
389 thread_sync_event.Wait(rtc::Event::kForever);
390 } else {
391 rtp_rtcp_module_->SetAsPartOfAllocation(false);
392 }
393 channel_send_->StartSend();
394 sending_ = true;
395 audio_state()->AddSendingStream(this, encoder_sample_rate_hz_,
396 encoder_num_channels_);
397 }
398
Stop()399 void AudioSendStream::Stop() {
400 RTC_DCHECK(worker_thread_checker_.IsCurrent());
401 if (!sending_) {
402 return;
403 }
404
405 RemoveBitrateObserver();
406 channel_send_->StopSend();
407 sending_ = false;
408 audio_state()->RemoveSendingStream(this);
409 }
410
SendAudioData(std::unique_ptr<AudioFrame> audio_frame)411 void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) {
412 RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_);
413 RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0);
414 double duration = static_cast<double>(audio_frame->samples_per_channel_) /
415 audio_frame->sample_rate_hz_;
416 {
417 // Note: SendAudioData() passes the frame further down the pipeline and it
418 // may eventually get sent. But this method is invoked even if we are not
419 // connected, as long as we have an AudioSendStream (created as a result of
420 // an O/A exchange). This means that we are calculating audio levels whether
421 // or not we are sending samples.
422 // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats
423 // should move from send-streams to the local audio sources or tracks; a
424 // send-stream should not be required to read the microphone audio levels.
425 MutexLock lock(&audio_level_lock_);
426 audio_level_.ComputeLevel(*audio_frame, duration);
427 }
428 channel_send_->ProcessAndEncodeAudio(std::move(audio_frame));
429 }
430
SendTelephoneEvent(int payload_type,int payload_frequency,int event,int duration_ms)431 bool AudioSendStream::SendTelephoneEvent(int payload_type,
432 int payload_frequency,
433 int event,
434 int duration_ms) {
435 RTC_DCHECK(worker_thread_checker_.IsCurrent());
436 channel_send_->SetSendTelephoneEventPayloadType(payload_type,
437 payload_frequency);
438 return channel_send_->SendTelephoneEventOutband(event, duration_ms);
439 }
440
SetMuted(bool muted)441 void AudioSendStream::SetMuted(bool muted) {
442 RTC_DCHECK(worker_thread_checker_.IsCurrent());
443 channel_send_->SetInputMute(muted);
444 }
445
GetStats() const446 webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
447 return GetStats(true);
448 }
449
GetStats(bool has_remote_tracks) const450 webrtc::AudioSendStream::Stats AudioSendStream::GetStats(
451 bool has_remote_tracks) const {
452 RTC_DCHECK(worker_thread_checker_.IsCurrent());
453 webrtc::AudioSendStream::Stats stats;
454 stats.local_ssrc = config_.rtp.ssrc;
455 stats.target_bitrate_bps = channel_send_->GetBitrate();
456
457 webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics();
458 stats.payload_bytes_sent = call_stats.payload_bytes_sent;
459 stats.header_and_padding_bytes_sent =
460 call_stats.header_and_padding_bytes_sent;
461 stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent;
462 stats.packets_sent = call_stats.packetsSent;
463 stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent;
464 // RTT isn't known until a RTCP report is received. Until then, VoiceEngine
465 // returns 0 to indicate an error value.
466 if (call_stats.rttMs > 0) {
467 stats.rtt_ms = call_stats.rttMs;
468 }
469 if (config_.send_codec_spec) {
470 const auto& spec = *config_.send_codec_spec;
471 stats.codec_name = spec.format.name;
472 stats.codec_payload_type = spec.payload_type;
473
474 // Get data from the last remote RTCP report.
475 for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) {
476 // Lookup report for send ssrc only.
477 if (block.source_SSRC == stats.local_ssrc) {
478 stats.packets_lost = block.cumulative_num_packets_lost;
479 stats.fraction_lost = Q8ToFloat(block.fraction_lost);
480 // Convert timestamps to milliseconds.
481 if (spec.format.clockrate_hz / 1000 > 0) {
482 stats.jitter_ms =
483 block.interarrival_jitter / (spec.format.clockrate_hz / 1000);
484 }
485 break;
486 }
487 }
488 }
489
490 {
491 MutexLock lock(&audio_level_lock_);
492 stats.audio_level = audio_level_.LevelFullRange();
493 stats.total_input_energy = audio_level_.TotalEnergy();
494 stats.total_input_duration = audio_level_.TotalDuration();
495 }
496
497 stats.typing_noise_detected = audio_state()->typing_noise_detected();
498 stats.ana_statistics = channel_send_->GetANAStatistics();
499
500 AudioProcessing* ap = audio_state_->audio_processing();
501 if (ap) {
502 stats.apm_statistics = ap->GetStatistics(has_remote_tracks);
503 }
504
505 stats.report_block_datas = std::move(call_stats.report_block_datas);
506
507 return stats;
508 }
509
DeliverRtcp(const uint8_t * packet,size_t length)510 void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
511 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
512 channel_send_->ReceivedRTCPPacket(packet, length);
513 worker_queue_->PostTask([&]() {
514 // Poll if overhead has changed, which it can do if ack triggers us to stop
515 // sending mid/rid.
516 MutexLock lock(&overhead_per_packet_lock_);
517 UpdateOverheadForEncoder();
518 });
519 }
520
OnBitrateUpdated(BitrateAllocationUpdate update)521 uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) {
522 RTC_DCHECK_RUN_ON(worker_queue_);
523
524 // Pick a target bitrate between the constraints. Overrules the allocator if
525 // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a
526 // higher than max to allow for e.g. extra FEC.
527 auto constraints = GetMinMaxBitrateConstraints();
528 update.target_bitrate.Clamp(constraints.min, constraints.max);
529 update.stable_target_bitrate.Clamp(constraints.min, constraints.max);
530
531 channel_send_->OnBitrateAllocation(update);
532
533 // The amount of audio protection is not exposed by the encoder, hence
534 // always returning 0.
535 return 0;
536 }
537
SetTransportOverhead(int transport_overhead_per_packet_bytes)538 void AudioSendStream::SetTransportOverhead(
539 int transport_overhead_per_packet_bytes) {
540 RTC_DCHECK(worker_thread_checker_.IsCurrent());
541 MutexLock lock(&overhead_per_packet_lock_);
542 transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes;
543 UpdateOverheadForEncoder();
544 }
545
UpdateOverheadForEncoder()546 void AudioSendStream::UpdateOverheadForEncoder() {
547 size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes();
548 if (overhead_per_packet_ == overhead_per_packet_bytes) {
549 return;
550 }
551 overhead_per_packet_ = overhead_per_packet_bytes;
552
553 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
554 encoder->OnReceivedOverhead(overhead_per_packet_bytes);
555 });
556 auto update_task = [this, overhead_per_packet_bytes] {
557 RTC_DCHECK_RUN_ON(worker_queue_);
558 if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) {
559 total_packet_overhead_bytes_ = overhead_per_packet_bytes;
560 if (registered_with_allocator_) {
561 ConfigureBitrateObserver();
562 }
563 }
564 };
565 if (worker_queue_->IsCurrent()) {
566 update_task();
567 } else {
568 worker_queue_->PostTask(update_task);
569 }
570 }
571
TestOnlyGetPerPacketOverheadBytes() const572 size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const {
573 MutexLock lock(&overhead_per_packet_lock_);
574 return GetPerPacketOverheadBytes();
575 }
576
GetPerPacketOverheadBytes() const577 size_t AudioSendStream::GetPerPacketOverheadBytes() const {
578 return transport_overhead_per_packet_bytes_ +
579 rtp_rtcp_module_->ExpectedPerPacketOverhead();
580 }
581
GetRtpState() const582 RtpState AudioSendStream::GetRtpState() const {
583 return rtp_rtcp_module_->GetRtpState();
584 }
585
GetChannel() const586 const voe::ChannelSendInterface* AudioSendStream::GetChannel() const {
587 return channel_send_.get();
588 }
589
audio_state()590 internal::AudioState* AudioSendStream::audio_state() {
591 internal::AudioState* audio_state =
592 static_cast<internal::AudioState*>(audio_state_.get());
593 RTC_DCHECK(audio_state);
594 return audio_state;
595 }
596
audio_state() const597 const internal::AudioState* AudioSendStream::audio_state() const {
598 internal::AudioState* audio_state =
599 static_cast<internal::AudioState*>(audio_state_.get());
600 RTC_DCHECK(audio_state);
601 return audio_state;
602 }
603
StoreEncoderProperties(int sample_rate_hz,size_t num_channels)604 void AudioSendStream::StoreEncoderProperties(int sample_rate_hz,
605 size_t num_channels) {
606 RTC_DCHECK(worker_thread_checker_.IsCurrent());
607 encoder_sample_rate_hz_ = sample_rate_hz;
608 encoder_num_channels_ = num_channels;
609 if (sending_) {
610 // Update AudioState's information about the stream.
611 audio_state()->AddSendingStream(this, sample_rate_hz, num_channels);
612 }
613 }
614
615 // Apply current codec settings to a single voe::Channel used for sending.
SetupSendCodec(const Config & new_config)616 bool AudioSendStream::SetupSendCodec(const Config& new_config) {
617 RTC_DCHECK(new_config.send_codec_spec);
618 const auto& spec = *new_config.send_codec_spec;
619
620 RTC_DCHECK(new_config.encoder_factory);
621 std::unique_ptr<AudioEncoder> encoder =
622 new_config.encoder_factory->MakeAudioEncoder(
623 spec.payload_type, spec.format, new_config.codec_pair_id);
624
625 if (!encoder) {
626 RTC_DLOG(LS_ERROR) << "Unable to create encoder for "
627 << rtc::ToString(spec.format);
628 return false;
629 }
630
631 // If a bitrate has been specified for the codec, use it over the
632 // codec's default.
633 if (spec.target_bitrate_bps) {
634 encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps);
635 }
636
637 // Enable ANA if configured (currently only used by Opus).
638 if (new_config.audio_network_adaptor_config) {
639 if (encoder->EnableAudioNetworkAdaptor(
640 *new_config.audio_network_adaptor_config, event_log_)) {
641 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
642 << new_config.rtp.ssrc;
643 } else {
644 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
645 << new_config.rtp.ssrc;
646 }
647 }
648
649 // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled.
650 if (spec.cng_payload_type) {
651 AudioEncoderCngConfig cng_config;
652 cng_config.num_channels = encoder->NumChannels();
653 cng_config.payload_type = *spec.cng_payload_type;
654 cng_config.speech_encoder = std::move(encoder);
655 cng_config.vad_mode = Vad::kVadNormal;
656 encoder = CreateComfortNoiseEncoder(std::move(cng_config));
657
658 RegisterCngPayloadType(*spec.cng_payload_type,
659 new_config.send_codec_spec->format.clockrate_hz);
660 }
661
662 // Wrap the encoder in a RED encoder, if RED is enabled.
663 if (spec.red_payload_type) {
664 AudioEncoderCopyRed::Config red_config;
665 red_config.payload_type = *spec.red_payload_type;
666 red_config.speech_encoder = std::move(encoder);
667 encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config));
668 }
669
670 // Set currently known overhead (used in ANA, opus only).
671 // If overhead changes later, it will be updated in UpdateOverheadForEncoder.
672 {
673 MutexLock lock(&overhead_per_packet_lock_);
674 size_t overhead = GetPerPacketOverheadBytes();
675 if (overhead > 0) {
676 encoder->OnReceivedOverhead(overhead);
677 }
678 }
679
680 StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels());
681 channel_send_->SetEncoder(new_config.send_codec_spec->payload_type,
682 std::move(encoder));
683
684 return true;
685 }
686
ReconfigureSendCodec(const Config & new_config)687 bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) {
688 const auto& old_config = config_;
689
690 if (!new_config.send_codec_spec) {
691 // We cannot de-configure a send codec. So we will do nothing.
692 // By design, the send codec should have not been configured.
693 RTC_DCHECK(!old_config.send_codec_spec);
694 return true;
695 }
696
697 if (new_config.send_codec_spec == old_config.send_codec_spec &&
698 new_config.audio_network_adaptor_config ==
699 old_config.audio_network_adaptor_config) {
700 return true;
701 }
702
703 // If we have no encoder, or the format or payload type's changed, create a
704 // new encoder.
705 if (!old_config.send_codec_spec ||
706 new_config.send_codec_spec->format !=
707 old_config.send_codec_spec->format ||
708 new_config.send_codec_spec->payload_type !=
709 old_config.send_codec_spec->payload_type) {
710 return SetupSendCodec(new_config);
711 }
712
713 const absl::optional<int>& new_target_bitrate_bps =
714 new_config.send_codec_spec->target_bitrate_bps;
715 // If a bitrate has been specified for the codec, use it over the
716 // codec's default.
717 if (new_target_bitrate_bps &&
718 new_target_bitrate_bps !=
719 old_config.send_codec_spec->target_bitrate_bps) {
720 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
721 encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps);
722 });
723 }
724
725 ReconfigureANA(new_config);
726 ReconfigureCNG(new_config);
727
728 return true;
729 }
730
ReconfigureANA(const Config & new_config)731 void AudioSendStream::ReconfigureANA(const Config& new_config) {
732 if (new_config.audio_network_adaptor_config ==
733 config_.audio_network_adaptor_config) {
734 return;
735 }
736 if (new_config.audio_network_adaptor_config) {
737 // This lock needs to be acquired before CallEncoder, since it aquires
738 // another lock and we need to maintain the same order at all call sites to
739 // avoid deadlock.
740 MutexLock lock(&overhead_per_packet_lock_);
741 size_t overhead = GetPerPacketOverheadBytes();
742 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
743 if (encoder->EnableAudioNetworkAdaptor(
744 *new_config.audio_network_adaptor_config, event_log_)) {
745 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
746 << new_config.rtp.ssrc;
747 if (overhead > 0) {
748 encoder->OnReceivedOverhead(overhead);
749 }
750 } else {
751 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
752 << new_config.rtp.ssrc;
753 }
754 });
755 } else {
756 channel_send_->CallEncoder(
757 [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); });
758 RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC "
759 << new_config.rtp.ssrc;
760 }
761 }
762
ReconfigureCNG(const Config & new_config)763 void AudioSendStream::ReconfigureCNG(const Config& new_config) {
764 if (new_config.send_codec_spec->cng_payload_type ==
765 config_.send_codec_spec->cng_payload_type) {
766 return;
767 }
768
769 // Register the CNG payload type if it's been added, don't do anything if CNG
770 // is removed. Payload types must not be redefined.
771 if (new_config.send_codec_spec->cng_payload_type) {
772 RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type,
773 new_config.send_codec_spec->format.clockrate_hz);
774 }
775
776 // Wrap or unwrap the encoder in an AudioEncoderCNG.
777 channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
778 std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr));
779 auto sub_encoders = old_encoder->ReclaimContainedEncoders();
780 if (!sub_encoders.empty()) {
781 // Replace enc with its sub encoder. We need to put the sub
782 // encoder in a temporary first, since otherwise the old value
783 // of enc would be destroyed before the new value got assigned,
784 // which would be bad since the new value is a part of the old
785 // value.
786 auto tmp = std::move(sub_encoders[0]);
787 old_encoder = std::move(tmp);
788 }
789 if (new_config.send_codec_spec->cng_payload_type) {
790 AudioEncoderCngConfig config;
791 config.speech_encoder = std::move(old_encoder);
792 config.num_channels = config.speech_encoder->NumChannels();
793 config.payload_type = *new_config.send_codec_spec->cng_payload_type;
794 config.vad_mode = Vad::kVadNormal;
795 *encoder_ptr = CreateComfortNoiseEncoder(std::move(config));
796 } else {
797 *encoder_ptr = std::move(old_encoder);
798 }
799 });
800 }
801
ReconfigureBitrateObserver(const webrtc::AudioSendStream::Config & new_config)802 void AudioSendStream::ReconfigureBitrateObserver(
803 const webrtc::AudioSendStream::Config& new_config) {
804 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
805 // Since the Config's default is for both of these to be -1, this test will
806 // allow us to configure the bitrate observer if the new config has bitrate
807 // limits set, but would only have us call RemoveBitrateObserver if we were
808 // previously configured with bitrate limits.
809 if (config_.min_bitrate_bps == new_config.min_bitrate_bps &&
810 config_.max_bitrate_bps == new_config.max_bitrate_bps &&
811 config_.bitrate_priority == new_config.bitrate_priority &&
812 (TransportSeqNumId(config_) == TransportSeqNumId(new_config) ||
813 !audio_send_side_bwe_) &&
814 config_.audio_network_adaptor_config ==
815 new_config.audio_network_adaptor_config) {
816 return;
817 }
818
819 if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 &&
820 new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) {
821 rtp_transport_->AccountForAudioPacketsInPacedSender(true);
822 if (send_side_bwe_with_overhead_)
823 rtp_transport_->IncludeOverheadInPacedSender();
824 rtc::Event thread_sync_event;
825 worker_queue_->PostTask([&] {
826 RTC_DCHECK_RUN_ON(worker_queue_);
827 // We may get a callback immediately as the observer is registered, so
828 // make
829 // sure the bitrate limits in config_ are up-to-date.
830 config_.min_bitrate_bps = new_config.min_bitrate_bps;
831 config_.max_bitrate_bps = new_config.max_bitrate_bps;
832
833 config_.bitrate_priority = new_config.bitrate_priority;
834 ConfigureBitrateObserver();
835 thread_sync_event.Set();
836 });
837 thread_sync_event.Wait(rtc::Event::kForever);
838 rtp_rtcp_module_->SetAsPartOfAllocation(true);
839 } else {
840 rtp_transport_->AccountForAudioPacketsInPacedSender(false);
841 RemoveBitrateObserver();
842 rtp_rtcp_module_->SetAsPartOfAllocation(false);
843 }
844 }
845
ConfigureBitrateObserver()846 void AudioSendStream::ConfigureBitrateObserver() {
847 // This either updates the current observer or adds a new observer.
848 // TODO(srte): Add overhead compensation here.
849 auto constraints = GetMinMaxBitrateConstraints();
850
851 DataRate priority_bitrate = allocation_settings_.priority_bitrate;
852 if (send_side_bwe_with_overhead_) {
853 if (use_legacy_overhead_calculation_) {
854 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
855 constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12;
856 const TimeDelta kMinPacketDuration = TimeDelta::Millis(20);
857 DataRate max_overhead =
858 DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration;
859 priority_bitrate += max_overhead;
860 } else {
861 RTC_DCHECK(frame_length_range_);
862 const DataSize overhead_per_packet =
863 DataSize::Bytes(total_packet_overhead_bytes_);
864 DataRate min_overhead = overhead_per_packet / frame_length_range_->second;
865 priority_bitrate += min_overhead;
866 }
867 }
868 if (allocation_settings_.priority_bitrate_raw)
869 priority_bitrate = *allocation_settings_.priority_bitrate_raw;
870
871 bitrate_allocator_->AddObserver(
872 this,
873 MediaStreamAllocationConfig{
874 constraints.min.bps<uint32_t>(), constraints.max.bps<uint32_t>(), 0,
875 priority_bitrate.bps(), true,
876 allocation_settings_.bitrate_priority.value_or(
877 config_.bitrate_priority)});
878 registered_with_allocator_ = true;
879 }
880
RemoveBitrateObserver()881 void AudioSendStream::RemoveBitrateObserver() {
882 RTC_DCHECK(worker_thread_checker_.IsCurrent());
883 rtc::Event thread_sync_event;
884 worker_queue_->PostTask([this, &thread_sync_event] {
885 RTC_DCHECK_RUN_ON(worker_queue_);
886 registered_with_allocator_ = false;
887 bitrate_allocator_->RemoveObserver(this);
888 thread_sync_event.Set();
889 });
890 thread_sync_event.Wait(rtc::Event::kForever);
891 }
892
893 AudioSendStream::TargetAudioBitrateConstraints
GetMinMaxBitrateConstraints() const894 AudioSendStream::GetMinMaxBitrateConstraints() const {
895 TargetAudioBitrateConstraints constraints{
896 DataRate::BitsPerSec(config_.min_bitrate_bps),
897 DataRate::BitsPerSec(config_.max_bitrate_bps)};
898
899 // If bitrates were explicitly overriden via field trial, use those values.
900 if (allocation_settings_.min_bitrate)
901 constraints.min = *allocation_settings_.min_bitrate;
902 if (allocation_settings_.max_bitrate)
903 constraints.max = *allocation_settings_.max_bitrate;
904
905 RTC_DCHECK_GE(constraints.min, DataRate::Zero());
906 RTC_DCHECK_GE(constraints.max, DataRate::Zero());
907 RTC_DCHECK_GE(constraints.max, constraints.min);
908 if (send_side_bwe_with_overhead_) {
909 if (use_legacy_overhead_calculation_) {
910 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
911 const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12);
912 const TimeDelta kMaxFrameLength =
913 TimeDelta::Millis(60); // Based on Opus spec
914 const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength;
915 constraints.min += kMinOverhead;
916 constraints.max += kMinOverhead;
917 } else {
918 RTC_DCHECK(frame_length_range_);
919 const DataSize kOverheadPerPacket =
920 DataSize::Bytes(total_packet_overhead_bytes_);
921 constraints.min += kOverheadPerPacket / frame_length_range_->second;
922 constraints.max += kOverheadPerPacket / frame_length_range_->first;
923 }
924 }
925 return constraints;
926 }
927
RegisterCngPayloadType(int payload_type,int clockrate_hz)928 void AudioSendStream::RegisterCngPayloadType(int payload_type,
929 int clockrate_hz) {
930 channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz);
931 }
932 } // namespace internal
933 } // namespace webrtc
934