1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "audio/audio_receive_stream.h"
12
13 #include <string>
14 #include <utility>
15
16 #include "absl/memory/memory.h"
17 #include "api/array_view.h"
18 #include "api/audio_codecs/audio_format.h"
19 #include "api/call/audio_sink.h"
20 #include "api/rtp_parameters.h"
21 #include "audio/audio_send_stream.h"
22 #include "audio/audio_state.h"
23 #include "audio/channel_receive.h"
24 #include "audio/conversion.h"
25 #include "call/rtp_config.h"
26 #include "call/rtp_stream_receiver_controller_interface.h"
27 #include "rtc_base/checks.h"
28 #include "rtc_base/logging.h"
29 #include "rtc_base/strings/string_builder.h"
30 #include "rtc_base/time_utils.h"
31
32 namespace webrtc {
33
ToString() const34 std::string AudioReceiveStream::Config::Rtp::ToString() const {
35 char ss_buf[1024];
36 rtc::SimpleStringBuilder ss(ss_buf);
37 ss << "{remote_ssrc: " << remote_ssrc;
38 ss << ", local_ssrc: " << local_ssrc;
39 ss << ", transport_cc: " << (transport_cc ? "on" : "off");
40 ss << ", nack: " << nack.ToString();
41 ss << ", extensions: [";
42 for (size_t i = 0; i < extensions.size(); ++i) {
43 ss << extensions[i].ToString();
44 if (i != extensions.size() - 1) {
45 ss << ", ";
46 }
47 }
48 ss << ']';
49 ss << ", rtcp_event_observer: "
50 << (rtcp_event_observer ? "(rtcp_event_observer)" : "nullptr");
51 ss << '}';
52 return ss.str();
53 }
54
ToString() const55 std::string AudioReceiveStream::Config::ToString() const {
56 char ss_buf[1024];
57 rtc::SimpleStringBuilder ss(ss_buf);
58 ss << "{rtp: " << rtp.ToString();
59 ss << ", rtcp_send_transport: "
60 << (rtcp_send_transport ? "(Transport)" : "null");
61 if (!sync_group.empty()) {
62 ss << ", sync_group: " << sync_group;
63 }
64 ss << '}';
65 return ss.str();
66 }
67
68 namespace internal {
69 namespace {
CreateChannelReceive(Clock * clock,webrtc::AudioState * audio_state,ProcessThread * module_process_thread,NetEqFactory * neteq_factory,const webrtc::AudioReceiveStream::Config & config,RtcEventLog * event_log)70 std::unique_ptr<voe::ChannelReceiveInterface> CreateChannelReceive(
71 Clock* clock,
72 webrtc::AudioState* audio_state,
73 ProcessThread* module_process_thread,
74 NetEqFactory* neteq_factory,
75 const webrtc::AudioReceiveStream::Config& config,
76 RtcEventLog* event_log) {
77 RTC_DCHECK(audio_state);
78 internal::AudioState* internal_audio_state =
79 static_cast<internal::AudioState*>(audio_state);
80 return voe::CreateChannelReceive(
81 clock, module_process_thread, neteq_factory,
82 internal_audio_state->audio_device_module(), config.rtcp_send_transport,
83 event_log, config.rtp.local_ssrc, config.rtp.remote_ssrc,
84 config.jitter_buffer_max_packets, config.jitter_buffer_fast_accelerate,
85 config.jitter_buffer_min_delay_ms,
86 config.jitter_buffer_enable_rtx_handling, config.decoder_factory,
87 config.codec_pair_id, config.frame_decryptor, config.crypto_options,
88 std::move(config.frame_transformer), config.rtp.rtcp_event_observer);
89 }
90 } // namespace
91
AudioReceiveStream(Clock * clock,RtpStreamReceiverControllerInterface * receiver_controller,PacketRouter * packet_router,ProcessThread * module_process_thread,NetEqFactory * neteq_factory,const webrtc::AudioReceiveStream::Config & config,const rtc::scoped_refptr<webrtc::AudioState> & audio_state,webrtc::RtcEventLog * event_log)92 AudioReceiveStream::AudioReceiveStream(
93 Clock* clock,
94 RtpStreamReceiverControllerInterface* receiver_controller,
95 PacketRouter* packet_router,
96 ProcessThread* module_process_thread,
97 NetEqFactory* neteq_factory,
98 const webrtc::AudioReceiveStream::Config& config,
99 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
100 webrtc::RtcEventLog* event_log)
101 : AudioReceiveStream(clock,
102 receiver_controller,
103 packet_router,
104 config,
105 audio_state,
106 event_log,
107 CreateChannelReceive(clock,
108 audio_state.get(),
109 module_process_thread,
110 neteq_factory,
111 config,
112 event_log)) {}
113
AudioReceiveStream(Clock * clock,RtpStreamReceiverControllerInterface * receiver_controller,PacketRouter * packet_router,const webrtc::AudioReceiveStream::Config & config,const rtc::scoped_refptr<webrtc::AudioState> & audio_state,webrtc::RtcEventLog * event_log,std::unique_ptr<voe::ChannelReceiveInterface> channel_receive)114 AudioReceiveStream::AudioReceiveStream(
115 Clock* clock,
116 RtpStreamReceiverControllerInterface* receiver_controller,
117 PacketRouter* packet_router,
118 const webrtc::AudioReceiveStream::Config& config,
119 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
120 webrtc::RtcEventLog* event_log,
121 std::unique_ptr<voe::ChannelReceiveInterface> channel_receive)
122 : audio_state_(audio_state),
123 channel_receive_(std::move(channel_receive)),
124 source_tracker_(clock) {
125 RTC_LOG(LS_INFO) << "AudioReceiveStream: " << config.rtp.remote_ssrc;
126 RTC_DCHECK(config.decoder_factory);
127 RTC_DCHECK(config.rtcp_send_transport);
128 RTC_DCHECK(audio_state_);
129 RTC_DCHECK(channel_receive_);
130
131 module_process_thread_checker_.Detach();
132
133 RTC_DCHECK(receiver_controller);
134 RTC_DCHECK(packet_router);
135 // Configure bandwidth estimation.
136 channel_receive_->RegisterReceiverCongestionControlObjects(packet_router);
137
138 // Register with transport.
139 rtp_stream_receiver_ = receiver_controller->CreateReceiver(
140 config.rtp.remote_ssrc, channel_receive_.get());
141 ConfigureStream(this, config, true);
142 }
143
~AudioReceiveStream()144 AudioReceiveStream::~AudioReceiveStream() {
145 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
146 RTC_LOG(LS_INFO) << "~AudioReceiveStream: " << config_.rtp.remote_ssrc;
147 Stop();
148 channel_receive_->SetAssociatedSendChannel(nullptr);
149 channel_receive_->ResetReceiverCongestionControlObjects();
150 }
151
Reconfigure(const webrtc::AudioReceiveStream::Config & config)152 void AudioReceiveStream::Reconfigure(
153 const webrtc::AudioReceiveStream::Config& config) {
154 RTC_DCHECK(worker_thread_checker_.IsCurrent());
155 ConfigureStream(this, config, false);
156 }
157
Start()158 void AudioReceiveStream::Start() {
159 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
160 if (playing_) {
161 return;
162 }
163 channel_receive_->StartPlayout();
164 playing_ = true;
165 audio_state()->AddReceivingStream(this);
166 }
167
Stop()168 void AudioReceiveStream::Stop() {
169 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
170 if (!playing_) {
171 return;
172 }
173 channel_receive_->StopPlayout();
174 playing_ = false;
175 audio_state()->RemoveReceivingStream(this);
176 }
177
GetStats(bool get_and_clear_legacy_stats) const178 webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats(
179 bool get_and_clear_legacy_stats) const {
180 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
181 webrtc::AudioReceiveStream::Stats stats;
182 stats.remote_ssrc = config_.rtp.remote_ssrc;
183
184 webrtc::CallReceiveStatistics call_stats =
185 channel_receive_->GetRTCPStatistics();
186 // TODO(solenberg): Don't return here if we can't get the codec - return the
187 // stats we *can* get.
188 auto receive_codec = channel_receive_->GetReceiveCodec();
189 if (!receive_codec) {
190 return stats;
191 }
192
193 stats.payload_bytes_rcvd = call_stats.payload_bytes_rcvd;
194 stats.header_and_padding_bytes_rcvd =
195 call_stats.header_and_padding_bytes_rcvd;
196 stats.packets_rcvd = call_stats.packetsReceived;
197 stats.packets_lost = call_stats.cumulativeLost;
198 stats.capture_start_ntp_time_ms = call_stats.capture_start_ntp_time_ms_;
199 stats.last_packet_received_timestamp_ms =
200 call_stats.last_packet_received_timestamp_ms;
201 stats.codec_name = receive_codec->second.name;
202 stats.codec_payload_type = receive_codec->first;
203 int clockrate_khz = receive_codec->second.clockrate_hz / 1000;
204 if (clockrate_khz > 0) {
205 stats.jitter_ms = call_stats.jitterSamples / clockrate_khz;
206 }
207 stats.delay_estimate_ms = channel_receive_->GetDelayEstimate();
208 stats.audio_level = channel_receive_->GetSpeechOutputLevelFullRange();
209 stats.total_output_energy = channel_receive_->GetTotalOutputEnergy();
210 stats.total_output_duration = channel_receive_->GetTotalOutputDuration();
211 stats.estimated_playout_ntp_timestamp_ms =
212 channel_receive_->GetCurrentEstimatedPlayoutNtpTimestampMs(
213 rtc::TimeMillis());
214
215 // Get jitter buffer and total delay (alg + jitter + playout) stats.
216 auto ns = channel_receive_->GetNetworkStatistics(get_and_clear_legacy_stats);
217 stats.packets_discarded = ns.packetsDiscarded;
218 stats.fec_packets_received = ns.fecPacketsReceived;
219 stats.fec_packets_discarded = ns.fecPacketsDiscarded;
220 stats.jitter_buffer_ms = ns.currentBufferSize;
221 stats.jitter_buffer_preferred_ms = ns.preferredBufferSize;
222 stats.total_samples_received = ns.totalSamplesReceived;
223 stats.concealed_samples = ns.concealedSamples;
224 stats.silent_concealed_samples = ns.silentConcealedSamples;
225 stats.concealment_events = ns.concealmentEvents;
226 stats.jitter_buffer_delay_seconds =
227 static_cast<double>(ns.jitterBufferDelayMs) /
228 static_cast<double>(rtc::kNumMillisecsPerSec);
229 stats.jitter_buffer_emitted_count = ns.jitterBufferEmittedCount;
230 stats.jitter_buffer_target_delay_seconds =
231 static_cast<double>(ns.jitterBufferTargetDelayMs) /
232 static_cast<double>(rtc::kNumMillisecsPerSec);
233 stats.inserted_samples_for_deceleration = ns.insertedSamplesForDeceleration;
234 stats.removed_samples_for_acceleration = ns.removedSamplesForAcceleration;
235 stats.expand_rate = Q14ToFloat(ns.currentExpandRate);
236 stats.speech_expand_rate = Q14ToFloat(ns.currentSpeechExpandRate);
237 stats.secondary_decoded_rate = Q14ToFloat(ns.currentSecondaryDecodedRate);
238 stats.secondary_discarded_rate = Q14ToFloat(ns.currentSecondaryDiscardedRate);
239 stats.accelerate_rate = Q14ToFloat(ns.currentAccelerateRate);
240 stats.preemptive_expand_rate = Q14ToFloat(ns.currentPreemptiveRate);
241 stats.jitter_buffer_flushes = ns.packetBufferFlushes;
242 stats.delayed_packet_outage_samples = ns.delayedPacketOutageSamples;
243 stats.relative_packet_arrival_delay_seconds =
244 static_cast<double>(ns.relativePacketArrivalDelayMs) /
245 static_cast<double>(rtc::kNumMillisecsPerSec);
246 stats.interruption_count = ns.interruptionCount;
247 stats.total_interruption_duration_ms = ns.totalInterruptionDurationMs;
248
249 auto ds = channel_receive_->GetDecodingCallStatistics();
250 stats.decoding_calls_to_silence_generator = ds.calls_to_silence_generator;
251 stats.decoding_calls_to_neteq = ds.calls_to_neteq;
252 stats.decoding_normal = ds.decoded_normal;
253 stats.decoding_plc = ds.decoded_neteq_plc;
254 stats.decoding_codec_plc = ds.decoded_codec_plc;
255 stats.decoding_cng = ds.decoded_cng;
256 stats.decoding_plc_cng = ds.decoded_plc_cng;
257 stats.decoding_muted_output = ds.decoded_muted_output;
258
259 stats.last_sender_report_timestamp_ms =
260 call_stats.last_sender_report_timestamp_ms;
261 stats.last_sender_report_remote_timestamp_ms =
262 call_stats.last_sender_report_remote_timestamp_ms;
263 stats.sender_reports_packets_sent = call_stats.sender_reports_packets_sent;
264 stats.sender_reports_bytes_sent = call_stats.sender_reports_bytes_sent;
265 stats.sender_reports_reports_count = call_stats.sender_reports_reports_count;
266
267 return stats;
268 }
269
SetSink(AudioSinkInterface * sink)270 void AudioReceiveStream::SetSink(AudioSinkInterface* sink) {
271 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
272 channel_receive_->SetSink(sink);
273 }
274
SetGain(float gain)275 void AudioReceiveStream::SetGain(float gain) {
276 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
277 channel_receive_->SetChannelOutputVolumeScaling(gain);
278 }
279
SetBaseMinimumPlayoutDelayMs(int delay_ms)280 bool AudioReceiveStream::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
281 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
282 return channel_receive_->SetBaseMinimumPlayoutDelayMs(delay_ms);
283 }
284
GetBaseMinimumPlayoutDelayMs() const285 int AudioReceiveStream::GetBaseMinimumPlayoutDelayMs() const {
286 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
287 return channel_receive_->GetBaseMinimumPlayoutDelayMs();
288 }
289
GetSources() const290 std::vector<RtpSource> AudioReceiveStream::GetSources() const {
291 return source_tracker_.GetSources();
292 }
293
GetAudioFrameWithInfo(int sample_rate_hz,AudioFrame * audio_frame)294 AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo(
295 int sample_rate_hz,
296 AudioFrame* audio_frame) {
297 AudioMixer::Source::AudioFrameInfo audio_frame_info =
298 channel_receive_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame);
299 if (audio_frame_info != AudioMixer::Source::AudioFrameInfo::kError) {
300 source_tracker_.OnFrameDelivered(audio_frame->packet_infos_);
301 }
302 return audio_frame_info;
303 }
304
Ssrc() const305 int AudioReceiveStream::Ssrc() const {
306 return config_.rtp.remote_ssrc;
307 }
308
PreferredSampleRate() const309 int AudioReceiveStream::PreferredSampleRate() const {
310 return channel_receive_->PreferredSampleRate();
311 }
312
id() const313 uint32_t AudioReceiveStream::id() const {
314 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
315 return config_.rtp.remote_ssrc;
316 }
317
GetInfo() const318 absl::optional<Syncable::Info> AudioReceiveStream::GetInfo() const {
319 RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
320 absl::optional<Syncable::Info> info = channel_receive_->GetSyncInfo();
321
322 if (!info)
323 return absl::nullopt;
324
325 info->current_delay_ms = channel_receive_->GetDelayEstimate();
326 return info;
327 }
328
GetPlayoutRtpTimestamp(uint32_t * rtp_timestamp,int64_t * time_ms) const329 bool AudioReceiveStream::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
330 int64_t* time_ms) const {
331 // Called on video capture thread.
332 return channel_receive_->GetPlayoutRtpTimestamp(rtp_timestamp, time_ms);
333 }
334
SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,int64_t time_ms)335 void AudioReceiveStream::SetEstimatedPlayoutNtpTimestampMs(
336 int64_t ntp_timestamp_ms,
337 int64_t time_ms) {
338 // Called on video capture thread.
339 channel_receive_->SetEstimatedPlayoutNtpTimestampMs(ntp_timestamp_ms,
340 time_ms);
341 }
342
SetMinimumPlayoutDelay(int delay_ms)343 bool AudioReceiveStream::SetMinimumPlayoutDelay(int delay_ms) {
344 RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
345 return channel_receive_->SetMinimumPlayoutDelay(delay_ms);
346 }
347
AssociateSendStream(AudioSendStream * send_stream)348 void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) {
349 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
350 channel_receive_->SetAssociatedSendChannel(
351 send_stream ? send_stream->GetChannel() : nullptr);
352 associated_send_stream_ = send_stream;
353 }
354
DeliverRtcp(const uint8_t * packet,size_t length)355 void AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) {
356 // TODO(solenberg): Tests call this function on a network thread, libjingle
357 // calls on the worker thread. We should move towards always using a network
358 // thread. Then this check can be enabled.
359 // RTC_DCHECK(!thread_checker_.IsCurrent());
360 channel_receive_->ReceivedRTCPPacket(packet, length);
361 }
362
config() const363 const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const {
364 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
365 return config_;
366 }
367
GetAssociatedSendStreamForTesting() const368 const AudioSendStream* AudioReceiveStream::GetAssociatedSendStreamForTesting()
369 const {
370 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
371 return associated_send_stream_;
372 }
373
audio_state() const374 internal::AudioState* AudioReceiveStream::audio_state() const {
375 auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get());
376 RTC_DCHECK(audio_state);
377 return audio_state;
378 }
379
ConfigureStream(AudioReceiveStream * stream,const Config & new_config,bool first_time)380 void AudioReceiveStream::ConfigureStream(AudioReceiveStream* stream,
381 const Config& new_config,
382 bool first_time) {
383 RTC_LOG(LS_INFO) << "AudioReceiveStream::ConfigureStream: "
384 << new_config.ToString();
385 RTC_DCHECK(stream);
386 const auto& channel_receive = stream->channel_receive_;
387 const auto& old_config = stream->config_;
388
389 // Configuration parameters which cannot be changed.
390 RTC_DCHECK(first_time ||
391 old_config.rtp.remote_ssrc == new_config.rtp.remote_ssrc);
392 RTC_DCHECK(first_time ||
393 old_config.rtcp_send_transport == new_config.rtcp_send_transport);
394 // Decoder factory cannot be changed because it is configured at
395 // voe::Channel construction time.
396 RTC_DCHECK(first_time ||
397 old_config.decoder_factory == new_config.decoder_factory);
398
399 if (!first_time) {
400 // SSRC can't be changed mid-stream.
401 RTC_DCHECK_EQ(old_config.rtp.local_ssrc, new_config.rtp.local_ssrc);
402 RTC_DCHECK_EQ(old_config.rtp.remote_ssrc, new_config.rtp.remote_ssrc);
403 }
404
405 // TODO(solenberg): Config NACK history window (which is a packet count),
406 // using the actual packet size for the configured codec.
407 if (first_time || old_config.rtp.nack.rtp_history_ms !=
408 new_config.rtp.nack.rtp_history_ms) {
409 channel_receive->SetNACKStatus(new_config.rtp.nack.rtp_history_ms != 0,
410 new_config.rtp.nack.rtp_history_ms / 20);
411 }
412 if (first_time || old_config.decoder_map != new_config.decoder_map) {
413 channel_receive->SetReceiveCodecs(new_config.decoder_map);
414 }
415
416 if (first_time ||
417 old_config.frame_transformer != new_config.frame_transformer) {
418 channel_receive->SetDepacketizerToDecoderFrameTransformer(
419 new_config.frame_transformer);
420 }
421
422 stream->config_ = new_config;
423 }
424 } // namespace internal
425 } // namespace webrtc
426