1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "audio/audio_receive_stream.h"
12
13 #include <string>
14 #include <utility>
15
16 #include "absl/memory/memory.h"
17 #include "api/array_view.h"
18 #include "api/audio_codecs/audio_format.h"
19 #include "api/call/audio_sink.h"
20 #include "api/rtp_parameters.h"
21 #include "audio/audio_send_stream.h"
22 #include "audio/audio_state.h"
23 #include "audio/channel_receive.h"
24 #include "audio/conversion.h"
25 #include "call/rtp_config.h"
26 #include "call/rtp_stream_receiver_controller_interface.h"
27 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/logging.h"
30 #include "rtc_base/strings/string_builder.h"
31 #include "rtc_base/time_utils.h"
32
33 namespace webrtc {
34
ToString() const35 std::string AudioReceiveStream::Config::Rtp::ToString() const {
36 char ss_buf[1024];
37 rtc::SimpleStringBuilder ss(ss_buf);
38 ss << "{remote_ssrc: " << remote_ssrc;
39 ss << ", local_ssrc: " << local_ssrc;
40 ss << ", transport_cc: " << (transport_cc ? "on" : "off");
41 ss << ", nack: " << nack.ToString();
42 ss << ", extensions: [";
43 for (size_t i = 0; i < extensions.size(); ++i) {
44 ss << extensions[i].ToString();
45 if (i != extensions.size() - 1) {
46 ss << ", ";
47 }
48 }
49 ss << ']';
50 ss << '}';
51 return ss.str();
52 }
53
ToString() const54 std::string AudioReceiveStream::Config::ToString() const {
55 char ss_buf[1024];
56 rtc::SimpleStringBuilder ss(ss_buf);
57 ss << "{rtp: " << rtp.ToString();
58 ss << ", rtcp_send_transport: "
59 << (rtcp_send_transport ? "(Transport)" : "null");
60 if (!sync_group.empty()) {
61 ss << ", sync_group: " << sync_group;
62 }
63 ss << '}';
64 return ss.str();
65 }
66
67 namespace internal {
68 namespace {
CreateChannelReceive(Clock * clock,webrtc::AudioState * audio_state,ProcessThread * module_process_thread,NetEqFactory * neteq_factory,const webrtc::AudioReceiveStream::Config & config,RtcEventLog * event_log)69 std::unique_ptr<voe::ChannelReceiveInterface> CreateChannelReceive(
70 Clock* clock,
71 webrtc::AudioState* audio_state,
72 ProcessThread* module_process_thread,
73 NetEqFactory* neteq_factory,
74 const webrtc::AudioReceiveStream::Config& config,
75 RtcEventLog* event_log) {
76 RTC_DCHECK(audio_state);
77 internal::AudioState* internal_audio_state =
78 static_cast<internal::AudioState*>(audio_state);
79 return voe::CreateChannelReceive(
80 clock, module_process_thread, neteq_factory,
81 internal_audio_state->audio_device_module(), config.rtcp_send_transport,
82 event_log, config.rtp.local_ssrc, config.rtp.remote_ssrc,
83 config.jitter_buffer_max_packets, config.jitter_buffer_fast_accelerate,
84 config.jitter_buffer_min_delay_ms,
85 config.jitter_buffer_enable_rtx_handling, config.decoder_factory,
86 config.codec_pair_id, config.frame_decryptor, config.crypto_options,
87 std::move(config.frame_transformer));
88 }
89 } // namespace
90
AudioReceiveStream(Clock * clock,RtpStreamReceiverControllerInterface * receiver_controller,PacketRouter * packet_router,ProcessThread * module_process_thread,NetEqFactory * neteq_factory,const webrtc::AudioReceiveStream::Config & config,const rtc::scoped_refptr<webrtc::AudioState> & audio_state,webrtc::RtcEventLog * event_log)91 AudioReceiveStream::AudioReceiveStream(
92 Clock* clock,
93 RtpStreamReceiverControllerInterface* receiver_controller,
94 PacketRouter* packet_router,
95 ProcessThread* module_process_thread,
96 NetEqFactory* neteq_factory,
97 const webrtc::AudioReceiveStream::Config& config,
98 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
99 webrtc::RtcEventLog* event_log)
100 : AudioReceiveStream(clock,
101 receiver_controller,
102 packet_router,
103 config,
104 audio_state,
105 event_log,
106 CreateChannelReceive(clock,
107 audio_state.get(),
108 module_process_thread,
109 neteq_factory,
110 config,
111 event_log)) {}
112
AudioReceiveStream(Clock * clock,RtpStreamReceiverControllerInterface * receiver_controller,PacketRouter * packet_router,const webrtc::AudioReceiveStream::Config & config,const rtc::scoped_refptr<webrtc::AudioState> & audio_state,webrtc::RtcEventLog * event_log,std::unique_ptr<voe::ChannelReceiveInterface> channel_receive)113 AudioReceiveStream::AudioReceiveStream(
114 Clock* clock,
115 RtpStreamReceiverControllerInterface* receiver_controller,
116 PacketRouter* packet_router,
117 const webrtc::AudioReceiveStream::Config& config,
118 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
119 webrtc::RtcEventLog* event_log,
120 std::unique_ptr<voe::ChannelReceiveInterface> channel_receive)
121 : audio_state_(audio_state),
122 source_tracker_(clock),
123 channel_receive_(std::move(channel_receive)) {
124 RTC_LOG(LS_INFO) << "AudioReceiveStream: " << config.rtp.remote_ssrc;
125 RTC_DCHECK(config.decoder_factory);
126 RTC_DCHECK(config.rtcp_send_transport);
127 RTC_DCHECK(audio_state_);
128 RTC_DCHECK(channel_receive_);
129
130 RTC_DCHECK(receiver_controller);
131 RTC_DCHECK(packet_router);
132 // Configure bandwidth estimation.
133 channel_receive_->RegisterReceiverCongestionControlObjects(packet_router);
134
135 // When output is muted, ChannelReceive will directly notify the source
136 // tracker of "delivered" frames, so RtpReceiver information will continue to
137 // be updated.
138 channel_receive_->SetSourceTracker(&source_tracker_);
139
140 // Register with transport.
141 rtp_stream_receiver_ = receiver_controller->CreateReceiver(
142 config.rtp.remote_ssrc, channel_receive_.get());
143 ConfigureStream(this, config, true);
144 }
145
~AudioReceiveStream()146 AudioReceiveStream::~AudioReceiveStream() {
147 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
148 RTC_LOG(LS_INFO) << "~AudioReceiveStream: " << config_.rtp.remote_ssrc;
149 Stop();
150 channel_receive_->SetAssociatedSendChannel(nullptr);
151 channel_receive_->ResetReceiverCongestionControlObjects();
152 }
153
Reconfigure(const webrtc::AudioReceiveStream::Config & config)154 void AudioReceiveStream::Reconfigure(
155 const webrtc::AudioReceiveStream::Config& config) {
156 RTC_DCHECK(worker_thread_checker_.IsCurrent());
157 ConfigureStream(this, config, false);
158 }
159
Start()160 void AudioReceiveStream::Start() {
161 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
162 if (playing_) {
163 return;
164 }
165 channel_receive_->StartPlayout();
166 playing_ = true;
167 audio_state()->AddReceivingStream(this);
168 }
169
Stop()170 void AudioReceiveStream::Stop() {
171 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
172 if (!playing_) {
173 return;
174 }
175 channel_receive_->StopPlayout();
176 playing_ = false;
177 audio_state()->RemoveReceivingStream(this);
178 }
179
IsRunning() const180 bool AudioReceiveStream::IsRunning() const {
181 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
182 return playing_;
183 }
184
GetStats(bool get_and_clear_legacy_stats) const185 webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats(
186 bool get_and_clear_legacy_stats) const {
187 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
188 webrtc::AudioReceiveStream::Stats stats;
189 stats.remote_ssrc = config_.rtp.remote_ssrc;
190
191 webrtc::CallReceiveStatistics call_stats =
192 channel_receive_->GetRTCPStatistics();
193 // TODO(solenberg): Don't return here if we can't get the codec - return the
194 // stats we *can* get.
195 auto receive_codec = channel_receive_->GetReceiveCodec();
196 if (!receive_codec) {
197 return stats;
198 }
199
200 stats.payload_bytes_rcvd = call_stats.payload_bytes_rcvd;
201 stats.header_and_padding_bytes_rcvd =
202 call_stats.header_and_padding_bytes_rcvd;
203 stats.packets_rcvd = call_stats.packetsReceived;
204 stats.packets_lost = call_stats.cumulativeLost;
205 stats.capture_start_ntp_time_ms = call_stats.capture_start_ntp_time_ms_;
206 stats.last_packet_received_timestamp_ms =
207 call_stats.last_packet_received_timestamp_ms;
208 stats.codec_name = receive_codec->second.name;
209 stats.codec_payload_type = receive_codec->first;
210 int clockrate_khz = receive_codec->second.clockrate_hz / 1000;
211 if (clockrate_khz > 0) {
212 stats.jitter_ms = call_stats.jitterSamples / clockrate_khz;
213 }
214 stats.delay_estimate_ms = channel_receive_->GetDelayEstimate();
215 stats.audio_level = channel_receive_->GetSpeechOutputLevelFullRange();
216 stats.total_output_energy = channel_receive_->GetTotalOutputEnergy();
217 stats.total_output_duration = channel_receive_->GetTotalOutputDuration();
218 stats.estimated_playout_ntp_timestamp_ms =
219 channel_receive_->GetCurrentEstimatedPlayoutNtpTimestampMs(
220 rtc::TimeMillis());
221
222 // Get jitter buffer and total delay (alg + jitter + playout) stats.
223 auto ns = channel_receive_->GetNetworkStatistics(get_and_clear_legacy_stats);
224 stats.fec_packets_received = ns.fecPacketsReceived;
225 stats.fec_packets_discarded = ns.fecPacketsDiscarded;
226 stats.jitter_buffer_ms = ns.currentBufferSize;
227 stats.jitter_buffer_preferred_ms = ns.preferredBufferSize;
228 stats.total_samples_received = ns.totalSamplesReceived;
229 stats.concealed_samples = ns.concealedSamples;
230 stats.silent_concealed_samples = ns.silentConcealedSamples;
231 stats.concealment_events = ns.concealmentEvents;
232 stats.jitter_buffer_delay_seconds =
233 static_cast<double>(ns.jitterBufferDelayMs) /
234 static_cast<double>(rtc::kNumMillisecsPerSec);
235 stats.jitter_buffer_emitted_count = ns.jitterBufferEmittedCount;
236 stats.jitter_buffer_target_delay_seconds =
237 static_cast<double>(ns.jitterBufferTargetDelayMs) /
238 static_cast<double>(rtc::kNumMillisecsPerSec);
239 stats.inserted_samples_for_deceleration = ns.insertedSamplesForDeceleration;
240 stats.removed_samples_for_acceleration = ns.removedSamplesForAcceleration;
241 stats.expand_rate = Q14ToFloat(ns.currentExpandRate);
242 stats.speech_expand_rate = Q14ToFloat(ns.currentSpeechExpandRate);
243 stats.secondary_decoded_rate = Q14ToFloat(ns.currentSecondaryDecodedRate);
244 stats.secondary_discarded_rate = Q14ToFloat(ns.currentSecondaryDiscardedRate);
245 stats.accelerate_rate = Q14ToFloat(ns.currentAccelerateRate);
246 stats.preemptive_expand_rate = Q14ToFloat(ns.currentPreemptiveRate);
247 stats.jitter_buffer_flushes = ns.packetBufferFlushes;
248 stats.delayed_packet_outage_samples = ns.delayedPacketOutageSamples;
249 stats.relative_packet_arrival_delay_seconds =
250 static_cast<double>(ns.relativePacketArrivalDelayMs) /
251 static_cast<double>(rtc::kNumMillisecsPerSec);
252 stats.interruption_count = ns.interruptionCount;
253 stats.total_interruption_duration_ms = ns.totalInterruptionDurationMs;
254
255 auto ds = channel_receive_->GetDecodingCallStatistics();
256 stats.decoding_calls_to_silence_generator = ds.calls_to_silence_generator;
257 stats.decoding_calls_to_neteq = ds.calls_to_neteq;
258 stats.decoding_normal = ds.decoded_normal;
259 stats.decoding_plc = ds.decoded_neteq_plc;
260 stats.decoding_codec_plc = ds.decoded_codec_plc;
261 stats.decoding_cng = ds.decoded_cng;
262 stats.decoding_plc_cng = ds.decoded_plc_cng;
263 stats.decoding_muted_output = ds.decoded_muted_output;
264
265 stats.last_sender_report_timestamp_ms =
266 call_stats.last_sender_report_timestamp_ms;
267 stats.last_sender_report_remote_timestamp_ms =
268 call_stats.last_sender_report_remote_timestamp_ms;
269 stats.sender_reports_packets_sent = call_stats.sender_reports_packets_sent;
270 stats.sender_reports_bytes_sent = call_stats.sender_reports_bytes_sent;
271 stats.sender_reports_reports_count = call_stats.sender_reports_reports_count;
272
273 return stats;
274 }
275
SetSink(AudioSinkInterface * sink)276 void AudioReceiveStream::SetSink(AudioSinkInterface* sink) {
277 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
278 channel_receive_->SetSink(sink);
279 }
280
SetGain(float gain)281 void AudioReceiveStream::SetGain(float gain) {
282 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
283 channel_receive_->SetChannelOutputVolumeScaling(gain);
284 }
285
SetBaseMinimumPlayoutDelayMs(int delay_ms)286 bool AudioReceiveStream::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
287 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
288 return channel_receive_->SetBaseMinimumPlayoutDelayMs(delay_ms);
289 }
290
GetBaseMinimumPlayoutDelayMs() const291 int AudioReceiveStream::GetBaseMinimumPlayoutDelayMs() const {
292 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
293 return channel_receive_->GetBaseMinimumPlayoutDelayMs();
294 }
295
GetSources() const296 std::vector<RtpSource> AudioReceiveStream::GetSources() const {
297 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
298 return source_tracker_.GetSources();
299 }
300
GetAudioFrameWithInfo(int sample_rate_hz,AudioFrame * audio_frame)301 AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo(
302 int sample_rate_hz,
303 AudioFrame* audio_frame) {
304 AudioMixer::Source::AudioFrameInfo audio_frame_info =
305 channel_receive_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame);
306 if (audio_frame_info != AudioMixer::Source::AudioFrameInfo::kError) {
307 source_tracker_.OnFrameDelivered(audio_frame->packet_infos_);
308 }
309 return audio_frame_info;
310 }
311
Ssrc() const312 int AudioReceiveStream::Ssrc() const {
313 return config_.rtp.remote_ssrc;
314 }
315
PreferredSampleRate() const316 int AudioReceiveStream::PreferredSampleRate() const {
317 return channel_receive_->PreferredSampleRate();
318 }
319
id() const320 uint32_t AudioReceiveStream::id() const {
321 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
322 return config_.rtp.remote_ssrc;
323 }
324
GetInfo() const325 absl::optional<Syncable::Info> AudioReceiveStream::GetInfo() const {
326 // TODO(bugs.webrtc.org/11993): This is called via RtpStreamsSynchronizer,
327 // expect to be called on the network thread.
328 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
329 return channel_receive_->GetSyncInfo();
330 }
331
GetPlayoutRtpTimestamp(uint32_t * rtp_timestamp,int64_t * time_ms) const332 bool AudioReceiveStream::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
333 int64_t* time_ms) const {
334 // Called on video capture thread.
335 return channel_receive_->GetPlayoutRtpTimestamp(rtp_timestamp, time_ms);
336 }
337
SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,int64_t time_ms)338 void AudioReceiveStream::SetEstimatedPlayoutNtpTimestampMs(
339 int64_t ntp_timestamp_ms,
340 int64_t time_ms) {
341 // Called on video capture thread.
342 channel_receive_->SetEstimatedPlayoutNtpTimestampMs(ntp_timestamp_ms,
343 time_ms);
344 }
345
SetMinimumPlayoutDelay(int delay_ms)346 bool AudioReceiveStream::SetMinimumPlayoutDelay(int delay_ms) {
347 // TODO(bugs.webrtc.org/11993): This is called via RtpStreamsSynchronizer,
348 // expect to be called on the network thread.
349 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
350 return channel_receive_->SetMinimumPlayoutDelay(delay_ms);
351 }
352
AssociateSendStream(AudioSendStream * send_stream)353 void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) {
354 // TODO(bugs.webrtc.org/11993): Expect to be called on the network thread.
355 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
356 channel_receive_->SetAssociatedSendChannel(
357 send_stream ? send_stream->GetChannel() : nullptr);
358 associated_send_stream_ = send_stream;
359 }
360
DeliverRtcp(const uint8_t * packet,size_t length)361 void AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) {
362 // TODO(solenberg): Tests call this function on a network thread, libjingle
363 // calls on the worker thread. We should move towards always using a network
364 // thread. Then this check can be enabled.
365 // RTC_DCHECK(!thread_checker_.IsCurrent());
366 channel_receive_->ReceivedRTCPPacket(packet, length);
367 }
368
config() const369 const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const {
370 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
371 return config_;
372 }
373
GetAssociatedSendStreamForTesting() const374 const AudioSendStream* AudioReceiveStream::GetAssociatedSendStreamForTesting()
375 const {
376 // TODO(bugs.webrtc.org/11993): Expect to be called on the network thread or
377 // remove test method and |associated_send_stream_| variable.
378 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
379 return associated_send_stream_;
380 }
381
audio_state() const382 internal::AudioState* AudioReceiveStream::audio_state() const {
383 auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get());
384 RTC_DCHECK(audio_state);
385 return audio_state;
386 }
387
ConfigureStream(AudioReceiveStream * stream,const Config & new_config,bool first_time)388 void AudioReceiveStream::ConfigureStream(AudioReceiveStream* stream,
389 const Config& new_config,
390 bool first_time) {
391 RTC_LOG(LS_INFO) << "AudioReceiveStream::ConfigureStream: "
392 << new_config.ToString();
393 RTC_DCHECK(stream);
394 const auto& channel_receive = stream->channel_receive_;
395 const auto& old_config = stream->config_;
396
397 // Configuration parameters which cannot be changed.
398 RTC_DCHECK(first_time ||
399 old_config.rtp.remote_ssrc == new_config.rtp.remote_ssrc);
400 RTC_DCHECK(first_time ||
401 old_config.rtcp_send_transport == new_config.rtcp_send_transport);
402 // Decoder factory cannot be changed because it is configured at
403 // voe::Channel construction time.
404 RTC_DCHECK(first_time ||
405 old_config.decoder_factory == new_config.decoder_factory);
406
407 if (!first_time) {
408 // SSRC can't be changed mid-stream.
409 RTC_DCHECK_EQ(old_config.rtp.local_ssrc, new_config.rtp.local_ssrc);
410 RTC_DCHECK_EQ(old_config.rtp.remote_ssrc, new_config.rtp.remote_ssrc);
411 }
412
413 // TODO(solenberg): Config NACK history window (which is a packet count),
414 // using the actual packet size for the configured codec.
415 if (first_time || old_config.rtp.nack.rtp_history_ms !=
416 new_config.rtp.nack.rtp_history_ms) {
417 channel_receive->SetNACKStatus(new_config.rtp.nack.rtp_history_ms != 0,
418 new_config.rtp.nack.rtp_history_ms / 20);
419 }
420 if (first_time || old_config.decoder_map != new_config.decoder_map) {
421 channel_receive->SetReceiveCodecs(new_config.decoder_map);
422 }
423
424 if (first_time ||
425 old_config.frame_transformer != new_config.frame_transformer) {
426 channel_receive->SetDepacketizerToDecoderFrameTransformer(
427 new_config.frame_transformer);
428 }
429
430 stream->config_ = new_config;
431 }
432 } // namespace internal
433 } // namespace webrtc
434