1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VOICE_ENGINE_CHANNEL_H_ 12 #define VOICE_ENGINE_CHANNEL_H_ 13 14 #include <memory> 15 16 #include "api/audio/audio_mixer.h" 17 #include "api/audio_codecs/audio_encoder.h" 18 #include "api/call/audio_sink.h" 19 #include "api/call/transport.h" 20 #include "api/optional.h" 21 #include "common_audio/resampler/include/push_resampler.h" 22 #include "common_types.h" // NOLINT(build/include) 23 #include "modules/audio_coding/include/audio_coding_module.h" 24 #include "modules/audio_processing/rms_level.h" 25 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" 26 #include "modules/rtp_rtcp/include/rtp_header_parser.h" 27 #include "modules/rtp_rtcp/include/rtp_receiver.h" 28 #include "modules/rtp_rtcp/include/rtp_rtcp.h" 29 #include "rtc_base/criticalsection.h" 30 #include "rtc_base/event.h" 31 #include "rtc_base/thread_checker.h" 32 #include "voice_engine/audio_level.h" 33 #include "voice_engine/include/voe_base.h" 34 #include "voice_engine/shared_data.h" 35 36 namespace rtc { 37 class TimestampWrapAroundHandler; 38 } 39 40 namespace webrtc { 41 42 class AudioDeviceModule; 43 class PacketRouter; 44 class ProcessThread; 45 class RateLimiter; 46 class ReceiveStatistics; 47 class RemoteNtpTimeEstimator; 48 class RtcEventLog; 49 class RtpPacketObserver; 50 class RTPPayloadRegistry; 51 class RTPReceiverAudio; 52 class RtpPacketReceived; 53 class RtpRtcp; 54 class RtpTransportControllerSendInterface; 55 class TelephoneEventHandler; 56 57 struct SenderInfo; 58 59 struct CallStatistics { 60 unsigned short fractionLost; 61 unsigned int cumulativeLost; 62 unsigned int extendedMax; 63 unsigned int jitterSamples; 64 int64_t rttMs; 65 size_t bytesSent; 66 int packetsSent; 67 size_t bytesReceived; 68 int packetsReceived; 69 // The capture ntp time (in local timebase) of the first played out audio 70 // frame. 71 int64_t capture_start_ntp_time_ms_; 72 73 uint32_t rtcp_sender_packets_sent; 74 uint32_t rtcp_sender_octets_sent; 75 NtpTime rtcp_sender_ntp_timestamp; 76 }; 77 78 // See section 6.4.2 in http://www.ietf.org/rfc/rfc3550.txt for details. 79 struct ReportBlock { 80 uint32_t sender_SSRC; // SSRC of sender 81 uint32_t source_SSRC; 82 uint8_t fraction_lost; 83 uint32_t cumulative_num_packets_lost; 84 uint32_t extended_highest_sequence_number; 85 uint32_t interarrival_jitter; 86 uint32_t last_SR_timestamp; 87 uint32_t delay_since_last_SR; 88 }; 89 90 namespace voe { 91 92 class RtcEventLogProxy; 93 class RtcpRttStatsProxy; 94 class RtpPacketSenderProxy; 95 class StatisticsProxy; 96 class TransportFeedbackProxy; 97 class TransportSequenceNumberProxy; 98 class VoERtcpObserver; 99 100 // Helper class to simplify locking scheme for members that are accessed from 101 // multiple threads. 102 // Example: a member can be set on thread T1 and read by an internal audio 103 // thread T2. Accessing the member via this class ensures that we are 104 // safe and also avoid TSan v2 warnings. 105 class ChannelState { 106 public: 107 struct State { 108 bool playing = false; 109 bool sending = false; 110 }; 111 ChannelState()112 ChannelState() {} ~ChannelState()113 virtual ~ChannelState() {} 114 Reset()115 void Reset() { 116 rtc::CritScope lock(&lock_); 117 state_ = State(); 118 } 119 Get()120 State Get() const { 121 rtc::CritScope lock(&lock_); 122 return state_; 123 } 124 SetPlaying(bool enable)125 void SetPlaying(bool enable) { 126 rtc::CritScope lock(&lock_); 127 state_.playing = enable; 128 } 129 SetSending(bool enable)130 void SetSending(bool enable) { 131 rtc::CritScope lock(&lock_); 132 state_.sending = enable; 133 } 134 135 private: 136 rtc::CriticalSection lock_; 137 State state_; 138 }; 139 140 class Channel 141 : public RtpData, 142 public RtpFeedback, 143 public Transport, 144 public AudioPacketizationCallback, // receive encoded packets from the 145 // ACM 146 public OverheadObserver { 147 public: 148 friend class VoERtcpObserver; 149 150 enum { KNumSocketThreads = 1 }; 151 enum { KNumberOfSocketBuffers = 8 }; 152 virtual ~Channel(); 153 static int32_t CreateChannel(Channel*& channel, 154 int32_t channelId, 155 uint32_t instanceId, 156 const VoEBase::ChannelConfig& config); 157 Channel(int32_t channelId, 158 uint32_t instanceId, 159 const VoEBase::ChannelConfig& config); 160 int32_t Init(); 161 void Terminate(); 162 int32_t SetEngineInformation(ProcessThread& moduleProcessThread, 163 AudioDeviceModule& audioDeviceModule, 164 rtc::TaskQueue* encoder_queue); 165 166 void SetSink(std::unique_ptr<AudioSinkInterface> sink); 167 168 // TODO(ossu): Don't use! It's only here to confirm that the decoder factory 169 // passed into AudioReceiveStream is the same as the one set when creating the 170 // ADM. Once Channel creation is moved into Audio{Send,Receive}Stream this can 171 // go. 172 const rtc::scoped_refptr<AudioDecoderFactory>& GetAudioDecoderFactory() const; 173 174 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs); 175 176 // Send using this encoder, with this payload type. 177 bool SetEncoder(int payload_type, std::unique_ptr<AudioEncoder> encoder); 178 void ModifyEncoder( 179 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier); 180 181 // API methods 182 183 // VoEBase 184 int32_t StartPlayout(); 185 int32_t StopPlayout(); 186 int32_t StartSend(); 187 void StopSend(); 188 189 // Codecs 190 struct EncoderProps { 191 int sample_rate_hz; 192 size_t num_channels; 193 }; 194 rtc::Optional<EncoderProps> GetEncoderProps() const; 195 int32_t GetRecCodec(CodecInst& codec); 196 void SetBitRate(int bitrate_bps, int64_t probing_interval_ms); 197 bool EnableAudioNetworkAdaptor(const std::string& config_string); 198 void DisableAudioNetworkAdaptor(); 199 void SetReceiverFrameLengthRange(int min_frame_length_ms, 200 int max_frame_length_ms); 201 202 // Network 203 void RegisterTransport(Transport* transport); 204 // TODO(nisse, solenberg): Delete when VoENetwork is deleted. 205 int32_t ReceivedRTCPPacket(const uint8_t* data, size_t length); 206 void OnRtpPacket(const RtpPacketReceived& packet); 207 208 // Muting, Volume and Level. 209 void SetInputMute(bool enable); 210 void SetChannelOutputVolumeScaling(float scaling); 211 int GetSpeechOutputLevel() const; 212 int GetSpeechOutputLevelFullRange() const; 213 // See description of "totalAudioEnergy" in the WebRTC stats spec: 214 // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy 215 double GetTotalOutputEnergy() const; 216 double GetTotalOutputDuration() const; 217 218 // Stats. 219 int GetNetworkStatistics(NetworkStatistics& stats); 220 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const; 221 ANAStats GetANAStatistics() const; 222 223 // Audio+Video Sync. 224 uint32_t GetDelayEstimate() const; 225 int SetMinimumPlayoutDelay(int delayMs); 226 int GetPlayoutTimestamp(unsigned int& timestamp); 227 int GetRtpRtcp(RtpRtcp** rtpRtcpModule, RtpReceiver** rtp_receiver) const; 228 229 // DTMF. 230 int SendTelephoneEventOutband(int event, int duration_ms); 231 int SetSendTelephoneEventPayloadType(int payload_type, int payload_frequency); 232 233 // RTP+RTCP 234 int SetLocalMID(const char* mid); 235 int SetLocalSSRC(unsigned int ssrc); 236 int SetSendAudioLevelIndicationStatus(bool enable, unsigned char id); 237 int SetSendMIDStatus(bool enable, unsigned char id); 238 int SetReceiveAudioLevelIndicationStatus(bool enable, unsigned char id, bool isLevelSsrc); 239 int SetReceiveCsrcAudioLevelIndicationStatus(bool enable, unsigned char id); 240 void EnableSendTransportSequenceNumber(int id); 241 void EnableReceiveTransportSequenceNumber(int id); 242 243 void RegisterSenderCongestionControlObjects( 244 RtpTransportControllerSendInterface* transport, 245 RtcpBandwidthObserver* bandwidth_observer); 246 void RegisterReceiverCongestionControlObjects(PacketRouter* packet_router); 247 void ResetSenderCongestionControlObjects(); 248 void ResetReceiverCongestionControlObjects(); 249 void SetRTCPStatus(bool enable); 250 int SetRTCP_CNAME(const char cName[256]); 251 int GetRTCPPacketTypeCounters(RtcpPacketTypeCounter& stats); 252 int GetRemoteRTCPReportBlocks(std::vector<ReportBlock>* report_blocks); 253 int GetRTPStatistics(CallStatistics& stats); 254 void SetNACKStatus(bool enable, int maxNumberOfPackets); 255 256 // From AudioPacketizationCallback in the ACM 257 int32_t SendData(FrameType frameType, 258 uint8_t payloadType, 259 uint32_t timeStamp, 260 const uint8_t* payloadData, 261 size_t payloadSize, 262 const RTPFragmentationHeader* fragmentation) override; 263 264 // From RtpData in the RTP/RTCP module 265 int32_t OnReceivedPayloadData(const uint8_t* payloadData, 266 size_t payloadSize, 267 const WebRtcRTPHeader* rtpHeader) override; 268 269 // From RtpFeedback in the RTP/RTCP module 270 int32_t OnInitializeDecoder(int payload_type, 271 const SdpAudioFormat& audio_format, 272 uint32_t rate) override; 273 void OnIncomingSSRCChanged(uint32_t ssrc) override; 274 void OnIncomingCSRCChanged(uint32_t CSRC, bool added) override; 275 276 void OnIncomingReceiverReports(const ReportBlockList& aReportBlocks, 277 const int64_t aRoundTripTime, 278 const int64_t aReceptionTime); 279 280 // From Transport (called by the RTP/RTCP module) 281 bool SendRtp(const uint8_t* data, 282 size_t len, 283 const PacketOptions& packet_options) override; 284 bool SendRtcp(const uint8_t* data, size_t len) override; 285 286 // From AudioMixer::Source. 287 AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( 288 int sample_rate_hz, 289 AudioFrame* audio_frame); 290 291 int PreferredSampleRate() const; 292 InstanceId()293 uint32_t InstanceId() const { return _instanceId; } ChannelId()294 int32_t ChannelId() const { return _channelId; } Playing()295 bool Playing() const { return channel_state_.Get().playing; } Sending()296 bool Sending() const { return channel_state_.Get().sending; } RtpRtcpModulePtr()297 RtpRtcp* RtpRtcpModulePtr() const { return _rtpRtcpModule.get(); } OutputEnergyLevel()298 int8_t OutputEnergyLevel() const { return _outputAudioLevel.Level(); } 299 300 // ProcessAndEncodeAudio() creates an audio frame copy and posts a task 301 // on the shared encoder task queue, wich in turn calls (on the queue) 302 // ProcessAndEncodeAudioOnTaskQueue() where the actual processing of the 303 // audio takes place. The processing mainly consists of encoding and preparing 304 // the result for sending by adding it to a send queue. 305 // The main reason for using a task queue here is to release the native, 306 // OS-specific, audio capture thread as soon as possible to ensure that it 307 // can go back to sleep and be prepared to deliver an new captured audio 308 // packet. 309 void ProcessAndEncodeAudio(const AudioFrame& audio_input); 310 311 // This version of ProcessAndEncodeAudio() is used by PushCaptureData() in 312 // VoEBase and the audio in |audio_data| has not been subject to any APM 313 // processing. Some extra steps are therfore needed when building up the 314 // audio frame copy before using the same task as in the default call to 315 // ProcessAndEncodeAudio(const AudioFrame& audio_input). 316 void ProcessAndEncodeAudio(const int16_t* audio_data, 317 int sample_rate, 318 size_t number_of_frames, 319 size_t number_of_channels); 320 321 // Associate to a send channel. 322 // Used for obtaining RTT for a receive-only channel. 323 void set_associate_send_channel(const ChannelOwner& channel); 324 // Disassociate a send channel if it was associated. 325 void DisassociateSendChannel(int channel_id); 326 327 // Set a RtcEventLog logging object. 328 void SetRtcEventLog(RtcEventLog* event_log); 329 330 void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats); 331 void SetTransportOverhead(size_t transport_overhead_per_packet); 332 333 // From OverheadObserver in the RTP/RTCP module 334 void OnOverheadChanged(size_t overhead_bytes_per_packet) override; 335 336 bool GetRTCPReceiverStatistics(int64_t* timestamp, 337 uint32_t* jitterMs, 338 uint32_t* cumulativeLost, 339 uint32_t* packetsReceived, 340 uint64_t* bytesReceived, 341 double* packetsFractionLost, 342 int64_t* rtt) const; 343 virtual void SetRtpPacketObserver(RtpPacketObserver* observer); 344 345 // The existence of this function alongside OnUplinkPacketLossRate is 346 // a compromise. We want the encoder to be agnostic of the PLR source, but 347 // we also don't want it to receive conflicting information from TWCC and 348 // from RTCP-XR. 349 void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate); 350 351 void OnRecoverableUplinkPacketLossRate(float recoverable_packet_loss_rate); 352 GetSources()353 std::vector<RtpSource> GetSources() const { 354 return rtp_receiver_->GetSources(); 355 } 356 GetPlayoutFrequency()357 int GetPlayoutFrequency() const { 358 if (audio_coding_) { 359 return audio_coding_->PlayoutFrequency(); 360 } 361 return 0; 362 } 363 364 void SetRtcpEventObserver(RtcpEventObserver* observer); 365 366 private: 367 class ProcessAndEncodeAudioTask; 368 369 void OnUplinkPacketLossRate(float packet_loss_rate); 370 bool InputMute() const; 371 bool OnRecoveredPacket(const uint8_t* packet, size_t packet_length); 372 373 bool ReceivePacket(const uint8_t* packet, 374 size_t packet_length, 375 const RTPHeader& header); 376 bool IsPacketInOrder(const RTPHeader& header) const; 377 bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const; 378 int ResendPackets(const uint16_t* sequence_numbers, int length); 379 void UpdatePlayoutTimestamp(bool rtcp); 380 void RegisterReceiveCodecsToRTPModule(); 381 382 int SetSendRtpHeaderExtension(bool enable, 383 RTPExtensionType type, 384 unsigned char id); 385 386 void UpdateOverheadForEncoder() 387 RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); 388 389 int GetRtpTimestampRateHz() const; 390 int64_t GetRTT(bool allow_associate_channel) const; 391 392 // Called on the encoder task queue when a new input audio frame is ready 393 // for encoding. 394 void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input); 395 396 uint32_t _instanceId; 397 int32_t _channelId; 398 399 rtc::CriticalSection _callbackCritSect; 400 rtc::CriticalSection volume_settings_critsect_; 401 402 ChannelState channel_state_; 403 404 std::unique_ptr<voe::RtcEventLogProxy> event_log_proxy_; 405 std::unique_ptr<voe::RtcpRttStatsProxy> rtcp_rtt_stats_proxy_; 406 407 std::unique_ptr<RtpHeaderParser> rtp_header_parser_; 408 std::unique_ptr<RTPPayloadRegistry> rtp_payload_registry_; 409 std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 410 std::unique_ptr<StatisticsProxy> statistics_proxy_; 411 std::unique_ptr<RtpReceiver> rtp_receiver_; 412 TelephoneEventHandler* telephone_event_handler_; 413 std::unique_ptr<RtpRtcp> _rtpRtcpModule; 414 std::unique_ptr<AudioCodingModule> audio_coding_; 415 std::unique_ptr<AudioSinkInterface> audio_sink_; 416 AudioLevel _outputAudioLevel; 417 // Downsamples to the codec rate if necessary. 418 PushResampler<int16_t> input_resampler_; 419 uint32_t _timeStamp RTC_ACCESS_ON(encoder_queue_); 420 421 RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_); 422 423 // Timestamp of the audio pulled from NetEq. 424 rtc::Optional<uint32_t> jitter_buffer_playout_timestamp_; 425 426 rtc::CriticalSection video_sync_lock_; 427 uint32_t playout_timestamp_rtp_ RTC_GUARDED_BY(video_sync_lock_); 428 uint32_t playout_delay_ms_ RTC_GUARDED_BY(video_sync_lock_); 429 uint16_t send_sequence_number_; 430 431 rtc::CriticalSection ts_stats_lock_; 432 433 std::unique_ptr<rtc::TimestampWrapAroundHandler> rtp_ts_wraparound_handler_; 434 // The rtp timestamp of the first played out audio frame. 435 int64_t capture_start_rtp_time_stamp_; 436 // The capture ntp time (in local timebase) of the first played out audio 437 // frame. 438 int64_t capture_start_ntp_time_ms_ RTC_GUARDED_BY(ts_stats_lock_); 439 440 // uses 441 ProcessThread* _moduleProcessThreadPtr; 442 AudioDeviceModule* _audioDeviceModulePtr; 443 Transport* _transportPtr; // WebRtc socket or external transport 444 RmsLevel rms_level_ RTC_ACCESS_ON(encoder_queue_); 445 bool input_mute_ RTC_GUARDED_BY(volume_settings_critsect_); 446 bool previous_frame_muted_ RTC_ACCESS_ON(encoder_queue_); 447 float _outputGain RTC_GUARDED_BY(volume_settings_critsect_); 448 // VoeRTP_RTCP 449 // TODO(henrika): can today be accessed on the main thread and on the 450 // task queue; hence potential race. 451 bool _includeAudioLevelIndication; 452 size_t transport_overhead_per_packet_ 453 RTC_GUARDED_BY(overhead_per_packet_lock_); 454 size_t rtp_overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_); 455 rtc::CriticalSection overhead_per_packet_lock_; 456 // VoENetwork 457 AudioFrame::SpeechType _outputSpeechType; 458 // RtcpBandwidthObserver 459 std::unique_ptr<VoERtcpObserver> rtcp_observer_; 460 // An associated send channel. 461 rtc::CriticalSection assoc_send_channel_lock_; 462 ChannelOwner associate_send_channel_ RTC_GUARDED_BY(assoc_send_channel_lock_); 463 464 bool pacing_enabled_; 465 PacketRouter* packet_router_ = nullptr; 466 std::unique_ptr<TransportFeedbackProxy> feedback_observer_proxy_; 467 std::unique_ptr<TransportSequenceNumberProxy> seq_num_allocator_proxy_; 468 std::unique_ptr<RtpPacketSenderProxy> rtp_packet_sender_proxy_; 469 std::unique_ptr<RateLimiter> retransmission_rate_limiter_; 470 471 // TODO(ossu): Remove once GetAudioDecoderFactory() is no longer needed. 472 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_; 473 474 RtpPacketObserver* rtp_source_observer_ = nullptr; 475 476 rtc::Optional<EncoderProps> cached_encoder_props_; 477 478 rtc::ThreadChecker construction_thread_; 479 480 const bool use_twcc_plr_for_ana_; 481 482 rtc::CriticalSection encoder_queue_lock_; 483 484 bool encoder_queue_is_active_ RTC_GUARDED_BY(encoder_queue_lock_) = false; 485 486 rtc::TaskQueue* encoder_queue_ = nullptr; 487 }; 488 489 } // namespace voe 490 } // namespace webrtc 491 492 #endif // VOICE_ENGINE_CHANNEL_H_ 493