1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VOICE_ENGINE_CHANNEL_H_ 12 #define VOICE_ENGINE_CHANNEL_H_ 13 14 #include <memory> 15 16 #include "api/audio/audio_mixer.h" 17 #include "api/audio_codecs/audio_encoder.h" 18 #include "api/call/audio_sink.h" 19 #include "api/call/transport.h" 20 #include "api/optional.h" 21 #include "common_audio/resampler/include/push_resampler.h" 22 #include "common_types.h" // NOLINT(build/include) 23 #include "modules/audio_coding/include/audio_coding_module.h" 24 #include "modules/audio_processing/rms_level.h" 25 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" 26 #include "modules/rtp_rtcp/include/rtp_header_parser.h" 27 #include "modules/rtp_rtcp/include/rtp_receiver.h" 28 #include "modules/rtp_rtcp/include/rtp_rtcp.h" 29 #include "rtc_base/criticalsection.h" 30 #include "rtc_base/event.h" 31 #include "rtc_base/thread_checker.h" 32 #include "voice_engine/audio_level.h" 33 #include "voice_engine/include/voe_base.h" 34 #include "voice_engine/shared_data.h" 35 36 namespace rtc { 37 class TimestampWrapAroundHandler; 38 } 39 40 namespace webrtc { 41 42 class AudioDeviceModule; 43 class PacketRouter; 44 class ProcessThread; 45 class RateLimiter; 46 class ReceiveStatistics; 47 class RemoteNtpTimeEstimator; 48 class RtcEventLog; 49 class RtpPacketObserver; 50 class RTPPayloadRegistry; 51 class RTPReceiverAudio; 52 class RtpPacketReceived; 53 class RtpRtcp; 54 class RtpTransportControllerSendInterface; 55 class TelephoneEventHandler; 56 57 struct SenderInfo; 58 59 struct CallStatistics { 60 unsigned short fractionLost; 61 unsigned int cumulativeLost; 62 unsigned int extendedMax; 63 unsigned int jitterSamples; 64 int64_t rttMs; 65 size_t bytesSent; 66 int packetsSent; 67 size_t bytesReceived; 68 int packetsReceived; 69 // The capture ntp time (in local timebase) of the first played out audio 70 // frame. 71 int64_t capture_start_ntp_time_ms_; 72 73 uint32_t rtcp_sender_packets_sent; 74 uint32_t rtcp_sender_octets_sent; 75 }; 76 77 // See section 6.4.2 in http://www.ietf.org/rfc/rfc3550.txt for details. 78 struct ReportBlock { 79 uint32_t sender_SSRC; // SSRC of sender 80 uint32_t source_SSRC; 81 uint8_t fraction_lost; 82 uint32_t cumulative_num_packets_lost; 83 uint32_t extended_highest_sequence_number; 84 uint32_t interarrival_jitter; 85 uint32_t last_SR_timestamp; 86 uint32_t delay_since_last_SR; 87 }; 88 89 namespace voe { 90 91 class RtcEventLogProxy; 92 class RtcpRttStatsProxy; 93 class RtpPacketSenderProxy; 94 class StatisticsProxy; 95 class TransportFeedbackProxy; 96 class TransportSequenceNumberProxy; 97 class VoERtcpObserver; 98 99 // Helper class to simplify locking scheme for members that are accessed from 100 // multiple threads. 101 // Example: a member can be set on thread T1 and read by an internal audio 102 // thread T2. Accessing the member via this class ensures that we are 103 // safe and also avoid TSan v2 warnings. 104 class ChannelState { 105 public: 106 struct State { 107 bool playing = false; 108 bool sending = false; 109 }; 110 ChannelState()111 ChannelState() {} ~ChannelState()112 virtual ~ChannelState() {} 113 Reset()114 void Reset() { 115 rtc::CritScope lock(&lock_); 116 state_ = State(); 117 } 118 Get()119 State Get() const { 120 rtc::CritScope lock(&lock_); 121 return state_; 122 } 123 SetPlaying(bool enable)124 void SetPlaying(bool enable) { 125 rtc::CritScope lock(&lock_); 126 state_.playing = enable; 127 } 128 SetSending(bool enable)129 void SetSending(bool enable) { 130 rtc::CritScope lock(&lock_); 131 state_.sending = enable; 132 } 133 134 private: 135 rtc::CriticalSection lock_; 136 State state_; 137 }; 138 139 class Channel 140 : public RtpData, 141 public RtpFeedback, 142 public Transport, 143 public AudioPacketizationCallback, // receive encoded packets from the 144 // ACM 145 public OverheadObserver { 146 public: 147 friend class VoERtcpObserver; 148 149 enum { KNumSocketThreads = 1 }; 150 enum { KNumberOfSocketBuffers = 8 }; 151 virtual ~Channel(); 152 static int32_t CreateChannel(Channel*& channel, 153 int32_t channelId, 154 uint32_t instanceId, 155 const VoEBase::ChannelConfig& config); 156 Channel(int32_t channelId, 157 uint32_t instanceId, 158 const VoEBase::ChannelConfig& config); 159 int32_t Init(); 160 void Terminate(); 161 int32_t SetEngineInformation(ProcessThread& moduleProcessThread, 162 AudioDeviceModule& audioDeviceModule, 163 rtc::TaskQueue* encoder_queue); 164 165 void SetSink(std::unique_ptr<AudioSinkInterface> sink); 166 167 // TODO(ossu): Don't use! It's only here to confirm that the decoder factory 168 // passed into AudioReceiveStream is the same as the one set when creating the 169 // ADM. Once Channel creation is moved into Audio{Send,Receive}Stream this can 170 // go. 171 const rtc::scoped_refptr<AudioDecoderFactory>& GetAudioDecoderFactory() const; 172 173 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs); 174 175 // Send using this encoder, with this payload type. 176 bool SetEncoder(int payload_type, std::unique_ptr<AudioEncoder> encoder); 177 void ModifyEncoder( 178 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier); 179 180 // API methods 181 182 // VoEBase 183 int32_t StartPlayout(); 184 int32_t StopPlayout(); 185 int32_t StartSend(); 186 void StopSend(); 187 188 // Codecs 189 struct EncoderProps { 190 int sample_rate_hz; 191 size_t num_channels; 192 }; 193 rtc::Optional<EncoderProps> GetEncoderProps() const; 194 int32_t GetRecCodec(CodecInst& codec); 195 void SetBitRate(int bitrate_bps, int64_t probing_interval_ms); 196 bool EnableAudioNetworkAdaptor(const std::string& config_string); 197 void DisableAudioNetworkAdaptor(); 198 void SetReceiverFrameLengthRange(int min_frame_length_ms, 199 int max_frame_length_ms); 200 201 // Network 202 void RegisterTransport(Transport* transport); 203 // TODO(nisse, solenberg): Delete when VoENetwork is deleted. 204 int32_t ReceivedRTCPPacket(const uint8_t* data, size_t length); 205 void OnRtpPacket(const RtpPacketReceived& packet); 206 207 // Muting, Volume and Level. 208 void SetInputMute(bool enable); 209 void SetChannelOutputVolumeScaling(float scaling); 210 int GetSpeechOutputLevel() const; 211 int GetSpeechOutputLevelFullRange() const; 212 // See description of "totalAudioEnergy" in the WebRTC stats spec: 213 // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy 214 double GetTotalOutputEnergy() const; 215 double GetTotalOutputDuration() const; 216 217 // Stats. 218 int GetNetworkStatistics(NetworkStatistics& stats); 219 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const; 220 ANAStats GetANAStatistics() const; 221 222 // Audio+Video Sync. 223 uint32_t GetDelayEstimate() const; 224 int SetMinimumPlayoutDelay(int delayMs); 225 int GetPlayoutTimestamp(unsigned int& timestamp); 226 int GetRtpRtcp(RtpRtcp** rtpRtcpModule, RtpReceiver** rtp_receiver) const; 227 228 // DTMF. 229 int SendTelephoneEventOutband(int event, int duration_ms); 230 int SetSendTelephoneEventPayloadType(int payload_type, int payload_frequency); 231 232 // RTP+RTCP 233 int SetLocalMID(const char* mid); 234 int SetLocalSSRC(unsigned int ssrc); 235 int SetSendAudioLevelIndicationStatus(bool enable, unsigned char id); 236 int SetSendMIDStatus(bool enable, unsigned char id); 237 int SetReceiveAudioLevelIndicationStatus(bool enable, unsigned char id, bool isLevelSsrc); 238 int SetReceiveCsrcAudioLevelIndicationStatus(bool enable, unsigned char id); 239 void EnableSendTransportSequenceNumber(int id); 240 void EnableReceiveTransportSequenceNumber(int id); 241 242 void RegisterSenderCongestionControlObjects( 243 RtpTransportControllerSendInterface* transport, 244 RtcpBandwidthObserver* bandwidth_observer); 245 void RegisterReceiverCongestionControlObjects(PacketRouter* packet_router); 246 void ResetSenderCongestionControlObjects(); 247 void ResetReceiverCongestionControlObjects(); 248 void SetRTCPStatus(bool enable); 249 int SetRTCP_CNAME(const char cName[256]); 250 int GetRTCPPacketTypeCounters(RtcpPacketTypeCounter& stats); 251 int GetRemoteRTCPReportBlocks(std::vector<ReportBlock>* report_blocks); 252 int GetRTPStatistics(CallStatistics& stats); 253 void SetNACKStatus(bool enable, int maxNumberOfPackets); 254 255 // From AudioPacketizationCallback in the ACM 256 int32_t SendData(FrameType frameType, 257 uint8_t payloadType, 258 uint32_t timeStamp, 259 const uint8_t* payloadData, 260 size_t payloadSize, 261 const RTPFragmentationHeader* fragmentation) override; 262 263 // From RtpData in the RTP/RTCP module 264 int32_t OnReceivedPayloadData(const uint8_t* payloadData, 265 size_t payloadSize, 266 const WebRtcRTPHeader* rtpHeader) override; 267 268 // From RtpFeedback in the RTP/RTCP module 269 int32_t OnInitializeDecoder(int payload_type, 270 const SdpAudioFormat& audio_format, 271 uint32_t rate) override; 272 void OnIncomingSSRCChanged(uint32_t ssrc) override; 273 void OnIncomingCSRCChanged(uint32_t CSRC, bool added) override; 274 275 void OnIncomingReceiverReports(const ReportBlockList& aReportBlocks, 276 const int64_t aRoundTripTime, 277 const int64_t aReceptionTime); 278 279 // From Transport (called by the RTP/RTCP module) 280 bool SendRtp(const uint8_t* data, 281 size_t len, 282 const PacketOptions& packet_options) override; 283 bool SendRtcp(const uint8_t* data, size_t len) override; 284 285 // From AudioMixer::Source. 286 AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( 287 int sample_rate_hz, 288 AudioFrame* audio_frame); 289 290 int PreferredSampleRate() const; 291 InstanceId()292 uint32_t InstanceId() const { return _instanceId; } ChannelId()293 int32_t ChannelId() const { return _channelId; } Playing()294 bool Playing() const { return channel_state_.Get().playing; } Sending()295 bool Sending() const { return channel_state_.Get().sending; } RtpRtcpModulePtr()296 RtpRtcp* RtpRtcpModulePtr() const { return _rtpRtcpModule.get(); } OutputEnergyLevel()297 int8_t OutputEnergyLevel() const { return _outputAudioLevel.Level(); } 298 299 // ProcessAndEncodeAudio() creates an audio frame copy and posts a task 300 // on the shared encoder task queue, wich in turn calls (on the queue) 301 // ProcessAndEncodeAudioOnTaskQueue() where the actual processing of the 302 // audio takes place. The processing mainly consists of encoding and preparing 303 // the result for sending by adding it to a send queue. 304 // The main reason for using a task queue here is to release the native, 305 // OS-specific, audio capture thread as soon as possible to ensure that it 306 // can go back to sleep and be prepared to deliver an new captured audio 307 // packet. 308 void ProcessAndEncodeAudio(const AudioFrame& audio_input); 309 310 // This version of ProcessAndEncodeAudio() is used by PushCaptureData() in 311 // VoEBase and the audio in |audio_data| has not been subject to any APM 312 // processing. Some extra steps are therfore needed when building up the 313 // audio frame copy before using the same task as in the default call to 314 // ProcessAndEncodeAudio(const AudioFrame& audio_input). 315 void ProcessAndEncodeAudio(const int16_t* audio_data, 316 int sample_rate, 317 size_t number_of_frames, 318 size_t number_of_channels); 319 320 // Associate to a send channel. 321 // Used for obtaining RTT for a receive-only channel. 322 void set_associate_send_channel(const ChannelOwner& channel); 323 // Disassociate a send channel if it was associated. 324 void DisassociateSendChannel(int channel_id); 325 326 // Set a RtcEventLog logging object. 327 void SetRtcEventLog(RtcEventLog* event_log); 328 329 void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats); 330 void SetTransportOverhead(size_t transport_overhead_per_packet); 331 332 // From OverheadObserver in the RTP/RTCP module 333 void OnOverheadChanged(size_t overhead_bytes_per_packet) override; 334 335 bool GetRTCPReceiverStatistics(int64_t* timestamp, 336 uint32_t* jitterMs, 337 uint32_t* cumulativeLost, 338 uint32_t* packetsReceived, 339 uint64_t* bytesReceived, 340 double* packetsFractionLost, 341 int64_t* rtt) const; 342 virtual void SetRtpPacketObserver(RtpPacketObserver* observer); 343 344 // The existence of this function alongside OnUplinkPacketLossRate is 345 // a compromise. We want the encoder to be agnostic of the PLR source, but 346 // we also don't want it to receive conflicting information from TWCC and 347 // from RTCP-XR. 348 void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate); 349 350 void OnRecoverableUplinkPacketLossRate(float recoverable_packet_loss_rate); 351 GetSources()352 std::vector<RtpSource> GetSources() const { 353 return rtp_receiver_->GetSources(); 354 } 355 GetPlayoutFrequency()356 int GetPlayoutFrequency() const { 357 if (audio_coding_) { 358 return audio_coding_->PlayoutFrequency(); 359 } 360 return 0; 361 } 362 363 void SetRtcpEventObserver(RtcpEventObserver* observer); 364 365 private: 366 class ProcessAndEncodeAudioTask; 367 368 void OnUplinkPacketLossRate(float packet_loss_rate); 369 bool InputMute() const; 370 bool OnRecoveredPacket(const uint8_t* packet, size_t packet_length); 371 372 bool ReceivePacket(const uint8_t* packet, 373 size_t packet_length, 374 const RTPHeader& header); 375 bool IsPacketInOrder(const RTPHeader& header) const; 376 bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const; 377 int ResendPackets(const uint16_t* sequence_numbers, int length); 378 void UpdatePlayoutTimestamp(bool rtcp); 379 void RegisterReceiveCodecsToRTPModule(); 380 381 int SetSendRtpHeaderExtension(bool enable, 382 RTPExtensionType type, 383 unsigned char id); 384 385 void UpdateOverheadForEncoder() 386 RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); 387 388 int GetRtpTimestampRateHz() const; 389 int64_t GetRTT(bool allow_associate_channel) const; 390 391 // Called on the encoder task queue when a new input audio frame is ready 392 // for encoding. 393 void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input); 394 395 uint32_t _instanceId; 396 int32_t _channelId; 397 398 rtc::CriticalSection _callbackCritSect; 399 rtc::CriticalSection volume_settings_critsect_; 400 401 ChannelState channel_state_; 402 403 std::unique_ptr<voe::RtcEventLogProxy> event_log_proxy_; 404 std::unique_ptr<voe::RtcpRttStatsProxy> rtcp_rtt_stats_proxy_; 405 406 std::unique_ptr<RtpHeaderParser> rtp_header_parser_; 407 std::unique_ptr<RTPPayloadRegistry> rtp_payload_registry_; 408 std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 409 std::unique_ptr<StatisticsProxy> statistics_proxy_; 410 std::unique_ptr<RtpReceiver> rtp_receiver_; 411 TelephoneEventHandler* telephone_event_handler_; 412 std::unique_ptr<RtpRtcp> _rtpRtcpModule; 413 std::unique_ptr<AudioCodingModule> audio_coding_; 414 std::unique_ptr<AudioSinkInterface> audio_sink_; 415 AudioLevel _outputAudioLevel; 416 // Downsamples to the codec rate if necessary. 417 PushResampler<int16_t> input_resampler_; 418 uint32_t _timeStamp RTC_ACCESS_ON(encoder_queue_); 419 420 RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_); 421 422 // Timestamp of the audio pulled from NetEq. 423 rtc::Optional<uint32_t> jitter_buffer_playout_timestamp_; 424 425 rtc::CriticalSection video_sync_lock_; 426 uint32_t playout_timestamp_rtp_ RTC_GUARDED_BY(video_sync_lock_); 427 uint32_t playout_delay_ms_ RTC_GUARDED_BY(video_sync_lock_); 428 uint16_t send_sequence_number_; 429 430 rtc::CriticalSection ts_stats_lock_; 431 432 std::unique_ptr<rtc::TimestampWrapAroundHandler> rtp_ts_wraparound_handler_; 433 // The rtp timestamp of the first played out audio frame. 434 int64_t capture_start_rtp_time_stamp_; 435 // The capture ntp time (in local timebase) of the first played out audio 436 // frame. 437 int64_t capture_start_ntp_time_ms_ RTC_GUARDED_BY(ts_stats_lock_); 438 439 // uses 440 ProcessThread* _moduleProcessThreadPtr; 441 AudioDeviceModule* _audioDeviceModulePtr; 442 Transport* _transportPtr; // WebRtc socket or external transport 443 RmsLevel rms_level_ RTC_ACCESS_ON(encoder_queue_); 444 bool input_mute_ RTC_GUARDED_BY(volume_settings_critsect_); 445 bool previous_frame_muted_ RTC_ACCESS_ON(encoder_queue_); 446 float _outputGain RTC_GUARDED_BY(volume_settings_critsect_); 447 // VoeRTP_RTCP 448 // TODO(henrika): can today be accessed on the main thread and on the 449 // task queue; hence potential race. 450 bool _includeAudioLevelIndication; 451 size_t transport_overhead_per_packet_ 452 RTC_GUARDED_BY(overhead_per_packet_lock_); 453 size_t rtp_overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_); 454 rtc::CriticalSection overhead_per_packet_lock_; 455 // VoENetwork 456 AudioFrame::SpeechType _outputSpeechType; 457 // RtcpBandwidthObserver 458 std::unique_ptr<VoERtcpObserver> rtcp_observer_; 459 // An associated send channel. 460 rtc::CriticalSection assoc_send_channel_lock_; 461 ChannelOwner associate_send_channel_ RTC_GUARDED_BY(assoc_send_channel_lock_); 462 463 bool pacing_enabled_; 464 PacketRouter* packet_router_ = nullptr; 465 std::unique_ptr<TransportFeedbackProxy> feedback_observer_proxy_; 466 std::unique_ptr<TransportSequenceNumberProxy> seq_num_allocator_proxy_; 467 std::unique_ptr<RtpPacketSenderProxy> rtp_packet_sender_proxy_; 468 std::unique_ptr<RateLimiter> retransmission_rate_limiter_; 469 470 // TODO(ossu): Remove once GetAudioDecoderFactory() is no longer needed. 471 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_; 472 473 RtpPacketObserver* rtp_source_observer_ = nullptr; 474 475 rtc::Optional<EncoderProps> cached_encoder_props_; 476 477 rtc::ThreadChecker construction_thread_; 478 479 const bool use_twcc_plr_for_ana_; 480 481 rtc::CriticalSection encoder_queue_lock_; 482 483 bool encoder_queue_is_active_ RTC_GUARDED_BY(encoder_queue_lock_) = false; 484 485 rtc::TaskQueue* encoder_queue_ = nullptr; 486 }; 487 488 } // namespace voe 489 } // namespace webrtc 490 491 #endif // VOICE_ENGINE_CHANNEL_H_ 492