1 /* 2 * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_ENGINE_WEBRTCVOICEENGINE_H_ 12 #define MEDIA_ENGINE_WEBRTCVOICEENGINE_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "api/audio_codecs/audio_encoder_factory.h" 20 #include "api/rtpreceiverinterface.h" 21 #include "call/audio_state.h" 22 #include "call/call.h" 23 #include "media/base/rtputils.h" 24 #include "media/engine/apm_helpers.h" 25 #include "media/engine/webrtcvoe.h" 26 #include "modules/audio_processing/include/audio_processing.h" 27 #include "pc/channel.h" 28 #include "rtc_base/buffer.h" 29 #include "rtc_base/constructormagic.h" 30 #include "rtc_base/networkroute.h" 31 #include "rtc_base/scoped_ref_ptr.h" 32 #include "rtc_base/task_queue.h" 33 #include "rtc_base/thread_checker.h" 34 35 namespace webrtc { 36 namespace voe { 37 class TransmitMixer; 38 } // namespace voe 39 } // namespace webrtc 40 41 namespace cricket { 42 43 class AudioDeviceModule; 44 class AudioMixer; 45 class AudioSource; 46 class VoEWrapper; 47 class WebRtcVoiceMediaChannel; 48 49 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine. 50 // It uses the WebRtc VoiceEngine library for audio handling. 51 class WebRtcVoiceEngine final { 52 friend class WebRtcVoiceMediaChannel; 53 public: 54 WebRtcVoiceEngine( 55 webrtc::AudioDeviceModule* adm, 56 const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory, 57 const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory, 58 rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer, 59 rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing); 60 // Dependency injection for testing. 61 WebRtcVoiceEngine( 62 webrtc::AudioDeviceModule* adm, 63 const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory, 64 const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory, 65 rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer, 66 rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing, 67 VoEWrapper* voe_wrapper); 68 ~WebRtcVoiceEngine(); 69 70 // Does initialization that needs to occur on the worker thread. 71 void Init(); 72 73 rtc::scoped_refptr<webrtc::AudioState> GetAudioState() const; 74 VoiceMediaChannel* CreateChannel(webrtc::Call* call, 75 const MediaConfig& config, 76 const AudioOptions& options); 77 78 int GetInputLevel(); 79 80 const std::vector<AudioCodec>& send_codecs() const; 81 const std::vector<AudioCodec>& recv_codecs() const; 82 RtpCapabilities GetCapabilities() const; 83 84 // For tracking WebRtc channels. Needed because we have to pause them 85 // all when switching devices. 86 // May only be called by WebRtcVoiceMediaChannel. 87 void RegisterChannel(WebRtcVoiceMediaChannel* channel); 88 void UnregisterChannel(WebRtcVoiceMediaChannel* channel); 89 voe()90 VoEWrapper* voe() { return voe_wrapper_.get(); } 91 92 // Starts AEC dump using an existing file. A maximum file size in bytes can be 93 // specified. When the maximum file size is reached, logging is stopped and 94 // the file is closed. If max_size_bytes is set to <= 0, no limit will be 95 // used. 96 bool StartAecDump(rtc::PlatformFile file, int64_t max_size_bytes); 97 98 // Stops AEC dump. 99 void StopAecDump(); 100 GetApmConfigForTest()101 const webrtc::AudioProcessing::Config GetApmConfigForTest() const { 102 return apm()->GetConfig(); 103 } 104 105 private: 106 // Every option that is "set" will be applied. Every option not "set" will be 107 // ignored. This allows us to selectively turn on and off different options 108 // easily at any time. 109 bool ApplyOptions(const AudioOptions& options); 110 111 void StartAecDump(const std::string& filename); 112 int CreateVoEChannel(); 113 114 std::unique_ptr<rtc::TaskQueue> low_priority_worker_queue_; 115 116 webrtc::AudioDeviceModule* adm(); 117 webrtc::AudioProcessing* apm() const; 118 webrtc::voe::TransmitMixer* transmit_mixer(); 119 120 AudioCodecs CollectCodecs( 121 const std::vector<webrtc::AudioCodecSpec>& specs) const; 122 123 rtc::ThreadChecker signal_thread_checker_; 124 rtc::ThreadChecker worker_thread_checker_; 125 126 // The audio device manager. 127 rtc::scoped_refptr<webrtc::AudioDeviceModule> adm_; 128 rtc::scoped_refptr<webrtc::AudioEncoderFactory> encoder_factory_; 129 rtc::scoped_refptr<webrtc::AudioDecoderFactory> decoder_factory_; 130 rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer_; 131 // Reference to the APM, owned by VoE. 132 rtc::scoped_refptr<webrtc::AudioProcessing> apm_; 133 // Reference to the TransmitMixer, owned by VoE. 134 webrtc::voe::TransmitMixer* transmit_mixer_ = nullptr; 135 // The primary instance of WebRtc VoiceEngine. 136 std::unique_ptr<VoEWrapper> voe_wrapper_; 137 rtc::scoped_refptr<webrtc::AudioState> audio_state_; 138 std::vector<AudioCodec> send_codecs_; 139 std::vector<AudioCodec> recv_codecs_; 140 std::vector<WebRtcVoiceMediaChannel*> channels_; 141 webrtc::VoEBase::ChannelConfig channel_config_; 142 bool is_dumping_aec_ = false; 143 bool initialized_ = false; 144 145 webrtc::AgcConfig default_agc_config_; 146 // Cache received extended_filter_aec, delay_agnostic_aec, experimental_ns 147 // level controller, and intelligibility_enhancer values, and apply them 148 // in case they are missing in the audio options. We need to do this because 149 // SetExtraOptions() will revert to defaults for options which are not 150 // provided. 151 rtc::Optional<bool> extended_filter_aec_; 152 rtc::Optional<bool> delay_agnostic_aec_; 153 rtc::Optional<bool> experimental_ns_; 154 rtc::Optional<bool> intelligibility_enhancer_; 155 rtc::Optional<bool> level_control_; 156 157 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcVoiceEngine); 158 }; 159 160 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses 161 // WebRtc Voice Engine. 162 class WebRtcVoiceMediaChannel final : public VoiceMediaChannel, 163 public webrtc::Transport { 164 public: 165 WebRtcVoiceMediaChannel(WebRtcVoiceEngine* engine, 166 const MediaConfig& config, 167 const AudioOptions& options, 168 webrtc::Call* call); 169 ~WebRtcVoiceMediaChannel() override; 170 options()171 const AudioOptions& options() const { return options_; } 172 173 rtc::DiffServCodePoint PreferredDscp() const override; 174 175 bool SetSendParameters(const AudioSendParameters& params) override; 176 bool SetRecvParameters(const AudioRecvParameters& params) override; 177 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; 178 bool SetRtpSendParameters(uint32_t ssrc, 179 const webrtc::RtpParameters& parameters) override; 180 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; 181 bool SetRtpReceiveParameters( 182 uint32_t ssrc, 183 const webrtc::RtpParameters& parameters) override; 184 185 void SetPlayout(bool playout) override; 186 void SetSend(bool send) override; 187 bool SetAudioSend(uint32_t ssrc, 188 bool enable, 189 const AudioOptions* options, 190 AudioSource* source) override; 191 bool AddSendStream(const StreamParams& sp) override; 192 bool RemoveSendStream(uint32_t ssrc) override; 193 bool AddRecvStream(const StreamParams& sp) override; 194 bool RemoveRecvStream(uint32_t ssrc) override; 195 bool GetActiveStreams(StreamList* actives) override; 196 int GetOutputLevel() override; 197 // SSRC=0 will apply the new volume to current and future unsignaled streams. 198 bool SetOutputVolume(uint32_t ssrc, double volume) override; 199 200 bool CanInsertDtmf() override; 201 bool InsertDtmf(uint32_t ssrc, int event, int duration) override; 202 203 void OnPacketReceived(rtc::CopyOnWriteBuffer* packet, 204 const rtc::PacketTime& packet_time) override; 205 void OnRtcpReceived(rtc::CopyOnWriteBuffer* packet, 206 const rtc::PacketTime& packet_time) override; 207 void OnNetworkRouteChanged(const std::string& transport_name, 208 const rtc::NetworkRoute& network_route) override; 209 void OnReadyToSend(bool ready) override; 210 bool GetStats(VoiceMediaInfo* info) override; 211 212 // SSRC=0 will set the audio sink on the latest unsignaled stream, future or 213 // current. Only one stream at a time will use the sink. 214 void SetRawAudioSink( 215 uint32_t ssrc, 216 std::unique_ptr<webrtc::AudioSinkInterface> sink) override; 217 218 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; 219 220 // implements Transport interface SendRtp(const uint8_t * data,size_t len,const webrtc::PacketOptions & options)221 bool SendRtp(const uint8_t* data, 222 size_t len, 223 const webrtc::PacketOptions& options) override { 224 rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); 225 rtc::PacketOptions rtc_options; 226 rtc_options.packet_id = options.packet_id; 227 return VoiceMediaChannel::SendPacket(&packet, rtc_options); 228 } 229 SendRtcp(const uint8_t * data,size_t len)230 bool SendRtcp(const uint8_t* data, size_t len) override { 231 rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); 232 return VoiceMediaChannel::SendRtcp(&packet, rtc::PacketOptions()); 233 } 234 235 int GetReceiveChannelId(uint32_t ssrc) const; 236 int GetSendChannelId(uint32_t ssrc) const; 237 238 private: 239 bool SetOptions(const AudioOptions& options); 240 bool SetRecvCodecs(const std::vector<AudioCodec>& codecs); 241 bool SetSendCodecs(const std::vector<AudioCodec>& codecs); 242 bool SetLocalSource(uint32_t ssrc, AudioSource* source); 243 bool MuteStream(uint32_t ssrc, bool mute); 244 engine()245 WebRtcVoiceEngine* engine() { return engine_; } 246 void ChangePlayout(bool playout); 247 int CreateVoEChannel(); 248 bool DeleteVoEChannel(int channel); 249 bool SetMaxSendBitrate(int bps); 250 bool ValidateRtpParameters(const webrtc::RtpParameters& parameters); 251 void SetupRecording(); 252 // Check if 'ssrc' is an unsignaled stream, and if so mark it as not being 253 // unsignaled anymore (i.e. it is now removed, or signaled), and return true. 254 bool MaybeDeregisterUnsignaledRecvStream(uint32_t ssrc); 255 256 rtc::ThreadChecker worker_thread_checker_; 257 258 WebRtcVoiceEngine* const engine_ = nullptr; 259 std::vector<AudioCodec> send_codecs_; 260 261 // TODO(kwiberg): decoder_map_ and recv_codecs_ store the exact same 262 // information, in slightly different formats. Eliminate recv_codecs_. 263 std::map<int, webrtc::SdpAudioFormat> decoder_map_; 264 std::vector<AudioCodec> recv_codecs_; 265 266 int max_send_bitrate_bps_ = 0; 267 AudioOptions options_; 268 rtc::Optional<int> dtmf_payload_type_; 269 int dtmf_payload_freq_ = -1; 270 bool recv_transport_cc_enabled_ = false; 271 bool recv_nack_enabled_ = false; 272 bool desired_playout_ = false; 273 bool playout_ = false; 274 bool send_ = false; 275 webrtc::Call* const call_ = nullptr; 276 277 // Queue of unsignaled SSRCs; oldest at the beginning. 278 std::vector<uint32_t> unsignaled_recv_ssrcs_; 279 280 // Volume for unsignaled streams, which may be set before the stream exists. 281 double default_recv_volume_ = 1.0; 282 // Sink for latest unsignaled stream - may be set before the stream exists. 283 std::unique_ptr<webrtc::AudioSinkInterface> default_sink_; 284 // Default SSRC to use for RTCP receiver reports in case of no signaled 285 // send streams. See: https://code.google.com/p/webrtc/issues/detail?id=4740 286 // and https://code.google.com/p/chromium/issues/detail?id=547661 287 uint32_t receiver_reports_ssrc_ = 0xFA17FA17u; 288 289 class WebRtcAudioSendStream; 290 std::map<uint32_t, WebRtcAudioSendStream*> send_streams_; 291 std::vector<webrtc::RtpExtension> send_rtp_extensions_; 292 293 class WebRtcAudioReceiveStream; 294 std::map<uint32_t, WebRtcAudioReceiveStream*> recv_streams_; 295 std::vector<webrtc::RtpExtension> recv_rtp_extensions_; 296 297 rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec> 298 send_codec_spec_; 299 300 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcVoiceMediaChannel); 301 }; 302 } // namespace cricket 303 304 #endif // MEDIA_ENGINE_WEBRTCVOICEENGINE_H_ 305