1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef AUDIO_AUDIO_SEND_STREAM_H_ 12 #define AUDIO_AUDIO_SEND_STREAM_H_ 13 14 #include <memory> 15 #include <utility> 16 #include <vector> 17 18 #include "api/sequence_checker.h" 19 #include "audio/audio_level.h" 20 #include "audio/channel_send.h" 21 #include "call/audio_send_stream.h" 22 #include "call/audio_state.h" 23 #include "call/bitrate_allocator.h" 24 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" 25 #include "rtc_base/experiments/struct_parameters_parser.h" 26 #include "rtc_base/race_checker.h" 27 #include "rtc_base/synchronization/mutex.h" 28 #include "rtc_base/task_queue.h" 29 30 namespace webrtc { 31 class RtcEventLog; 32 class RtcpBandwidthObserver; 33 class RtcpRttStats; 34 class RtpTransportControllerSendInterface; 35 36 struct AudioAllocationConfig { 37 static constexpr char kKey[] = "WebRTC-Audio-Allocation"; 38 // Field Trial configured bitrates to use as overrides over default/user 39 // configured bitrate range when audio bitrate allocation is enabled. 40 absl::optional<DataRate> min_bitrate; 41 absl::optional<DataRate> max_bitrate; 42 DataRate priority_bitrate = DataRate::Zero(); 43 // By default the priority_bitrate is compensated for packet overhead. 44 // Use this flag to configure a raw value instead. 45 absl::optional<DataRate> priority_bitrate_raw; 46 absl::optional<double> bitrate_priority; 47 48 std::unique_ptr<StructParametersParser> Parser(); 49 AudioAllocationConfig(); 50 }; 51 namespace internal { 52 class AudioState; 53 54 class AudioSendStream final : public webrtc::AudioSendStream, 55 public webrtc::BitrateAllocatorObserver { 56 public: 57 AudioSendStream(Clock* clock, 58 const webrtc::AudioSendStream::Config& config, 59 const rtc::scoped_refptr<webrtc::AudioState>& audio_state, 60 TaskQueueFactory* task_queue_factory, 61 ProcessThread* module_process_thread, 62 RtpTransportControllerSendInterface* rtp_transport, 63 BitrateAllocatorInterface* bitrate_allocator, 64 RtcEventLog* event_log, 65 RtcpRttStats* rtcp_rtt_stats, 66 const absl::optional<RtpState>& suspended_rtp_state); 67 // For unit tests, which need to supply a mock ChannelSend. 68 AudioSendStream(Clock* clock, 69 const webrtc::AudioSendStream::Config& config, 70 const rtc::scoped_refptr<webrtc::AudioState>& audio_state, 71 TaskQueueFactory* task_queue_factory, 72 RtpTransportControllerSendInterface* rtp_transport, 73 BitrateAllocatorInterface* bitrate_allocator, 74 RtcEventLog* event_log, 75 const absl::optional<RtpState>& suspended_rtp_state, 76 std::unique_ptr<voe::ChannelSendInterface> channel_send); 77 78 AudioSendStream() = delete; 79 AudioSendStream(const AudioSendStream&) = delete; 80 AudioSendStream& operator=(const AudioSendStream&) = delete; 81 82 ~AudioSendStream() override; 83 84 // webrtc::AudioSendStream implementation. 85 const webrtc::AudioSendStream::Config& GetConfig() const override; 86 void Reconfigure(const webrtc::AudioSendStream::Config& config) override; 87 void Start() override; 88 void Stop() override; 89 void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override; 90 bool SendTelephoneEvent(int payload_type, 91 int payload_frequency, 92 int event, 93 int duration_ms) override; 94 void SetMuted(bool muted) override; 95 webrtc::AudioSendStream::Stats GetStats() const override; 96 webrtc::AudioSendStream::Stats GetStats( 97 bool has_remote_tracks) const override; 98 99 void DeliverRtcp(const uint8_t* packet, size_t length); 100 101 // Implements BitrateAllocatorObserver. 102 uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; 103 104 void SetTransportOverhead(int transport_overhead_per_packet_bytes); 105 106 RtpState GetRtpState() const; 107 const voe::ChannelSendInterface* GetChannel() const; 108 109 // Returns combined per-packet overhead. 110 size_t TestOnlyGetPerPacketOverheadBytes() const 111 RTC_LOCKS_EXCLUDED(overhead_per_packet_lock_); 112 113 private: 114 class TimedTransport; 115 // Constraints including overhead. 116 struct TargetAudioBitrateConstraints { 117 DataRate min; 118 DataRate max; 119 }; 120 121 internal::AudioState* audio_state(); 122 const internal::AudioState* audio_state() const; 123 124 void StoreEncoderProperties(int sample_rate_hz, size_t num_channels) 125 RTC_RUN_ON(worker_thread_checker_); 126 127 void ConfigureStream(const Config& new_config, bool first_time) 128 RTC_RUN_ON(worker_thread_checker_); 129 bool SetupSendCodec(const Config& new_config) 130 RTC_RUN_ON(worker_thread_checker_); 131 bool ReconfigureSendCodec(const Config& new_config) 132 RTC_RUN_ON(worker_thread_checker_); 133 void ReconfigureANA(const Config& new_config) 134 RTC_RUN_ON(worker_thread_checker_); 135 void ReconfigureCNG(const Config& new_config) 136 RTC_RUN_ON(worker_thread_checker_); 137 void ReconfigureBitrateObserver(const Config& new_config) 138 RTC_RUN_ON(worker_thread_checker_); 139 140 void ConfigureBitrateObserver() RTC_RUN_ON(worker_thread_checker_); 141 void RemoveBitrateObserver() RTC_RUN_ON(worker_thread_checker_); 142 143 // Returns bitrate constraints, maybe including overhead when enabled by 144 // field trial. 145 absl::optional<TargetAudioBitrateConstraints> GetMinMaxBitrateConstraints() 146 const RTC_RUN_ON(worker_thread_checker_); 147 148 // Sets per-packet overhead on encoded (for ANA) based on current known values 149 // of transport and packetization overheads. 150 void UpdateOverheadForEncoder() 151 RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); 152 153 // Returns combined per-packet overhead. 154 size_t GetPerPacketOverheadBytes() const 155 RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); 156 157 void RegisterCngPayloadType(int payload_type, int clockrate_hz) 158 RTC_RUN_ON(worker_thread_checker_); 159 160 void UpdateCachedTargetAudioBitrateConstraints() 161 RTC_RUN_ON(worker_thread_checker_); 162 163 Clock* clock_; 164 165 SequenceChecker worker_thread_checker_; 166 SequenceChecker pacer_thread_checker_; 167 rtc::RaceChecker audio_capture_race_checker_; 168 rtc::TaskQueue* worker_queue_; 169 170 const bool allocate_audio_without_feedback_; 171 const bool force_no_audio_feedback_ = allocate_audio_without_feedback_; 172 const bool enable_audio_alr_probing_; 173 const bool send_side_bwe_with_overhead_; 174 const AudioAllocationConfig allocation_settings_; 175 176 webrtc::AudioSendStream::Config config_ 177 RTC_GUARDED_BY(worker_thread_checker_); 178 rtc::scoped_refptr<webrtc::AudioState> audio_state_; 179 const std::unique_ptr<voe::ChannelSendInterface> channel_send_; 180 RtcEventLog* const event_log_; 181 const bool use_legacy_overhead_calculation_; 182 183 int encoder_sample_rate_hz_ RTC_GUARDED_BY(worker_thread_checker_) = 0; 184 size_t encoder_num_channels_ RTC_GUARDED_BY(worker_thread_checker_) = 0; 185 bool sending_ RTC_GUARDED_BY(worker_thread_checker_) = false; 186 mutable Mutex audio_level_lock_; 187 // Keeps track of audio level, total audio energy and total samples duration. 188 // https://w3c.github.io/webrtc-stats/#dom-rtcaudiohandlerstats-totalaudioenergy 189 webrtc::voe::AudioLevel audio_level_ RTC_GUARDED_BY(audio_level_lock_); 190 191 BitrateAllocatorInterface* const bitrate_allocator_ 192 RTC_GUARDED_BY(worker_queue_); 193 // Constrains cached to be accessed from |worker_queue_|. 194 absl::optional<AudioSendStream::TargetAudioBitrateConstraints> 195 cached_constraints_ RTC_GUARDED_BY(worker_queue_) = absl::nullopt; 196 RtpTransportControllerSendInterface* const rtp_transport_; 197 198 RtpRtcpInterface* const rtp_rtcp_module_; 199 absl::optional<RtpState> const suspended_rtp_state_; 200 201 // RFC 5285: Each distinct extension MUST have a unique ID. The value 0 is 202 // reserved for padding and MUST NOT be used as a local identifier. 203 // So it should be safe to use 0 here to indicate "not configured". 204 struct ExtensionIds { 205 int audio_level = 0; 206 int abs_send_time = 0; 207 int abs_capture_time = 0; 208 int transport_sequence_number = 0; 209 int mid = 0; 210 int rid = 0; 211 int repaired_rid = 0; 212 }; 213 static ExtensionIds FindExtensionIds( 214 const std::vector<RtpExtension>& extensions); 215 static int TransportSeqNumId(const Config& config); 216 217 mutable Mutex overhead_per_packet_lock_; 218 size_t overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_) = 0; 219 220 // Current transport overhead (ICE, TURN, etc.) 221 size_t transport_overhead_per_packet_bytes_ 222 RTC_GUARDED_BY(overhead_per_packet_lock_) = 0; 223 224 bool registered_with_allocator_ RTC_GUARDED_BY(worker_thread_checker_) = 225 false; 226 size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_thread_checker_) = 227 0; 228 absl::optional<std::pair<TimeDelta, TimeDelta>> frame_length_range_ 229 RTC_GUARDED_BY(worker_thread_checker_); 230 }; 231 } // namespace internal 232 } // namespace webrtc 233 234 #endif // AUDIO_AUDIO_SEND_STREAM_H_ 235