1 /*
2  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef AUDIO_AUDIO_SEND_STREAM_H_
12 #define AUDIO_AUDIO_SEND_STREAM_H_
13 
14 #include <memory>
15 #include <utility>
16 #include <vector>
17 
18 #include "api/sequence_checker.h"
19 #include "audio/audio_level.h"
20 #include "audio/channel_send.h"
21 #include "call/audio_send_stream.h"
22 #include "call/audio_state.h"
23 #include "call/bitrate_allocator.h"
24 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h"
25 #include "rtc_base/experiments/struct_parameters_parser.h"
26 #include "rtc_base/race_checker.h"
27 #include "rtc_base/synchronization/mutex.h"
28 #include "rtc_base/task_queue.h"
29 
30 namespace webrtc {
31 class RtcEventLog;
32 class RtcpBandwidthObserver;
33 class RtcpRttStats;
34 class RtpTransportControllerSendInterface;
35 
36 struct AudioAllocationConfig {
37   static constexpr char kKey[] = "WebRTC-Audio-Allocation";
38   // Field Trial configured bitrates to use as overrides over default/user
39   // configured bitrate range when audio bitrate allocation is enabled.
40   absl::optional<DataRate> min_bitrate;
41   absl::optional<DataRate> max_bitrate;
42   DataRate priority_bitrate = DataRate::Zero();
43   // By default the priority_bitrate is compensated for packet overhead.
44   // Use this flag to configure a raw value instead.
45   absl::optional<DataRate> priority_bitrate_raw;
46   absl::optional<double> bitrate_priority;
47 
48   std::unique_ptr<StructParametersParser> Parser();
49   AudioAllocationConfig();
50 };
51 namespace internal {
52 class AudioState;
53 
54 class AudioSendStream final : public webrtc::AudioSendStream,
55                               public webrtc::BitrateAllocatorObserver {
56  public:
57   AudioSendStream(Clock* clock,
58                   const webrtc::AudioSendStream::Config& config,
59                   const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
60                   TaskQueueFactory* task_queue_factory,
61                   ProcessThread* module_process_thread,
62                   RtpTransportControllerSendInterface* rtp_transport,
63                   BitrateAllocatorInterface* bitrate_allocator,
64                   RtcEventLog* event_log,
65                   RtcpRttStats* rtcp_rtt_stats,
66                   const absl::optional<RtpState>& suspended_rtp_state);
67   // For unit tests, which need to supply a mock ChannelSend.
68   AudioSendStream(Clock* clock,
69                   const webrtc::AudioSendStream::Config& config,
70                   const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
71                   TaskQueueFactory* task_queue_factory,
72                   RtpTransportControllerSendInterface* rtp_transport,
73                   BitrateAllocatorInterface* bitrate_allocator,
74                   RtcEventLog* event_log,
75                   const absl::optional<RtpState>& suspended_rtp_state,
76                   std::unique_ptr<voe::ChannelSendInterface> channel_send);
77 
78   AudioSendStream() = delete;
79   AudioSendStream(const AudioSendStream&) = delete;
80   AudioSendStream& operator=(const AudioSendStream&) = delete;
81 
82   ~AudioSendStream() override;
83 
84   // webrtc::AudioSendStream implementation.
85   const webrtc::AudioSendStream::Config& GetConfig() const override;
86   void Reconfigure(const webrtc::AudioSendStream::Config& config) override;
87   void Start() override;
88   void Stop() override;
89   void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override;
90   bool SendTelephoneEvent(int payload_type,
91                           int payload_frequency,
92                           int event,
93                           int duration_ms) override;
94   void SetMuted(bool muted) override;
95   webrtc::AudioSendStream::Stats GetStats() const override;
96   webrtc::AudioSendStream::Stats GetStats(
97       bool has_remote_tracks) const override;
98 
99   void DeliverRtcp(const uint8_t* packet, size_t length);
100 
101   // Implements BitrateAllocatorObserver.
102   uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override;
103 
104   void SetTransportOverhead(int transport_overhead_per_packet_bytes);
105 
106   RtpState GetRtpState() const;
107   const voe::ChannelSendInterface* GetChannel() const;
108 
109   // Returns combined per-packet overhead.
110   size_t TestOnlyGetPerPacketOverheadBytes() const
111       RTC_LOCKS_EXCLUDED(overhead_per_packet_lock_);
112 
113  private:
114   class TimedTransport;
115   // Constraints including overhead.
116   struct TargetAudioBitrateConstraints {
117     DataRate min;
118     DataRate max;
119   };
120 
121   internal::AudioState* audio_state();
122   const internal::AudioState* audio_state() const;
123 
124   void StoreEncoderProperties(int sample_rate_hz, size_t num_channels)
125       RTC_RUN_ON(worker_thread_checker_);
126 
127   void ConfigureStream(const Config& new_config, bool first_time)
128       RTC_RUN_ON(worker_thread_checker_);
129   bool SetupSendCodec(const Config& new_config)
130       RTC_RUN_ON(worker_thread_checker_);
131   bool ReconfigureSendCodec(const Config& new_config)
132       RTC_RUN_ON(worker_thread_checker_);
133   void ReconfigureANA(const Config& new_config)
134       RTC_RUN_ON(worker_thread_checker_);
135   void ReconfigureCNG(const Config& new_config)
136       RTC_RUN_ON(worker_thread_checker_);
137   void ReconfigureBitrateObserver(const Config& new_config)
138       RTC_RUN_ON(worker_thread_checker_);
139 
140   void ConfigureBitrateObserver() RTC_RUN_ON(worker_thread_checker_);
141   void RemoveBitrateObserver() RTC_RUN_ON(worker_thread_checker_);
142 
143   // Returns bitrate constraints, maybe including overhead when enabled by
144   // field trial.
145   absl::optional<TargetAudioBitrateConstraints> GetMinMaxBitrateConstraints()
146       const RTC_RUN_ON(worker_thread_checker_);
147 
148   // Sets per-packet overhead on encoded (for ANA) based on current known values
149   // of transport and packetization overheads.
150   void UpdateOverheadForEncoder()
151       RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_);
152 
153   // Returns combined per-packet overhead.
154   size_t GetPerPacketOverheadBytes() const
155       RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_);
156 
157   void RegisterCngPayloadType(int payload_type, int clockrate_hz)
158       RTC_RUN_ON(worker_thread_checker_);
159 
160   void UpdateCachedTargetAudioBitrateConstraints()
161       RTC_RUN_ON(worker_thread_checker_);
162 
163   Clock* clock_;
164 
165   SequenceChecker worker_thread_checker_;
166   SequenceChecker pacer_thread_checker_;
167   rtc::RaceChecker audio_capture_race_checker_;
168   rtc::TaskQueue* worker_queue_;
169 
170   const bool allocate_audio_without_feedback_;
171   const bool force_no_audio_feedback_ = allocate_audio_without_feedback_;
172   const bool enable_audio_alr_probing_;
173   const bool send_side_bwe_with_overhead_;
174   const AudioAllocationConfig allocation_settings_;
175 
176   webrtc::AudioSendStream::Config config_
177       RTC_GUARDED_BY(worker_thread_checker_);
178   rtc::scoped_refptr<webrtc::AudioState> audio_state_;
179   const std::unique_ptr<voe::ChannelSendInterface> channel_send_;
180   RtcEventLog* const event_log_;
181   const bool use_legacy_overhead_calculation_;
182 
183   int encoder_sample_rate_hz_ RTC_GUARDED_BY(worker_thread_checker_) = 0;
184   size_t encoder_num_channels_ RTC_GUARDED_BY(worker_thread_checker_) = 0;
185   bool sending_ RTC_GUARDED_BY(worker_thread_checker_) = false;
186   mutable Mutex audio_level_lock_;
187   // Keeps track of audio level, total audio energy and total samples duration.
188   // https://w3c.github.io/webrtc-stats/#dom-rtcaudiohandlerstats-totalaudioenergy
189   webrtc::voe::AudioLevel audio_level_ RTC_GUARDED_BY(audio_level_lock_);
190 
191   BitrateAllocatorInterface* const bitrate_allocator_
192       RTC_GUARDED_BY(worker_queue_);
193   // Constrains cached to be accessed from |worker_queue_|.
194   absl::optional<AudioSendStream::TargetAudioBitrateConstraints>
195       cached_constraints_ RTC_GUARDED_BY(worker_queue_) = absl::nullopt;
196   RtpTransportControllerSendInterface* const rtp_transport_;
197 
198   RtpRtcpInterface* const rtp_rtcp_module_;
199   absl::optional<RtpState> const suspended_rtp_state_;
200 
201   // RFC 5285: Each distinct extension MUST have a unique ID. The value 0 is
202   // reserved for padding and MUST NOT be used as a local identifier.
203   // So it should be safe to use 0 here to indicate "not configured".
204   struct ExtensionIds {
205     int audio_level = 0;
206     int abs_send_time = 0;
207     int abs_capture_time = 0;
208     int transport_sequence_number = 0;
209     int mid = 0;
210     int rid = 0;
211     int repaired_rid = 0;
212   };
213   static ExtensionIds FindExtensionIds(
214       const std::vector<RtpExtension>& extensions);
215   static int TransportSeqNumId(const Config& config);
216 
217   mutable Mutex overhead_per_packet_lock_;
218   size_t overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_) = 0;
219 
220   // Current transport overhead (ICE, TURN, etc.)
221   size_t transport_overhead_per_packet_bytes_
222       RTC_GUARDED_BY(overhead_per_packet_lock_) = 0;
223 
224   bool registered_with_allocator_ RTC_GUARDED_BY(worker_thread_checker_) =
225       false;
226   size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_thread_checker_) =
227       0;
228   absl::optional<std::pair<TimeDelta, TimeDelta>> frame_length_range_
229       RTC_GUARDED_BY(worker_thread_checker_);
230 };
231 }  // namespace internal
232 }  // namespace webrtc
233 
234 #endif  // AUDIO_AUDIO_SEND_STREAM_H_
235