1 /*
2  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef AUDIO_AUDIO_SEND_STREAM_H_
12 #define AUDIO_AUDIO_SEND_STREAM_H_
13 
14 #include <memory>
15 #include <utility>
16 #include <vector>
17 
18 #include "audio/audio_level.h"
19 #include "audio/channel_send.h"
20 #include "call/audio_send_stream.h"
21 #include "call/audio_state.h"
22 #include "call/bitrate_allocator.h"
23 #include "modules/rtp_rtcp/include/rtp_rtcp.h"
24 #include "rtc_base/constructor_magic.h"
25 #include "rtc_base/experiments/struct_parameters_parser.h"
26 #include "rtc_base/race_checker.h"
27 #include "rtc_base/task_queue.h"
28 #include "rtc_base/thread_checker.h"
29 
30 namespace webrtc {
31 class RtcEventLog;
32 class RtcpBandwidthObserver;
33 class RtcpRttStats;
34 class RtpTransportControllerSendInterface;
35 
36 struct AudioAllocationConfig {
37   static constexpr char kKey[] = "WebRTC-Audio-Allocation";
38   // Field Trial configured bitrates to use as overrides over default/user
39   // configured bitrate range when audio bitrate allocation is enabled.
40   absl::optional<DataRate> min_bitrate;
41   absl::optional<DataRate> max_bitrate;
42   DataRate priority_bitrate = DataRate::Zero();
43   // By default the priority_bitrate is compensated for packet overhead.
44   // Use this flag to configure a raw value instead.
45   absl::optional<DataRate> priority_bitrate_raw;
46   absl::optional<double> bitrate_priority;
47 
48   std::unique_ptr<StructParametersParser> Parser();
49   AudioAllocationConfig();
50 };
51 namespace internal {
52 class AudioState;
53 
54 class AudioSendStream final : public webrtc::AudioSendStream,
55                               public webrtc::BitrateAllocatorObserver,
56                               public webrtc::OverheadObserver {
57  public:
58   AudioSendStream(Clock* clock,
59                   const webrtc::AudioSendStream::Config& config,
60                   const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
61                   TaskQueueFactory* task_queue_factory,
62                   ProcessThread* module_process_thread,
63                   RtpTransportControllerSendInterface* rtp_transport,
64                   BitrateAllocatorInterface* bitrate_allocator,
65                   RtcEventLog* event_log,
66                   RtcpRttStats* rtcp_rtt_stats,
67                   const absl::optional<RtpState>& suspended_rtp_state);
68   // For unit tests, which need to supply a mock ChannelSend.
69   AudioSendStream(Clock* clock,
70                   const webrtc::AudioSendStream::Config& config,
71                   const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
72                   TaskQueueFactory* task_queue_factory,
73                   RtpTransportControllerSendInterface* rtp_transport,
74                   BitrateAllocatorInterface* bitrate_allocator,
75                   RtcEventLog* event_log,
76                   RtcpRttStats* rtcp_rtt_stats,
77                   const absl::optional<RtpState>& suspended_rtp_state,
78                   std::unique_ptr<voe::ChannelSendInterface> channel_send);
79   ~AudioSendStream() override;
80 
81   // webrtc::AudioSendStream implementation.
82   const webrtc::AudioSendStream::Config& GetConfig() const override;
83   void Reconfigure(const webrtc::AudioSendStream::Config& config) override;
84   void Start() override;
85   void Stop() override;
86   void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override;
87   bool SendTelephoneEvent(int payload_type,
88                           int payload_frequency,
89                           int event,
90                           int duration_ms) override;
91   void SetMuted(bool muted) override;
92   webrtc::AudioSendStream::Stats GetStats() const override;
93   webrtc::AudioSendStream::Stats GetStats(
94       bool has_remote_tracks) const override;
95 
96   void DeliverRtcp(const uint8_t* packet, size_t length);
97 
98   // Implements BitrateAllocatorObserver.
99   uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override;
100 
101   void SetTransportOverhead(int transport_overhead_per_packet_bytes);
102 
103   // OverheadObserver override reports audio packetization overhead from
104   // RTP/RTCP module or Media Transport.
105   void OnOverheadChanged(size_t overhead_bytes_per_packet_bytes) override;
106 
107   RtpState GetRtpState() const;
108   const voe::ChannelSendInterface* GetChannel() const;
109 
110   // Returns combined per-packet overhead.
111   size_t TestOnlyGetPerPacketOverheadBytes() const
112       RTC_LOCKS_EXCLUDED(overhead_per_packet_lock_);
113 
114  private:
115   class TimedTransport;
116   // Constraints including overhead.
117   struct TargetAudioBitrateConstraints {
118     DataRate min;
119     DataRate max;
120   };
121 
122   internal::AudioState* audio_state();
123   const internal::AudioState* audio_state() const;
124 
125   void StoreEncoderProperties(int sample_rate_hz, size_t num_channels);
126 
127   void ConfigureStream(const Config& new_config, bool first_time);
128   bool SetupSendCodec(const Config& new_config);
129   bool ReconfigureSendCodec(const Config& new_config);
130   void ReconfigureANA(const Config& new_config);
131   void ReconfigureCNG(const Config& new_config);
132   void ReconfigureBitrateObserver(const Config& new_config);
133 
134   void ConfigureBitrateObserver() RTC_RUN_ON(worker_queue_);
135   void RemoveBitrateObserver();
136 
137   // Returns bitrate constraints, maybe including overhead when enabled by
138   // field trial.
139   TargetAudioBitrateConstraints GetMinMaxBitrateConstraints() const
140       RTC_RUN_ON(worker_queue_);
141 
142   // Sets per-packet overhead on encoded (for ANA) based on current known values
143   // of transport and packetization overheads.
144   void UpdateOverheadForEncoder()
145       RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_);
146 
147   // Returns combined per-packet overhead.
148   size_t GetPerPacketOverheadBytes() const
149       RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_);
150 
151   void RegisterCngPayloadType(int payload_type, int clockrate_hz);
152   Clock* clock_;
153 
154   rtc::ThreadChecker worker_thread_checker_;
155   rtc::ThreadChecker pacer_thread_checker_;
156   rtc::RaceChecker audio_capture_race_checker_;
157   rtc::TaskQueue* worker_queue_;
158 
159   const bool audio_send_side_bwe_;
160   const bool allocate_audio_without_feedback_;
161   const bool force_no_audio_feedback_ = allocate_audio_without_feedback_;
162   const bool enable_audio_alr_probing_;
163   const bool send_side_bwe_with_overhead_;
164   const AudioAllocationConfig allocation_settings_;
165 
166   webrtc::AudioSendStream::Config config_;
167   rtc::scoped_refptr<webrtc::AudioState> audio_state_;
168   const std::unique_ptr<voe::ChannelSendInterface> channel_send_;
169   RtcEventLog* const event_log_;
170   const bool use_legacy_overhead_calculation_;
171 
172   int encoder_sample_rate_hz_ = 0;
173   size_t encoder_num_channels_ = 0;
174   bool sending_ = false;
175   rtc::CriticalSection audio_level_lock_;
176   // Keeps track of audio level, total audio energy and total samples duration.
177   // https://w3c.github.io/webrtc-stats/#dom-rtcaudiohandlerstats-totalaudioenergy
178   webrtc::voe::AudioLevel audio_level_;
179 
180   BitrateAllocatorInterface* const bitrate_allocator_
181       RTC_GUARDED_BY(worker_queue_);
182   RtpTransportControllerSendInterface* const rtp_transport_;
183 
184   RtpRtcp* const rtp_rtcp_module_;
185   absl::optional<RtpState> const suspended_rtp_state_;
186 
187   // RFC 5285: Each distinct extension MUST have a unique ID. The value 0 is
188   // reserved for padding and MUST NOT be used as a local identifier.
189   // So it should be safe to use 0 here to indicate "not configured".
190   struct ExtensionIds {
191     int audio_level = 0;
192     int abs_send_time = 0;
193     int abs_capture_time = 0;
194     int transport_sequence_number = 0;
195     int mid = 0;
196     int rid = 0;
197     int repaired_rid = 0;
198   };
199   static ExtensionIds FindExtensionIds(
200       const std::vector<RtpExtension>& extensions);
201   static int TransportSeqNumId(const Config& config);
202 
203   rtc::CriticalSection overhead_per_packet_lock_;
204 
205   // Current transport overhead (ICE, TURN, etc.)
206   size_t transport_overhead_per_packet_bytes_
207       RTC_GUARDED_BY(overhead_per_packet_lock_) = 0;
208 
209   // Current audio packetization overhead (RTP or Media Transport).
210   size_t audio_overhead_per_packet_bytes_
211       RTC_GUARDED_BY(overhead_per_packet_lock_) = 0;
212 
213   bool registered_with_allocator_ RTC_GUARDED_BY(worker_queue_) = false;
214   size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_queue_) = 0;
215   absl::optional<std::pair<TimeDelta, TimeDelta>> frame_length_range_
216       RTC_GUARDED_BY(worker_queue_);
217 
218   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioSendStream);
219 };
220 }  // namespace internal
221 }  // namespace webrtc
222 
223 #endif  // AUDIO_AUDIO_SEND_STREAM_H_
224