1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/codecs/opus/audio_encoder_opus.h"
12
13 #include <algorithm>
14 #include <iterator>
15 #include <memory>
16 #include <string>
17 #include <utility>
18
19 #include "absl/strings/match.h"
20 #include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
21 #include "modules/audio_coding/audio_network_adaptor/controller_manager.h"
22 #include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h"
23 #include "modules/audio_coding/codecs/opus/opus_interface.h"
24 #include "rtc_base/arraysize.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/logging.h"
27 #include "rtc_base/numerics/exp_filter.h"
28 #include "rtc_base/numerics/safe_conversions.h"
29 #include "rtc_base/numerics/safe_minmax.h"
30 #include "rtc_base/string_encode.h"
31 #include "rtc_base/string_to_number.h"
32 #include "rtc_base/time_utils.h"
33 #include "system_wrappers/include/field_trial.h"
34
35 namespace webrtc {
36
37 namespace {
38
39 // Codec parameters for Opus.
40 // draft-spittka-payload-rtp-opus-03
41
42 // Recommended bitrates:
43 // 8-12 kb/s for NB speech,
44 // 16-20 kb/s for WB speech,
45 // 28-40 kb/s for FB speech,
46 // 48-64 kb/s for FB mono music, and
47 // 64-128 kb/s for FB stereo music.
48 // The current implementation applies the following values to mono signals,
49 // and multiplies them by 2 for stereo.
50 constexpr int kOpusBitrateNbBps = 12000;
51 constexpr int kOpusBitrateWbBps = 20000;
52 constexpr int kOpusBitrateFbBps = 32000;
53
54 constexpr int kRtpTimestampRateHz = 48000;
55 constexpr int kDefaultMaxPlaybackRate = 48000;
56
57 // These two lists must be sorted from low to high
58 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
59 constexpr int kANASupportedFrameLengths[] = {20, 40, 60, 120};
60 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
61 #else
62 constexpr int kANASupportedFrameLengths[] = {20, 40, 60};
63 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
64 #endif
65
66 // PacketLossFractionSmoother uses an exponential filter with a time constant
67 // of -1.0 / ln(0.9999) = 10000 ms.
68 constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
69 constexpr float kMaxPacketLossFraction = 0.2f;
70
CalculateDefaultBitrate(int max_playback_rate,size_t num_channels)71 int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) {
72 const int bitrate = [&] {
73 if (max_playback_rate <= 8000) {
74 return kOpusBitrateNbBps * rtc::dchecked_cast<int>(num_channels);
75 } else if (max_playback_rate <= 16000) {
76 return kOpusBitrateWbBps * rtc::dchecked_cast<int>(num_channels);
77 } else {
78 return kOpusBitrateFbBps * rtc::dchecked_cast<int>(num_channels);
79 }
80 }();
81 RTC_DCHECK_GE(bitrate, AudioEncoderOpusConfig::kMinBitrateBps);
82 RTC_DCHECK_LE(bitrate, AudioEncoderOpusConfig::kMaxBitrateBps);
83 return bitrate;
84 }
85
86 // Get the maxaveragebitrate parameter in string-form, so we can properly figure
87 // out how invalid it is and accurately log invalid values.
CalculateBitrate(int max_playback_rate_hz,size_t num_channels,absl::optional<std::string> bitrate_param)88 int CalculateBitrate(int max_playback_rate_hz,
89 size_t num_channels,
90 absl::optional<std::string> bitrate_param) {
91 const int default_bitrate =
92 CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
93
94 if (bitrate_param) {
95 const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
96 if (bitrate) {
97 const int chosen_bitrate =
98 std::max(AudioEncoderOpusConfig::kMinBitrateBps,
99 std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps));
100 if (bitrate != chosen_bitrate) {
101 RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
102 << " clamped to " << chosen_bitrate;
103 }
104 return chosen_bitrate;
105 }
106 RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
107 << "\" replaced by default bitrate " << default_bitrate;
108 }
109
110 return default_bitrate;
111 }
112
GetChannelCount(const SdpAudioFormat & format)113 int GetChannelCount(const SdpAudioFormat& format) {
114 const auto param = GetFormatParameter(format, "stereo");
115 if (param == "1") {
116 return 2;
117 } else {
118 return 1;
119 }
120 }
121
GetMaxPlaybackRate(const SdpAudioFormat & format)122 int GetMaxPlaybackRate(const SdpAudioFormat& format) {
123 const auto param = GetFormatParameter<int>(format, "maxplaybackrate");
124 if (param && *param >= 8000) {
125 return std::min(*param, kDefaultMaxPlaybackRate);
126 }
127 return kDefaultMaxPlaybackRate;
128 }
129
GetFrameSizeMs(const SdpAudioFormat & format)130 int GetFrameSizeMs(const SdpAudioFormat& format) {
131 const auto ptime = GetFormatParameter<int>(format, "ptime");
132 if (ptime) {
133 // Pick the next highest supported frame length from
134 // kOpusSupportedFrameLengths.
135 for (const int supported_frame_length : kOpusSupportedFrameLengths) {
136 if (supported_frame_length >= *ptime) {
137 return supported_frame_length;
138 }
139 }
140 // If none was found, return the largest supported frame length.
141 return *(std::end(kOpusSupportedFrameLengths) - 1);
142 }
143
144 return AudioEncoderOpusConfig::kDefaultFrameSizeMs;
145 }
146
FindSupportedFrameLengths(int min_frame_length_ms,int max_frame_length_ms,std::vector<int> * out)147 void FindSupportedFrameLengths(int min_frame_length_ms,
148 int max_frame_length_ms,
149 std::vector<int>* out) {
150 out->clear();
151 std::copy_if(std::begin(kANASupportedFrameLengths),
152 std::end(kANASupportedFrameLengths), std::back_inserter(*out),
153 [&](int frame_length_ms) {
154 return frame_length_ms >= min_frame_length_ms &&
155 frame_length_ms <= max_frame_length_ms;
156 });
157 RTC_DCHECK(std::is_sorted(out->begin(), out->end()));
158 }
159
GetBitrateBps(const AudioEncoderOpusConfig & config)160 int GetBitrateBps(const AudioEncoderOpusConfig& config) {
161 RTC_DCHECK(config.IsOk());
162 return *config.bitrate_bps;
163 }
164
GetBitrateMultipliers()165 std::vector<float> GetBitrateMultipliers() {
166 constexpr char kBitrateMultipliersName[] =
167 "WebRTC-Audio-OpusBitrateMultipliers";
168 const bool use_bitrate_multipliers =
169 webrtc::field_trial::IsEnabled(kBitrateMultipliersName);
170 if (use_bitrate_multipliers) {
171 const std::string field_trial_string =
172 webrtc::field_trial::FindFullName(kBitrateMultipliersName);
173 std::vector<std::string> pieces;
174 rtc::tokenize(field_trial_string, '-', &pieces);
175 if (pieces.size() < 2 || pieces[0] != "Enabled") {
176 RTC_LOG(LS_WARNING) << "Invalid parameters for "
177 << kBitrateMultipliersName
178 << ", not using custom values.";
179 return std::vector<float>();
180 }
181 std::vector<float> multipliers(pieces.size() - 1);
182 for (size_t i = 1; i < pieces.size(); i++) {
183 if (!rtc::FromString(pieces[i], &multipliers[i - 1])) {
184 RTC_LOG(LS_WARNING)
185 << "Invalid parameters for " << kBitrateMultipliersName
186 << ", not using custom values.";
187 return std::vector<float>();
188 }
189 }
190 RTC_LOG(LS_INFO) << "Using custom bitrate multipliers: "
191 << field_trial_string;
192 return multipliers;
193 }
194 return std::vector<float>();
195 }
196
GetMultipliedBitrate(int bitrate,const std::vector<float> & multipliers)197 int GetMultipliedBitrate(int bitrate, const std::vector<float>& multipliers) {
198 // The multipliers are valid from 5 kbps.
199 const size_t bitrate_kbps = static_cast<size_t>(bitrate / 1000);
200 if (bitrate_kbps < 5 || bitrate_kbps >= multipliers.size() + 5) {
201 return bitrate;
202 }
203 return static_cast<int>(multipliers[bitrate_kbps - 5] * bitrate);
204 }
205 } // namespace
206
AppendSupportedEncoders(std::vector<AudioCodecSpec> * specs)207 void AudioEncoderOpusImpl::AppendSupportedEncoders(
208 std::vector<AudioCodecSpec>* specs) {
209 const SdpAudioFormat fmt = {"opus",
210 kRtpTimestampRateHz,
211 2,
212 {{"minptime", "10"}, {"useinbandfec", "1"}}};
213 const AudioCodecInfo info = QueryAudioEncoder(*SdpToConfig(fmt));
214 specs->push_back({fmt, info});
215 }
216
QueryAudioEncoder(const AudioEncoderOpusConfig & config)217 AudioCodecInfo AudioEncoderOpusImpl::QueryAudioEncoder(
218 const AudioEncoderOpusConfig& config) {
219 RTC_DCHECK(config.IsOk());
220 AudioCodecInfo info(config.sample_rate_hz, config.num_channels,
221 *config.bitrate_bps,
222 AudioEncoderOpusConfig::kMinBitrateBps,
223 AudioEncoderOpusConfig::kMaxBitrateBps);
224 info.allow_comfort_noise = false;
225 info.supports_network_adaption = true;
226 return info;
227 }
228
MakeAudioEncoder(const AudioEncoderOpusConfig & config,int payload_type)229 std::unique_ptr<AudioEncoder> AudioEncoderOpusImpl::MakeAudioEncoder(
230 const AudioEncoderOpusConfig& config,
231 int payload_type) {
232 RTC_DCHECK(config.IsOk());
233 return std::make_unique<AudioEncoderOpusImpl>(config, payload_type);
234 }
235
SdpToConfig(const SdpAudioFormat & format)236 absl::optional<AudioEncoderOpusConfig> AudioEncoderOpusImpl::SdpToConfig(
237 const SdpAudioFormat& format) {
238 if (!absl::EqualsIgnoreCase(format.name, "opus") ||
239 format.clockrate_hz != kRtpTimestampRateHz || format.num_channels != 2) {
240 return absl::nullopt;
241 }
242
243 AudioEncoderOpusConfig config;
244 config.num_channels = GetChannelCount(format);
245 config.frame_size_ms = GetFrameSizeMs(format);
246 config.max_playback_rate_hz = GetMaxPlaybackRate(format);
247 config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
248 config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
249 config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1");
250 config.bitrate_bps =
251 CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
252 GetFormatParameter(format, "maxaveragebitrate"));
253 config.application = config.num_channels == 1
254 ? AudioEncoderOpusConfig::ApplicationMode::kVoip
255 : AudioEncoderOpusConfig::ApplicationMode::kAudio;
256
257 constexpr int kMinANAFrameLength = kANASupportedFrameLengths[0];
258 constexpr int kMaxANAFrameLength =
259 kANASupportedFrameLengths[arraysize(kANASupportedFrameLengths) - 1];
260
261 // For now, minptime and maxptime are only used with ANA. If ptime is outside
262 // of this range, it will get adjusted once ANA takes hold. Ideally, we'd know
263 // if ANA was to be used when setting up the config, and adjust accordingly.
264 const int min_frame_length_ms =
265 GetFormatParameter<int>(format, "minptime").value_or(kMinANAFrameLength);
266 const int max_frame_length_ms =
267 GetFormatParameter<int>(format, "maxptime").value_or(kMaxANAFrameLength);
268
269 FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
270 &config.supported_frame_lengths_ms);
271 RTC_DCHECK(config.IsOk());
272 return config;
273 }
274
GetNewComplexity(const AudioEncoderOpusConfig & config)275 absl::optional<int> AudioEncoderOpusImpl::GetNewComplexity(
276 const AudioEncoderOpusConfig& config) {
277 RTC_DCHECK(config.IsOk());
278 const int bitrate_bps = GetBitrateBps(config);
279 if (bitrate_bps >= config.complexity_threshold_bps -
280 config.complexity_threshold_window_bps &&
281 bitrate_bps <= config.complexity_threshold_bps +
282 config.complexity_threshold_window_bps) {
283 // Within the hysteresis window; make no change.
284 return absl::nullopt;
285 } else {
286 return bitrate_bps <= config.complexity_threshold_bps
287 ? config.low_rate_complexity
288 : config.complexity;
289 }
290 }
291
GetNewBandwidth(const AudioEncoderOpusConfig & config,OpusEncInst * inst)292 absl::optional<int> AudioEncoderOpusImpl::GetNewBandwidth(
293 const AudioEncoderOpusConfig& config,
294 OpusEncInst* inst) {
295 constexpr int kMinWidebandBitrate = 8000;
296 constexpr int kMaxNarrowbandBitrate = 9000;
297 constexpr int kAutomaticThreshold = 11000;
298 RTC_DCHECK(config.IsOk());
299 const int bitrate = GetBitrateBps(config);
300 if (bitrate > kAutomaticThreshold) {
301 return absl::optional<int>(OPUS_AUTO);
302 }
303 const int bandwidth = WebRtcOpus_GetBandwidth(inst);
304 RTC_DCHECK_GE(bandwidth, 0);
305 if (bitrate > kMaxNarrowbandBitrate && bandwidth < OPUS_BANDWIDTH_WIDEBAND) {
306 return absl::optional<int>(OPUS_BANDWIDTH_WIDEBAND);
307 } else if (bitrate < kMinWidebandBitrate &&
308 bandwidth > OPUS_BANDWIDTH_NARROWBAND) {
309 return absl::optional<int>(OPUS_BANDWIDTH_NARROWBAND);
310 }
311 return absl::optional<int>();
312 }
313
314 class AudioEncoderOpusImpl::PacketLossFractionSmoother {
315 public:
PacketLossFractionSmoother()316 explicit PacketLossFractionSmoother()
317 : last_sample_time_ms_(rtc::TimeMillis()),
318 smoother_(kAlphaForPacketLossFractionSmoother) {}
319
320 // Gets the smoothed packet loss fraction.
GetAverage() const321 float GetAverage() const {
322 float value = smoother_.filtered();
323 return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value;
324 }
325
326 // Add new observation to the packet loss fraction smoother.
AddSample(float packet_loss_fraction)327 void AddSample(float packet_loss_fraction) {
328 int64_t now_ms = rtc::TimeMillis();
329 smoother_.Apply(static_cast<float>(now_ms - last_sample_time_ms_),
330 packet_loss_fraction);
331 last_sample_time_ms_ = now_ms;
332 }
333
334 private:
335 int64_t last_sample_time_ms_;
336
337 // An exponential filter is used to smooth the packet loss fraction.
338 rtc::ExpFilter smoother_;
339 };
340
AudioEncoderOpusImpl(const AudioEncoderOpusConfig & config,int payload_type)341 AudioEncoderOpusImpl::AudioEncoderOpusImpl(const AudioEncoderOpusConfig& config,
342 int payload_type)
343 : AudioEncoderOpusImpl(
344 config,
345 payload_type,
346 [this](const std::string& config_string, RtcEventLog* event_log) {
347 return DefaultAudioNetworkAdaptorCreator(config_string, event_log);
348 },
349 // We choose 5sec as initial time constant due to empirical data.
350 std::make_unique<SmoothingFilterImpl>(5000)) {}
351
AudioEncoderOpusImpl(const AudioEncoderOpusConfig & config,int payload_type,const AudioNetworkAdaptorCreator & audio_network_adaptor_creator,std::unique_ptr<SmoothingFilter> bitrate_smoother)352 AudioEncoderOpusImpl::AudioEncoderOpusImpl(
353 const AudioEncoderOpusConfig& config,
354 int payload_type,
355 const AudioNetworkAdaptorCreator& audio_network_adaptor_creator,
356 std::unique_ptr<SmoothingFilter> bitrate_smoother)
357 : payload_type_(payload_type),
358 send_side_bwe_with_overhead_(
359 !webrtc::field_trial::IsDisabled("WebRTC-SendSideBwe-WithOverhead")),
360 use_stable_target_for_adaptation_(!webrtc::field_trial::IsDisabled(
361 "WebRTC-Audio-StableTargetAdaptation")),
362 adjust_bandwidth_(
363 webrtc::field_trial::IsEnabled("WebRTC-AdjustOpusBandwidth")),
364 bitrate_changed_(true),
365 bitrate_multipliers_(GetBitrateMultipliers()),
366 packet_loss_rate_(0.0),
367 inst_(nullptr),
368 packet_loss_fraction_smoother_(new PacketLossFractionSmoother()),
369 audio_network_adaptor_creator_(audio_network_adaptor_creator),
370 bitrate_smoother_(std::move(bitrate_smoother)),
371 consecutive_dtx_frames_(0) {
372 RTC_DCHECK(0 <= payload_type && payload_type <= 127);
373
374 // Sanity check of the redundant payload type field that we want to get rid
375 // of. See https://bugs.chromium.org/p/webrtc/issues/detail?id=7847
376 RTC_CHECK(config.payload_type == -1 || config.payload_type == payload_type);
377
378 RTC_CHECK(RecreateEncoderInstance(config));
379 SetProjectedPacketLossRate(packet_loss_rate_);
380 }
381
AudioEncoderOpusImpl(int payload_type,const SdpAudioFormat & format)382 AudioEncoderOpusImpl::AudioEncoderOpusImpl(int payload_type,
383 const SdpAudioFormat& format)
384 : AudioEncoderOpusImpl(*SdpToConfig(format), payload_type) {}
385
~AudioEncoderOpusImpl()386 AudioEncoderOpusImpl::~AudioEncoderOpusImpl() {
387 RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
388 }
389
SampleRateHz() const390 int AudioEncoderOpusImpl::SampleRateHz() const {
391 return config_.sample_rate_hz;
392 }
393
NumChannels() const394 size_t AudioEncoderOpusImpl::NumChannels() const {
395 return config_.num_channels;
396 }
397
RtpTimestampRateHz() const398 int AudioEncoderOpusImpl::RtpTimestampRateHz() const {
399 return kRtpTimestampRateHz;
400 }
401
Num10MsFramesInNextPacket() const402 size_t AudioEncoderOpusImpl::Num10MsFramesInNextPacket() const {
403 return Num10msFramesPerPacket();
404 }
405
Max10MsFramesInAPacket() const406 size_t AudioEncoderOpusImpl::Max10MsFramesInAPacket() const {
407 return Num10msFramesPerPacket();
408 }
409
GetTargetBitrate() const410 int AudioEncoderOpusImpl::GetTargetBitrate() const {
411 return GetBitrateBps(config_);
412 }
413
Reset()414 void AudioEncoderOpusImpl::Reset() {
415 RTC_CHECK(RecreateEncoderInstance(config_));
416 }
417
SetFec(bool enable)418 bool AudioEncoderOpusImpl::SetFec(bool enable) {
419 if (enable) {
420 RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
421 } else {
422 RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
423 }
424 config_.fec_enabled = enable;
425 return true;
426 }
427
SetDtx(bool enable)428 bool AudioEncoderOpusImpl::SetDtx(bool enable) {
429 if (enable) {
430 RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
431 } else {
432 RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
433 }
434 config_.dtx_enabled = enable;
435 return true;
436 }
437
GetDtx() const438 bool AudioEncoderOpusImpl::GetDtx() const {
439 return config_.dtx_enabled;
440 }
441
SetApplication(Application application)442 bool AudioEncoderOpusImpl::SetApplication(Application application) {
443 auto conf = config_;
444 switch (application) {
445 case Application::kSpeech:
446 conf.application = AudioEncoderOpusConfig::ApplicationMode::kVoip;
447 break;
448 case Application::kAudio:
449 conf.application = AudioEncoderOpusConfig::ApplicationMode::kAudio;
450 break;
451 }
452 return RecreateEncoderInstance(conf);
453 }
454
SetMaxPlaybackRate(int frequency_hz)455 void AudioEncoderOpusImpl::SetMaxPlaybackRate(int frequency_hz) {
456 auto conf = config_;
457 conf.max_playback_rate_hz = frequency_hz;
458 RTC_CHECK(RecreateEncoderInstance(conf));
459 }
460
EnableAudioNetworkAdaptor(const std::string & config_string,RtcEventLog * event_log)461 bool AudioEncoderOpusImpl::EnableAudioNetworkAdaptor(
462 const std::string& config_string,
463 RtcEventLog* event_log) {
464 audio_network_adaptor_ =
465 audio_network_adaptor_creator_(config_string, event_log);
466 return audio_network_adaptor_.get() != nullptr;
467 }
468
DisableAudioNetworkAdaptor()469 void AudioEncoderOpusImpl::DisableAudioNetworkAdaptor() {
470 audio_network_adaptor_.reset(nullptr);
471 }
472
OnReceivedUplinkPacketLossFraction(float uplink_packet_loss_fraction)473 void AudioEncoderOpusImpl::OnReceivedUplinkPacketLossFraction(
474 float uplink_packet_loss_fraction) {
475 if (audio_network_adaptor_) {
476 audio_network_adaptor_->SetUplinkPacketLossFraction(
477 uplink_packet_loss_fraction);
478 ApplyAudioNetworkAdaptor();
479 }
480 packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction);
481 float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage();
482 SetProjectedPacketLossRate(average_fraction_loss);
483 }
484
OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps)485 void AudioEncoderOpusImpl::OnReceivedTargetAudioBitrate(
486 int target_audio_bitrate_bps) {
487 SetTargetBitrate(target_audio_bitrate_bps);
488 }
489
OnReceivedUplinkBandwidth(int target_audio_bitrate_bps,absl::optional<int64_t> bwe_period_ms,absl::optional<int64_t> stable_target_bitrate_bps)490 void AudioEncoderOpusImpl::OnReceivedUplinkBandwidth(
491 int target_audio_bitrate_bps,
492 absl::optional<int64_t> bwe_period_ms,
493 absl::optional<int64_t> stable_target_bitrate_bps) {
494 if (audio_network_adaptor_) {
495 audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps);
496 if (use_stable_target_for_adaptation_) {
497 if (stable_target_bitrate_bps)
498 audio_network_adaptor_->SetUplinkBandwidth(*stable_target_bitrate_bps);
499 } else {
500 // We give smoothed bitrate allocation to audio network adaptor as
501 // the uplink bandwidth.
502 // The BWE spikes should not affect the bitrate smoother more than 25%.
503 // To simplify the calculations we use a step response as input signal.
504 // The step response of an exponential filter is
505 // u(t) = 1 - e^(-t / time_constant).
506 // In order to limit the affect of a BWE spike within 25% of its value
507 // before
508 // the next BWE update, we would choose a time constant that fulfills
509 // 1 - e^(-bwe_period_ms / time_constant) < 0.25
510 // Then 4 * bwe_period_ms is a good choice.
511 if (bwe_period_ms)
512 bitrate_smoother_->SetTimeConstantMs(*bwe_period_ms * 4);
513 bitrate_smoother_->AddSample(target_audio_bitrate_bps);
514 }
515
516 ApplyAudioNetworkAdaptor();
517 } else if (send_side_bwe_with_overhead_) {
518 if (!overhead_bytes_per_packet_) {
519 RTC_LOG(LS_INFO)
520 << "AudioEncoderOpusImpl: Overhead unknown, target audio bitrate "
521 << target_audio_bitrate_bps << " bps is ignored.";
522 return;
523 }
524 const int overhead_bps = static_cast<int>(
525 *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket());
526 SetTargetBitrate(
527 std::min(AudioEncoderOpusConfig::kMaxBitrateBps,
528 std::max(AudioEncoderOpusConfig::kMinBitrateBps,
529 target_audio_bitrate_bps - overhead_bps)));
530 } else {
531 SetTargetBitrate(target_audio_bitrate_bps);
532 }
533 }
OnReceivedUplinkBandwidth(int target_audio_bitrate_bps,absl::optional<int64_t> bwe_period_ms)534 void AudioEncoderOpusImpl::OnReceivedUplinkBandwidth(
535 int target_audio_bitrate_bps,
536 absl::optional<int64_t> bwe_period_ms) {
537 OnReceivedUplinkBandwidth(target_audio_bitrate_bps, bwe_period_ms,
538 absl::nullopt);
539 }
540
OnReceivedUplinkAllocation(BitrateAllocationUpdate update)541 void AudioEncoderOpusImpl::OnReceivedUplinkAllocation(
542 BitrateAllocationUpdate update) {
543 OnReceivedUplinkBandwidth(update.target_bitrate.bps(), update.bwe_period.ms(),
544 update.stable_target_bitrate.bps());
545 }
546
OnReceivedRtt(int rtt_ms)547 void AudioEncoderOpusImpl::OnReceivedRtt(int rtt_ms) {
548 if (!audio_network_adaptor_)
549 return;
550 audio_network_adaptor_->SetRtt(rtt_ms);
551 ApplyAudioNetworkAdaptor();
552 }
553
OnReceivedOverhead(size_t overhead_bytes_per_packet)554 void AudioEncoderOpusImpl::OnReceivedOverhead(
555 size_t overhead_bytes_per_packet) {
556 if (audio_network_adaptor_) {
557 audio_network_adaptor_->SetOverhead(overhead_bytes_per_packet);
558 ApplyAudioNetworkAdaptor();
559 } else {
560 overhead_bytes_per_packet_ = overhead_bytes_per_packet;
561 }
562 }
563
SetReceiverFrameLengthRange(int min_frame_length_ms,int max_frame_length_ms)564 void AudioEncoderOpusImpl::SetReceiverFrameLengthRange(
565 int min_frame_length_ms,
566 int max_frame_length_ms) {
567 // Ensure that |SetReceiverFrameLengthRange| is called before
568 // |EnableAudioNetworkAdaptor|, otherwise we need to recreate
569 // |audio_network_adaptor_|, which is not a needed use case.
570 RTC_DCHECK(!audio_network_adaptor_);
571 FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
572 &config_.supported_frame_lengths_ms);
573 }
574
EncodeImpl(uint32_t rtp_timestamp,rtc::ArrayView<const int16_t> audio,rtc::Buffer * encoded)575 AudioEncoder::EncodedInfo AudioEncoderOpusImpl::EncodeImpl(
576 uint32_t rtp_timestamp,
577 rtc::ArrayView<const int16_t> audio,
578 rtc::Buffer* encoded) {
579 MaybeUpdateUplinkBandwidth();
580
581 if (input_buffer_.empty())
582 first_timestamp_in_buffer_ = rtp_timestamp;
583
584 input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend());
585 if (input_buffer_.size() <
586 (Num10msFramesPerPacket() * SamplesPer10msFrame())) {
587 return EncodedInfo();
588 }
589 RTC_CHECK_EQ(input_buffer_.size(),
590 Num10msFramesPerPacket() * SamplesPer10msFrame());
591
592 const size_t max_encoded_bytes = SufficientOutputBufferSize();
593 EncodedInfo info;
594 info.encoded_bytes = encoded->AppendData(
595 max_encoded_bytes, [&](rtc::ArrayView<uint8_t> encoded) {
596 int status = WebRtcOpus_Encode(
597 inst_, &input_buffer_[0],
598 rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels),
599 rtc::saturated_cast<int16_t>(max_encoded_bytes), encoded.data());
600
601 RTC_CHECK_GE(status, 0); // Fails only if fed invalid data.
602
603 return static_cast<size_t>(status);
604 });
605 input_buffer_.clear();
606
607 bool dtx_frame = (info.encoded_bytes <= 2);
608
609 // Will use new packet size for next encoding.
610 config_.frame_size_ms = next_frame_length_ms_;
611
612 if (adjust_bandwidth_ && bitrate_changed_) {
613 const auto bandwidth = GetNewBandwidth(config_, inst_);
614 if (bandwidth) {
615 RTC_CHECK_EQ(0, WebRtcOpus_SetBandwidth(inst_, *bandwidth));
616 }
617 bitrate_changed_ = false;
618 }
619
620 info.encoded_timestamp = first_timestamp_in_buffer_;
621 info.payload_type = payload_type_;
622 info.send_even_if_empty = true; // Allows Opus to send empty packets.
623 // After 20 DTX frames (MAX_CONSECUTIVE_DTX) Opus will send a frame
624 // coding the background noise. Avoid flagging this frame as speech
625 // (even though there is a probability of the frame being speech).
626 info.speech = !dtx_frame && (consecutive_dtx_frames_ != 20);
627 info.encoder_type = CodecType::kOpus;
628
629 // Increase or reset DTX counter.
630 consecutive_dtx_frames_ = (dtx_frame) ? (consecutive_dtx_frames_ + 1) : (0);
631
632 return info;
633 }
634
Num10msFramesPerPacket() const635 size_t AudioEncoderOpusImpl::Num10msFramesPerPacket() const {
636 return static_cast<size_t>(rtc::CheckedDivExact(config_.frame_size_ms, 10));
637 }
638
SamplesPer10msFrame() const639 size_t AudioEncoderOpusImpl::SamplesPer10msFrame() const {
640 return rtc::CheckedDivExact(config_.sample_rate_hz, 100) *
641 config_.num_channels;
642 }
643
SufficientOutputBufferSize() const644 size_t AudioEncoderOpusImpl::SufficientOutputBufferSize() const {
645 // Calculate the number of bytes we expect the encoder to produce,
646 // then multiply by two to give a wide margin for error.
647 const size_t bytes_per_millisecond =
648 static_cast<size_t>(GetBitrateBps(config_) / (1000 * 8) + 1);
649 const size_t approx_encoded_bytes =
650 Num10msFramesPerPacket() * 10 * bytes_per_millisecond;
651 return 2 * approx_encoded_bytes;
652 }
653
654 // If the given config is OK, recreate the Opus encoder instance with those
655 // settings, save the config, and return true. Otherwise, do nothing and return
656 // false.
RecreateEncoderInstance(const AudioEncoderOpusConfig & config)657 bool AudioEncoderOpusImpl::RecreateEncoderInstance(
658 const AudioEncoderOpusConfig& config) {
659 if (!config.IsOk())
660 return false;
661 config_ = config;
662 if (inst_)
663 RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
664 input_buffer_.clear();
665 input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame());
666 RTC_CHECK_EQ(0, WebRtcOpus_EncoderCreate(
667 &inst_, config.num_channels,
668 config.application ==
669 AudioEncoderOpusConfig::ApplicationMode::kVoip
670 ? 0
671 : 1,
672 config.sample_rate_hz));
673 const int bitrate = GetBitrateBps(config);
674 RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate));
675 RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
676 if (config.fec_enabled) {
677 RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
678 } else {
679 RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
680 }
681 RTC_CHECK_EQ(
682 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz));
683 // Use the default complexity if the start bitrate is within the hysteresis
684 // window.
685 complexity_ = GetNewComplexity(config).value_or(config.complexity);
686 RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
687 bitrate_changed_ = true;
688 if (config.dtx_enabled) {
689 RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
690 } else {
691 RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
692 }
693 RTC_CHECK_EQ(0,
694 WebRtcOpus_SetPacketLossRate(
695 inst_, static_cast<int32_t>(packet_loss_rate_ * 100 + .5)));
696 if (config.cbr_enabled) {
697 RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_));
698 } else {
699 RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_));
700 }
701 num_channels_to_encode_ = NumChannels();
702 next_frame_length_ms_ = config_.frame_size_ms;
703 return true;
704 }
705
SetFrameLength(int frame_length_ms)706 void AudioEncoderOpusImpl::SetFrameLength(int frame_length_ms) {
707 next_frame_length_ms_ = frame_length_ms;
708 }
709
SetNumChannelsToEncode(size_t num_channels_to_encode)710 void AudioEncoderOpusImpl::SetNumChannelsToEncode(
711 size_t num_channels_to_encode) {
712 RTC_DCHECK_GT(num_channels_to_encode, 0);
713 RTC_DCHECK_LE(num_channels_to_encode, config_.num_channels);
714
715 if (num_channels_to_encode_ == num_channels_to_encode)
716 return;
717
718 RTC_CHECK_EQ(0, WebRtcOpus_SetForceChannels(inst_, num_channels_to_encode));
719 num_channels_to_encode_ = num_channels_to_encode;
720 }
721
SetProjectedPacketLossRate(float fraction)722 void AudioEncoderOpusImpl::SetProjectedPacketLossRate(float fraction) {
723 fraction = std::min(std::max(fraction, 0.0f), kMaxPacketLossFraction);
724 if (packet_loss_rate_ != fraction) {
725 packet_loss_rate_ = fraction;
726 RTC_CHECK_EQ(
727 0, WebRtcOpus_SetPacketLossRate(
728 inst_, static_cast<int32_t>(packet_loss_rate_ * 100 + .5)));
729 }
730 }
731
SetTargetBitrate(int bits_per_second)732 void AudioEncoderOpusImpl::SetTargetBitrate(int bits_per_second) {
733 const int new_bitrate = rtc::SafeClamp<int>(
734 bits_per_second, AudioEncoderOpusConfig::kMinBitrateBps,
735 AudioEncoderOpusConfig::kMaxBitrateBps);
736 if (config_.bitrate_bps && *config_.bitrate_bps != new_bitrate) {
737 config_.bitrate_bps = new_bitrate;
738 RTC_DCHECK(config_.IsOk());
739 const int bitrate = GetBitrateBps(config_);
740 RTC_CHECK_EQ(
741 0, WebRtcOpus_SetBitRate(
742 inst_, GetMultipliedBitrate(bitrate, bitrate_multipliers_)));
743 RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
744 bitrate_changed_ = true;
745 }
746
747 const auto new_complexity = GetNewComplexity(config_);
748 if (new_complexity && complexity_ != *new_complexity) {
749 complexity_ = *new_complexity;
750 RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
751 }
752 }
753
ApplyAudioNetworkAdaptor()754 void AudioEncoderOpusImpl::ApplyAudioNetworkAdaptor() {
755 auto config = audio_network_adaptor_->GetEncoderRuntimeConfig();
756
757 if (config.bitrate_bps)
758 SetTargetBitrate(*config.bitrate_bps);
759 if (config.frame_length_ms)
760 SetFrameLength(*config.frame_length_ms);
761 if (config.enable_dtx)
762 SetDtx(*config.enable_dtx);
763 if (config.num_channels)
764 SetNumChannelsToEncode(*config.num_channels);
765 }
766
767 std::unique_ptr<AudioNetworkAdaptor>
DefaultAudioNetworkAdaptorCreator(const std::string & config_string,RtcEventLog * event_log) const768 AudioEncoderOpusImpl::DefaultAudioNetworkAdaptorCreator(
769 const std::string& config_string,
770 RtcEventLog* event_log) const {
771 AudioNetworkAdaptorImpl::Config config;
772 config.event_log = event_log;
773 return std::unique_ptr<AudioNetworkAdaptor>(new AudioNetworkAdaptorImpl(
774 config, ControllerManagerImpl::Create(
775 config_string, NumChannels(), supported_frame_lengths_ms(),
776 AudioEncoderOpusConfig::kMinBitrateBps,
777 num_channels_to_encode_, next_frame_length_ms_,
778 GetTargetBitrate(), config_.fec_enabled, GetDtx())));
779 }
780
MaybeUpdateUplinkBandwidth()781 void AudioEncoderOpusImpl::MaybeUpdateUplinkBandwidth() {
782 if (audio_network_adaptor_ && !use_stable_target_for_adaptation_) {
783 int64_t now_ms = rtc::TimeMillis();
784 if (!bitrate_smoother_last_update_time_ ||
785 now_ms - *bitrate_smoother_last_update_time_ >=
786 config_.uplink_bandwidth_update_interval_ms) {
787 absl::optional<float> smoothed_bitrate = bitrate_smoother_->GetAverage();
788 if (smoothed_bitrate)
789 audio_network_adaptor_->SetUplinkBandwidth(*smoothed_bitrate);
790 bitrate_smoother_last_update_time_ = now_ms;
791 }
792 }
793 }
794
GetANAStats() const795 ANAStats AudioEncoderOpusImpl::GetANAStats() const {
796 if (audio_network_adaptor_) {
797 return audio_network_adaptor_->GetStats();
798 }
799 return ANAStats();
800 }
801
802 absl::optional<std::pair<TimeDelta, TimeDelta> >
GetFrameLengthRange() const803 AudioEncoderOpusImpl::GetFrameLengthRange() const {
804 if (config_.supported_frame_lengths_ms.empty()) {
805 return absl::nullopt;
806 } else if (audio_network_adaptor_) {
807 return {{TimeDelta::Millis(config_.supported_frame_lengths_ms.front()),
808 TimeDelta::Millis(config_.supported_frame_lengths_ms.back())}};
809 } else {
810 return {{TimeDelta::Millis(config_.frame_size_ms),
811 TimeDelta::Millis(config_.frame_size_ms)}};
812 }
813 }
814
815 } // namespace webrtc
816