1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 /*
12 * LEFT TO DO:
13 * - WRITE TESTS for the stuff in this file.
14 * - Check the creation, maybe make it safer by returning an empty optional or
15 * unique_ptr. --- It looks OK, but RecreateEncoderInstance can perhaps crash
16 * on a valid config. Can run it in the fuzzer for some time. Should prbl also
17 * fuzz the config.
18 */
19
20 #include "modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h"
21
22 #include <algorithm>
23 #include <memory>
24 #include <string>
25 #include <vector>
26
27 #include "absl/strings/match.h"
28 #include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h"
29 #include "rtc_base/arraysize.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/logging.h"
32 #include "rtc_base/string_to_number.h"
33
34 namespace webrtc {
35
36 namespace {
37
38 // Recommended bitrates for one channel:
39 // 8-12 kb/s for NB speech,
40 // 16-20 kb/s for WB speech,
41 // 28-40 kb/s for FB speech,
42 // 48-64 kb/s for FB mono music, and
43 // 64-128 kb/s for FB stereo music.
44 // The current implementation multiplies these values by the number of channels.
45 constexpr int kOpusBitrateNbBps = 12000;
46 constexpr int kOpusBitrateWbBps = 20000;
47 constexpr int kOpusBitrateFbBps = 32000;
48
49 constexpr int kDefaultMaxPlaybackRate = 48000;
50 // These two lists must be sorted from low to high
51 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
52 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
53 #else
54 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
55 #endif
56
GetBitrateBps(const AudioEncoderMultiChannelOpusConfig & config)57 int GetBitrateBps(const AudioEncoderMultiChannelOpusConfig& config) {
58 RTC_DCHECK(config.IsOk());
59 return config.bitrate_bps;
60 }
GetMaxPlaybackRate(const SdpAudioFormat & format)61 int GetMaxPlaybackRate(const SdpAudioFormat& format) {
62 const auto param = GetFormatParameter<int>(format, "maxplaybackrate");
63 if (param && *param >= 8000) {
64 return std::min(*param, kDefaultMaxPlaybackRate);
65 }
66 return kDefaultMaxPlaybackRate;
67 }
68
GetFrameSizeMs(const SdpAudioFormat & format)69 int GetFrameSizeMs(const SdpAudioFormat& format) {
70 const auto ptime = GetFormatParameter<int>(format, "ptime");
71 if (ptime.has_value()) {
72 // Pick the next highest supported frame length from
73 // kOpusSupportedFrameLengths.
74 for (const int supported_frame_length : kOpusSupportedFrameLengths) {
75 if (supported_frame_length >= *ptime) {
76 return supported_frame_length;
77 }
78 }
79 // If none was found, return the largest supported frame length.
80 return *(std::end(kOpusSupportedFrameLengths) - 1);
81 }
82
83 return AudioEncoderOpusConfig::kDefaultFrameSizeMs;
84 }
85
CalculateDefaultBitrate(int max_playback_rate,size_t num_channels)86 int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) {
87 const int bitrate = [&] {
88 if (max_playback_rate <= 8000) {
89 return kOpusBitrateNbBps * rtc::dchecked_cast<int>(num_channels);
90 } else if (max_playback_rate <= 16000) {
91 return kOpusBitrateWbBps * rtc::dchecked_cast<int>(num_channels);
92 } else {
93 return kOpusBitrateFbBps * rtc::dchecked_cast<int>(num_channels);
94 }
95 }();
96 RTC_DCHECK_GE(bitrate, AudioEncoderMultiChannelOpusConfig::kMinBitrateBps);
97 return bitrate;
98 }
99
100 // Get the maxaveragebitrate parameter in string-form, so we can properly figure
101 // out how invalid it is and accurately log invalid values.
CalculateBitrate(int max_playback_rate_hz,size_t num_channels,absl::optional<std::string> bitrate_param)102 int CalculateBitrate(int max_playback_rate_hz,
103 size_t num_channels,
104 absl::optional<std::string> bitrate_param) {
105 const int default_bitrate =
106 CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
107
108 if (bitrate_param) {
109 const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
110 if (bitrate) {
111 const int chosen_bitrate =
112 std::max(AudioEncoderOpusConfig::kMinBitrateBps,
113 std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps));
114 if (bitrate != chosen_bitrate) {
115 RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
116 << " clamped to " << chosen_bitrate;
117 }
118 return chosen_bitrate;
119 }
120 RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
121 << "\" replaced by default bitrate " << default_bitrate;
122 }
123
124 return default_bitrate;
125 }
126
127 } // namespace
128
129 std::unique_ptr<AudioEncoder>
MakeAudioEncoder(const AudioEncoderMultiChannelOpusConfig & config,int payload_type)130 AudioEncoderMultiChannelOpusImpl::MakeAudioEncoder(
131 const AudioEncoderMultiChannelOpusConfig& config,
132 int payload_type) {
133 if (!config.IsOk()) {
134 return nullptr;
135 }
136 return std::make_unique<AudioEncoderMultiChannelOpusImpl>(config,
137 payload_type);
138 }
139
AudioEncoderMultiChannelOpusImpl(const AudioEncoderMultiChannelOpusConfig & config,int payload_type)140 AudioEncoderMultiChannelOpusImpl::AudioEncoderMultiChannelOpusImpl(
141 const AudioEncoderMultiChannelOpusConfig& config,
142 int payload_type)
143 : payload_type_(payload_type), inst_(nullptr) {
144 RTC_DCHECK(0 <= payload_type && payload_type <= 127);
145
146 RTC_CHECK(RecreateEncoderInstance(config));
147 }
148
~AudioEncoderMultiChannelOpusImpl()149 AudioEncoderMultiChannelOpusImpl::~AudioEncoderMultiChannelOpusImpl() {
150 RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
151 }
152
SufficientOutputBufferSize() const153 size_t AudioEncoderMultiChannelOpusImpl::SufficientOutputBufferSize() const {
154 // Calculate the number of bytes we expect the encoder to produce,
155 // then multiply by two to give a wide margin for error.
156 const size_t bytes_per_millisecond =
157 static_cast<size_t>(GetBitrateBps(config_) / (1000 * 8) + 1);
158 const size_t approx_encoded_bytes =
159 Num10msFramesPerPacket() * 10 * bytes_per_millisecond;
160 return 2 * approx_encoded_bytes;
161 }
162
Reset()163 void AudioEncoderMultiChannelOpusImpl::Reset() {
164 RTC_CHECK(RecreateEncoderInstance(config_));
165 }
166
167 absl::optional<std::pair<TimeDelta, TimeDelta>>
GetFrameLengthRange() const168 AudioEncoderMultiChannelOpusImpl::GetFrameLengthRange() const {
169 return {{TimeDelta::Millis(config_.frame_size_ms),
170 TimeDelta::Millis(config_.frame_size_ms)}};
171 }
172
173 // If the given config is OK, recreate the Opus encoder instance with those
174 // settings, save the config, and return true. Otherwise, do nothing and return
175 // false.
RecreateEncoderInstance(const AudioEncoderMultiChannelOpusConfig & config)176 bool AudioEncoderMultiChannelOpusImpl::RecreateEncoderInstance(
177 const AudioEncoderMultiChannelOpusConfig& config) {
178 if (!config.IsOk())
179 return false;
180 config_ = config;
181 if (inst_)
182 RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
183 input_buffer_.clear();
184 input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame());
185 RTC_CHECK_EQ(
186 0, WebRtcOpus_MultistreamEncoderCreate(
187 &inst_, config.num_channels,
188 config.application ==
189 AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
190 ? 0
191 : 1,
192 config.num_streams, config.coupled_streams,
193 config.channel_mapping.data()));
194 const int bitrate = GetBitrateBps(config);
195 RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate));
196 RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
197 if (config.fec_enabled) {
198 RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
199 RTC_LOG(LS_VERBOSE) << "Opus enable FEC";
200 } else {
201 RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
202 RTC_LOG(LS_VERBOSE) << "Opus disable FEC";
203 }
204 RTC_CHECK_EQ(
205 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz));
206 RTC_LOG(LS_VERBOSE) << "Set Opus playback rate to "
207 << config.max_playback_rate_hz << " hz.";
208
209 // Use the DEFAULT complexity.
210 RTC_CHECK_EQ(
211 0, WebRtcOpus_SetComplexity(inst_, AudioEncoderOpusConfig().complexity));
212 RTC_LOG(LS_VERBOSE) << "Set Opus coding complexity to "
213 << AudioEncoderOpusConfig().complexity;
214
215 if (config.dtx_enabled) {
216 RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
217 RTC_LOG(LS_VERBOSE) << "Opus enable DTX";
218 } else {
219 RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
220 RTC_LOG(LS_VERBOSE) << "Opus disable DTX";
221 }
222
223 if (config.cbr_enabled) {
224 RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_));
225 RTC_LOG(LS_VERBOSE) << "Opus enable CBR";
226 } else {
227 RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_));
228 RTC_LOG(LS_VERBOSE) << "Opus disable CBR";
229 }
230 num_channels_to_encode_ = NumChannels();
231 next_frame_length_ms_ = config_.frame_size_ms;
232 RTC_LOG(LS_VERBOSE) << "Set Opus frame length to " << config_.frame_size_ms
233 << " ms";
234 return true;
235 }
236
237 absl::optional<AudioEncoderMultiChannelOpusConfig>
SdpToConfig(const SdpAudioFormat & format)238 AudioEncoderMultiChannelOpusImpl::SdpToConfig(const SdpAudioFormat& format) {
239 if (!absl::EqualsIgnoreCase(format.name, "multiopus") ||
240 format.clockrate_hz != 48000) {
241 return absl::nullopt;
242 }
243
244 AudioEncoderMultiChannelOpusConfig config;
245 config.num_channels = format.num_channels;
246 config.frame_size_ms = GetFrameSizeMs(format);
247 config.max_playback_rate_hz = GetMaxPlaybackRate(format);
248 config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
249 config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
250 config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1");
251 config.bitrate_bps =
252 CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
253 GetFormatParameter(format, "maxaveragebitrate"));
254 config.application =
255 config.num_channels == 1
256 ? AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
257 : AudioEncoderMultiChannelOpusConfig::ApplicationMode::kAudio;
258
259 config.supported_frame_lengths_ms.clear();
260 std::copy(std::begin(kOpusSupportedFrameLengths),
261 std::end(kOpusSupportedFrameLengths),
262 std::back_inserter(config.supported_frame_lengths_ms));
263
264 auto num_streams = GetFormatParameter<int>(format, "num_streams");
265 if (!num_streams.has_value()) {
266 return absl::nullopt;
267 }
268 config.num_streams = *num_streams;
269
270 auto coupled_streams = GetFormatParameter<int>(format, "coupled_streams");
271 if (!coupled_streams.has_value()) {
272 return absl::nullopt;
273 }
274 config.coupled_streams = *coupled_streams;
275
276 auto channel_mapping =
277 GetFormatParameter<std::vector<unsigned char>>(format, "channel_mapping");
278 if (!channel_mapping.has_value()) {
279 return absl::nullopt;
280 }
281 config.channel_mapping = *channel_mapping;
282
283 return config;
284 }
285
QueryAudioEncoder(const AudioEncoderMultiChannelOpusConfig & config)286 AudioCodecInfo AudioEncoderMultiChannelOpusImpl::QueryAudioEncoder(
287 const AudioEncoderMultiChannelOpusConfig& config) {
288 RTC_DCHECK(config.IsOk());
289 AudioCodecInfo info(48000, config.num_channels, config.bitrate_bps,
290 AudioEncoderOpusConfig::kMinBitrateBps,
291 AudioEncoderOpusConfig::kMaxBitrateBps);
292 info.allow_comfort_noise = false;
293 info.supports_network_adaption = false;
294 return info;
295 }
296
Num10msFramesPerPacket() const297 size_t AudioEncoderMultiChannelOpusImpl::Num10msFramesPerPacket() const {
298 return static_cast<size_t>(rtc::CheckedDivExact(config_.frame_size_ms, 10));
299 }
SamplesPer10msFrame() const300 size_t AudioEncoderMultiChannelOpusImpl::SamplesPer10msFrame() const {
301 return rtc::CheckedDivExact(48000, 100) * config_.num_channels;
302 }
SampleRateHz() const303 int AudioEncoderMultiChannelOpusImpl::SampleRateHz() const {
304 return 48000;
305 }
NumChannels() const306 size_t AudioEncoderMultiChannelOpusImpl::NumChannels() const {
307 return config_.num_channels;
308 }
Num10MsFramesInNextPacket() const309 size_t AudioEncoderMultiChannelOpusImpl::Num10MsFramesInNextPacket() const {
310 return Num10msFramesPerPacket();
311 }
Max10MsFramesInAPacket() const312 size_t AudioEncoderMultiChannelOpusImpl::Max10MsFramesInAPacket() const {
313 return Num10msFramesPerPacket();
314 }
GetTargetBitrate() const315 int AudioEncoderMultiChannelOpusImpl::GetTargetBitrate() const {
316 return GetBitrateBps(config_);
317 }
318
EncodeImpl(uint32_t rtp_timestamp,rtc::ArrayView<const int16_t> audio,rtc::Buffer * encoded)319 AudioEncoder::EncodedInfo AudioEncoderMultiChannelOpusImpl::EncodeImpl(
320 uint32_t rtp_timestamp,
321 rtc::ArrayView<const int16_t> audio,
322 rtc::Buffer* encoded) {
323 if (input_buffer_.empty())
324 first_timestamp_in_buffer_ = rtp_timestamp;
325
326 input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend());
327 if (input_buffer_.size() <
328 (Num10msFramesPerPacket() * SamplesPer10msFrame())) {
329 return EncodedInfo();
330 }
331 RTC_CHECK_EQ(input_buffer_.size(),
332 Num10msFramesPerPacket() * SamplesPer10msFrame());
333
334 const size_t max_encoded_bytes = SufficientOutputBufferSize();
335 EncodedInfo info;
336 info.encoded_bytes = encoded->AppendData(
337 max_encoded_bytes, [&](rtc::ArrayView<uint8_t> encoded) {
338 int status = WebRtcOpus_Encode(
339 inst_, &input_buffer_[0],
340 rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels),
341 rtc::saturated_cast<int16_t>(max_encoded_bytes), encoded.data());
342
343 RTC_CHECK_GE(status, 0); // Fails only if fed invalid data.
344
345 return static_cast<size_t>(status);
346 });
347 input_buffer_.clear();
348
349 // Will use new packet size for next encoding.
350 config_.frame_size_ms = next_frame_length_ms_;
351
352 info.encoded_timestamp = first_timestamp_in_buffer_;
353 info.payload_type = payload_type_;
354 info.send_even_if_empty = true; // Allows Opus to send empty packets.
355
356 info.speech = true;
357 info.encoder_type = CodecType::kOther;
358
359 return info;
360 }
361
362 } // namespace webrtc
363