1 /*
2  *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 /*
12  * LEFT TO DO:
13  * - WRITE TESTS for the stuff in this  file.
14  * - Check the creation, maybe make it safer by returning an empty optional or
15  *   unique_ptr. --- It looks OK, but RecreateEncoderInstance can perhaps crash
16  *   on a valid config. Can run it in the fuzzer for some time. Should prbl also
17  *   fuzz the config.
18  */
19 
20 #include "modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h"
21 
22 #include <algorithm>
23 #include <memory>
24 #include <string>
25 #include <vector>
26 
27 #include "absl/strings/match.h"
28 #include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h"
29 #include "rtc_base/arraysize.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/logging.h"
32 #include "rtc_base/string_to_number.h"
33 
34 namespace webrtc {
35 
36 namespace {
37 
38 // Recommended bitrates for one channel:
39 // 8-12 kb/s for NB speech,
40 // 16-20 kb/s for WB speech,
41 // 28-40 kb/s for FB speech,
42 // 48-64 kb/s for FB mono music, and
43 // 64-128 kb/s for FB stereo music.
44 // The current implementation multiplies these values by the number of channels.
45 constexpr int kOpusBitrateNbBps = 12000;
46 constexpr int kOpusBitrateWbBps = 20000;
47 constexpr int kOpusBitrateFbBps = 32000;
48 
49 constexpr int kDefaultMaxPlaybackRate = 48000;
50 // These two lists must be sorted from low to high
51 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
52 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
53 #else
54 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
55 #endif
56 
GetBitrateBps(const AudioEncoderMultiChannelOpusConfig & config)57 int GetBitrateBps(const AudioEncoderMultiChannelOpusConfig& config) {
58   RTC_DCHECK(config.IsOk());
59   return config.bitrate_bps;
60 }
GetMaxPlaybackRate(const SdpAudioFormat & format)61 int GetMaxPlaybackRate(const SdpAudioFormat& format) {
62   const auto param = GetFormatParameter<int>(format, "maxplaybackrate");
63   if (param && *param >= 8000) {
64     return std::min(*param, kDefaultMaxPlaybackRate);
65   }
66   return kDefaultMaxPlaybackRate;
67 }
68 
GetFrameSizeMs(const SdpAudioFormat & format)69 int GetFrameSizeMs(const SdpAudioFormat& format) {
70   const auto ptime = GetFormatParameter<int>(format, "ptime");
71   if (ptime.has_value()) {
72     // Pick the next highest supported frame length from
73     // kOpusSupportedFrameLengths.
74     for (const int supported_frame_length : kOpusSupportedFrameLengths) {
75       if (supported_frame_length >= *ptime) {
76         return supported_frame_length;
77       }
78     }
79     // If none was found, return the largest supported frame length.
80     return *(std::end(kOpusSupportedFrameLengths) - 1);
81   }
82 
83   return AudioEncoderOpusConfig::kDefaultFrameSizeMs;
84 }
85 
CalculateDefaultBitrate(int max_playback_rate,size_t num_channels)86 int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) {
87   const int bitrate = [&] {
88     if (max_playback_rate <= 8000) {
89       return kOpusBitrateNbBps * rtc::dchecked_cast<int>(num_channels);
90     } else if (max_playback_rate <= 16000) {
91       return kOpusBitrateWbBps * rtc::dchecked_cast<int>(num_channels);
92     } else {
93       return kOpusBitrateFbBps * rtc::dchecked_cast<int>(num_channels);
94     }
95   }();
96   RTC_DCHECK_GE(bitrate, AudioEncoderMultiChannelOpusConfig::kMinBitrateBps);
97   return bitrate;
98 }
99 
100 // Get the maxaveragebitrate parameter in string-form, so we can properly figure
101 // out how invalid it is and accurately log invalid values.
CalculateBitrate(int max_playback_rate_hz,size_t num_channels,absl::optional<std::string> bitrate_param)102 int CalculateBitrate(int max_playback_rate_hz,
103                      size_t num_channels,
104                      absl::optional<std::string> bitrate_param) {
105   const int default_bitrate =
106       CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
107 
108   if (bitrate_param) {
109     const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
110     if (bitrate) {
111       const int chosen_bitrate =
112           std::max(AudioEncoderOpusConfig::kMinBitrateBps,
113                    std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps));
114       if (bitrate != chosen_bitrate) {
115         RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
116                             << " clamped to " << chosen_bitrate;
117       }
118       return chosen_bitrate;
119     }
120     RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
121                         << "\" replaced by default bitrate " << default_bitrate;
122   }
123 
124   return default_bitrate;
125 }
126 
127 }  // namespace
128 
129 std::unique_ptr<AudioEncoder>
MakeAudioEncoder(const AudioEncoderMultiChannelOpusConfig & config,int payload_type)130 AudioEncoderMultiChannelOpusImpl::MakeAudioEncoder(
131     const AudioEncoderMultiChannelOpusConfig& config,
132     int payload_type) {
133   if (!config.IsOk()) {
134     return nullptr;
135   }
136   return std::make_unique<AudioEncoderMultiChannelOpusImpl>(config,
137                                                             payload_type);
138 }
139 
AudioEncoderMultiChannelOpusImpl(const AudioEncoderMultiChannelOpusConfig & config,int payload_type)140 AudioEncoderMultiChannelOpusImpl::AudioEncoderMultiChannelOpusImpl(
141     const AudioEncoderMultiChannelOpusConfig& config,
142     int payload_type)
143     : payload_type_(payload_type), inst_(nullptr) {
144   RTC_DCHECK(0 <= payload_type && payload_type <= 127);
145 
146   RTC_CHECK(RecreateEncoderInstance(config));
147 }
148 
~AudioEncoderMultiChannelOpusImpl()149 AudioEncoderMultiChannelOpusImpl::~AudioEncoderMultiChannelOpusImpl() {
150   RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
151 }
152 
SufficientOutputBufferSize() const153 size_t AudioEncoderMultiChannelOpusImpl::SufficientOutputBufferSize() const {
154   // Calculate the number of bytes we expect the encoder to produce,
155   // then multiply by two to give a wide margin for error.
156   const size_t bytes_per_millisecond =
157       static_cast<size_t>(GetBitrateBps(config_) / (1000 * 8) + 1);
158   const size_t approx_encoded_bytes =
159       Num10msFramesPerPacket() * 10 * bytes_per_millisecond;
160   return 2 * approx_encoded_bytes;
161 }
162 
Reset()163 void AudioEncoderMultiChannelOpusImpl::Reset() {
164   RTC_CHECK(RecreateEncoderInstance(config_));
165 }
166 
167 absl::optional<std::pair<TimeDelta, TimeDelta>>
GetFrameLengthRange() const168 AudioEncoderMultiChannelOpusImpl::GetFrameLengthRange() const {
169   return {{TimeDelta::Millis(config_.frame_size_ms),
170            TimeDelta::Millis(config_.frame_size_ms)}};
171 }
172 
173 // If the given config is OK, recreate the Opus encoder instance with those
174 // settings, save the config, and return true. Otherwise, do nothing and return
175 // false.
RecreateEncoderInstance(const AudioEncoderMultiChannelOpusConfig & config)176 bool AudioEncoderMultiChannelOpusImpl::RecreateEncoderInstance(
177     const AudioEncoderMultiChannelOpusConfig& config) {
178   if (!config.IsOk())
179     return false;
180   config_ = config;
181   if (inst_)
182     RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
183   input_buffer_.clear();
184   input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame());
185   RTC_CHECK_EQ(
186       0, WebRtcOpus_MultistreamEncoderCreate(
187              &inst_, config.num_channels,
188              config.application ==
189                      AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
190                  ? 0
191                  : 1,
192              config.num_streams, config.coupled_streams,
193              config.channel_mapping.data()));
194   const int bitrate = GetBitrateBps(config);
195   RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate));
196   RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
197   if (config.fec_enabled) {
198     RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
199     RTC_LOG(LS_VERBOSE) << "Opus enable FEC";
200   } else {
201     RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
202     RTC_LOG(LS_VERBOSE) << "Opus disable FEC";
203   }
204   RTC_CHECK_EQ(
205       0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz));
206   RTC_LOG(LS_VERBOSE) << "Set Opus playback rate to "
207                       << config.max_playback_rate_hz << " hz.";
208 
209   // Use the DEFAULT complexity.
210   RTC_CHECK_EQ(
211       0, WebRtcOpus_SetComplexity(inst_, AudioEncoderOpusConfig().complexity));
212   RTC_LOG(LS_VERBOSE) << "Set Opus coding complexity to "
213                       << AudioEncoderOpusConfig().complexity;
214 
215   if (config.dtx_enabled) {
216     RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
217     RTC_LOG(LS_VERBOSE) << "Opus enable DTX";
218   } else {
219     RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
220     RTC_LOG(LS_VERBOSE) << "Opus disable DTX";
221   }
222 
223   if (config.cbr_enabled) {
224     RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_));
225     RTC_LOG(LS_VERBOSE) << "Opus enable CBR";
226   } else {
227     RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_));
228     RTC_LOG(LS_VERBOSE) << "Opus disable CBR";
229   }
230   num_channels_to_encode_ = NumChannels();
231   next_frame_length_ms_ = config_.frame_size_ms;
232   RTC_LOG(LS_VERBOSE) << "Set Opus frame length to " << config_.frame_size_ms
233                       << " ms";
234   return true;
235 }
236 
237 absl::optional<AudioEncoderMultiChannelOpusConfig>
SdpToConfig(const SdpAudioFormat & format)238 AudioEncoderMultiChannelOpusImpl::SdpToConfig(const SdpAudioFormat& format) {
239   if (!absl::EqualsIgnoreCase(format.name, "multiopus") ||
240       format.clockrate_hz != 48000) {
241     return absl::nullopt;
242   }
243 
244   AudioEncoderMultiChannelOpusConfig config;
245   config.num_channels = format.num_channels;
246   config.frame_size_ms = GetFrameSizeMs(format);
247   config.max_playback_rate_hz = GetMaxPlaybackRate(format);
248   config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
249   config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
250   config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1");
251   config.bitrate_bps =
252       CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
253                        GetFormatParameter(format, "maxaveragebitrate"));
254   config.application =
255       config.num_channels == 1
256           ? AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
257           : AudioEncoderMultiChannelOpusConfig::ApplicationMode::kAudio;
258 
259   config.supported_frame_lengths_ms.clear();
260   std::copy(std::begin(kOpusSupportedFrameLengths),
261             std::end(kOpusSupportedFrameLengths),
262             std::back_inserter(config.supported_frame_lengths_ms));
263 
264   auto num_streams = GetFormatParameter<int>(format, "num_streams");
265   if (!num_streams.has_value()) {
266     return absl::nullopt;
267   }
268   config.num_streams = *num_streams;
269 
270   auto coupled_streams = GetFormatParameter<int>(format, "coupled_streams");
271   if (!coupled_streams.has_value()) {
272     return absl::nullopt;
273   }
274   config.coupled_streams = *coupled_streams;
275 
276   auto channel_mapping =
277       GetFormatParameter<std::vector<unsigned char>>(format, "channel_mapping");
278   if (!channel_mapping.has_value()) {
279     return absl::nullopt;
280   }
281   config.channel_mapping = *channel_mapping;
282 
283   return config;
284 }
285 
QueryAudioEncoder(const AudioEncoderMultiChannelOpusConfig & config)286 AudioCodecInfo AudioEncoderMultiChannelOpusImpl::QueryAudioEncoder(
287     const AudioEncoderMultiChannelOpusConfig& config) {
288   RTC_DCHECK(config.IsOk());
289   AudioCodecInfo info(48000, config.num_channels, config.bitrate_bps,
290                       AudioEncoderOpusConfig::kMinBitrateBps,
291                       AudioEncoderOpusConfig::kMaxBitrateBps);
292   info.allow_comfort_noise = false;
293   info.supports_network_adaption = false;
294   return info;
295 }
296 
Num10msFramesPerPacket() const297 size_t AudioEncoderMultiChannelOpusImpl::Num10msFramesPerPacket() const {
298   return static_cast<size_t>(rtc::CheckedDivExact(config_.frame_size_ms, 10));
299 }
SamplesPer10msFrame() const300 size_t AudioEncoderMultiChannelOpusImpl::SamplesPer10msFrame() const {
301   return rtc::CheckedDivExact(48000, 100) * config_.num_channels;
302 }
SampleRateHz() const303 int AudioEncoderMultiChannelOpusImpl::SampleRateHz() const {
304   return 48000;
305 }
NumChannels() const306 size_t AudioEncoderMultiChannelOpusImpl::NumChannels() const {
307   return config_.num_channels;
308 }
Num10MsFramesInNextPacket() const309 size_t AudioEncoderMultiChannelOpusImpl::Num10MsFramesInNextPacket() const {
310   return Num10msFramesPerPacket();
311 }
Max10MsFramesInAPacket() const312 size_t AudioEncoderMultiChannelOpusImpl::Max10MsFramesInAPacket() const {
313   return Num10msFramesPerPacket();
314 }
GetTargetBitrate() const315 int AudioEncoderMultiChannelOpusImpl::GetTargetBitrate() const {
316   return GetBitrateBps(config_);
317 }
318 
EncodeImpl(uint32_t rtp_timestamp,rtc::ArrayView<const int16_t> audio,rtc::Buffer * encoded)319 AudioEncoder::EncodedInfo AudioEncoderMultiChannelOpusImpl::EncodeImpl(
320     uint32_t rtp_timestamp,
321     rtc::ArrayView<const int16_t> audio,
322     rtc::Buffer* encoded) {
323   if (input_buffer_.empty())
324     first_timestamp_in_buffer_ = rtp_timestamp;
325 
326   input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend());
327   if (input_buffer_.size() <
328       (Num10msFramesPerPacket() * SamplesPer10msFrame())) {
329     return EncodedInfo();
330   }
331   RTC_CHECK_EQ(input_buffer_.size(),
332                Num10msFramesPerPacket() * SamplesPer10msFrame());
333 
334   const size_t max_encoded_bytes = SufficientOutputBufferSize();
335   EncodedInfo info;
336   info.encoded_bytes = encoded->AppendData(
337       max_encoded_bytes, [&](rtc::ArrayView<uint8_t> encoded) {
338         int status = WebRtcOpus_Encode(
339             inst_, &input_buffer_[0],
340             rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels),
341             rtc::saturated_cast<int16_t>(max_encoded_bytes), encoded.data());
342 
343         RTC_CHECK_GE(status, 0);  // Fails only if fed invalid data.
344 
345         return static_cast<size_t>(status);
346       });
347   input_buffer_.clear();
348 
349   // Will use new packet size for next encoding.
350   config_.frame_size_ms = next_frame_length_ms_;
351 
352   info.encoded_timestamp = first_timestamp_in_buffer_;
353   info.payload_type = payload_type_;
354   info.send_even_if_empty = true;  // Allows Opus to send empty packets.
355 
356   info.speech = true;
357   info.encoder_type = CodecType::kOther;
358 
359   return info;
360 }
361 
362 }  // namespace webrtc
363