1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_processing/audio_buffer.h" 12 13 #include <string.h> 14 15 #include <cstdint> 16 17 #include "common_audio/channel_buffer.h" 18 #include "common_audio/include/audio_util.h" 19 #include "common_audio/resampler/push_sinc_resampler.h" 20 #include "modules/audio_processing/splitting_filter.h" 21 #include "rtc_base/checks.h" 22 23 namespace webrtc { 24 namespace { 25 26 constexpr size_t kSamplesPer32kHzChannel = 320; 27 constexpr size_t kSamplesPer48kHzChannel = 480; 28 constexpr size_t kMaxSamplesPerChannel = AudioBuffer::kMaxSampleRate / 100; 29 30 size_t NumBandsFromFramesPerChannel(size_t num_frames) { 31 if (num_frames == kSamplesPer32kHzChannel) { 32 return 2; 33 } 34 if (num_frames == kSamplesPer48kHzChannel) { 35 return 3; 36 } 37 return 1; 38 } 39 40 } // namespace 41 42 AudioBuffer::AudioBuffer(size_t input_rate, 43 size_t input_num_channels, 44 size_t buffer_rate, 45 size_t buffer_num_channels, 46 size_t output_rate, 47 size_t output_num_channels) 48 : AudioBuffer(static_cast<int>(input_rate) / 100, 49 input_num_channels, 50 static_cast<int>(buffer_rate) / 100, 51 buffer_num_channels, 52 static_cast<int>(output_rate) / 100) {} 53 54 AudioBuffer::AudioBuffer(size_t input_num_frames, 55 size_t input_num_channels, 56 size_t buffer_num_frames, 57 size_t buffer_num_channels, 58 size_t output_num_frames) 59 : input_num_frames_(input_num_frames), 60 input_num_channels_(input_num_channels), 61 buffer_num_frames_(buffer_num_frames), 62 buffer_num_channels_(buffer_num_channels), 63 output_num_frames_(output_num_frames), 64 output_num_channels_(0), 65 num_channels_(buffer_num_channels), 66 num_bands_(NumBandsFromFramesPerChannel(buffer_num_frames_)), 67 num_split_frames_(rtc::CheckedDivExact(buffer_num_frames_, num_bands_)), 68 data_( 69 new ChannelBuffer<float>(buffer_num_frames_, buffer_num_channels_)) { 70 RTC_DCHECK_GT(input_num_frames_, 0); 71 RTC_DCHECK_GT(buffer_num_frames_, 0); 72 RTC_DCHECK_GT(output_num_frames_, 0); 73 RTC_DCHECK_GT(input_num_channels_, 0); 74 RTC_DCHECK_GT(buffer_num_channels_, 0); 75 RTC_DCHECK_LE(buffer_num_channels_, input_num_channels_); 76 77 const bool input_resampling_needed = input_num_frames_ != buffer_num_frames_; 78 const bool output_resampling_needed = 79 output_num_frames_ != buffer_num_frames_; 80 if (input_resampling_needed) { 81 for (size_t i = 0; i < buffer_num_channels_; ++i) { 82 input_resamplers_.push_back(std::unique_ptr<PushSincResampler>( 83 new PushSincResampler(input_num_frames_, buffer_num_frames_))); 84 } 85 } 86 87 if (output_resampling_needed) { 88 for (size_t i = 0; i < buffer_num_channels_; ++i) { 89 output_resamplers_.push_back(std::unique_ptr<PushSincResampler>( 90 new PushSincResampler(buffer_num_frames_, output_num_frames_))); 91 } 92 } 93 94 if (num_bands_ > 1) { 95 split_data_.reset(new ChannelBuffer<float>( 96 buffer_num_frames_, buffer_num_channels_, num_bands_)); 97 splitting_filter_.reset(new SplittingFilter( 98 buffer_num_channels_, num_bands_, buffer_num_frames_)); 99 } 100 } 101 102 AudioBuffer::~AudioBuffer() {} 103 104 void AudioBuffer::set_downmixing_to_specific_channel(size_t channel) { 105 downmix_by_averaging_ = false; 106 RTC_DCHECK_GT(input_num_channels_, channel); 107 channel_for_downmixing_ = std::min(channel, input_num_channels_ - 1); 108 } 109 110 void AudioBuffer::set_downmixing_by_averaging() { 111 downmix_by_averaging_ = true; 112 } 113 114 void AudioBuffer::CopyFrom(const float* const* stacked_data, 115 const StreamConfig& stream_config) { 116 RTC_DCHECK_EQ(stream_config.num_frames(), input_num_frames_); 117 RTC_DCHECK_EQ(stream_config.num_channels(), input_num_channels_); 118 RestoreNumChannels(); 119 const bool downmix_needed = input_num_channels_ > 1 && num_channels_ == 1; 120 121 const bool resampling_needed = input_num_frames_ != buffer_num_frames_; 122 123 if (downmix_needed) { 124 RTC_DCHECK_GE(kMaxSamplesPerChannel, input_num_frames_); 125 126 std::array<float, kMaxSamplesPerChannel> downmix; 127 if (downmix_by_averaging_) { 128 const float kOneByNumChannels = 1.f / input_num_channels_; 129 for (size_t i = 0; i < input_num_frames_; ++i) { 130 float value = stacked_data[0][i]; 131 for (size_t j = 1; j < input_num_channels_; ++j) { 132 value += stacked_data[j][i]; 133 } 134 downmix[i] = value * kOneByNumChannels; 135 } 136 } 137 const float* downmixed_data = downmix_by_averaging_ 138 ? downmix.data() 139 : stacked_data[channel_for_downmixing_]; 140 141 if (resampling_needed) { 142 input_resamplers_[0]->Resample(downmixed_data, input_num_frames_, 143 data_->channels()[0], buffer_num_frames_); 144 } 145 const float* data_to_convert = 146 resampling_needed ? data_->channels()[0] : downmixed_data; 147 FloatToFloatS16(data_to_convert, buffer_num_frames_, data_->channels()[0]); 148 } else { 149 if (resampling_needed) { 150 for (size_t i = 0; i < num_channels_; ++i) { 151 input_resamplers_[i]->Resample(stacked_data[i], input_num_frames_, 152 data_->channels()[i], 153 buffer_num_frames_); 154 FloatToFloatS16(data_->channels()[i], buffer_num_frames_, 155 data_->channels()[i]); 156 } 157 } else { 158 for (size_t i = 0; i < num_channels_; ++i) { 159 FloatToFloatS16(stacked_data[i], buffer_num_frames_, 160 data_->channels()[i]); 161 } 162 } 163 } 164 } 165 166 void AudioBuffer::CopyTo(const StreamConfig& stream_config, 167 float* const* stacked_data) { 168 RTC_DCHECK_EQ(stream_config.num_frames(), output_num_frames_); 169 170 const bool resampling_needed = output_num_frames_ != buffer_num_frames_; 171 if (resampling_needed) { 172 for (size_t i = 0; i < num_channels_; ++i) { 173 FloatS16ToFloat(data_->channels()[i], buffer_num_frames_, 174 data_->channels()[i]); 175 output_resamplers_[i]->Resample(data_->channels()[i], buffer_num_frames_, 176 stacked_data[i], output_num_frames_); 177 } 178 } else { 179 for (size_t i = 0; i < num_channels_; ++i) { 180 FloatS16ToFloat(data_->channels()[i], buffer_num_frames_, 181 stacked_data[i]); 182 } 183 } 184 185 for (size_t i = num_channels_; i < stream_config.num_channels(); ++i) { 186 memcpy(stacked_data[i], stacked_data[0], 187 output_num_frames_ * sizeof(**stacked_data)); 188 } 189 } 190 191 void AudioBuffer::CopyTo(AudioBuffer* buffer) const { 192 RTC_DCHECK_EQ(buffer->num_frames(), output_num_frames_); 193 194 const bool resampling_needed = output_num_frames_ != buffer_num_frames_; 195 if (resampling_needed) { 196 for (size_t i = 0; i < num_channels_; ++i) { 197 output_resamplers_[i]->Resample(data_->channels()[i], buffer_num_frames_, 198 buffer->channels()[i], 199 buffer->num_frames()); 200 } 201 } else { 202 for (size_t i = 0; i < num_channels_; ++i) { 203 memcpy(buffer->channels()[i], data_->channels()[i], 204 buffer_num_frames_ * sizeof(**buffer->channels())); 205 } 206 } 207 208 for (size_t i = num_channels_; i < buffer->num_channels(); ++i) { 209 memcpy(buffer->channels()[i], buffer->channels()[0], 210 output_num_frames_ * sizeof(**buffer->channels())); 211 } 212 } 213 214 void AudioBuffer::RestoreNumChannels() { 215 num_channels_ = buffer_num_channels_; 216 data_->set_num_channels(buffer_num_channels_); 217 if (split_data_.get()) { 218 split_data_->set_num_channels(buffer_num_channels_); 219 } 220 } 221 222 void AudioBuffer::set_num_channels(size_t num_channels) { 223 RTC_DCHECK_GE(buffer_num_channels_, num_channels); 224 num_channels_ = num_channels; 225 data_->set_num_channels(num_channels); 226 if (split_data_.get()) { 227 split_data_->set_num_channels(num_channels); 228 } 229 } 230 231 // The resampler is only for supporting 48kHz to 16kHz in the reverse stream. 232 void AudioBuffer::CopyFrom(const int16_t* const interleaved_data, 233 const StreamConfig& stream_config) { 234 RTC_DCHECK_EQ(stream_config.num_channels(), input_num_channels_); 235 RTC_DCHECK_EQ(stream_config.num_frames(), input_num_frames_); 236 RestoreNumChannels(); 237 238 const bool resampling_required = input_num_frames_ != buffer_num_frames_; 239 240 const int16_t* interleaved = interleaved_data; 241 if (num_channels_ == 1) { 242 if (input_num_channels_ == 1) { 243 if (resampling_required) { 244 std::array<float, kMaxSamplesPerChannel> float_buffer; 245 S16ToFloatS16(interleaved, input_num_frames_, float_buffer.data()); 246 input_resamplers_[0]->Resample(float_buffer.data(), input_num_frames_, 247 data_->channels()[0], 248 buffer_num_frames_); 249 } else { 250 S16ToFloatS16(interleaved, input_num_frames_, data_->channels()[0]); 251 } 252 } else { 253 std::array<float, kMaxSamplesPerChannel> float_buffer; 254 float* downmixed_data = 255 resampling_required ? float_buffer.data() : data_->channels()[0]; 256 if (downmix_by_averaging_) { 257 for (size_t j = 0, k = 0; j < input_num_frames_; ++j) { 258 int32_t sum = 0; 259 for (size_t i = 0; i < input_num_channels_; ++i, ++k) { 260 sum += interleaved[k]; 261 } 262 downmixed_data[j] = sum / static_cast<int16_t>(input_num_channels_); 263 } 264 } else { 265 for (size_t j = 0, k = channel_for_downmixing_; j < input_num_frames_; 266 ++j, k += input_num_channels_) { 267 downmixed_data[j] = interleaved[k]; 268 } 269 } 270 271 if (resampling_required) { 272 input_resamplers_[0]->Resample(downmixed_data, input_num_frames_, 273 data_->channels()[0], 274 buffer_num_frames_); 275 } 276 } 277 } else { 278 auto deinterleave_channel = [](size_t channel, size_t num_channels, 279 size_t samples_per_channel, const int16_t* x, 280 float* y) { 281 for (size_t j = 0, k = channel; j < samples_per_channel; 282 ++j, k += num_channels) { 283 y[j] = x[k]; 284 } 285 }; 286 287 if (resampling_required) { 288 std::array<float, kMaxSamplesPerChannel> float_buffer; 289 for (size_t i = 0; i < num_channels_; ++i) { 290 deinterleave_channel(i, num_channels_, input_num_frames_, interleaved, 291 float_buffer.data()); 292 input_resamplers_[i]->Resample(float_buffer.data(), input_num_frames_, 293 data_->channels()[i], 294 buffer_num_frames_); 295 } 296 } else { 297 for (size_t i = 0; i < num_channels_; ++i) { 298 deinterleave_channel(i, num_channels_, input_num_frames_, interleaved, 299 data_->channels()[i]); 300 } 301 } 302 } 303 } 304 305 void AudioBuffer::CopyTo(const StreamConfig& stream_config, 306 int16_t* const interleaved_data) { 307 const size_t config_num_channels = stream_config.num_channels(); 308 309 RTC_DCHECK(config_num_channels == num_channels_ || num_channels_ == 1); 310 RTC_DCHECK_EQ(stream_config.num_frames(), output_num_frames_); 311 312 const bool resampling_required = buffer_num_frames_ != output_num_frames_; 313 314 int16_t* interleaved = interleaved_data; 315 if (num_channels_ == 1) { 316 std::array<float, kMaxSamplesPerChannel> float_buffer; 317 318 if (resampling_required) { 319 output_resamplers_[0]->Resample(data_->channels()[0], buffer_num_frames_, 320 float_buffer.data(), output_num_frames_); 321 } 322 const float* deinterleaved = 323 resampling_required ? float_buffer.data() : data_->channels()[0]; 324 325 if (config_num_channels == 1) { 326 for (size_t j = 0; j < output_num_frames_; ++j) { 327 interleaved[j] = FloatS16ToS16(deinterleaved[j]); 328 } 329 } else { 330 for (size_t i = 0, k = 0; i < output_num_frames_; ++i) { 331 float tmp = FloatS16ToS16(deinterleaved[i]); 332 for (size_t j = 0; j < config_num_channels; ++j, ++k) { 333 interleaved[k] = tmp; 334 } 335 } 336 } 337 } else { 338 auto interleave_channel = [](size_t channel, size_t num_channels, 339 size_t samples_per_channel, const float* x, 340 int16_t* y) { 341 for (size_t k = 0, j = channel; k < samples_per_channel; 342 ++k, j += num_channels) { 343 y[j] = FloatS16ToS16(x[k]); 344 } 345 }; 346 347 if (resampling_required) { 348 for (size_t i = 0; i < num_channels_; ++i) { 349 std::array<float, kMaxSamplesPerChannel> float_buffer; 350 output_resamplers_[i]->Resample(data_->channels()[i], 351 buffer_num_frames_, float_buffer.data(), 352 output_num_frames_); 353 interleave_channel(i, config_num_channels, output_num_frames_, 354 float_buffer.data(), interleaved); 355 } 356 } else { 357 for (size_t i = 0; i < num_channels_; ++i) { 358 interleave_channel(i, config_num_channels, output_num_frames_, 359 data_->channels()[i], interleaved); 360 } 361 } 362 363 for (size_t i = num_channels_; i < config_num_channels; ++i) { 364 for (size_t j = 0, k = i, n = num_channels_; j < output_num_frames_; 365 ++j, k += config_num_channels, n += config_num_channels) { 366 interleaved[k] = interleaved[n]; 367 } 368 } 369 } 370 } 371 372 void AudioBuffer::SplitIntoFrequencyBands() { 373 splitting_filter_->Analysis(data_.get(), split_data_.get()); 374 } 375 376 void AudioBuffer::MergeFrequencyBands() { 377 splitting_filter_->Synthesis(split_data_.get(), data_.get()); 378 } 379 380 void AudioBuffer::ExportSplitChannelData( 381 size_t channel, 382 int16_t* const* split_band_data) const { 383 for (size_t k = 0; k < num_bands(); ++k) { 384 const float* band_data = split_bands_const(channel)[k]; 385 386 RTC_DCHECK(split_band_data[k]); 387 RTC_DCHECK(band_data); 388 for (size_t i = 0; i < num_frames_per_band(); ++i) { 389 split_band_data[k][i] = FloatS16ToS16(band_data[i]); 390 } 391 } 392 } 393 394 void AudioBuffer::ImportSplitChannelData( 395 size_t channel, 396 const int16_t* const* split_band_data) { 397 for (size_t k = 0; k < num_bands(); ++k) { 398 float* band_data = split_bands(channel)[k]; 399 RTC_DCHECK(split_band_data[k]); 400 RTC_DCHECK(band_data); 401 for (size_t i = 0; i < num_frames_per_band(); ++i) { 402 band_data[i] = split_band_data[k][i]; 403 } 404 } 405 } 406 407 } // namespace webrtc 408