1 /*
2 * Copyright © 2016 Mozilla Foundation
3 *
4 * This program is made available under an ISC-style license. See the
5 * accompanying file LICENSE for details.
6 */
7
8 #if !defined(CUBEB_RESAMPLER_INTERNAL)
9 #define CUBEB_RESAMPLER_INTERNAL
10
11 #include <algorithm>
12 #include <cassert>
13 #include <cmath>
14 #include <memory>
15 #ifdef CUBEB_GECKO_BUILD
16 #include "mozilla/UniquePtr.h"
17 // In libc++, symbols such as std::unique_ptr may be defined in std::__1.
18 // The _LIBCPP_BEGIN_NAMESPACE_STD and _LIBCPP_END_NAMESPACE_STD macros
19 // will expand to the correct namespace.
20 #ifdef _LIBCPP_BEGIN_NAMESPACE_STD
21 #define MOZ_BEGIN_STD_NAMESPACE _LIBCPP_BEGIN_NAMESPACE_STD
22 #define MOZ_END_STD_NAMESPACE _LIBCPP_END_NAMESPACE_STD
23 #else
24 #define MOZ_BEGIN_STD_NAMESPACE namespace std {
25 #define MOZ_END_STD_NAMESPACE }
26 #endif
27 MOZ_BEGIN_STD_NAMESPACE
28 using mozilla::DefaultDelete;
29 using mozilla::UniquePtr;
30 #define default_delete DefaultDelete
31 #define unique_ptr UniquePtr
32 MOZ_END_STD_NAMESPACE
33 #endif
34 #include "cubeb-speex-resampler.h"
35 #include "cubeb/cubeb.h"
36 #include "cubeb_log.h"
37 #include "cubeb_resampler.h"
38 #include "cubeb_utils.h"
39 #include <stdio.h>
40
41 /* This header file contains the internal C++ API of the resamplers, for
42 * testing. */
43
44 // When dropping audio input frames to prevent building
45 // an input delay, this function returns the number of frames
46 // to keep in the buffer.
47 // @parameter sample_rate The sample rate of the stream.
48 // @return A number of frames to keep.
49 uint32_t
50 min_buffered_audio_frame(uint32_t sample_rate);
51
52 int
53 to_speex_quality(cubeb_resampler_quality q);
54
55 struct cubeb_resampler {
56 virtual long fill(void * input_buffer, long * input_frames_count,
57 void * output_buffer, long frames_needed) = 0;
58 virtual long latency() = 0;
~cubeb_resamplercubeb_resampler59 virtual ~cubeb_resampler() {}
60 };
61
62 /** Base class for processors. This is just used to share methods for now. */
63 class processor {
64 public:
processor(uint32_t channels)65 explicit processor(uint32_t channels) : channels(channels) {}
66
67 protected:
frames_to_samples(size_t frames)68 size_t frames_to_samples(size_t frames) const { return frames * channels; }
samples_to_frames(size_t samples)69 size_t samples_to_frames(size_t samples) const
70 {
71 assert(!(samples % channels));
72 return samples / channels;
73 }
74 /** The number of channel of the audio buffers to be resampled. */
75 const uint32_t channels;
76 };
77
78 template <typename T>
79 class passthrough_resampler : public cubeb_resampler, public processor {
80 public:
81 passthrough_resampler(cubeb_stream * s, cubeb_data_callback cb, void * ptr,
82 uint32_t input_channels, uint32_t sample_rate);
83
84 virtual long fill(void * input_buffer, long * input_frames_count,
85 void * output_buffer, long output_frames);
86
latency()87 virtual long latency() { return 0; }
88
drop_audio_if_needed()89 void drop_audio_if_needed()
90 {
91 uint32_t to_keep = min_buffered_audio_frame(sample_rate);
92 uint32_t available = samples_to_frames(internal_input_buffer.length());
93 if (available > to_keep) {
94 internal_input_buffer.pop(nullptr,
95 frames_to_samples(available - to_keep));
96 }
97 }
98
99 private:
100 cubeb_stream * const stream;
101 const cubeb_data_callback data_callback;
102 void * const user_ptr;
103 /* This allows to buffer some input to account for the fact that we buffer
104 * some inputs. */
105 auto_array<T> internal_input_buffer;
106 uint32_t sample_rate;
107 };
108
109 /** Bidirectional resampler, can resample an input and an output stream, or just
110 * an input stream or output stream. In this case a delay is inserted in the
111 * opposite direction to keep the streams synchronized. */
112 template <typename T, typename InputProcessing, typename OutputProcessing>
113 class cubeb_resampler_speex : public cubeb_resampler {
114 public:
115 cubeb_resampler_speex(InputProcessing * input_processor,
116 OutputProcessing * output_processor, cubeb_stream * s,
117 cubeb_data_callback cb, void * ptr);
118
119 virtual ~cubeb_resampler_speex();
120
121 virtual long fill(void * input_buffer, long * input_frames_count,
122 void * output_buffer, long output_frames_needed);
123
latency()124 virtual long latency()
125 {
126 if (input_processor && output_processor) {
127 assert(input_processor->latency() == output_processor->latency());
128 return input_processor->latency();
129 } else if (input_processor) {
130 return input_processor->latency();
131 } else {
132 return output_processor->latency();
133 }
134 }
135
136 private:
137 typedef long (cubeb_resampler_speex::*processing_callback)(
138 T * input_buffer, long * input_frames_count, T * output_buffer,
139 long output_frames_needed);
140
141 long fill_internal_duplex(T * input_buffer, long * input_frames_count,
142 T * output_buffer, long output_frames_needed);
143 long fill_internal_input(T * input_buffer, long * input_frames_count,
144 T * output_buffer, long output_frames_needed);
145 long fill_internal_output(T * input_buffer, long * input_frames_count,
146 T * output_buffer, long output_frames_needed);
147
148 std::unique_ptr<InputProcessing> input_processor;
149 std::unique_ptr<OutputProcessing> output_processor;
150 processing_callback fill_internal;
151 cubeb_stream * const stream;
152 const cubeb_data_callback data_callback;
153 void * const user_ptr;
154 bool draining = false;
155 };
156
157 /** Handles one way of a (possibly) duplex resampler, working on interleaved
158 * audio buffers of type T. This class is designed so that the number of frames
159 * coming out of the resampler can be precisely controled. It manages its own
160 * input buffer, and can use the caller's output buffer, or allocate its own. */
161 template <typename T> class cubeb_resampler_speex_one_way : public processor {
162 public:
163 /** The sample type of this resampler, either 16-bit integers or 32-bit
164 * floats. */
165 typedef T sample_type;
166 /** Construct a resampler resampling from #source_rate to #target_rate, that
167 * can be arbitrary, strictly positive number.
168 * @parameter channels The number of channels this resampler will resample.
169 * @parameter source_rate The sample-rate of the audio input.
170 * @parameter target_rate The sample-rate of the audio output.
171 * @parameter quality A number between 0 (fast, low quality) and 10 (slow,
172 * high quality). */
cubeb_resampler_speex_one_way(uint32_t channels,uint32_t source_rate,uint32_t target_rate,int quality)173 cubeb_resampler_speex_one_way(uint32_t channels, uint32_t source_rate,
174 uint32_t target_rate, int quality)
175 : processor(channels),
176 resampling_ratio(static_cast<float>(source_rate) / target_rate),
177 source_rate(source_rate), additional_latency(0), leftover_samples(0)
178 {
179 int r;
180 speex_resampler =
181 speex_resampler_init(channels, source_rate, target_rate, quality, &r);
182 assert(r == RESAMPLER_ERR_SUCCESS && "resampler allocation failure");
183
184 uint32_t input_latency = speex_resampler_get_input_latency(speex_resampler);
185 const size_t LATENCY_SAMPLES = 8192;
186 T input_buffer[LATENCY_SAMPLES] = {};
187 T output_buffer[LATENCY_SAMPLES] = {};
188 uint32_t input_frame_count = input_latency;
189 uint32_t output_frame_count = LATENCY_SAMPLES;
190 assert(input_latency * channels <= LATENCY_SAMPLES);
191 speex_resample(input_buffer, &input_frame_count, output_buffer,
192 &output_frame_count);
193 }
194
195 /** Destructor, deallocate the resampler */
~cubeb_resampler_speex_one_way()196 virtual ~cubeb_resampler_speex_one_way()
197 {
198 speex_resampler_destroy(speex_resampler);
199 }
200
201 /* Fill the resampler with `input_frame_count` frames. */
input(T * input_buffer,size_t input_frame_count)202 void input(T * input_buffer, size_t input_frame_count)
203 {
204 resampling_in_buffer.push(input_buffer,
205 frames_to_samples(input_frame_count));
206 }
207
208 /** Outputs exactly `output_frame_count` into `output_buffer`.
209 * `output_buffer` has to be at least `output_frame_count` long. */
output(T * output_buffer,size_t output_frame_count)210 size_t output(T * output_buffer, size_t output_frame_count)
211 {
212 uint32_t in_len = samples_to_frames(resampling_in_buffer.length());
213 uint32_t out_len = output_frame_count;
214
215 speex_resample(resampling_in_buffer.data(), &in_len, output_buffer,
216 &out_len);
217
218 /* This shifts back any unresampled samples to the beginning of the input
219 buffer. */
220 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len));
221
222 return out_len;
223 }
224
output_for_input(uint32_t input_frames)225 size_t output_for_input(uint32_t input_frames)
226 {
227 return (size_t)floorf(
228 (input_frames + samples_to_frames(resampling_in_buffer.length())) /
229 resampling_ratio);
230 }
231
232 /** Returns a buffer containing exactly `output_frame_count` resampled frames.
233 * The consumer should not hold onto the pointer. */
output(size_t output_frame_count,size_t * input_frames_used)234 T * output(size_t output_frame_count, size_t * input_frames_used)
235 {
236 if (resampling_out_buffer.capacity() <
237 frames_to_samples(output_frame_count)) {
238 resampling_out_buffer.reserve(frames_to_samples(output_frame_count));
239 }
240
241 uint32_t in_len = samples_to_frames(resampling_in_buffer.length());
242 uint32_t out_len = output_frame_count;
243
244 speex_resample(resampling_in_buffer.data(), &in_len,
245 resampling_out_buffer.data(), &out_len);
246
247 if (out_len < output_frame_count) {
248 LOGV("underrun during resampling: got %u frames, expected %zu",
249 (unsigned)out_len, output_frame_count);
250 // silence the rightmost part
251 T * data = resampling_out_buffer.data();
252 for (uint32_t i = frames_to_samples(out_len);
253 i < frames_to_samples(output_frame_count); i++) {
254 data[i] = 0;
255 }
256 }
257
258 /* This shifts back any unresampled samples to the beginning of the input
259 buffer. */
260 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len));
261 *input_frames_used = in_len;
262
263 return resampling_out_buffer.data();
264 }
265
266 /** Get the latency of the resampler, in output frames. */
latency()267 uint32_t latency() const
268 {
269 /* The documentation of the resampler talks about "samples" here, but it
270 * only consider a single channel here so it's the same number of frames. */
271 int latency = 0;
272
273 latency = speex_resampler_get_output_latency(speex_resampler) +
274 additional_latency;
275
276 assert(latency >= 0);
277
278 return latency;
279 }
280
281 /** Returns the number of frames to pass in the input of the resampler to have
282 * exactly `output_frame_count` resampled frames. This can return a number
283 * slightly bigger than what is strictly necessary, but it guaranteed that the
284 * number of output frames will be exactly equal. */
input_needed_for_output(int32_t output_frame_count)285 uint32_t input_needed_for_output(int32_t output_frame_count) const
286 {
287 assert(output_frame_count >= 0); // Check overflow
288 int32_t unresampled_frames_left =
289 samples_to_frames(resampling_in_buffer.length());
290 int32_t resampled_frames_left =
291 samples_to_frames(resampling_out_buffer.length());
292 float input_frames_needed =
293 (output_frame_count - unresampled_frames_left) * resampling_ratio -
294 resampled_frames_left;
295 if (input_frames_needed < 0) {
296 return 0;
297 }
298 return (uint32_t)ceilf(input_frames_needed);
299 }
300
301 /** Returns a pointer to the input buffer, that contains empty space for at
302 * least `frame_count` elements. This is useful so that consumer can directly
303 * write into the input buffer of the resampler. The pointer returned is
304 * adjusted so that leftover data are not overwritten.
305 */
input_buffer(size_t frame_count)306 T * input_buffer(size_t frame_count)
307 {
308 leftover_samples = resampling_in_buffer.length();
309 resampling_in_buffer.reserve(leftover_samples +
310 frames_to_samples(frame_count));
311 return resampling_in_buffer.data() + leftover_samples;
312 }
313
314 /** This method works with `input_buffer`, and allows to inform the processor
315 how much frames have been written in the provided buffer. */
written(size_t written_frames)316 void written(size_t written_frames)
317 {
318 resampling_in_buffer.set_length(leftover_samples +
319 frames_to_samples(written_frames));
320 }
321
drop_audio_if_needed()322 void drop_audio_if_needed()
323 {
324 // Keep at most 100ms buffered.
325 uint32_t available = samples_to_frames(resampling_in_buffer.length());
326 uint32_t to_keep = min_buffered_audio_frame(source_rate);
327 if (available > to_keep) {
328 resampling_in_buffer.pop(nullptr, frames_to_samples(available - to_keep));
329 }
330 }
331
332 private:
333 /** Wrapper for the speex resampling functions to have a typed
334 * interface. */
speex_resample(float * input_buffer,uint32_t * input_frame_count,float * output_buffer,uint32_t * output_frame_count)335 void speex_resample(float * input_buffer, uint32_t * input_frame_count,
336 float * output_buffer, uint32_t * output_frame_count)
337 {
338 #ifndef NDEBUG
339 int rv;
340 rv =
341 #endif
342 speex_resampler_process_interleaved_float(
343 speex_resampler, input_buffer, input_frame_count, output_buffer,
344 output_frame_count);
345 assert(rv == RESAMPLER_ERR_SUCCESS);
346 }
347
speex_resample(short * input_buffer,uint32_t * input_frame_count,short * output_buffer,uint32_t * output_frame_count)348 void speex_resample(short * input_buffer, uint32_t * input_frame_count,
349 short * output_buffer, uint32_t * output_frame_count)
350 {
351 #ifndef NDEBUG
352 int rv;
353 rv =
354 #endif
355 speex_resampler_process_interleaved_int(
356 speex_resampler, input_buffer, input_frame_count, output_buffer,
357 output_frame_count);
358 assert(rv == RESAMPLER_ERR_SUCCESS);
359 }
360 /** The state for the speex resampler used internaly. */
361 SpeexResamplerState * speex_resampler;
362 /** Source rate / target rate. */
363 const float resampling_ratio;
364 const uint32_t source_rate;
365 /** Storage for the input frames, to be resampled. Also contains
366 * any unresampled frames after resampling. */
367 auto_array<T> resampling_in_buffer;
368 /* Storage for the resampled frames, to be passed back to the caller. */
369 auto_array<T> resampling_out_buffer;
370 /** Additional latency inserted into the pipeline for synchronisation. */
371 uint32_t additional_latency;
372 /** When `input_buffer` is called, this allows tracking the number of samples
373 that were in the buffer. */
374 uint32_t leftover_samples;
375 };
376
377 /** This class allows delaying an audio stream by `frames` frames. */
378 template <typename T> class delay_line : public processor {
379 public:
380 /** Constructor
381 * @parameter frames the number of frames of delay.
382 * @parameter channels the number of channels of this delay line.
383 * @parameter sample_rate sample-rate of the audio going through this delay
384 * line */
delay_line(uint32_t frames,uint32_t channels,uint32_t sample_rate)385 delay_line(uint32_t frames, uint32_t channels, uint32_t sample_rate)
386 : processor(channels), length(frames), leftover_samples(0),
387 sample_rate(sample_rate)
388 {
389 /* Fill the delay line with some silent frames to add latency. */
390 delay_input_buffer.push_silence(frames * channels);
391 }
392 /** Push some frames into the delay line.
393 * @parameter buffer the frames to push.
394 * @parameter frame_count the number of frames in #buffer. */
input(T * buffer,uint32_t frame_count)395 void input(T * buffer, uint32_t frame_count)
396 {
397 delay_input_buffer.push(buffer, frames_to_samples(frame_count));
398 }
399 /** Pop some frames from the internal buffer, into a internal output buffer.
400 * @parameter frames_needed the number of frames to be returned.
401 * @return a buffer containing the delayed frames. The consumer should not
402 * hold onto the pointer. */
output(uint32_t frames_needed,size_t * input_frames_used)403 T * output(uint32_t frames_needed, size_t * input_frames_used)
404 {
405 if (delay_output_buffer.capacity() < frames_to_samples(frames_needed)) {
406 delay_output_buffer.reserve(frames_to_samples(frames_needed));
407 }
408
409 delay_output_buffer.clear();
410 delay_output_buffer.push(delay_input_buffer.data(),
411 frames_to_samples(frames_needed));
412 delay_input_buffer.pop(nullptr, frames_to_samples(frames_needed));
413 *input_frames_used = frames_needed;
414
415 return delay_output_buffer.data();
416 }
417 /** Get a pointer to the first writable location in the input buffer>
418 * @parameter frames_needed the number of frames the user needs to write into
419 * the buffer.
420 * @returns a pointer to a location in the input buffer where #frames_needed
421 * can be writen. */
input_buffer(uint32_t frames_needed)422 T * input_buffer(uint32_t frames_needed)
423 {
424 leftover_samples = delay_input_buffer.length();
425 delay_input_buffer.reserve(leftover_samples +
426 frames_to_samples(frames_needed));
427 return delay_input_buffer.data() + leftover_samples;
428 }
429 /** This method works with `input_buffer`, and allows to inform the processor
430 how much frames have been written in the provided buffer. */
written(size_t frames_written)431 void written(size_t frames_written)
432 {
433 delay_input_buffer.set_length(leftover_samples +
434 frames_to_samples(frames_written));
435 }
436 /** Drains the delay line, emptying the buffer.
437 * @parameter output_buffer the buffer in which the frames are written.
438 * @parameter frames_needed the maximum number of frames to write.
439 * @return the actual number of frames written. */
output(T * output_buffer,uint32_t frames_needed)440 size_t output(T * output_buffer, uint32_t frames_needed)
441 {
442 uint32_t in_len = samples_to_frames(delay_input_buffer.length());
443 uint32_t out_len = frames_needed;
444
445 uint32_t to_pop = std::min(in_len, out_len);
446
447 delay_input_buffer.pop(output_buffer, frames_to_samples(to_pop));
448
449 return to_pop;
450 }
451 /** Returns the number of frames one needs to input into the delay line to get
452 * #frames_needed frames back.
453 * @parameter frames_needed the number of frames one want to write into the
454 * delay_line
455 * @returns the number of frames one will get. */
input_needed_for_output(int32_t frames_needed)456 uint32_t input_needed_for_output(int32_t frames_needed) const
457 {
458 assert(frames_needed >= 0); // Check overflow
459 return frames_needed;
460 }
461 /** Returns the number of frames produces for `input_frames` frames in input
462 */
output_for_input(uint32_t input_frames)463 size_t output_for_input(uint32_t input_frames) { return input_frames; }
464 /** The number of frames this delay line delays the stream by.
465 * @returns The number of frames of delay. */
latency()466 size_t latency() { return length; }
467
drop_audio_if_needed()468 void drop_audio_if_needed()
469 {
470 size_t available = samples_to_frames(delay_input_buffer.length());
471 uint32_t to_keep = min_buffered_audio_frame(sample_rate);
472 if (available > to_keep) {
473 delay_input_buffer.pop(nullptr, frames_to_samples(available - to_keep));
474 }
475 }
476
477 private:
478 /** The length, in frames, of this delay line */
479 uint32_t length;
480 /** When `input_buffer` is called, this allows tracking the number of samples
481 that where in the buffer. */
482 uint32_t leftover_samples;
483 /** The input buffer, where the delay is applied. */
484 auto_array<T> delay_input_buffer;
485 /** The output buffer. This is only ever used if using the ::output with a
486 * single argument. */
487 auto_array<T> delay_output_buffer;
488 uint32_t sample_rate;
489 };
490
491 /** This sits behind the C API and is more typed. */
492 template <typename T>
493 cubeb_resampler *
cubeb_resampler_create_internal(cubeb_stream * stream,cubeb_stream_params * input_params,cubeb_stream_params * output_params,unsigned int target_rate,cubeb_data_callback callback,void * user_ptr,cubeb_resampler_quality quality)494 cubeb_resampler_create_internal(cubeb_stream * stream,
495 cubeb_stream_params * input_params,
496 cubeb_stream_params * output_params,
497 unsigned int target_rate,
498 cubeb_data_callback callback, void * user_ptr,
499 cubeb_resampler_quality quality)
500 {
501 std::unique_ptr<cubeb_resampler_speex_one_way<T>> input_resampler = nullptr;
502 std::unique_ptr<cubeb_resampler_speex_one_way<T>> output_resampler = nullptr;
503 std::unique_ptr<delay_line<T>> input_delay = nullptr;
504 std::unique_ptr<delay_line<T>> output_delay = nullptr;
505
506 assert((input_params || output_params) &&
507 "need at least one valid parameter pointer.");
508
509 /* All the streams we have have a sample rate that matches the target
510 sample rate, use a no-op resampler, that simply forwards the buffers to the
511 callback. */
512 if (((input_params && input_params->rate == target_rate) &&
513 (output_params && output_params->rate == target_rate)) ||
514 (input_params && !output_params && (input_params->rate == target_rate)) ||
515 (output_params && !input_params &&
516 (output_params->rate == target_rate))) {
517 LOG("Input and output sample-rate match, target rate of %dHz", target_rate);
518 return new passthrough_resampler<T>(
519 stream, callback, user_ptr, input_params ? input_params->channels : 0,
520 target_rate);
521 }
522
523 /* Determine if we need to resampler one or both directions, and create the
524 resamplers. */
525 if (output_params && (output_params->rate != target_rate)) {
526 output_resampler.reset(new cubeb_resampler_speex_one_way<T>(
527 output_params->channels, target_rate, output_params->rate,
528 to_speex_quality(quality)));
529 if (!output_resampler) {
530 return NULL;
531 }
532 }
533
534 if (input_params && (input_params->rate != target_rate)) {
535 input_resampler.reset(new cubeb_resampler_speex_one_way<T>(
536 input_params->channels, input_params->rate, target_rate,
537 to_speex_quality(quality)));
538 if (!input_resampler) {
539 return NULL;
540 }
541 }
542
543 /* If we resample only one direction but we have a duplex stream, insert a
544 * delay line with a length equal to the resampler latency of the
545 * other direction so that the streams are synchronized. */
546 if (input_resampler && !output_resampler && input_params && output_params) {
547 output_delay.reset(new delay_line<T>(input_resampler->latency(),
548 output_params->channels,
549 output_params->rate));
550 if (!output_delay) {
551 return NULL;
552 }
553 } else if (output_resampler && !input_resampler && input_params &&
554 output_params) {
555 input_delay.reset(new delay_line<T>(output_resampler->latency(),
556 input_params->channels,
557 output_params->rate));
558 if (!input_delay) {
559 return NULL;
560 }
561 }
562
563 if (input_resampler && output_resampler) {
564 LOG("Resampling input (%d) and output (%d) to target rate of %dHz",
565 input_params->rate, output_params->rate, target_rate);
566 return new cubeb_resampler_speex<T, cubeb_resampler_speex_one_way<T>,
567 cubeb_resampler_speex_one_way<T>>(
568 input_resampler.release(), output_resampler.release(), stream, callback,
569 user_ptr);
570 } else if (input_resampler) {
571 LOG("Resampling input (%d) to target and output rate of %dHz",
572 input_params->rate, target_rate);
573 return new cubeb_resampler_speex<T, cubeb_resampler_speex_one_way<T>,
574 delay_line<T>>(input_resampler.release(),
575 output_delay.release(),
576 stream, callback, user_ptr);
577 } else {
578 LOG("Resampling output (%dHz) to target and input rate of %dHz",
579 output_params->rate, target_rate);
580 return new cubeb_resampler_speex<T, delay_line<T>,
581 cubeb_resampler_speex_one_way<T>>(
582 input_delay.release(), output_resampler.release(), stream, callback,
583 user_ptr);
584 }
585 }
586
587 #endif /* CUBEB_RESAMPLER_INTERNAL */
588