1 /*
2 * Copyright © 2016 Mozilla Foundation
3 *
4 * This program is made available under an ISC-style license. See the
5 * accompanying file LICENSE for details.
6 */
7
8 #if !defined(CUBEB_RESAMPLER_INTERNAL)
9 #define CUBEB_RESAMPLER_INTERNAL
10
11 #include <cmath>
12 #include <cassert>
13 #include <algorithm>
14 #include <memory>
15 #ifdef CUBEB_GECKO_BUILD
16 #include "mozilla/UniquePtr.h"
17 // In libc++, symbols such as std::unique_ptr may be defined in std::__1.
18 // The _LIBCPP_BEGIN_NAMESPACE_STD and _LIBCPP_END_NAMESPACE_STD macros
19 // will expand to the correct namespace.
20 #ifdef _LIBCPP_BEGIN_NAMESPACE_STD
21 #define MOZ_BEGIN_STD_NAMESPACE _LIBCPP_BEGIN_NAMESPACE_STD
22 #define MOZ_END_STD_NAMESPACE _LIBCPP_END_NAMESPACE_STD
23 #else
24 #define MOZ_BEGIN_STD_NAMESPACE namespace std {
25 #define MOZ_END_STD_NAMESPACE }
26 #endif
27 MOZ_BEGIN_STD_NAMESPACE
28 using mozilla::DefaultDelete;
29 using mozilla::UniquePtr;
30 #define default_delete DefaultDelete
31 #define unique_ptr UniquePtr
32 MOZ_END_STD_NAMESPACE
33 #endif
34 #include "cubeb/cubeb.h"
35 #include "cubeb_utils.h"
36 #include "cubeb-speex-resampler.h"
37 #include "cubeb_resampler.h"
38 #include <stdio.h>
39
40 /* This header file contains the internal C++ API of the resamplers, for testing. */
41
42 // When dropping audio input frames to prevent building
43 // an input delay, this function returns the number of frames
44 // to keep in the buffer.
45 // @parameter sample_rate The sample rate of the stream.
46 // @return A number of frames to keep.
47 uint32_t min_buffered_audio_frame(uint32_t sample_rate);
48
49 int to_speex_quality(cubeb_resampler_quality q);
50
51 struct cubeb_resampler {
52 virtual long fill(void * input_buffer, long * input_frames_count,
53 void * output_buffer, long frames_needed) = 0;
54 virtual long latency() = 0;
~cubeb_resamplercubeb_resampler55 virtual ~cubeb_resampler() {}
56 };
57
58 /** Base class for processors. This is just used to share methods for now. */
59 class processor {
60 public:
processor(uint32_t channels)61 explicit processor(uint32_t channels)
62 : channels(channels)
63 {}
64 protected:
frames_to_samples(size_t frames)65 size_t frames_to_samples(size_t frames) const
66 {
67 return frames * channels;
68 }
samples_to_frames(size_t samples)69 size_t samples_to_frames(size_t samples) const
70 {
71 assert(!(samples % channels));
72 return samples / channels;
73 }
74 /** The number of channel of the audio buffers to be resampled. */
75 const uint32_t channels;
76 };
77
78 template<typename T>
79 class passthrough_resampler : public cubeb_resampler
80 , public processor {
81 public:
82 passthrough_resampler(cubeb_stream * s,
83 cubeb_data_callback cb,
84 void * ptr,
85 uint32_t input_channels,
86 uint32_t sample_rate);
87
88 virtual long fill(void * input_buffer, long * input_frames_count,
89 void * output_buffer, long output_frames);
90
latency()91 virtual long latency()
92 {
93 return 0;
94 }
95
drop_audio_if_needed()96 void drop_audio_if_needed()
97 {
98 uint32_t to_keep = min_buffered_audio_frame(sample_rate);
99 uint32_t available = samples_to_frames(internal_input_buffer.length());
100 if (available > to_keep) {
101 internal_input_buffer.pop(nullptr, frames_to_samples(available - to_keep));
102 }
103 }
104
105 private:
106 cubeb_stream * const stream;
107 const cubeb_data_callback data_callback;
108 void * const user_ptr;
109 /* This allows to buffer some input to account for the fact that we buffer
110 * some inputs. */
111 auto_array<T> internal_input_buffer;
112 uint32_t sample_rate;
113 };
114
115 /** Bidirectional resampler, can resample an input and an output stream, or just
116 * an input stream or output stream. In this case a delay is inserted in the
117 * opposite direction to keep the streams synchronized. */
118 template<typename T, typename InputProcessing, typename OutputProcessing>
119 class cubeb_resampler_speex : public cubeb_resampler {
120 public:
121 cubeb_resampler_speex(InputProcessing * input_processor,
122 OutputProcessing * output_processor,
123 cubeb_stream * s,
124 cubeb_data_callback cb,
125 void * ptr);
126
127 virtual ~cubeb_resampler_speex();
128
129 virtual long fill(void * input_buffer, long * input_frames_count,
130 void * output_buffer, long output_frames_needed);
131
latency()132 virtual long latency()
133 {
134 if (input_processor && output_processor) {
135 assert(input_processor->latency() == output_processor->latency());
136 return input_processor->latency();
137 } else if (input_processor) {
138 return input_processor->latency();
139 } else {
140 return output_processor->latency();
141 }
142 }
143
144 private:
145 typedef long(cubeb_resampler_speex::*processing_callback)(T * input_buffer, long * input_frames_count, T * output_buffer, long output_frames_needed);
146
147 long fill_internal_duplex(T * input_buffer, long * input_frames_count,
148 T * output_buffer, long output_frames_needed);
149 long fill_internal_input(T * input_buffer, long * input_frames_count,
150 T * output_buffer, long output_frames_needed);
151 long fill_internal_output(T * input_buffer, long * input_frames_count,
152 T * output_buffer, long output_frames_needed);
153
154 std::unique_ptr<InputProcessing> input_processor;
155 std::unique_ptr<OutputProcessing> output_processor;
156 processing_callback fill_internal;
157 cubeb_stream * const stream;
158 const cubeb_data_callback data_callback;
159 void * const user_ptr;
160 bool draining = false;
161 };
162
163 /** Handles one way of a (possibly) duplex resampler, working on interleaved
164 * audio buffers of type T. This class is designed so that the number of frames
165 * coming out of the resampler can be precisely controled. It manages its own
166 * input buffer, and can use the caller's output buffer, or allocate its own. */
167 template<typename T>
168 class cubeb_resampler_speex_one_way : public processor {
169 public:
170 /** The sample type of this resampler, either 16-bit integers or 32-bit
171 * floats. */
172 typedef T sample_type;
173 /** Construct a resampler resampling from #source_rate to #target_rate, that
174 * can be arbitrary, strictly positive number.
175 * @parameter channels The number of channels this resampler will resample.
176 * @parameter source_rate The sample-rate of the audio input.
177 * @parameter target_rate The sample-rate of the audio output.
178 * @parameter quality A number between 0 (fast, low quality) and 10 (slow,
179 * high quality). */
cubeb_resampler_speex_one_way(uint32_t channels,uint32_t source_rate,uint32_t target_rate,int quality)180 cubeb_resampler_speex_one_way(uint32_t channels,
181 uint32_t source_rate,
182 uint32_t target_rate,
183 int quality)
184 : processor(channels)
185 , resampling_ratio(static_cast<float>(source_rate) / target_rate)
186 , source_rate(source_rate)
187 , additional_latency(0)
188 , leftover_samples(0)
189 {
190 int r;
191 speex_resampler = speex_resampler_init(channels, source_rate,
192 target_rate, quality, &r);
193 assert(r == RESAMPLER_ERR_SUCCESS && "resampler allocation failure");
194 }
195
196 /** Destructor, deallocate the resampler */
~cubeb_resampler_speex_one_way()197 virtual ~cubeb_resampler_speex_one_way()
198 {
199 speex_resampler_destroy(speex_resampler);
200 }
201
202 /** Sometimes, it is necessary to add latency on one way of a two-way
203 * resampler so that the stream are synchronized. This must be called only on
204 * a fresh resampler, otherwise, silent samples will be inserted in the
205 * stream.
206 * @param frames the number of frames of latency to add. */
add_latency(size_t frames)207 void add_latency(size_t frames)
208 {
209 additional_latency += frames;
210 resampling_in_buffer.push_silence(frames_to_samples(frames));
211 }
212
213 /* Fill the resampler with `input_frame_count` frames. */
input(T * input_buffer,size_t input_frame_count)214 void input(T * input_buffer, size_t input_frame_count)
215 {
216 resampling_in_buffer.push(input_buffer,
217 frames_to_samples(input_frame_count));
218 }
219
220 /** Outputs exactly `output_frame_count` into `output_buffer`.
221 * `output_buffer` has to be at least `output_frame_count` long. */
output(T * output_buffer,size_t output_frame_count)222 size_t output(T * output_buffer, size_t output_frame_count)
223 {
224 uint32_t in_len = samples_to_frames(resampling_in_buffer.length());
225 uint32_t out_len = output_frame_count;
226
227 speex_resample(resampling_in_buffer.data(), &in_len,
228 output_buffer, &out_len);
229
230 /* This shifts back any unresampled samples to the beginning of the input
231 buffer. */
232 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len));
233
234 return out_len;
235 }
236
output_for_input(uint32_t input_frames)237 size_t output_for_input(uint32_t input_frames)
238 {
239 return (size_t)floorf((input_frames + samples_to_frames(resampling_in_buffer.length()))
240 / resampling_ratio);
241 }
242
243 /** Returns a buffer containing exactly `output_frame_count` resampled frames.
244 * The consumer should not hold onto the pointer. */
output(size_t output_frame_count,size_t * input_frames_used)245 T * output(size_t output_frame_count, size_t * input_frames_used)
246 {
247 if (resampling_out_buffer.capacity() < frames_to_samples(output_frame_count)) {
248 resampling_out_buffer.reserve(frames_to_samples(output_frame_count));
249 }
250
251 uint32_t in_len = samples_to_frames(resampling_in_buffer.length());
252 uint32_t out_len = output_frame_count;
253
254 speex_resample(resampling_in_buffer.data(), &in_len,
255 resampling_out_buffer.data(), &out_len);
256
257 assert(out_len == output_frame_count);
258
259 /* This shifts back any unresampled samples to the beginning of the input
260 buffer. */
261 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len));
262 *input_frames_used = in_len;
263
264 return resampling_out_buffer.data();
265 }
266
267 /** Get the latency of the resampler, in output frames. */
latency()268 uint32_t latency() const
269 {
270 /* The documentation of the resampler talks about "samples" here, but it
271 * only consider a single channel here so it's the same number of frames. */
272 int latency = 0;
273
274 latency =
275 speex_resampler_get_output_latency(speex_resampler) + additional_latency;
276
277 assert(latency >= 0);
278
279 return latency;
280 }
281
282 /** Returns the number of frames to pass in the input of the resampler to have
283 * exactly `output_frame_count` resampled frames. This can return a number
284 * slightly bigger than what is strictly necessary, but it guaranteed that the
285 * number of output frames will be exactly equal. */
input_needed_for_output(uint32_t output_frame_count)286 uint32_t input_needed_for_output(uint32_t output_frame_count) const
287 {
288 int32_t unresampled_frames_left = samples_to_frames(resampling_in_buffer.length());
289 int32_t resampled_frames_left = samples_to_frames(resampling_out_buffer.length());
290 float input_frames_needed =
291 (output_frame_count - unresampled_frames_left) * resampling_ratio
292 - resampled_frames_left;
293 if (input_frames_needed < 0) {
294 return 0;
295 }
296 return (uint32_t)ceilf(input_frames_needed);
297 }
298
299 /** Returns a pointer to the input buffer, that contains empty space for at
300 * least `frame_count` elements. This is useful so that consumer can directly
301 * write into the input buffer of the resampler. The pointer returned is
302 * adjusted so that leftover data are not overwritten.
303 */
input_buffer(size_t frame_count)304 T * input_buffer(size_t frame_count)
305 {
306 leftover_samples = resampling_in_buffer.length();
307 resampling_in_buffer.reserve(leftover_samples +
308 frames_to_samples(frame_count));
309 return resampling_in_buffer.data() + leftover_samples;
310 }
311
312 /** This method works with `input_buffer`, and allows to inform the processor
313 how much frames have been written in the provided buffer. */
written(size_t written_frames)314 void written(size_t written_frames)
315 {
316 resampling_in_buffer.set_length(leftover_samples +
317 frames_to_samples(written_frames));
318 }
319
drop_audio_if_needed()320 void drop_audio_if_needed()
321 {
322 // Keep at most 100ms buffered.
323 uint32_t available = samples_to_frames(resampling_in_buffer.length());
324 uint32_t to_keep = min_buffered_audio_frame(source_rate);
325 if (available > to_keep) {
326 resampling_in_buffer.pop(nullptr, frames_to_samples(available - to_keep));
327 }
328 }
329 private:
330 /** Wrapper for the speex resampling functions to have a typed
331 * interface. */
speex_resample(float * input_buffer,uint32_t * input_frame_count,float * output_buffer,uint32_t * output_frame_count)332 void speex_resample(float * input_buffer, uint32_t * input_frame_count,
333 float * output_buffer, uint32_t * output_frame_count)
334 {
335 #ifndef NDEBUG
336 int rv;
337 rv =
338 #endif
339 speex_resampler_process_interleaved_float(speex_resampler,
340 input_buffer,
341 input_frame_count,
342 output_buffer,
343 output_frame_count);
344 assert(rv == RESAMPLER_ERR_SUCCESS);
345 }
346
speex_resample(short * input_buffer,uint32_t * input_frame_count,short * output_buffer,uint32_t * output_frame_count)347 void speex_resample(short * input_buffer, uint32_t * input_frame_count,
348 short * output_buffer, uint32_t * output_frame_count)
349 {
350 #ifndef NDEBUG
351 int rv;
352 rv =
353 #endif
354 speex_resampler_process_interleaved_int(speex_resampler,
355 input_buffer,
356 input_frame_count,
357 output_buffer,
358 output_frame_count);
359 assert(rv == RESAMPLER_ERR_SUCCESS);
360 }
361 /** The state for the speex resampler used internaly. */
362 SpeexResamplerState * speex_resampler;
363 /** Source rate / target rate. */
364 const float resampling_ratio;
365 const uint32_t source_rate;
366 /** Storage for the input frames, to be resampled. Also contains
367 * any unresampled frames after resampling. */
368 auto_array<T> resampling_in_buffer;
369 /* Storage for the resampled frames, to be passed back to the caller. */
370 auto_array<T> resampling_out_buffer;
371 /** Additional latency inserted into the pipeline for synchronisation. */
372 uint32_t additional_latency;
373 /** When `input_buffer` is called, this allows tracking the number of samples
374 that were in the buffer. */
375 uint32_t leftover_samples;
376 };
377
378 /** This class allows delaying an audio stream by `frames` frames. */
379 template<typename T>
380 class delay_line : public processor {
381 public:
382 /** Constructor
383 * @parameter frames the number of frames of delay.
384 * @parameter channels the number of channels of this delay line.
385 * @parameter sample_rate sample-rate of the audio going through this delay line */
delay_line(uint32_t frames,uint32_t channels,uint32_t sample_rate)386 delay_line(uint32_t frames, uint32_t channels, uint32_t sample_rate)
387 : processor(channels)
388 , length(frames)
389 , leftover_samples(0)
390 , sample_rate(sample_rate)
391 {
392 /* Fill the delay line with some silent frames to add latency. */
393 delay_input_buffer.push_silence(frames * channels);
394 }
395 /* Add some latency to the delay line.
396 * @param frames the number of frames of latency to add. */
add_latency(size_t frames)397 void add_latency(size_t frames)
398 {
399 length += frames;
400 delay_input_buffer.push_silence(frames_to_samples(frames));
401 }
402 /** Push some frames into the delay line.
403 * @parameter buffer the frames to push.
404 * @parameter frame_count the number of frames in #buffer. */
input(T * buffer,uint32_t frame_count)405 void input(T * buffer, uint32_t frame_count)
406 {
407 delay_input_buffer.push(buffer, frames_to_samples(frame_count));
408 }
409 /** Pop some frames from the internal buffer, into a internal output buffer.
410 * @parameter frames_needed the number of frames to be returned.
411 * @return a buffer containing the delayed frames. The consumer should not
412 * hold onto the pointer. */
output(uint32_t frames_needed,size_t * input_frames_used)413 T * output(uint32_t frames_needed, size_t * input_frames_used)
414 {
415 if (delay_output_buffer.capacity() < frames_to_samples(frames_needed)) {
416 delay_output_buffer.reserve(frames_to_samples(frames_needed));
417 }
418
419 delay_output_buffer.clear();
420 delay_output_buffer.push(delay_input_buffer.data(),
421 frames_to_samples(frames_needed));
422 delay_input_buffer.pop(nullptr, frames_to_samples(frames_needed));
423 *input_frames_used = frames_needed;
424
425 return delay_output_buffer.data();
426 }
427 /** Get a pointer to the first writable location in the input buffer>
428 * @parameter frames_needed the number of frames the user needs to write into
429 * the buffer.
430 * @returns a pointer to a location in the input buffer where #frames_needed
431 * can be writen. */
input_buffer(uint32_t frames_needed)432 T * input_buffer(uint32_t frames_needed)
433 {
434 leftover_samples = delay_input_buffer.length();
435 delay_input_buffer.reserve(leftover_samples + frames_to_samples(frames_needed));
436 return delay_input_buffer.data() + leftover_samples;
437 }
438 /** This method works with `input_buffer`, and allows to inform the processor
439 how much frames have been written in the provided buffer. */
written(size_t frames_written)440 void written(size_t frames_written)
441 {
442 delay_input_buffer.set_length(leftover_samples +
443 frames_to_samples(frames_written));
444 }
445 /** Drains the delay line, emptying the buffer.
446 * @parameter output_buffer the buffer in which the frames are written.
447 * @parameter frames_needed the maximum number of frames to write.
448 * @return the actual number of frames written. */
output(T * output_buffer,uint32_t frames_needed)449 size_t output(T * output_buffer, uint32_t frames_needed)
450 {
451 uint32_t in_len = samples_to_frames(delay_input_buffer.length());
452 uint32_t out_len = frames_needed;
453
454 uint32_t to_pop = std::min(in_len, out_len);
455
456 delay_input_buffer.pop(output_buffer, frames_to_samples(to_pop));
457
458 return to_pop;
459 }
460 /** Returns the number of frames one needs to input into the delay line to get
461 * #frames_needed frames back.
462 * @parameter frames_needed the number of frames one want to write into the
463 * delay_line
464 * @returns the number of frames one will get. */
input_needed_for_output(uint32_t frames_needed)465 size_t input_needed_for_output(uint32_t frames_needed) const
466 {
467 return frames_needed;
468 }
469 /** Returns the number of frames produces for `input_frames` frames in input */
output_for_input(uint32_t input_frames)470 size_t output_for_input(uint32_t input_frames)
471 {
472 return input_frames;
473 }
474 /** The number of frames this delay line delays the stream by.
475 * @returns The number of frames of delay. */
latency()476 size_t latency()
477 {
478 return length;
479 }
480
drop_audio_if_needed()481 void drop_audio_if_needed()
482 {
483 size_t available = samples_to_frames(delay_input_buffer.length());
484 uint32_t to_keep = min_buffered_audio_frame(sample_rate);
485 if (available > to_keep) {
486 delay_input_buffer.pop(nullptr, frames_to_samples(available - to_keep));
487 }
488 }
489 private:
490 /** The length, in frames, of this delay line */
491 uint32_t length;
492 /** When `input_buffer` is called, this allows tracking the number of samples
493 that where in the buffer. */
494 uint32_t leftover_samples;
495 /** The input buffer, where the delay is applied. */
496 auto_array<T> delay_input_buffer;
497 /** The output buffer. This is only ever used if using the ::output with a
498 * single argument. */
499 auto_array<T> delay_output_buffer;
500 uint32_t sample_rate;
501 };
502
503 /** This sits behind the C API and is more typed. */
504 template<typename T>
505 cubeb_resampler *
cubeb_resampler_create_internal(cubeb_stream * stream,cubeb_stream_params * input_params,cubeb_stream_params * output_params,unsigned int target_rate,cubeb_data_callback callback,void * user_ptr,cubeb_resampler_quality quality)506 cubeb_resampler_create_internal(cubeb_stream * stream,
507 cubeb_stream_params * input_params,
508 cubeb_stream_params * output_params,
509 unsigned int target_rate,
510 cubeb_data_callback callback,
511 void * user_ptr,
512 cubeb_resampler_quality quality)
513 {
514 std::unique_ptr<cubeb_resampler_speex_one_way<T>> input_resampler = nullptr;
515 std::unique_ptr<cubeb_resampler_speex_one_way<T>> output_resampler = nullptr;
516 std::unique_ptr<delay_line<T>> input_delay = nullptr;
517 std::unique_ptr<delay_line<T>> output_delay = nullptr;
518
519 assert((input_params || output_params) &&
520 "need at least one valid parameter pointer.");
521
522 /* All the streams we have have a sample rate that matches the target
523 sample rate, use a no-op resampler, that simply forwards the buffers to the
524 callback. */
525 if (((input_params && input_params->rate == target_rate) &&
526 (output_params && output_params->rate == target_rate)) ||
527 (input_params && !output_params && (input_params->rate == target_rate)) ||
528 (output_params && !input_params && (output_params->rate == target_rate))) {
529 return new passthrough_resampler<T>(stream, callback,
530 user_ptr,
531 input_params ? input_params->channels : 0,
532 target_rate);
533 }
534
535 /* Determine if we need to resampler one or both directions, and create the
536 resamplers. */
537 if (output_params && (output_params->rate != target_rate)) {
538 output_resampler.reset(
539 new cubeb_resampler_speex_one_way<T>(output_params->channels,
540 target_rate,
541 output_params->rate,
542 to_speex_quality(quality)));
543 if (!output_resampler) {
544 return NULL;
545 }
546 }
547
548 if (input_params && (input_params->rate != target_rate)) {
549 input_resampler.reset(
550 new cubeb_resampler_speex_one_way<T>(input_params->channels,
551 input_params->rate,
552 target_rate,
553 to_speex_quality(quality)));
554 if (!input_resampler) {
555 return NULL;
556 }
557 }
558
559 /* If we resample only one direction but we have a duplex stream, insert a
560 * delay line with a length equal to the resampler latency of the
561 * other direction so that the streams are synchronized. */
562 if (input_resampler && !output_resampler && input_params && output_params) {
563 output_delay.reset(new delay_line<T>(input_resampler->latency(),
564 output_params->channels,
565 output_params->rate));
566 if (!output_delay) {
567 return NULL;
568 }
569 } else if (output_resampler && !input_resampler && input_params && output_params) {
570 input_delay.reset(new delay_line<T>(output_resampler->latency(),
571 input_params->channels,
572 output_params->rate));
573 if (!input_delay) {
574 return NULL;
575 }
576 }
577
578 if (input_resampler && output_resampler) {
579 return new cubeb_resampler_speex<T,
580 cubeb_resampler_speex_one_way<T>,
581 cubeb_resampler_speex_one_way<T>>
582 (input_resampler.release(),
583 output_resampler.release(),
584 stream, callback, user_ptr);
585 } else if (input_resampler) {
586 return new cubeb_resampler_speex<T,
587 cubeb_resampler_speex_one_way<T>,
588 delay_line<T>>
589 (input_resampler.release(),
590 output_delay.release(),
591 stream, callback, user_ptr);
592 } else {
593 return new cubeb_resampler_speex<T,
594 delay_line<T>,
595 cubeb_resampler_speex_one_way<T>>
596 (input_delay.release(),
597 output_resampler.release(),
598 stream, callback, user_ptr);
599 }
600 }
601
602 #endif /* CUBEB_RESAMPLER_INTERNAL */
603