1 /*
2 * Copyright (C) 2019-2021 Alexandros Theodotou <alex at zrythm dot org>
3 *
4 * This file is part of Zrythm
5 *
6 * Zrythm is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU Affero General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * Zrythm is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Affero General Public License for more details.
15 *
16 * You should have received a copy of the GNU Affero General Public License
17 * along with Zrythm. If not, see <https://www.gnu.org/licenses/>.
18 *
19 * This file incorporates work covered by the following copyright and
20 * permission notice:
21 *
22 * Copyright (C) 2018 Robin Gareus <robin@gareus.org>
23 *
24 * This program is free software; you can redistribute it and/or modify
25 * it under the terms of the GNU General Public License as published by
26 * the Free Software Foundation; either version 2, or (at your option)
27 * any later version.
28 *
29 * This program is distributed in the hope that it will be useful,
30 * but WITHOUT ANY WARRANTY; without even the implied warranty of
31 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32 * GNU General Public License for more details.
33 *
34 * You should have received a copy of the GNU General Public License
35 * along with this program. If not, see <http://www.gnu.org/licenses/>.
36 */
37
38 #include <math.h>
39 #include <stdlib.h>
40 #include <string.h>
41
42 #include "audio/stretcher.h"
43 #include "utils/math.h"
44 #include "utils/mem.h"
45 #include "utils/objects.h"
46
47 #include <gtk/gtk.h>
48
49 /**
50 * Create a new Stretcher using the rubberband
51 * backend.
52 *
53 * @param samplerate The new samplerate.
54 * @param time_ratio The ratio to multiply time by
55 * (eg if the BPM is doubled, this will be 0.5).
56 * @param pitch_ratio The ratio to pitch by. This
57 * will normally be 1.0 when time-stretching).
58 * @param realtime Whether to perform realtime
59 * stretching (lower quality but fast enough to
60 * be used real-time).
61 */
62 Stretcher *
stretcher_new_rubberband(unsigned int samplerate,unsigned int channels,double time_ratio,double pitch_ratio,bool realtime)63 stretcher_new_rubberband (
64 unsigned int samplerate,
65 unsigned int channels,
66 double time_ratio,
67 double pitch_ratio,
68 bool realtime)
69 {
70 Stretcher * self = object_new (Stretcher);
71
72 self->backend = STRETCHER_BACKEND_RUBBERBAND;
73 self->samplerate = samplerate;
74 self->channels = channels;
75 self->is_realtime = realtime;
76 if (realtime)
77 {
78 self->block_size = 16000;
79 self->rubberband_state =
80 rubberband_new (
81 samplerate, channels,
82 RubberBandOptionProcessRealTime |
83 RubberBandOptionTransientsCrisp |
84 RubberBandOptionDetectorCompound |
85 RubberBandOptionPhaseLaminar |
86 RubberBandOptionThreadingAlways |
87 RubberBandOptionWindowStandard |
88 RubberBandOptionSmoothingOff |
89 RubberBandOptionFormantShifted |
90 RubberBandOptionPitchHighSpeed |
91 RubberBandOptionChannelsApart,
92 time_ratio, pitch_ratio);
93
94 /* feed it samples so it is ready to use */
95 #if 0
96 unsigned int samples_required =
97 rubberband_get_samples_required (
98 self->rubberband_state);
99 float in_samples_l[samples_required];
100 float in_samples_r[samples_required];
101 for (unsigned int i; i < samples_required; i++)
102 {
103 in_samples_l[i] = 0.f;
104 in_samples_r[i] = 0.f;
105 }
106 const float * in_samples[channels];
107 in_samples[0] = in_samples_l;
108 if (channels == 2)
109 in_samples[1] = in_samples_r;
110 rubberband_process (
111 self->rubberband_state, in_samples,
112 samples_required, false);
113 g_usleep (1000);
114 int avail =
115 rubberband_available (
116 self->rubberband_state);
117 float * out_samples[2] = {
118 in_samples_l, in_samples_r };
119 size_t retrieved_out_samples =
120 rubberband_retrieve (
121 self->rubberband_state, out_samples,
122 (unsigned int) avail);
123 g_message (
124 "%s: required: %u, available %d, "
125 "retrieved %zu",
126 __func__, samples_required, avail,
127 retrieved_out_samples);
128 samples_required =
129 rubberband_get_samples_required (
130 self->rubberband_state);
131 rubberband_process (
132 self->rubberband_state, in_samples,
133 samples_required, false);
134 g_usleep (1000);
135 avail =
136 rubberband_available (
137 self->rubberband_state);
138 retrieved_out_samples =
139 rubberband_retrieve (
140 self->rubberband_state, out_samples,
141 (unsigned int) avail);
142 g_message (
143 "%s: required: %u, available %d, "
144 "retrieved %zu",
145 __func__, samples_required, avail,
146 retrieved_out_samples);
147 #endif
148 }
149 else
150 {
151 self->block_size = 6000;
152 self->rubberband_state =
153 rubberband_new (
154 samplerate, channels,
155 RubberBandOptionProcessOffline |
156 RubberBandOptionStretchElastic |
157 RubberBandOptionTransientsCrisp |
158 RubberBandOptionDetectorCompound |
159 RubberBandOptionPhaseLaminar |
160 RubberBandOptionThreadingNever |
161 RubberBandOptionWindowStandard |
162 RubberBandOptionSmoothingOff |
163 RubberBandOptionFormantShifted |
164 RubberBandOptionPitchHighQuality |
165 RubberBandOptionChannelsApart,
166 time_ratio, pitch_ratio);
167 rubberband_set_max_process_size (
168 self->rubberband_state, self->block_size);
169 }
170 rubberband_set_default_debug_level (0);
171
172 g_message (
173 "%s: time ratio: %f, latency: %u",
174 __func__, time_ratio,
175 stretcher_get_latency (self));
176
177 return self;
178 }
179
180 /**
181 * Perform stretching.
182 *
183 * @param in_samples_l The left samples.
184 * @param in_samples_r The right channel samples. If
185 * this is NULL, the audio is assumed to be mono.
186 * @param in_samples_size The number of input samples
187 * per channel.
188 */
189 ssize_t
stretcher_stretch(Stretcher * self,float * in_samples_l,float * in_samples_r,size_t in_samples_size,float * out_samples_l,float * out_samples_r,size_t out_samples_wanted)190 stretcher_stretch (
191 Stretcher * self,
192 float * in_samples_l,
193 float * in_samples_r,
194 size_t in_samples_size,
195 float * out_samples_l,
196 float * out_samples_r,
197 size_t out_samples_wanted)
198 {
199 g_message (
200 "%s: in samples size: %zu",
201 __func__, in_samples_size);
202 g_return_val_if_fail (in_samples_l, -1);
203
204 /*rubberband_reset (self->rubberband_state);*/
205
206 /* create the de-interleaved array */
207 unsigned int channels = in_samples_r ? 2 : 1;
208 g_return_val_if_fail (
209 self->channels == channels, -1);
210 const float * in_samples[channels];
211 in_samples[0] = in_samples_l;
212 if (channels == 2)
213 in_samples[1] = in_samples_r;
214 float * out_samples[2] = {
215 out_samples_l, out_samples_r };
216
217 if (self->is_realtime)
218 {
219 rubberband_set_max_process_size (
220 self->rubberband_state, in_samples_size);
221 }
222 else
223 {
224 /* tell rubberband how many input samples it
225 * will receive */
226 rubberband_set_expected_input_duration (
227 self->rubberband_state, in_samples_size);
228
229 rubberband_study (
230 self->rubberband_state, in_samples,
231 in_samples_size, 1);
232 }
233 unsigned int samples_required =
234 rubberband_get_samples_required (
235 self->rubberband_state);
236 g_message (
237 "%s: samples required: %u, latency: %u",
238 __func__, samples_required,
239 rubberband_get_latency (
240 self->rubberband_state));
241 rubberband_process (
242 self->rubberband_state, in_samples,
243 in_samples_size, false);
244
245 /* get the output data */
246 int avail =
247 rubberband_available (self->rubberband_state);
248
249 /* if the wanted amount of samples are not ready,
250 * fill with silence */
251 if (avail < (int) out_samples_wanted)
252 {
253 g_message (
254 "%s: not enough samples available",
255 __func__);
256 return (ssize_t) out_samples_wanted;
257 }
258
259 g_message (
260 "%s: samples wanted %zu (avail %u)",
261 __func__, out_samples_wanted, avail);
262 size_t retrieved_out_samples =
263 rubberband_retrieve (
264 self->rubberband_state, out_samples,
265 out_samples_wanted);
266 g_warn_if_fail (
267 retrieved_out_samples == out_samples_wanted);
268
269 g_message (
270 "%s: out samples size: %zu",
271 __func__, retrieved_out_samples);
272
273 return (ssize_t) retrieved_out_samples;
274 }
275
276 void
stretcher_set_time_ratio(Stretcher * self,double ratio)277 stretcher_set_time_ratio (
278 Stretcher * self,
279 double ratio)
280 {
281 rubberband_set_time_ratio (
282 self->rubberband_state, ratio);
283 }
284
285 /**
286 * Get latency in number of samples.
287 */
288 unsigned int
stretcher_get_latency(Stretcher * self)289 stretcher_get_latency (
290 Stretcher * self)
291 {
292 return
293 rubberband_get_latency (self->rubberband_state);
294 }
295
296 /**
297 * Perform stretching.
298 *
299 * @note This must only be used offline.
300 *
301 * @param in_samples_size The number of input samples
302 * per channel.
303 *
304 * @return The number of output samples generated per
305 * channel.
306 */
307 ssize_t
stretcher_stretch_interleaved(Stretcher * self,float * in_samples,size_t in_samples_size,float ** _out_samples)308 stretcher_stretch_interleaved (
309 Stretcher * self,
310 float * in_samples,
311 size_t in_samples_size,
312 float ** _out_samples)
313 {
314 g_return_val_if_fail (in_samples, -1);
315
316 g_message ("input samples: %zu", in_samples_size);
317
318 /* create the de-interleaved array */
319 unsigned int channels = self->channels;
320 float in_buffers_l[in_samples_size];
321 float in_buffers_r[in_samples_size];
322 for (size_t i = 0; i < in_samples_size; i++)
323 {
324 in_buffers_l[i] = in_samples[i * channels];
325 if (channels == 2)
326 in_buffers_r[i] =
327 in_samples[i * channels + 1];
328 }
329 const float * in_buffers[2] = {
330 in_buffers_l, in_buffers_r };
331
332 /* tell rubberband how many input samples it will
333 * receive */
334 rubberband_set_expected_input_duration (
335 self->rubberband_state, in_samples_size);
336
337 /* study first */
338 size_t samples_to_read = in_samples_size;
339 while (samples_to_read > 0)
340 {
341 /* samples to read now */
342 unsigned int read_now =
343 (unsigned int)
344 MIN (
345 (size_t) self->block_size,
346 samples_to_read);
347
348 /* read */
349 rubberband_study (
350 self->rubberband_state, in_buffers,
351 read_now, read_now == samples_to_read);
352
353 /* remaining samples to read */
354 samples_to_read -= read_now;
355 }
356 g_warn_if_fail (samples_to_read == 0);
357
358 /* create the out sample arrays */
359 float * out_samples[channels];
360 size_t out_samples_size =
361 (size_t)
362 math_round_double_to_size_t (
363 rubberband_get_time_ratio (
364 self->rubberband_state) * in_samples_size);
365 for (unsigned int i = 0; i < channels; i++)
366 {
367 out_samples[i] =
368 object_new_n (out_samples_size, float);
369 }
370
371 /* process */
372 size_t processed = 0;
373 size_t total_out_frames = 0;
374 while (processed < in_samples_size)
375 {
376 size_t in_chunk_size =
377 rubberband_get_samples_required (
378 self->rubberband_state);
379 size_t samples_left =
380 in_samples_size - processed;
381
382 if (samples_left < in_chunk_size)
383 {
384 in_chunk_size = samples_left;
385 }
386
387 /* move the in buffers */
388 const float * tmp_in_arrays[2] = {
389 in_buffers[0] + processed,
390 in_buffers[1] + processed };
391
392 /* process */
393 rubberband_process (
394 self->rubberband_state, tmp_in_arrays,
395 in_chunk_size,
396 samples_left == in_chunk_size);
397
398 processed += in_chunk_size;
399
400 /*g_message ("processed %lu, in samples %lu",*/
401 /*processed, in_samples_size);*/
402
403 size_t avail =
404 (size_t)
405 rubberband_available (
406 self->rubberband_state);
407
408 /* retrieve the output data in temporary
409 * arrays */
410 float tmp_out_l[avail];
411 float tmp_out_r[avail];
412 float * tmp_out_arrays[2] = {
413 tmp_out_l, tmp_out_r };
414 size_t out_chunk_size =
415 rubberband_retrieve (
416 self->rubberband_state,
417 tmp_out_arrays, avail);
418
419 /* save the result */
420 for (size_t i = 0; i < channels; i++)
421 {
422 for (size_t j = 0; j < out_chunk_size;
423 j++)
424 {
425 out_samples[i][j + total_out_frames] =
426 tmp_out_arrays[i][j];
427 }
428 }
429
430 total_out_frames += out_chunk_size;
431 }
432
433 g_message (
434 "retrieved %zu samples (expected %zu)",
435 total_out_frames, out_samples_size);
436 g_warn_if_fail (
437 /* allow 1 sample earlier */
438 total_out_frames <= out_samples_size &&
439 total_out_frames >= out_samples_size - 1);
440
441 /* store the output data in the given arrays */
442 * _out_samples =
443 g_realloc (
444 * _out_samples,
445 (size_t) channels * total_out_frames *
446 sizeof (float));
447 for (unsigned int ch = 0; ch < channels; ch++)
448 {
449 for (size_t i = 0; i < total_out_frames; i++)
450 {
451 (*_out_samples)[
452 i * (size_t) channels + ch] =
453 out_samples[ch][i];
454 }
455 }
456
457 return (ssize_t) total_out_frames;
458 }
459
460 /**
461 * Frees the resampler.
462 */
463 void
stretcher_free(Stretcher * self)464 stretcher_free (
465 Stretcher * self)
466 {
467 if (self->rubberband_state)
468 rubberband_delete (self->rubberband_state);
469
470 free (self);
471 }
472