1 /*
2 * GStreamer
3 * Copyright (C) 2008 Rov Juvano <rovjuvano@users.sourceforge.net>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
19 */
20
21 /**
22 * SECTION:element-scaletempo
23 *
24 * Scale tempo while maintaining pitch
25 * (WSOLA-like technique with cross correlation)
26 * Inspired by SoundTouch library by Olli Parviainen
27 *
28 * Use Sceletempo to apply playback rates without the chipmunk effect.
29 *
30 * <refsect2>
31 * <title>Example pipelines</title>
32 * <para>
33 * |[
34 * filesrc location=media.ext ! decodebin name=d \
35 * d. ! queue ! audioconvert ! audioresample ! scaletempo ! audioconvert ! audioresample ! autoaudiosink \
36 * d. ! queue ! videoconvert ! autovideosink
37 * ]|
38 * OR
39 * |[
40 * playbin uri=... audio_sink="scaletempo ! audioconvert ! audioresample ! autoaudiosink"
41 * ]|
42 * When an application sends a seek event with rate != 1.0, Scaletempo applies
43 * the rate change by scaling the tempo without scaling the pitch.
44 *
45 * Scaletempo works by producing audio in constant sized chunks
46 * (#GstScaletempo:stride) but consuming chunks proportional to the playback
47 * rate.
48 *
49 * Scaletempo then smooths the output by blending the end of one stride with
50 * the next (#GstScaletempo:overlap).
51 *
52 * Scaletempo smooths the overlap further by searching within the input buffer
53 * for the best overlap position. Scaletempo uses a statistical cross
54 * correlation (roughly a dot-product). Scaletempo consumes most of its CPU
55 * cycles here. One can use the #GstScaletempo:search propery to tune how far
56 * the algoritm looks.
57 * </para>
58 * </refsect2>
59 */
60
61 /*
62 * Note: frame = audio key unit (i.e. one sample for each channel)
63 */
64
65 #ifdef HAVE_CONFIG_H
66 #include "config.h"
67 #endif
68
69 #include <gst/gst.h>
70 #include <gst/base/gstbasetransform.h>
71 #include <gst/audio/audio.h>
72 #include <string.h> /* for memset */
73
74 #include "gstscaletempo.h"
75
76 GST_DEBUG_CATEGORY_STATIC (gst_scaletempo_debug);
77 #define GST_CAT_DEFAULT gst_scaletempo_debug
78
79 /* Filter signals and args */
80 enum
81 {
82 LAST_SIGNAL
83 };
84
85 enum
86 {
87 PROP_0,
88 PROP_RATE,
89 PROP_STRIDE,
90 PROP_OVERLAP,
91 PROP_SEARCH,
92 };
93
94 #define SUPPORTED_CAPS \
95 GST_STATIC_CAPS ( \
96 GST_AUDIO_CAPS_MAKE (GST_AUDIO_NE (F32)) ", layout=(string)interleaved; " \
97 GST_AUDIO_CAPS_MAKE (GST_AUDIO_NE (F64)) ", layout=(string)interleaved; " \
98 GST_AUDIO_CAPS_MAKE (GST_AUDIO_NE (S16)) ", layout=(string)interleaved" \
99 )
100
101 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
102 GST_PAD_SINK,
103 GST_PAD_ALWAYS,
104 SUPPORTED_CAPS);
105
106 static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
107 GST_PAD_SRC,
108 GST_PAD_ALWAYS,
109 SUPPORTED_CAPS);
110
111 #define DEBUG_INIT(bla) GST_DEBUG_CATEGORY_INIT (gst_scaletempo_debug, "scaletempo", 0, "scaletempo element");
112
113 #define gst_scaletempo_parent_class parent_class
114 G_DEFINE_TYPE_WITH_CODE (GstScaletempo, gst_scaletempo,
115 GST_TYPE_BASE_TRANSFORM, DEBUG_INIT (0));
116
117 #define CREATE_BEST_OVERLAP_OFFSET_FLOAT_FUNC(type) \
118 static guint \
119 best_overlap_offset_##type (GstScaletempo * st) \
120 { \
121 g##type *pw, *po, *ppc, *search_start; \
122 g##type best_corr = G_MININT; \
123 guint best_off = 0; \
124 gint i, off; \
125 \
126 pw = st->table_window; \
127 po = st->buf_overlap; \
128 po += st->samples_per_frame; \
129 ppc = st->buf_pre_corr; \
130 for (i = st->samples_per_frame; i < st->samples_overlap; i++) { \
131 *ppc++ = *pw++ * *po++; \
132 } \
133 \
134 search_start = (g##type *) st->buf_queue + st->samples_per_frame; \
135 for (off = 0; off < st->frames_search; off++) { \
136 g##type corr = 0; \
137 g##type *ps = search_start; \
138 ppc = st->buf_pre_corr; \
139 for (i = st->samples_per_frame; i < st->samples_overlap; i++) { \
140 corr += *ppc++ * *ps++; \
141 } \
142 if (corr > best_corr) { \
143 best_corr = corr; \
144 best_off = off; \
145 } \
146 search_start += st->samples_per_frame; \
147 } \
148 \
149 return best_off * st->bytes_per_frame; \
150 }
151
152 CREATE_BEST_OVERLAP_OFFSET_FLOAT_FUNC (float);
153 CREATE_BEST_OVERLAP_OFFSET_FLOAT_FUNC (double);
154
155 /* buffer padding for loop optimization: sizeof(gint32) * (loop_size - 1) */
156 #define UNROLL_PADDING (4*3)
157 static guint
best_overlap_offset_s16(GstScaletempo * st)158 best_overlap_offset_s16 (GstScaletempo * st)
159 {
160 gint32 *pw, *ppc;
161 gint16 *po, *search_start;
162 gint64 best_corr = G_MININT64;
163 guint best_off = 0;
164 guint off;
165 glong i;
166
167 pw = st->table_window;
168 po = st->buf_overlap;
169 po += st->samples_per_frame;
170 ppc = st->buf_pre_corr;
171 for (i = st->samples_per_frame; i < st->samples_overlap; i++) {
172 *ppc++ = (*pw++ * *po++) >> 15;
173 }
174
175 search_start = (gint16 *) st->buf_queue + st->samples_per_frame;
176 for (off = 0; off < st->frames_search; off++) {
177 gint64 corr = 0;
178 gint16 *ps = search_start;
179 ppc = st->buf_pre_corr;
180 ppc += st->samples_overlap - st->samples_per_frame;
181 ps += st->samples_overlap - st->samples_per_frame;
182 i = -((glong) st->samples_overlap - (glong) st->samples_per_frame);
183 do {
184 corr += ppc[i + 0] * ps[i + 0];
185 corr += ppc[i + 1] * ps[i + 1];
186 corr += ppc[i + 2] * ps[i + 2];
187 corr += ppc[i + 3] * ps[i + 3];
188 i += 4;
189 } while (i < 0);
190 if (corr > best_corr) {
191 best_corr = corr;
192 best_off = off;
193 }
194 search_start += st->samples_per_frame;
195 }
196
197 return best_off * st->bytes_per_frame;
198 }
199
200 #define CREATE_OUTPUT_OVERLAP_FLOAT_FUNC(type) \
201 static void \
202 output_overlap_##type (GstScaletempo * st, gpointer buf_out, guint bytes_off) \
203 { \
204 g##type *pout = buf_out; \
205 g##type *pb = st->table_blend; \
206 g##type *po = st->buf_overlap; \
207 g##type *pin = (g##type *) (st->buf_queue + bytes_off); \
208 gint i; \
209 for (i = 0; i < st->samples_overlap; i++) { \
210 *pout++ = *po - *pb++ * (*po - *pin++); \
211 po++; \
212 } \
213 }
214
215 CREATE_OUTPUT_OVERLAP_FLOAT_FUNC (float);
216 CREATE_OUTPUT_OVERLAP_FLOAT_FUNC (double);
217
218 static void
output_overlap_s16(GstScaletempo * st,gpointer buf_out,guint bytes_off)219 output_overlap_s16 (GstScaletempo * st, gpointer buf_out, guint bytes_off)
220 {
221 gint16 *pout = buf_out;
222 gint32 *pb = st->table_blend;
223 gint16 *po = st->buf_overlap;
224 gint16 *pin = (gint16 *) (st->buf_queue + bytes_off);
225 gint i;
226 for (i = 0; i < st->samples_overlap; i++) {
227 *pout++ = *po - ((*pb++ * (*po - *pin++)) >> 16);
228 po++;
229 }
230 }
231
232 static guint
fill_queue(GstScaletempo * st,GstBuffer * buf_in,guint offset)233 fill_queue (GstScaletempo * st, GstBuffer * buf_in, guint offset)
234 {
235 guint bytes_in = gst_buffer_get_size (buf_in) - offset;
236 guint offset_unchanged = offset;
237 GstMapInfo map;
238
239 gst_buffer_map (buf_in, &map, GST_MAP_READ);
240 if (st->bytes_to_slide > 0) {
241 if (st->bytes_to_slide < st->bytes_queued) {
242 guint bytes_in_move = st->bytes_queued - st->bytes_to_slide;
243 memmove (st->buf_queue, st->buf_queue + st->bytes_to_slide,
244 bytes_in_move);
245 st->bytes_to_slide = 0;
246 st->bytes_queued = bytes_in_move;
247 } else {
248 guint bytes_in_skip;
249 st->bytes_to_slide -= st->bytes_queued;
250 bytes_in_skip = MIN (st->bytes_to_slide, bytes_in);
251 st->bytes_queued = 0;
252 st->bytes_to_slide -= bytes_in_skip;
253 offset += bytes_in_skip;
254 bytes_in -= bytes_in_skip;
255 }
256 }
257
258 if (bytes_in > 0) {
259 guint bytes_in_copy =
260 MIN (st->bytes_queue_max - st->bytes_queued, bytes_in);
261 memcpy (st->buf_queue + st->bytes_queued, map.data + offset, bytes_in_copy);
262 st->bytes_queued += bytes_in_copy;
263 offset += bytes_in_copy;
264 }
265 gst_buffer_unmap (buf_in, &map);
266
267 return offset - offset_unchanged;
268 }
269
270 static void
reinit_buffers(GstScaletempo * st)271 reinit_buffers (GstScaletempo * st)
272 {
273 gint i, j;
274 guint frames_overlap;
275 guint new_size;
276 GstClockTime latency;
277
278 guint frames_stride = st->ms_stride * st->sample_rate / 1000.0;
279 st->bytes_stride = frames_stride * st->bytes_per_frame;
280
281 /* overlap */
282 frames_overlap = frames_stride * st->percent_overlap;
283 if (frames_overlap < 1) { /* if no overlap */
284 st->bytes_overlap = 0;
285 st->bytes_standing = st->bytes_stride;
286 st->samples_standing = st->bytes_standing / st->bytes_per_sample;
287 st->output_overlap = NULL;
288 } else {
289 guint prev_overlap = st->bytes_overlap;
290 st->bytes_overlap = frames_overlap * st->bytes_per_frame;
291 st->samples_overlap = frames_overlap * st->samples_per_frame;
292 st->bytes_standing = st->bytes_stride - st->bytes_overlap;
293 st->samples_standing = st->bytes_standing / st->bytes_per_sample;
294 st->buf_overlap = g_realloc (st->buf_overlap, st->bytes_overlap);
295 /* S16 uses gint32 blend table, floats/doubles use their respective type */
296 st->table_blend =
297 g_realloc (st->table_blend,
298 st->samples_overlap * (st->format ==
299 GST_AUDIO_FORMAT_S16 ? 4 : st->bytes_per_sample));
300 if (st->bytes_overlap > prev_overlap) {
301 memset ((guint8 *) st->buf_overlap + prev_overlap, 0,
302 st->bytes_overlap - prev_overlap);
303 }
304 if (st->format == GST_AUDIO_FORMAT_S16) {
305 gint32 *pb = st->table_blend;
306 gint64 blend = 0;
307 for (i = 0; i < frames_overlap; i++) {
308 gint32 v = blend / frames_overlap;
309 for (j = 0; j < st->samples_per_frame; j++) {
310 *pb++ = v;
311 }
312 blend += 65535; /* 2^16 */
313 }
314 st->output_overlap = output_overlap_s16;
315 } else if (st->format == GST_AUDIO_FORMAT_F32) {
316 gfloat *pb = st->table_blend;
317 gfloat t = (gfloat) frames_overlap;
318 for (i = 0; i < frames_overlap; i++) {
319 gfloat v = i / t;
320 for (j = 0; j < st->samples_per_frame; j++) {
321 *pb++ = v;
322 }
323 }
324 st->output_overlap = output_overlap_float;
325 } else {
326 gdouble *pb = st->table_blend;
327 gdouble t = (gdouble) frames_overlap;
328 for (i = 0; i < frames_overlap; i++) {
329 gdouble v = i / t;
330 for (j = 0; j < st->samples_per_frame; j++) {
331 *pb++ = v;
332 }
333 }
334 st->output_overlap = output_overlap_double;
335 }
336 }
337
338 /* best overlap */
339 st->frames_search =
340 (frames_overlap <= 1) ? 0 : st->ms_search * st->sample_rate / 1000.0;
341 if (st->frames_search < 1) { /* if no search */
342 st->best_overlap_offset = NULL;
343 } else {
344 /* S16 uses gint32 buffer, floats/doubles use their respective type */
345 guint bytes_pre_corr =
346 (st->samples_overlap - st->samples_per_frame) * (st->format ==
347 GST_AUDIO_FORMAT_S16 ? 4 : st->bytes_per_sample);
348 st->buf_pre_corr =
349 g_realloc (st->buf_pre_corr, bytes_pre_corr + UNROLL_PADDING);
350 st->table_window = g_realloc (st->table_window, bytes_pre_corr);
351 if (st->format == GST_AUDIO_FORMAT_S16) {
352 gint64 t = frames_overlap;
353 gint32 n = 8589934588LL / (t * t); /* 4 * (2^31 - 1) / t^2 */
354 gint32 *pw;
355
356 memset ((guint8 *) st->buf_pre_corr + bytes_pre_corr, 0, UNROLL_PADDING);
357 pw = st->table_window;
358 for (i = 1; i < frames_overlap; i++) {
359 gint32 v = (i * (t - i) * n) >> 15;
360 for (j = 0; j < st->samples_per_frame; j++) {
361 *pw++ = v;
362 }
363 }
364 st->best_overlap_offset = best_overlap_offset_s16;
365 } else if (st->format == GST_AUDIO_FORMAT_F32) {
366 gfloat *pw = st->table_window;
367 for (i = 1; i < frames_overlap; i++) {
368 gfloat v = i * (frames_overlap - i);
369 for (j = 0; j < st->samples_per_frame; j++) {
370 *pw++ = v;
371 }
372 }
373 st->best_overlap_offset = best_overlap_offset_float;
374 } else {
375 gdouble *pw = st->table_window;
376 for (i = 1; i < frames_overlap; i++) {
377 gdouble v = i * (frames_overlap - i);
378 for (j = 0; j < st->samples_per_frame; j++) {
379 *pw++ = v;
380 }
381 }
382 st->best_overlap_offset = best_overlap_offset_double;
383 }
384 }
385
386 new_size =
387 (st->frames_search + frames_stride +
388 frames_overlap) * st->bytes_per_frame;
389 if (st->bytes_queued > new_size) {
390 if (st->bytes_to_slide > st->bytes_queued) {
391 st->bytes_to_slide -= st->bytes_queued;
392 st->bytes_queued = 0;
393 } else {
394 guint new_queued = MIN (st->bytes_queued - st->bytes_to_slide, new_size);
395 memmove (st->buf_queue,
396 st->buf_queue + st->bytes_queued - new_queued, new_queued);
397 st->bytes_to_slide = 0;
398 st->bytes_queued = new_queued;
399 }
400 }
401
402 st->bytes_queue_max = new_size;
403 st->buf_queue = g_realloc (st->buf_queue, st->bytes_queue_max);
404
405 latency =
406 gst_util_uint64_scale (st->bytes_queue_max, GST_SECOND,
407 st->bytes_per_frame * st->sample_rate);
408 if (st->latency != latency) {
409 st->latency = latency;
410 gst_element_post_message (GST_ELEMENT (st),
411 gst_message_new_latency (GST_OBJECT (st)));
412 }
413
414 st->bytes_stride_scaled = st->bytes_stride * st->scale;
415 st->frames_stride_scaled = st->bytes_stride_scaled / st->bytes_per_frame;
416
417 GST_DEBUG
418 ("%.3f scale, %.3f stride_in, %i stride_out, %i standing, %i overlap, %i search, %i queue, %s mode",
419 st->scale, st->frames_stride_scaled,
420 (gint) (st->bytes_stride / st->bytes_per_frame),
421 (gint) (st->bytes_standing / st->bytes_per_frame),
422 (gint) (st->bytes_overlap / st->bytes_per_frame), st->frames_search,
423 (gint) (st->bytes_queue_max / st->bytes_per_frame),
424 gst_audio_format_to_string (st->format));
425
426 st->reinit_buffers = FALSE;
427 }
428
429 static GstBuffer *
reverse_buffer(GstScaletempo * st,GstBuffer * inbuf)430 reverse_buffer (GstScaletempo * st, GstBuffer * inbuf)
431 {
432 GstBuffer *outbuf;
433 GstMapInfo imap, omap;
434
435 gst_buffer_map (inbuf, &imap, GST_MAP_READ);
436 outbuf = gst_buffer_new_and_alloc (imap.size);
437 gst_buffer_map (outbuf, &omap, GST_MAP_WRITE);
438
439 if (st->format == GST_AUDIO_FORMAT_F64) {
440 const gint64 *ip = (const gint64 *) imap.data;
441 gint64 *op = (gint64 *) (omap.data + omap.size - 8 * st->samples_per_frame);
442 guint i, n = imap.size / (8 * st->samples_per_frame);
443 guint j, c = st->samples_per_frame;
444
445 for (i = 0; i < n; i++) {
446 for (j = 0; j < c; j++)
447 op[j] = ip[j];
448 op -= c;
449 ip += c;
450 }
451 } else {
452 const gint32 *ip = (const gint32 *) imap.data;
453 gint32 *op = (gint32 *) (omap.data + omap.size - 4 * st->samples_per_frame);
454 guint i, n = imap.size / (4 * st->samples_per_frame);
455 guint j, c = st->samples_per_frame;
456
457 for (i = 0; i < n; i++) {
458 for (j = 0; j < c; j++)
459 op[j] = ip[j];
460 op -= c;
461 ip += c;
462 }
463 }
464
465 gst_buffer_unmap (inbuf, &imap);
466 gst_buffer_unmap (outbuf, &omap);
467
468 return outbuf;
469 }
470
471 /* GstBaseTransform vmethod implementations */
472 static GstFlowReturn
gst_scaletempo_transform(GstBaseTransform * trans,GstBuffer * inbuf,GstBuffer * outbuf)473 gst_scaletempo_transform (GstBaseTransform * trans,
474 GstBuffer * inbuf, GstBuffer * outbuf)
475 {
476 GstScaletempo *st = GST_SCALETEMPO (trans);
477 gint8 *pout;
478 guint offset_in, bytes_out;
479 GstMapInfo omap;
480 GstClockTime timestamp;
481 GstBuffer *tmpbuf = NULL;
482
483 if (st->reverse)
484 tmpbuf = reverse_buffer (st, inbuf);
485
486 gst_buffer_map (outbuf, &omap, GST_MAP_WRITE);
487 pout = (gint8 *) omap.data;
488 bytes_out = omap.size;
489
490 offset_in = fill_queue (st, tmpbuf ? tmpbuf : inbuf, 0);
491 bytes_out = 0;
492 while (st->bytes_queued >= st->bytes_queue_max) {
493 guint bytes_off = 0;
494 gdouble frames_to_slide;
495 guint frames_to_stride_whole;
496
497 /* output stride */
498 if (st->output_overlap) {
499 if (st->best_overlap_offset) {
500 bytes_off = st->best_overlap_offset (st);
501 }
502 st->output_overlap (st, pout, bytes_off);
503 }
504 memcpy (pout + st->bytes_overlap,
505 st->buf_queue + bytes_off + st->bytes_overlap, st->bytes_standing);
506 pout += st->bytes_stride;
507 bytes_out += st->bytes_stride;
508
509 /* input stride */
510 memcpy (st->buf_overlap,
511 st->buf_queue + bytes_off + st->bytes_stride, st->bytes_overlap);
512 frames_to_slide = st->frames_stride_scaled + st->frames_stride_error;
513 frames_to_stride_whole = (gint) frames_to_slide;
514 st->bytes_to_slide = frames_to_stride_whole * st->bytes_per_frame;
515 st->frames_stride_error = frames_to_slide - frames_to_stride_whole;
516
517 offset_in += fill_queue (st, tmpbuf ? tmpbuf : inbuf, offset_in);
518 }
519 gst_buffer_unmap (outbuf, &omap);
520
521 if (st->reverse) {
522 timestamp = st->in_segment.stop - GST_BUFFER_TIMESTAMP (inbuf);
523 if (timestamp < st->latency)
524 timestamp = 0;
525 else
526 timestamp -= st->latency;
527 } else {
528 timestamp = GST_BUFFER_TIMESTAMP (inbuf) - st->in_segment.start;
529 if (timestamp < st->latency)
530 timestamp = 0;
531 else
532 timestamp -= st->latency;
533 }
534 GST_BUFFER_TIMESTAMP (outbuf) = timestamp / st->scale + st->in_segment.start;
535 GST_BUFFER_DURATION (outbuf) =
536 gst_util_uint64_scale (bytes_out, GST_SECOND,
537 st->bytes_per_frame * st->sample_rate);
538 gst_buffer_set_size (outbuf, bytes_out);
539
540 if (tmpbuf)
541 gst_buffer_unref (tmpbuf);
542
543 return GST_FLOW_OK;
544 }
545
546 static GstFlowReturn
gst_scaletempo_submit_input_buffer(GstBaseTransform * trans,gboolean is_discont,GstBuffer * input)547 gst_scaletempo_submit_input_buffer (GstBaseTransform * trans,
548 gboolean is_discont, GstBuffer * input)
549 {
550 GstScaletempo *scaletempo = GST_SCALETEMPO (trans);
551
552 if (scaletempo->in_segment.format == GST_FORMAT_TIME) {
553 input =
554 gst_audio_buffer_clip (input, &scaletempo->in_segment,
555 scaletempo->sample_rate, scaletempo->bytes_per_frame);
556 if (!input)
557 return GST_FLOW_OK;
558 }
559
560 return GST_BASE_TRANSFORM_CLASS (parent_class)->submit_input_buffer (trans,
561 is_discont, input);
562 }
563
564 static gboolean
gst_scaletempo_transform_size(GstBaseTransform * trans,GstPadDirection direction,GstCaps * caps,gsize size,GstCaps * othercaps,gsize * othersize)565 gst_scaletempo_transform_size (GstBaseTransform * trans,
566 GstPadDirection direction,
567 GstCaps * caps, gsize size, GstCaps * othercaps, gsize * othersize)
568 {
569 if (direction == GST_PAD_SINK) {
570 GstScaletempo *scaletempo = GST_SCALETEMPO (trans);
571 gint bytes_to_out;
572
573 if (scaletempo->reinit_buffers)
574 reinit_buffers (scaletempo);
575
576 bytes_to_out = size + scaletempo->bytes_queued - scaletempo->bytes_to_slide;
577 if (bytes_to_out < (gint) scaletempo->bytes_queue_max) {
578 *othersize = 0;
579 } else {
580 /* while (total_buffered - stride_length * n >= queue_max) n++ */
581 *othersize = scaletempo->bytes_stride * ((guint) (
582 (bytes_to_out - scaletempo->bytes_queue_max +
583 /* rounding protection */ scaletempo->bytes_per_frame)
584 / scaletempo->bytes_stride_scaled) + 1);
585 }
586
587 return TRUE;
588 }
589 return FALSE;
590 }
591
592 static gboolean
gst_scaletempo_sink_event(GstBaseTransform * trans,GstEvent * event)593 gst_scaletempo_sink_event (GstBaseTransform * trans, GstEvent * event)
594 {
595 GstScaletempo *scaletempo = GST_SCALETEMPO (trans);
596
597 if (GST_EVENT_TYPE (event) == GST_EVENT_SEGMENT) {
598 GstSegment segment;
599
600 gst_event_copy_segment (event, &segment);
601
602 if (segment.format != GST_FORMAT_TIME
603 || scaletempo->scale != ABS (segment.rate)
604 || ! !scaletempo->reverse != ! !(segment.rate < 0.0)) {
605 if (segment.format != GST_FORMAT_TIME || ABS (segment.rate - 1.0) < 1e-10) {
606 scaletempo->scale = 1.0;
607 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (scaletempo),
608 TRUE);
609 } else {
610 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (scaletempo),
611 FALSE);
612 scaletempo->scale = ABS (segment.rate);
613 scaletempo->reverse = segment.rate < 0.0;
614 scaletempo->bytes_stride_scaled =
615 scaletempo->bytes_stride * scaletempo->scale;
616 scaletempo->frames_stride_scaled =
617 scaletempo->bytes_stride_scaled / scaletempo->bytes_per_frame;
618 GST_DEBUG ("%.3f scale, %.3f stride_in, %i stride_out",
619 scaletempo->scale, scaletempo->frames_stride_scaled,
620 (gint) (scaletempo->bytes_stride / scaletempo->bytes_per_frame));
621
622 scaletempo->bytes_to_slide = 0;
623 }
624 }
625
626 scaletempo->in_segment = segment;
627 scaletempo->out_segment = segment;
628
629 if (scaletempo->scale != 1.0 || scaletempo->reverse) {
630 guint32 seqnum;
631
632 segment.applied_rate = segment.rate;
633 segment.rate = 1.0;
634
635 if (segment.stop != -1) {
636 segment.stop =
637 (segment.stop - segment.start) / ABS (segment.applied_rate) +
638 segment.start;
639 }
640
641 scaletempo->out_segment = segment;
642
643 seqnum = gst_event_get_seqnum (event);
644 gst_event_unref (event);
645
646 event = gst_event_new_segment (&segment);
647 gst_event_set_seqnum (event, seqnum);
648
649 return gst_pad_push_event (GST_BASE_TRANSFORM_SRC_PAD (trans), event);
650 }
651 } else if (GST_EVENT_TYPE (event) == GST_EVENT_FLUSH_STOP) {
652 gst_segment_init (&scaletempo->in_segment, GST_FORMAT_UNDEFINED);
653 gst_segment_init (&scaletempo->out_segment, GST_FORMAT_UNDEFINED);
654 } else if (GST_EVENT_TYPE (event) == GST_EVENT_GAP) {
655 if (scaletempo->scale != 1.0) {
656 GstClockTime gap_ts, gap_duration;
657 gst_event_parse_gap (event, &gap_ts, &gap_duration);
658 if (scaletempo->reverse) {
659 gap_ts = scaletempo->in_segment.stop - gap_ts;
660 } else {
661 gap_ts = gap_ts - scaletempo->in_segment.start;
662 }
663 gap_ts = gap_ts / scaletempo->scale + scaletempo->in_segment.start;
664 if (GST_CLOCK_TIME_IS_VALID (gap_duration)) {
665 gap_duration = gap_duration / ABS (scaletempo->scale);
666 }
667 gst_event_unref (event);
668 event = gst_event_new_gap (gap_ts, gap_duration);
669 }
670 }
671
672 return GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (trans, event);
673 }
674
675 static gboolean
gst_scaletempo_set_caps(GstBaseTransform * trans,GstCaps * incaps,GstCaps * outcaps)676 gst_scaletempo_set_caps (GstBaseTransform * trans,
677 GstCaps * incaps, GstCaps * outcaps)
678 {
679 GstScaletempo *scaletempo = GST_SCALETEMPO (trans);
680
681 gint width, bps, nch, rate;
682 GstAudioInfo info;
683 GstAudioFormat format;
684
685 if (!gst_audio_info_from_caps (&info, incaps))
686 return FALSE;
687
688 nch = GST_AUDIO_INFO_CHANNELS (&info);
689 rate = GST_AUDIO_INFO_RATE (&info);
690 width = GST_AUDIO_INFO_WIDTH (&info);
691 format = GST_AUDIO_INFO_FORMAT (&info);
692
693 bps = width / 8;
694
695 GST_DEBUG ("caps: %" GST_PTR_FORMAT ", %d bps", incaps, bps);
696
697 if (rate != scaletempo->sample_rate
698 || nch != scaletempo->samples_per_frame
699 || bps != scaletempo->bytes_per_sample || format != scaletempo->format) {
700 scaletempo->sample_rate = rate;
701 scaletempo->samples_per_frame = nch;
702 scaletempo->bytes_per_sample = bps;
703 scaletempo->bytes_per_frame = nch * bps;
704 scaletempo->format = format;
705 scaletempo->reinit_buffers = TRUE;
706 }
707
708 return TRUE;
709 }
710
711 static gboolean
gst_scaletempo_start(GstBaseTransform * trans)712 gst_scaletempo_start (GstBaseTransform * trans)
713 {
714 GstScaletempo *scaletempo = GST_SCALETEMPO (trans);
715
716 gst_segment_init (&scaletempo->in_segment, GST_FORMAT_UNDEFINED);
717 gst_segment_init (&scaletempo->out_segment, GST_FORMAT_UNDEFINED);
718 scaletempo->reinit_buffers = TRUE;
719
720 return TRUE;
721 }
722
723 static gboolean
gst_scaletempo_stop(GstBaseTransform * trans)724 gst_scaletempo_stop (GstBaseTransform * trans)
725 {
726 GstScaletempo *scaletempo = GST_SCALETEMPO (trans);
727
728 g_free (scaletempo->buf_queue);
729 scaletempo->buf_queue = NULL;
730 g_free (scaletempo->buf_overlap);
731 scaletempo->buf_overlap = NULL;
732 g_free (scaletempo->table_blend);
733 scaletempo->table_blend = NULL;
734 g_free (scaletempo->buf_pre_corr);
735 scaletempo->buf_pre_corr = NULL;
736 g_free (scaletempo->table_window);
737 scaletempo->table_window = NULL;
738 scaletempo->reinit_buffers = TRUE;
739
740 return TRUE;
741 }
742
743 static gboolean
gst_scaletempo_query(GstBaseTransform * trans,GstPadDirection direction,GstQuery * query)744 gst_scaletempo_query (GstBaseTransform * trans, GstPadDirection direction,
745 GstQuery * query)
746 {
747 GstScaletempo *scaletempo = GST_SCALETEMPO (trans);
748
749 if (direction == GST_PAD_SRC) {
750 switch (GST_QUERY_TYPE (query)) {
751 case GST_QUERY_SEGMENT:
752 {
753 GstFormat format;
754 gint64 start, stop;
755
756 format = scaletempo->out_segment.format;
757
758 start =
759 gst_segment_to_stream_time (&scaletempo->out_segment, format,
760 scaletempo->out_segment.start);
761 if ((stop = scaletempo->out_segment.stop) == -1)
762 stop = scaletempo->out_segment.duration;
763 else
764 stop =
765 gst_segment_to_stream_time (&scaletempo->out_segment, format,
766 stop);
767
768 gst_query_set_segment (query, scaletempo->out_segment.rate, format,
769 start, stop);
770 return TRUE;
771 }
772 case GST_QUERY_LATENCY:{
773 GstPad *peer;
774
775 if ((peer = gst_pad_get_peer (GST_BASE_TRANSFORM_SINK_PAD (trans)))) {
776 if ((gst_pad_query (peer, query))) {
777 GstClockTime min, max;
778 gboolean live;
779
780 gst_query_parse_latency (query, &live, &min, &max);
781
782 GST_DEBUG_OBJECT (scaletempo, "Peer latency: min %"
783 GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
784 GST_TIME_ARGS (min), GST_TIME_ARGS (max));
785
786 /* add our own latency */
787 GST_DEBUG_OBJECT (scaletempo, "Our latency: %" GST_TIME_FORMAT,
788 GST_TIME_ARGS (scaletempo->latency));
789 min += scaletempo->latency;
790 if (max != GST_CLOCK_TIME_NONE)
791 max += scaletempo->latency;
792
793 GST_DEBUG_OBJECT (scaletempo, "Calculated total latency : min %"
794 GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
795 GST_TIME_ARGS (min), GST_TIME_ARGS (max));
796 gst_query_set_latency (query, live, min, max);
797 }
798 gst_object_unref (peer);
799 }
800
801 return TRUE;
802 }
803 default:{
804 return GST_BASE_TRANSFORM_CLASS (parent_class)->query (trans, direction,
805 query);
806 }
807 }
808 } else {
809 return GST_BASE_TRANSFORM_CLASS (parent_class)->query (trans, direction,
810 query);
811 }
812 }
813
814 /* GObject vmethod implementations */
815 static void
gst_scaletempo_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)816 gst_scaletempo_get_property (GObject * object,
817 guint prop_id, GValue * value, GParamSpec * pspec)
818 {
819 GstScaletempo *scaletempo = GST_SCALETEMPO (object);
820
821 switch (prop_id) {
822 case PROP_RATE:
823 g_value_set_double (value, scaletempo->scale);
824 break;
825 case PROP_STRIDE:
826 g_value_set_uint (value, scaletempo->ms_stride);
827 break;
828 case PROP_OVERLAP:
829 g_value_set_double (value, scaletempo->percent_overlap);
830 break;
831 case PROP_SEARCH:
832 g_value_set_uint (value, scaletempo->ms_search);
833 break;
834 default:
835 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
836 break;
837 }
838 }
839
840 static void
gst_scaletempo_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)841 gst_scaletempo_set_property (GObject * object,
842 guint prop_id, const GValue * value, GParamSpec * pspec)
843 {
844 GstScaletempo *scaletempo = GST_SCALETEMPO (object);
845
846 switch (prop_id) {
847 case PROP_STRIDE:{
848 guint new_value = g_value_get_uint (value);
849 if (scaletempo->ms_stride != new_value) {
850 scaletempo->ms_stride = new_value;
851 scaletempo->reinit_buffers = TRUE;
852 }
853 break;
854 }
855 case PROP_OVERLAP:{
856 gdouble new_value = g_value_get_double (value);
857 if (scaletempo->percent_overlap != new_value) {
858 scaletempo->percent_overlap = new_value;
859 scaletempo->reinit_buffers = TRUE;
860 }
861 break;
862 }
863 case PROP_SEARCH:{
864 guint new_value = g_value_get_uint (value);
865 if (scaletempo->ms_search != new_value) {
866 scaletempo->ms_search = new_value;
867 scaletempo->reinit_buffers = TRUE;
868 }
869 break;
870 }
871 default:
872 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
873 break;
874 }
875 }
876
877 static void
gst_scaletempo_class_init(GstScaletempoClass * klass)878 gst_scaletempo_class_init (GstScaletempoClass * klass)
879 {
880 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
881 GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);
882 GstBaseTransformClass *basetransform_class = GST_BASE_TRANSFORM_CLASS (klass);
883
884 gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_scaletempo_get_property);
885 gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_scaletempo_set_property);
886
887 g_object_class_install_property (gobject_class, PROP_RATE,
888 g_param_spec_double ("rate", "Playback Rate", "Current playback rate",
889 G_MININT, G_MAXINT, 1.0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
890
891 g_object_class_install_property (gobject_class, PROP_STRIDE,
892 g_param_spec_uint ("stride", "Stride Length",
893 "Length in milliseconds to output each stride", 1, 5000, 30,
894 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
895
896 g_object_class_install_property (gobject_class, PROP_OVERLAP,
897 g_param_spec_double ("overlap", "Overlap Length",
898 "Percentage of stride to overlap", 0, 1, .2,
899 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
900
901 g_object_class_install_property (gobject_class, PROP_SEARCH,
902 g_param_spec_uint ("search", "Search Length",
903 "Length in milliseconds to search for best overlap position", 0, 500,
904 14, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
905
906 gst_element_class_add_static_pad_template (gstelement_class, &src_template);
907 gst_element_class_add_static_pad_template (gstelement_class, &sink_template);
908 gst_element_class_set_static_metadata (gstelement_class, "Scaletempo",
909 "Filter/Effect/Rate/Audio",
910 "Sync audio tempo with playback rate",
911 "Rov Juvano <rovjuvano@users.sourceforge.net>");
912
913 basetransform_class->sink_event =
914 GST_DEBUG_FUNCPTR (gst_scaletempo_sink_event);
915 basetransform_class->set_caps = GST_DEBUG_FUNCPTR (gst_scaletempo_set_caps);
916 basetransform_class->transform_size =
917 GST_DEBUG_FUNCPTR (gst_scaletempo_transform_size);
918 basetransform_class->transform = GST_DEBUG_FUNCPTR (gst_scaletempo_transform);
919 basetransform_class->query = GST_DEBUG_FUNCPTR (gst_scaletempo_query);
920 basetransform_class->start = GST_DEBUG_FUNCPTR (gst_scaletempo_start);
921 basetransform_class->stop = GST_DEBUG_FUNCPTR (gst_scaletempo_stop);
922 basetransform_class->submit_input_buffer =
923 GST_DEBUG_FUNCPTR (gst_scaletempo_submit_input_buffer);
924 }
925
926 static void
gst_scaletempo_init(GstScaletempo * scaletempo)927 gst_scaletempo_init (GstScaletempo * scaletempo)
928 {
929 /* defaults */
930 scaletempo->ms_stride = 30;
931 scaletempo->percent_overlap = .2;
932 scaletempo->ms_search = 14;
933
934 /* uninitialized */
935 scaletempo->scale = 0;
936 scaletempo->sample_rate = 0;
937 scaletempo->frames_stride_error = 0;
938 scaletempo->bytes_stride = 0;
939 scaletempo->bytes_queued = 0;
940 scaletempo->bytes_to_slide = 0;
941 gst_segment_init (&scaletempo->in_segment, GST_FORMAT_UNDEFINED);
942 gst_segment_init (&scaletempo->out_segment, GST_FORMAT_UNDEFINED);
943 }
944