1 /*
2 * GStreamer
3 * Copyright (C) 2015 Vivia Nikolaidou <vivia@toolsonair.com>
4 *
5 * Based on gstlevel.c:
6 * Copyright (C) 2000,2001,2002,2003,2005
7 * Thomas Vander Stichele <thomas at apestaart dot org>
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Library General Public
11 * License as published by the Free Software Foundation; either
12 * version 2 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Library General Public License for more details.
18 *
19 * You should have received a copy of the GNU Library General Public
20 * License along with this library; if not, write to the
21 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
22 * Boston, MA 02110-1301, USA.
23 */
24
25 /**
26 * SECTION:element-videoframe-audiolevel
27 * @title: videoframe-audiolevel
28 *
29 * This element acts like a synchronized audio/video "level". It gathers
30 * all audio buffers sent between two video frames, and then sends a message
31 * that contains the RMS value of all samples for these buffers.
32 *
33 * ## Example launch line
34 * |[
35 * gst-launch-1.0 -m filesrc location="file.mkv" ! decodebin name=d ! "audio/x-raw" ! videoframe-audiolevel name=l ! autoaudiosink d. ! "video/x-raw" ! l. l. ! queue ! autovideosink ]|
36 *
37 */
38
39 #ifdef HAVE_CONFIG_H
40 #include "config.h"
41 #endif
42
43 /* FIXME 2.0: suppress warnings for deprecated API such as GValueArray
44 * with newer GLib versions (>= 2.31.0) */
45 #define GLIB_DISABLE_DEPRECATION_WARNINGS
46
47 #include "gstvideoframe-audiolevel.h"
48 #include <math.h>
49
50 #define GST_CAT_DEFAULT gst_videoframe_audiolevel_debug
51 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
52 # define FORMATS "{ S8, S16LE, S32LE, F32LE, F64LE }"
53 #else
54 # define FORMATS "{ S8, S16BE, S32BE, F32BE, F64BE }"
55 #endif
56 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
57
58 static GstStaticPadTemplate audio_sink_template =
59 GST_STATIC_PAD_TEMPLATE ("asink",
60 GST_PAD_SINK,
61 GST_PAD_ALWAYS,
62 GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (FORMATS))
63 );
64
65 static GstStaticPadTemplate audio_src_template =
66 GST_STATIC_PAD_TEMPLATE ("asrc",
67 GST_PAD_SRC,
68 GST_PAD_ALWAYS,
69 GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (FORMATS))
70 );
71
72 static GstStaticPadTemplate video_sink_template =
73 GST_STATIC_PAD_TEMPLATE ("vsink",
74 GST_PAD_SINK,
75 GST_PAD_ALWAYS,
76 GST_STATIC_CAPS ("video/x-raw")
77 );
78
79 static GstStaticPadTemplate video_src_template =
80 GST_STATIC_PAD_TEMPLATE ("vsrc",
81 GST_PAD_SRC,
82 GST_PAD_ALWAYS,
83 GST_STATIC_CAPS ("video/x-raw")
84 );
85
86 #define parent_class gst_videoframe_audiolevel_parent_class
87 G_DEFINE_TYPE (GstVideoFrameAudioLevel, gst_videoframe_audiolevel,
88 GST_TYPE_ELEMENT);
89
90 static GstFlowReturn gst_videoframe_audiolevel_asink_chain (GstPad * pad,
91 GstObject * parent, GstBuffer * inbuf);
92 static GstFlowReturn gst_videoframe_audiolevel_vsink_chain (GstPad * pad,
93 GstObject * parent, GstBuffer * inbuf);
94 static gboolean gst_videoframe_audiolevel_asink_event (GstPad * pad,
95 GstObject * parent, GstEvent * event);
96 static gboolean gst_videoframe_audiolevel_vsink_event (GstPad * pad,
97 GstObject * parent, GstEvent * event);
98 static GstIterator *gst_videoframe_audiolevel_iterate_internal_links (GstPad *
99 pad, GstObject * parent);
100
101 static void gst_videoframe_audiolevel_finalize (GObject * gobject);
102
103 static GstStateChangeReturn gst_videoframe_audiolevel_change_state (GstElement *
104 element, GstStateChange transition);
105
106 static void
gst_videoframe_audiolevel_class_init(GstVideoFrameAudioLevelClass * klass)107 gst_videoframe_audiolevel_class_init (GstVideoFrameAudioLevelClass * klass)
108 {
109 GstElementClass *gstelement_class;
110 GObjectClass *gobject_class = (GObjectClass *) klass;
111
112 GST_DEBUG_CATEGORY_INIT (gst_videoframe_audiolevel_debug,
113 "videoframe-audiolevel", 0, "Synchronized audio/video level");
114
115 gstelement_class = (GstElementClass *) klass;
116
117 gst_element_class_set_static_metadata (gstelement_class,
118 "Video-frame audio level", "Filter/Analyzer/Audio",
119 "Synchronized audio/video RMS Level messenger for audio/raw",
120 "Vivia Nikolaidou <vivia@toolsonair.com>");
121
122 gobject_class->finalize = gst_videoframe_audiolevel_finalize;
123 gstelement_class->change_state = gst_videoframe_audiolevel_change_state;
124
125 gst_element_class_add_static_pad_template (gstelement_class,
126 &audio_src_template);
127 gst_element_class_add_static_pad_template (gstelement_class,
128 &audio_sink_template);
129
130 gst_element_class_add_static_pad_template (gstelement_class,
131 &video_src_template);
132 gst_element_class_add_static_pad_template (gstelement_class,
133 &video_sink_template);
134 }
135
136 static void
gst_videoframe_audiolevel_init(GstVideoFrameAudioLevel * self)137 gst_videoframe_audiolevel_init (GstVideoFrameAudioLevel * self)
138 {
139 self->asinkpad =
140 gst_pad_new_from_static_template (&audio_sink_template, "asink");
141 gst_pad_set_chain_function (self->asinkpad,
142 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_asink_chain));
143 gst_pad_set_event_function (self->asinkpad,
144 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_asink_event));
145 gst_pad_set_iterate_internal_links_function (self->asinkpad,
146 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
147 gst_element_add_pad (GST_ELEMENT (self), self->asinkpad);
148
149 self->vsinkpad =
150 gst_pad_new_from_static_template (&video_sink_template, "vsink");
151 gst_pad_set_chain_function (self->vsinkpad,
152 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_vsink_chain));
153 gst_pad_set_event_function (self->vsinkpad,
154 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_vsink_event));
155 gst_pad_set_iterate_internal_links_function (self->vsinkpad,
156 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
157 gst_element_add_pad (GST_ELEMENT (self), self->vsinkpad);
158
159 self->asrcpad =
160 gst_pad_new_from_static_template (&audio_src_template, "asrc");
161 gst_pad_set_iterate_internal_links_function (self->asrcpad,
162 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
163 gst_element_add_pad (GST_ELEMENT (self), self->asrcpad);
164
165 self->vsrcpad =
166 gst_pad_new_from_static_template (&video_src_template, "vsrc");
167 gst_pad_set_iterate_internal_links_function (self->vsrcpad,
168 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
169 gst_element_add_pad (GST_ELEMENT (self), self->vsrcpad);
170
171 GST_PAD_SET_PROXY_CAPS (self->asinkpad);
172 GST_PAD_SET_PROXY_ALLOCATION (self->asinkpad);
173
174 GST_PAD_SET_PROXY_CAPS (self->asrcpad);
175 GST_PAD_SET_PROXY_SCHEDULING (self->asrcpad);
176
177 GST_PAD_SET_PROXY_CAPS (self->vsinkpad);
178 GST_PAD_SET_PROXY_ALLOCATION (self->vsinkpad);
179
180 GST_PAD_SET_PROXY_CAPS (self->vsrcpad);
181 GST_PAD_SET_PROXY_SCHEDULING (self->vsrcpad);
182
183 self->adapter = gst_adapter_new ();
184
185 g_queue_init (&self->vtimeq);
186 self->first_time = GST_CLOCK_TIME_NONE;
187 self->total_frames = 0;
188 /* alignment_threshold and discont_wait should become properties if needed */
189 self->alignment_threshold = 40 * GST_MSECOND;
190 self->discont_time = GST_CLOCK_TIME_NONE;
191 self->next_offset = -1;
192 self->discont_wait = 1 * GST_SECOND;
193
194 self->video_eos_flag = FALSE;
195 self->audio_flush_flag = FALSE;
196 self->shutdown_flag = FALSE;
197
198 g_mutex_init (&self->mutex);
199 g_cond_init (&self->cond);
200 }
201
202 static GstStateChangeReturn
gst_videoframe_audiolevel_change_state(GstElement * element,GstStateChange transition)203 gst_videoframe_audiolevel_change_state (GstElement * element,
204 GstStateChange transition)
205 {
206 GstStateChangeReturn ret;
207 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (element);
208
209 switch (transition) {
210 case GST_STATE_CHANGE_PAUSED_TO_READY:
211 g_mutex_lock (&self->mutex);
212 self->shutdown_flag = TRUE;
213 g_cond_signal (&self->cond);
214 g_mutex_unlock (&self->mutex);
215 break;
216 case GST_STATE_CHANGE_READY_TO_PAUSED:
217 g_mutex_lock (&self->mutex);
218 self->shutdown_flag = FALSE;
219 self->video_eos_flag = FALSE;
220 self->audio_flush_flag = FALSE;
221 g_mutex_unlock (&self->mutex);
222 default:
223 break;
224 }
225
226 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
227
228 switch (transition) {
229 case GST_STATE_CHANGE_PAUSED_TO_READY:
230 g_mutex_lock (&self->mutex);
231 self->first_time = GST_CLOCK_TIME_NONE;
232 self->total_frames = 0;
233 gst_segment_init (&self->asegment, GST_FORMAT_UNDEFINED);
234 gst_segment_init (&self->vsegment, GST_FORMAT_UNDEFINED);
235 self->vsegment.position = GST_CLOCK_TIME_NONE;
236 gst_adapter_clear (self->adapter);
237 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
238 g_queue_clear (&self->vtimeq);
239 if (self->CS) {
240 g_free (self->CS);
241 self->CS = NULL;
242 }
243 g_mutex_unlock (&self->mutex);
244 break;
245 default:
246 break;
247 }
248
249 return ret;
250 }
251
252 static void
gst_videoframe_audiolevel_finalize(GObject * object)253 gst_videoframe_audiolevel_finalize (GObject * object)
254 {
255 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (object);
256
257 if (self->adapter) {
258 g_object_unref (self->adapter);
259 self->adapter = NULL;
260 }
261 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
262 g_queue_clear (&self->vtimeq);
263 self->first_time = GST_CLOCK_TIME_NONE;
264 self->total_frames = 0;
265 if (self->CS) {
266 g_free (self->CS);
267 self->CS = NULL;
268 }
269
270 g_mutex_clear (&self->mutex);
271 g_cond_clear (&self->cond);
272
273 G_OBJECT_CLASS (parent_class)->finalize (object);
274 }
275
276 #define DEFINE_INT_LEVEL_CALCULATOR(TYPE, RESOLUTION) \
277 static void inline \
278 gst_videoframe_audiolevel_calculate_##TYPE (gpointer data, guint num, guint channels, \
279 gdouble *NCS) \
280 { \
281 TYPE * in = (TYPE *)data; \
282 register guint j; \
283 gdouble squaresum = 0.0; /* square sum of the input samples */ \
284 register gdouble square = 0.0; /* Square */ \
285 gdouble normalizer; /* divisor to get a [-1.0, 1.0] range */ \
286 \
287 /* *NCS = 0.0; Normalized Cumulative Square */ \
288 \
289 for (j = 0; j < num; j += channels) { \
290 square = ((gdouble) in[j]) * in[j]; \
291 squaresum += square; \
292 } \
293 \
294 normalizer = (gdouble) (G_GINT64_CONSTANT(1) << (RESOLUTION * 2)); \
295 *NCS = squaresum / normalizer; \
296 }
297
298 DEFINE_INT_LEVEL_CALCULATOR (gint32, 31);
299 DEFINE_INT_LEVEL_CALCULATOR (gint16, 15);
300 DEFINE_INT_LEVEL_CALCULATOR (gint8, 7);
301
302 #define DEFINE_FLOAT_LEVEL_CALCULATOR(TYPE) \
303 static void inline \
304 gst_videoframe_audiolevel_calculate_##TYPE (gpointer data, guint num, guint channels, \
305 gdouble *NCS) \
306 { \
307 TYPE * in = (TYPE *)data; \
308 register guint j; \
309 gdouble squaresum = 0.0; /* square sum of the input samples */ \
310 register gdouble square = 0.0; /* Square */ \
311 \
312 /* *NCS = 0.0; Normalized Cumulative Square */ \
313 \
314 for (j = 0; j < num; j += channels) { \
315 square = ((gdouble) in[j]) * in[j]; \
316 squaresum += square; \
317 } \
318 \
319 *NCS = squaresum; \
320 }
321
322 DEFINE_FLOAT_LEVEL_CALCULATOR (gfloat);
323 DEFINE_FLOAT_LEVEL_CALCULATOR (gdouble);
324
325 static gboolean
gst_videoframe_audiolevel_vsink_event(GstPad * pad,GstObject * parent,GstEvent * event)326 gst_videoframe_audiolevel_vsink_event (GstPad * pad, GstObject * parent,
327 GstEvent * event)
328 {
329 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
330 GST_LOG_OBJECT (pad, "Got %s event", GST_EVENT_TYPE_NAME (event));
331
332 switch (GST_EVENT_TYPE (event)) {
333 case GST_EVENT_SEGMENT:
334 g_mutex_lock (&self->mutex);
335 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
336 g_queue_clear (&self->vtimeq);
337 g_mutex_unlock (&self->mutex);
338 gst_event_copy_segment (event, &self->vsegment);
339 if (self->vsegment.format != GST_FORMAT_TIME)
340 return FALSE;
341 self->vsegment.position = GST_CLOCK_TIME_NONE;
342 break;
343 case GST_EVENT_GAP:
344 return TRUE;
345 case GST_EVENT_EOS:
346 g_mutex_lock (&self->mutex);
347 self->video_eos_flag = TRUE;
348 g_cond_signal (&self->cond);
349 g_mutex_unlock (&self->mutex);
350 break;
351 case GST_EVENT_FLUSH_STOP:
352 g_mutex_lock (&self->mutex);
353 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
354 g_queue_clear (&self->vtimeq);
355 gst_segment_init (&self->vsegment, GST_FORMAT_UNDEFINED);
356 g_cond_signal (&self->cond);
357 g_mutex_unlock (&self->mutex);
358 self->vsegment.position = GST_CLOCK_TIME_NONE;
359 break;
360 default:
361 break;
362 }
363 return gst_pad_event_default (pad, parent, event);
364 }
365
366 static gboolean
gst_videoframe_audiolevel_asink_event(GstPad * pad,GstObject * parent,GstEvent * event)367 gst_videoframe_audiolevel_asink_event (GstPad * pad, GstObject * parent,
368 GstEvent * event)
369 {
370 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
371 GST_LOG_OBJECT (pad, "Got %s event", GST_EVENT_TYPE_NAME (event));
372
373 switch (GST_EVENT_TYPE (event)) {
374 case GST_EVENT_SEGMENT:
375 self->first_time = GST_CLOCK_TIME_NONE;
376 self->total_frames = 0;
377 gst_adapter_clear (self->adapter);
378 gst_event_copy_segment (event, &self->asegment);
379 if (self->asegment.format != GST_FORMAT_TIME)
380 return FALSE;
381 break;
382 case GST_EVENT_FLUSH_START:
383 g_mutex_lock (&self->mutex);
384 self->audio_flush_flag = TRUE;
385 g_cond_signal (&self->cond);
386 g_mutex_unlock (&self->mutex);
387 break;
388 case GST_EVENT_FLUSH_STOP:
389 self->audio_flush_flag = FALSE;
390 self->total_frames = 0;
391 self->first_time = GST_CLOCK_TIME_NONE;
392 gst_adapter_clear (self->adapter);
393 gst_segment_init (&self->asegment, GST_FORMAT_UNDEFINED);
394 break;
395 case GST_EVENT_CAPS:{
396 GstCaps *caps;
397 gint channels;
398 gst_event_parse_caps (event, &caps);
399 GST_DEBUG_OBJECT (self, "Got caps %" GST_PTR_FORMAT, caps);
400 if (!gst_audio_info_from_caps (&self->ainfo, caps))
401 return FALSE;
402 switch (GST_AUDIO_INFO_FORMAT (&self->ainfo)) {
403 case GST_AUDIO_FORMAT_S8:
404 self->process = gst_videoframe_audiolevel_calculate_gint8;
405 break;
406 case GST_AUDIO_FORMAT_S16:
407 self->process = gst_videoframe_audiolevel_calculate_gint16;
408 break;
409 case GST_AUDIO_FORMAT_S32:
410 self->process = gst_videoframe_audiolevel_calculate_gint32;
411 break;
412 case GST_AUDIO_FORMAT_F32:
413 self->process = gst_videoframe_audiolevel_calculate_gfloat;
414 break;
415 case GST_AUDIO_FORMAT_F64:
416 self->process = gst_videoframe_audiolevel_calculate_gdouble;
417 break;
418 default:
419 self->process = NULL;
420 break;
421 }
422 gst_adapter_clear (self->adapter);
423 channels = GST_AUDIO_INFO_CHANNELS (&self->ainfo);
424 self->first_time = GST_CLOCK_TIME_NONE;
425 self->total_frames = 0;
426 if (self->CS)
427 g_free (self->CS);
428 self->CS = g_new0 (gdouble, channels);
429 break;
430 }
431 default:
432 break;
433 }
434
435 return gst_pad_event_default (pad, parent, event);
436 }
437
438 static GstMessage *
update_rms_from_buffer(GstVideoFrameAudioLevel * self,GstBuffer * inbuf)439 update_rms_from_buffer (GstVideoFrameAudioLevel * self, GstBuffer * inbuf)
440 {
441 GstMapInfo map;
442 guint8 *in_data;
443 gsize in_size;
444 gdouble CS;
445 guint i;
446 guint num_frames, frames;
447 guint num_int_samples = 0; /* number of interleaved samples
448 * ie. total count for all channels combined */
449 gint channels, rate, bps;
450 GValue v = G_VALUE_INIT;
451 GValue va = G_VALUE_INIT;
452 GValueArray *a;
453 GstStructure *s;
454 GstMessage *msg;
455 GstClockTime duration, running_time;
456
457 channels = GST_AUDIO_INFO_CHANNELS (&self->ainfo);
458 bps = GST_AUDIO_INFO_BPS (&self->ainfo);
459 rate = GST_AUDIO_INFO_RATE (&self->ainfo);
460
461 gst_buffer_map (inbuf, &map, GST_MAP_READ);
462 in_data = map.data;
463 in_size = map.size;
464
465 num_int_samples = in_size / bps;
466
467 GST_LOG_OBJECT (self, "analyzing %u sample frames at ts %" GST_TIME_FORMAT,
468 num_int_samples, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (inbuf)));
469
470 g_return_val_if_fail (num_int_samples % channels == 0, NULL);
471
472 num_frames = num_int_samples / channels;
473 frames = num_frames;
474 duration = GST_FRAMES_TO_CLOCK_TIME (frames, rate);
475 if (num_frames > 0) {
476 for (i = 0; i < channels; ++i) {
477 self->process (in_data + (bps * i), num_int_samples, channels, &CS);
478 GST_LOG_OBJECT (self,
479 "[%d]: cumulative squares %lf, over %d samples/%d channels",
480 i, CS, num_int_samples, channels);
481 self->CS[i] += CS;
482 }
483 in_data += num_frames * bps;
484
485 self->total_frames += num_frames;
486 }
487 running_time =
488 self->first_time + gst_util_uint64_scale (self->total_frames, GST_SECOND,
489 rate);
490
491 a = g_value_array_new (channels);
492 s = gst_structure_new ("videoframe-audiolevel", "running-time", G_TYPE_UINT64,
493 running_time, "duration", G_TYPE_UINT64, duration, NULL);
494
495 g_value_init (&v, G_TYPE_DOUBLE);
496 g_value_init (&va, G_TYPE_VALUE_ARRAY);
497 for (i = 0; i < channels; i++) {
498 gdouble rms;
499 if (frames == 0 || self->CS[i] == 0) {
500 rms = 0; /* empty buffer */
501 } else {
502 rms = sqrt (self->CS[i] / frames);
503 }
504 self->CS[i] = 0.0;
505 g_value_set_double (&v, rms);
506 g_value_array_append (a, &v);
507 }
508 g_value_take_boxed (&va, a);
509 gst_structure_take_value (s, "rms", &va);
510 msg = gst_message_new_element (GST_OBJECT (self), s);
511
512 gst_buffer_unmap (inbuf, &map);
513
514 return msg;
515 }
516
517 static GstFlowReturn
gst_videoframe_audiolevel_vsink_chain(GstPad * pad,GstObject * parent,GstBuffer * inbuf)518 gst_videoframe_audiolevel_vsink_chain (GstPad * pad, GstObject * parent,
519 GstBuffer * inbuf)
520 {
521 GstClockTime timestamp;
522 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
523 GstClockTime duration;
524 GstClockTime *ptrtime = g_new (GstClockTime, 1);
525
526 timestamp = GST_BUFFER_TIMESTAMP (inbuf);
527 *ptrtime =
528 gst_segment_to_running_time (&self->vsegment, GST_FORMAT_TIME, timestamp);
529 g_mutex_lock (&self->mutex);
530 self->vsegment.position = timestamp;
531 duration = GST_BUFFER_DURATION (inbuf);
532 if (duration != GST_CLOCK_TIME_NONE)
533 self->vsegment.position += duration;
534 g_queue_push_tail (&self->vtimeq, ptrtime);
535 g_cond_signal (&self->cond);
536 GST_DEBUG_OBJECT (pad, "Pushed a frame");
537 g_mutex_unlock (&self->mutex);
538 return gst_pad_push (self->vsrcpad, inbuf);
539 }
540
541 static GstFlowReturn
gst_videoframe_audiolevel_asink_chain(GstPad * pad,GstObject * parent,GstBuffer * inbuf)542 gst_videoframe_audiolevel_asink_chain (GstPad * pad, GstObject * parent,
543 GstBuffer * inbuf)
544 {
545 GstClockTime timestamp, cur_time;
546 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
547 GstBuffer *buf;
548 gsize inbuf_size;
549 guint64 start_offset, end_offset;
550 GstClockTime running_time;
551 gint rate, bpf;
552 gboolean discont = FALSE;
553
554 timestamp = GST_BUFFER_TIMESTAMP (inbuf);
555 running_time =
556 gst_segment_to_running_time (&self->asegment, GST_FORMAT_TIME, timestamp);
557
558 rate = GST_AUDIO_INFO_RATE (&self->ainfo);
559 bpf = GST_AUDIO_INFO_BPF (&self->ainfo);
560 start_offset = gst_util_uint64_scale (timestamp, rate, GST_SECOND);
561 inbuf_size = gst_buffer_get_size (inbuf);
562 end_offset = start_offset + inbuf_size / bpf;
563
564 g_mutex_lock (&self->mutex);
565
566 if (GST_BUFFER_IS_DISCONT (inbuf)
567 || GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_RESYNC)
568 || self->first_time == GST_CLOCK_TIME_NONE) {
569 discont = TRUE;
570 } else {
571 guint64 diff, max_sample_diff;
572
573 /* Check discont, based on audiobasesink */
574 if (start_offset <= self->next_offset)
575 diff = self->next_offset - start_offset;
576 else
577 diff = start_offset - self->next_offset;
578
579 max_sample_diff =
580 gst_util_uint64_scale_int (self->alignment_threshold, rate, GST_SECOND);
581
582 /* Discont! */
583 if (G_UNLIKELY (diff >= max_sample_diff)) {
584 if (self->discont_wait > 0) {
585 if (self->discont_time == GST_CLOCK_TIME_NONE) {
586 self->discont_time = timestamp;
587 } else if (timestamp - self->discont_time >= self->discont_wait) {
588 discont = TRUE;
589 self->discont_time = GST_CLOCK_TIME_NONE;
590 }
591 } else {
592 discont = TRUE;
593 }
594 } else if (G_UNLIKELY (self->discont_time != GST_CLOCK_TIME_NONE)) {
595 /* we have had a discont, but are now back on track! */
596 self->discont_time = GST_CLOCK_TIME_NONE;
597 }
598 }
599
600 if (discont) {
601 /* Have discont, need resync */
602 if (self->next_offset != -1)
603 GST_INFO_OBJECT (pad, "Have discont. Expected %"
604 G_GUINT64_FORMAT ", got %" G_GUINT64_FORMAT,
605 self->next_offset, start_offset);
606 self->total_frames = 0;
607 self->first_time = running_time;
608 self->next_offset = end_offset;
609 } else {
610 self->next_offset += inbuf_size / bpf;
611 }
612
613 gst_adapter_push (self->adapter, gst_buffer_ref (inbuf));
614
615 GST_DEBUG_OBJECT (self, "Queue length %i",
616 g_queue_get_length (&self->vtimeq));
617
618 while (TRUE) {
619 GstClockTime *vt0, *vt1;
620 GstClockTime vtemp;
621 GstMessage *msg;
622 gsize bytes, available_bytes;
623
624 vtemp = GST_CLOCK_TIME_NONE;
625
626 while (!(g_queue_get_length (&self->vtimeq) >= 2 || self->video_eos_flag
627 || self->audio_flush_flag || self->shutdown_flag))
628 g_cond_wait (&self->cond, &self->mutex);
629
630 if (self->audio_flush_flag || self->shutdown_flag) {
631 g_mutex_unlock (&self->mutex);
632 gst_buffer_unref (inbuf);
633 return GST_FLOW_FLUSHING;
634 } else if (self->video_eos_flag) {
635 GST_DEBUG_OBJECT (self, "Video EOS flag alert");
636 /* nothing to do here if queue is empty */
637 if (g_queue_get_length (&self->vtimeq) == 0)
638 break;
639
640 if (g_queue_get_length (&self->vtimeq) < 2) {
641 vtemp = self->vsegment.position;
642 } else if (self->vsegment.position == GST_CLOCK_TIME_NONE) {
643 /* g_queue_get_length is surely >= 2 at this point
644 * so the adapter isn't empty */
645 buf =
646 gst_adapter_take_buffer (self->adapter,
647 gst_adapter_available (self->adapter));
648 if (buf != NULL) {
649 GstMessage *msg;
650 msg = update_rms_from_buffer (self, buf);
651 g_mutex_unlock (&self->mutex);
652 gst_element_post_message (GST_ELEMENT (self), msg);
653 gst_buffer_unref (buf);
654 g_mutex_lock (&self->mutex); /* we unlock again later */
655 }
656 break;
657 }
658 } else if (g_queue_get_length (&self->vtimeq) < 2) {
659 continue;
660 }
661
662 vt0 = g_queue_pop_head (&self->vtimeq);
663 if (vtemp == GST_CLOCK_TIME_NONE)
664 vt1 = g_queue_peek_head (&self->vtimeq);
665 else
666 vt1 = &vtemp;
667
668 cur_time =
669 self->first_time + gst_util_uint64_scale (self->total_frames,
670 GST_SECOND, rate);
671 GST_DEBUG_OBJECT (self,
672 "Processing: current time is %" GST_TIME_FORMAT,
673 GST_TIME_ARGS (cur_time));
674 GST_DEBUG_OBJECT (self, "Total frames is %i with a rate of %d",
675 self->total_frames, rate);
676 GST_DEBUG_OBJECT (self, "Start time is %" GST_TIME_FORMAT,
677 GST_TIME_ARGS (self->first_time));
678 GST_DEBUG_OBJECT (self, "Time on top is %" GST_TIME_FORMAT,
679 GST_TIME_ARGS (*vt0));
680
681 if (cur_time < *vt0) {
682 guint num_frames =
683 gst_util_uint64_scale (*vt0 - cur_time, rate, GST_SECOND);
684 bytes = num_frames * GST_AUDIO_INFO_BPF (&self->ainfo);
685 available_bytes = gst_adapter_available (self->adapter);
686 if (available_bytes == 0) {
687 g_queue_push_head (&self->vtimeq, vt0);
688 break;
689 }
690 if (bytes == 0) {
691 cur_time = *vt0;
692 } else {
693 GST_DEBUG_OBJECT (self,
694 "Flushed %" G_GSIZE_FORMAT " out of %" G_GSIZE_FORMAT " bytes",
695 bytes, available_bytes);
696 gst_adapter_flush (self->adapter, MIN (bytes, available_bytes));
697 self->total_frames += num_frames;
698 if (available_bytes <= bytes) {
699 g_queue_push_head (&self->vtimeq, vt0);
700 break;
701 }
702 cur_time =
703 self->first_time + gst_util_uint64_scale (self->total_frames,
704 GST_SECOND, rate);
705 }
706 }
707 if (*vt1 > cur_time) {
708 bytes =
709 GST_AUDIO_INFO_BPF (&self->ainfo) * gst_util_uint64_scale (*vt1 -
710 cur_time, rate, GST_SECOND);
711 } else {
712 bytes = 0; /* We just need to discard vt0 */
713 }
714 available_bytes = gst_adapter_available (self->adapter);
715 GST_DEBUG_OBJECT (self,
716 "Adapter contains %" G_GSIZE_FORMAT " out of %" G_GSIZE_FORMAT " bytes",
717 available_bytes, bytes);
718
719 if (available_bytes < bytes) {
720 g_queue_push_head (&self->vtimeq, vt0);
721 goto done;
722 }
723
724 if (bytes > 0) {
725 buf = gst_adapter_take_buffer (self->adapter, bytes);
726 g_assert (buf != NULL);
727 } else {
728 /* Just an empty buffer */
729 buf = gst_buffer_new ();
730 }
731 msg = update_rms_from_buffer (self, buf);
732 g_mutex_unlock (&self->mutex);
733 gst_element_post_message (GST_ELEMENT (self), msg);
734 g_mutex_lock (&self->mutex);
735
736 gst_buffer_unref (buf);
737 g_free (vt0);
738 if (available_bytes == bytes)
739 break;
740 }
741 done:
742 g_mutex_unlock (&self->mutex);
743 return gst_pad_push (self->asrcpad, inbuf);
744 }
745
746 static GstIterator *
gst_videoframe_audiolevel_iterate_internal_links(GstPad * pad,GstObject * parent)747 gst_videoframe_audiolevel_iterate_internal_links (GstPad * pad,
748 GstObject * parent)
749 {
750 GstIterator *it = NULL;
751 GstPad *opad;
752 GValue val = { 0, };
753 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
754
755 if (self->asinkpad == pad)
756 opad = gst_object_ref (self->asrcpad);
757 else if (self->asrcpad == pad)
758 opad = gst_object_ref (self->asinkpad);
759 else if (self->vsinkpad == pad)
760 opad = gst_object_ref (self->vsrcpad);
761 else if (self->vsrcpad == pad)
762 opad = gst_object_ref (self->vsinkpad);
763 else
764 goto out;
765
766 g_value_init (&val, GST_TYPE_PAD);
767 g_value_set_object (&val, opad);
768 it = gst_iterator_new_single (GST_TYPE_PAD, &val);
769 g_value_unset (&val);
770
771 gst_object_unref (opad);
772
773 out:
774 return it;
775 }
776
777 static gboolean
gst_videoframe_audiolevel_plugin_init(GstPlugin * plugin)778 gst_videoframe_audiolevel_plugin_init (GstPlugin * plugin)
779 {
780 return gst_element_register (plugin, "videoframe-audiolevel",
781 GST_RANK_NONE, GST_TYPE_VIDEOFRAME_AUDIOLEVEL);
782 }
783
784 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
785 GST_VERSION_MINOR,
786 videoframe_audiolevel,
787 "Video frame-synchronized audio level",
788 gst_videoframe_audiolevel_plugin_init, VERSION, GST_LICENSE,
789 GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN);
790