1 /* GStreamer
2 * Copyright (C) 2011 David Schleef <ds@entropywave.com>
3 * Copyright (C) 2014 Sebastian Dröge <sebastian@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
18 * Boston, MA 02110-1335, USA.
19 */
20 /**
21 * SECTION:element-decklinkaudiosrc
22 * @short_description: Inputs Audio from a BlackMagic DeckLink Device
23 * @see_also: decklinkvideosrc
24 *
25 * Capture Video and Audio from a BlackMagic DeckLink Device. Can only be used
26 * in conjunction with decklinkvideosink.
27 *
28 * ## Sample pipeline
29 * |[
30 * gst-launch-1.0 \
31 * decklinkvideosrc device-number=0 mode=1080p25 ! autovideosink \
32 * decklinkaudiosrc device-number=0 ! autoaudiosink
33 * ]|
34 * Capturing 1080p25 video and audio from the SDI-In of Card 0. Devices are numbered
35 * starting with 0.
36 */
37
38 #ifdef HAVE_CONFIG_H
39 #include "config.h"
40 #endif
41
42 #include "gstdecklinkaudiosrc.h"
43 #include "gstdecklinkvideosrc.h"
44 #include <string.h>
45
46 GST_DEBUG_CATEGORY_STATIC (gst_decklink_audio_src_debug);
47 #define GST_CAT_DEFAULT gst_decklink_audio_src_debug
48
49 #define DEFAULT_CONNECTION (GST_DECKLINK_AUDIO_CONNECTION_AUTO)
50 #define DEFAULT_BUFFER_SIZE (5)
51
52 #define DEFAULT_ALIGNMENT_THRESHOLD (40 * GST_MSECOND)
53 #define DEFAULT_DISCONT_WAIT (1 * GST_SECOND)
54 #define DEFAULT_CHANNELS (GST_DECKLINK_AUDIO_CHANNELS_2)
55
56 #ifndef ABSDIFF
57 #define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )
58 #endif
59
60 enum
61 {
62 PROP_0,
63 PROP_CONNECTION,
64 PROP_DEVICE_NUMBER,
65 PROP_ALIGNMENT_THRESHOLD,
66 PROP_DISCONT_WAIT,
67 PROP_BUFFER_SIZE,
68 PROP_CHANNELS,
69 PROP_HW_SERIAL_NUMBER
70 };
71
72 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("src",
73 GST_PAD_SRC,
74 GST_PAD_ALWAYS,
75 GST_STATIC_CAPS
76 ("audio/x-raw, format={S16LE,S32LE}, channels=2, rate=48000, "
77 "layout=interleaved;"
78 "audio/x-raw, format={S16LE,S32LE}, channels={8,16}, channel-mask=(bitmask)0, rate=48000, "
79 "layout=interleaved")
80 );
81
82 typedef struct
83 {
84 IDeckLinkAudioInputPacket *packet;
85 GstClockTime timestamp;
86 GstClockTime stream_timestamp;
87 GstClockTime stream_duration;
88 GstClockTime hardware_timestamp;
89 GstClockTime hardware_duration;
90 gboolean no_signal;
91 } CapturePacket;
92
93 static void
capture_packet_clear(CapturePacket * packet)94 capture_packet_clear (CapturePacket * packet)
95 {
96 packet->packet->Release ();
97 memset (packet, 0, sizeof (*packet));
98 }
99
100 typedef struct
101 {
102 IDeckLinkAudioInputPacket *packet;
103 IDeckLinkInput *input;
104 } AudioPacket;
105
106 static void
audio_packet_free(void * data)107 audio_packet_free (void *data)
108 {
109 AudioPacket *packet = (AudioPacket *) data;
110
111 packet->packet->Release ();
112 packet->input->Release ();
113 g_free (packet);
114 }
115
116 static void gst_decklink_audio_src_set_property (GObject * object,
117 guint property_id, const GValue * value, GParamSpec * pspec);
118 static void gst_decklink_audio_src_get_property (GObject * object,
119 guint property_id, GValue * value, GParamSpec * pspec);
120 static void gst_decklink_audio_src_finalize (GObject * object);
121
122 static GstStateChangeReturn
123 gst_decklink_audio_src_change_state (GstElement * element,
124 GstStateChange transition);
125
126 static gboolean gst_decklink_audio_src_unlock (GstBaseSrc * bsrc);
127 static gboolean gst_decklink_audio_src_unlock_stop (GstBaseSrc * bsrc);
128 static gboolean gst_decklink_audio_src_query (GstBaseSrc * bsrc,
129 GstQuery * query);
130
131 static GstFlowReturn gst_decklink_audio_src_create (GstPushSrc * psrc,
132 GstBuffer ** buffer);
133
134 static gboolean gst_decklink_audio_src_open (GstDecklinkAudioSrc * self);
135 static gboolean gst_decklink_audio_src_close (GstDecklinkAudioSrc * self);
136
137 static gboolean gst_decklink_audio_src_stop (GstDecklinkAudioSrc * self);
138
139 #define parent_class gst_decklink_audio_src_parent_class
140 G_DEFINE_TYPE (GstDecklinkAudioSrc, gst_decklink_audio_src, GST_TYPE_PUSH_SRC);
141
142 static void
gst_decklink_audio_src_class_init(GstDecklinkAudioSrcClass * klass)143 gst_decklink_audio_src_class_init (GstDecklinkAudioSrcClass * klass)
144 {
145 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
146 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
147 GstBaseSrcClass *basesrc_class = GST_BASE_SRC_CLASS (klass);
148 GstPushSrcClass *pushsrc_class = GST_PUSH_SRC_CLASS (klass);
149
150 gobject_class->set_property = gst_decklink_audio_src_set_property;
151 gobject_class->get_property = gst_decklink_audio_src_get_property;
152 gobject_class->finalize = gst_decklink_audio_src_finalize;
153
154 element_class->change_state =
155 GST_DEBUG_FUNCPTR (gst_decklink_audio_src_change_state);
156
157 basesrc_class->query = GST_DEBUG_FUNCPTR (gst_decklink_audio_src_query);
158 basesrc_class->negotiate = NULL;
159 basesrc_class->unlock = GST_DEBUG_FUNCPTR (gst_decklink_audio_src_unlock);
160 basesrc_class->unlock_stop =
161 GST_DEBUG_FUNCPTR (gst_decklink_audio_src_unlock_stop);
162
163 pushsrc_class->create = GST_DEBUG_FUNCPTR (gst_decklink_audio_src_create);
164
165 g_object_class_install_property (gobject_class, PROP_CONNECTION,
166 g_param_spec_enum ("connection", "Connection",
167 "Audio input connection to use",
168 GST_TYPE_DECKLINK_AUDIO_CONNECTION, DEFAULT_CONNECTION,
169 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
170 G_PARAM_CONSTRUCT)));
171
172 g_object_class_install_property (gobject_class, PROP_DEVICE_NUMBER,
173 g_param_spec_int ("device-number", "Device number",
174 "Output device instance to use", 0, G_MAXINT, 0,
175 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
176 G_PARAM_CONSTRUCT)));
177
178 g_object_class_install_property (gobject_class, PROP_ALIGNMENT_THRESHOLD,
179 g_param_spec_uint64 ("alignment-threshold", "Alignment Threshold",
180 "Timestamp alignment threshold in nanoseconds", 0,
181 G_MAXUINT64 - 1, DEFAULT_ALIGNMENT_THRESHOLD,
182 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
183
184 g_object_class_install_property (gobject_class, PROP_DISCONT_WAIT,
185 g_param_spec_uint64 ("discont-wait", "Discont Wait",
186 "Window of time in nanoseconds to wait before "
187 "creating a discontinuity", 0,
188 G_MAXUINT64 - 1, DEFAULT_DISCONT_WAIT,
189 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
190
191 g_object_class_install_property (gobject_class, PROP_BUFFER_SIZE,
192 g_param_spec_uint ("buffer-size", "Buffer Size",
193 "Size of internal buffer in number of video frames", 1,
194 G_MAXINT, DEFAULT_BUFFER_SIZE,
195 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
196
197 g_object_class_install_property (gobject_class, PROP_CHANNELS,
198 g_param_spec_enum ("channels", "Channels",
199 "Audio channels",
200 GST_TYPE_DECKLINK_AUDIO_CHANNELS, DEFAULT_CHANNELS,
201 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
202 G_PARAM_CONSTRUCT)));
203
204 g_object_class_install_property (gobject_class, PROP_HW_SERIAL_NUMBER,
205 g_param_spec_string ("hw-serial-number", "Hardware serial number",
206 "The serial number (hardware ID) of the Decklink card",
207 NULL, (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
208
209 gst_element_class_add_static_pad_template (element_class, &sink_template);
210
211 gst_element_class_set_static_metadata (element_class, "Decklink Audio Source",
212 "Audio/Source/Hardware", "Decklink Source",
213 "David Schleef <ds@entropywave.com>, "
214 "Sebastian Dröge <sebastian@centricular.com>");
215
216 GST_DEBUG_CATEGORY_INIT (gst_decklink_audio_src_debug, "decklinkaudiosrc",
217 0, "debug category for decklinkaudiosrc element");
218 }
219
220 static void
gst_decklink_audio_src_init(GstDecklinkAudioSrc * self)221 gst_decklink_audio_src_init (GstDecklinkAudioSrc * self)
222 {
223 self->device_number = 0;
224 self->alignment_threshold = DEFAULT_ALIGNMENT_THRESHOLD;
225 self->discont_wait = DEFAULT_DISCONT_WAIT;
226 self->buffer_size = DEFAULT_BUFFER_SIZE;
227 self->channels = DEFAULT_CHANNELS;
228
229 gst_base_src_set_live (GST_BASE_SRC (self), TRUE);
230 gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME);
231
232 gst_pad_use_fixed_caps (GST_BASE_SRC_PAD (self));
233
234 g_mutex_init (&self->lock);
235 g_cond_init (&self->cond);
236
237 self->current_packets =
238 gst_queue_array_new_for_struct (sizeof (CapturePacket),
239 DEFAULT_BUFFER_SIZE);
240 }
241
242 void
gst_decklink_audio_src_set_property(GObject * object,guint property_id,const GValue * value,GParamSpec * pspec)243 gst_decklink_audio_src_set_property (GObject * object, guint property_id,
244 const GValue * value, GParamSpec * pspec)
245 {
246 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (object);
247
248 switch (property_id) {
249 case PROP_CONNECTION:
250 self->connection =
251 (GstDecklinkAudioConnectionEnum) g_value_get_enum (value);
252 break;
253 case PROP_DEVICE_NUMBER:
254 self->device_number = g_value_get_int (value);
255 break;
256 case PROP_ALIGNMENT_THRESHOLD:
257 self->alignment_threshold = g_value_get_uint64 (value);
258 break;
259 case PROP_DISCONT_WAIT:
260 self->discont_wait = g_value_get_uint64 (value);
261 break;
262 case PROP_BUFFER_SIZE:
263 self->buffer_size = g_value_get_uint (value);
264 break;
265 case PROP_CHANNELS:
266 self->channels = (GstDecklinkAudioChannelsEnum) g_value_get_enum (value);
267 break;
268 default:
269 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
270 break;
271 }
272 }
273
274 void
gst_decklink_audio_src_get_property(GObject * object,guint property_id,GValue * value,GParamSpec * pspec)275 gst_decklink_audio_src_get_property (GObject * object, guint property_id,
276 GValue * value, GParamSpec * pspec)
277 {
278 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (object);
279
280 switch (property_id) {
281 case PROP_CONNECTION:
282 g_value_set_enum (value, self->connection);
283 break;
284 case PROP_DEVICE_NUMBER:
285 g_value_set_int (value, self->device_number);
286 break;
287 case PROP_ALIGNMENT_THRESHOLD:
288 g_value_set_uint64 (value, self->alignment_threshold);
289 break;
290 case PROP_DISCONT_WAIT:
291 g_value_set_uint64 (value, self->discont_wait);
292 break;
293 case PROP_BUFFER_SIZE:
294 g_value_set_uint (value, self->buffer_size);
295 break;
296 case PROP_CHANNELS:
297 g_value_set_enum (value, self->channels);
298 break;
299 case PROP_HW_SERIAL_NUMBER:
300 if (self->input)
301 g_value_set_string (value, self->input->hw_serial_number);
302 else
303 g_value_set_string (value, NULL);
304 break;
305 default:
306 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
307 break;
308 }
309 }
310
311 void
gst_decklink_audio_src_finalize(GObject * object)312 gst_decklink_audio_src_finalize (GObject * object)
313 {
314 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (object);
315
316 g_mutex_clear (&self->lock);
317 g_cond_clear (&self->cond);
318 if (self->current_packets) {
319 while (gst_queue_array_get_length (self->current_packets) > 0) {
320 CapturePacket *tmp = (CapturePacket *)
321 gst_queue_array_pop_head_struct (self->current_packets);
322 capture_packet_clear (tmp);
323 }
324 gst_queue_array_free (self->current_packets);
325 self->current_packets = NULL;
326 }
327
328 G_OBJECT_CLASS (parent_class)->finalize (object);
329 }
330
331 static gboolean
gst_decklink_audio_src_start(GstDecklinkAudioSrc * self)332 gst_decklink_audio_src_start (GstDecklinkAudioSrc * self)
333 {
334 BMDAudioSampleType sample_depth;
335 HRESULT ret;
336 BMDAudioConnection conn = (BMDAudioConnection) - 1;
337 GstCaps *allowed_caps, *caps;
338
339 g_mutex_lock (&self->input->lock);
340 if (self->input->audio_enabled) {
341 g_mutex_unlock (&self->input->lock);
342 return TRUE;
343 }
344 g_mutex_unlock (&self->input->lock);
345
346 /* Negotiate the format / sample depth with downstream */
347 allowed_caps = gst_pad_get_allowed_caps (GST_BASE_SRC_PAD (self));
348 if (!allowed_caps)
349 allowed_caps = gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (self));
350
351 sample_depth = bmdAudioSampleType32bitInteger;
352 if (!gst_caps_is_empty (allowed_caps)) {
353 GstStructure *s;
354
355 allowed_caps = gst_caps_simplify (allowed_caps);
356
357 s = gst_caps_get_structure (allowed_caps, 0);
358
359 /* If it's not a string then both formats are supported */
360 if (gst_structure_has_field_typed (s, "format", G_TYPE_STRING)) {
361 const gchar *format = gst_structure_get_string (s, "format");
362 if (g_str_equal (format, "S16LE")) {
363 sample_depth = bmdAudioSampleType16bitInteger;
364 }
365 }
366 }
367 gst_caps_unref (allowed_caps);
368
369 switch (self->connection) {
370 case GST_DECKLINK_AUDIO_CONNECTION_AUTO:{
371 GstElement *videosrc = NULL;
372 GstDecklinkConnectionEnum vconn;
373
374 // Try to get the connection from the videosrc and try
375 // to select a sensible audio connection based on that
376 g_mutex_lock (&self->input->lock);
377 if (self->input->videosrc)
378 videosrc = GST_ELEMENT_CAST (gst_object_ref (self->input->videosrc));
379 g_mutex_unlock (&self->input->lock);
380
381 if (videosrc) {
382 g_object_get (videosrc, "connection", &vconn, NULL);
383 gst_object_unref (videosrc);
384
385 switch (vconn) {
386 case GST_DECKLINK_CONNECTION_SDI:
387 conn = bmdAudioConnectionEmbedded;
388 break;
389 case GST_DECKLINK_CONNECTION_HDMI:
390 conn = bmdAudioConnectionEmbedded;
391 break;
392 case GST_DECKLINK_CONNECTION_OPTICAL_SDI:
393 conn = bmdAudioConnectionEmbedded;
394 break;
395 case GST_DECKLINK_CONNECTION_COMPONENT:
396 conn = bmdAudioConnectionAnalog;
397 break;
398 case GST_DECKLINK_CONNECTION_COMPOSITE:
399 conn = bmdAudioConnectionAnalog;
400 break;
401 case GST_DECKLINK_CONNECTION_SVIDEO:
402 conn = bmdAudioConnectionAnalog;
403 break;
404 default:
405 // Use default
406 break;
407 }
408 }
409
410 break;
411 }
412 case GST_DECKLINK_AUDIO_CONNECTION_EMBEDDED:
413 conn = bmdAudioConnectionEmbedded;
414 break;
415 case GST_DECKLINK_AUDIO_CONNECTION_AES_EBU:
416 conn = bmdAudioConnectionAESEBU;
417 break;
418 case GST_DECKLINK_AUDIO_CONNECTION_ANALOG:
419 conn = bmdAudioConnectionAnalog;
420 break;
421 case GST_DECKLINK_AUDIO_CONNECTION_ANALOG_XLR:
422 conn = bmdAudioConnectionAnalogXLR;
423 break;
424 case GST_DECKLINK_AUDIO_CONNECTION_ANALOG_RCA:
425 conn = bmdAudioConnectionAnalogRCA;
426 break;
427 default:
428 g_assert_not_reached ();
429 break;
430 }
431
432 if (conn != (BMDAudioConnection) - 1) {
433 ret =
434 self->input->config->SetInt (bmdDeckLinkConfigAudioInputConnection,
435 conn);
436 if (ret != S_OK) {
437 GST_ERROR ("set configuration (audio input connection): 0x%08lx",
438 (unsigned long) ret);
439 return FALSE;
440 }
441 }
442
443 ret = self->input->input->EnableAudioInput (bmdAudioSampleRate48kHz,
444 sample_depth, self->channels_found);
445 if (ret != S_OK) {
446 GST_WARNING_OBJECT (self, "Failed to enable audio input: 0x%08lx",
447 (unsigned long) ret);
448 return FALSE;
449 }
450 gst_audio_info_set_format (&self->info,
451 sample_depth ==
452 bmdAudioSampleType16bitInteger ? GST_AUDIO_FORMAT_S16LE :
453 GST_AUDIO_FORMAT_S32LE, 48000, self->channels_found, NULL);
454
455 g_mutex_lock (&self->input->lock);
456 self->input->audio_enabled = TRUE;
457 if (self->input->start_streams && self->input->videosrc)
458 self->input->start_streams (self->input->videosrc);
459 g_mutex_unlock (&self->input->lock);
460
461 caps = gst_audio_info_to_caps (&self->info);
462 if (!gst_base_src_set_caps (GST_BASE_SRC (self), caps)) {
463 gst_caps_unref (caps);
464 GST_WARNING_OBJECT (self, "Failed to set caps");
465 return FALSE;
466 }
467 gst_caps_unref (caps);
468
469 return TRUE;
470 }
471
472 static void
gst_decklink_audio_src_got_packet(GstElement * element,IDeckLinkAudioInputPacket * packet,GstClockTime capture_time,GstClockTime stream_time,GstClockTime stream_duration,GstClockTime hardware_time,GstClockTime hardware_duration,gboolean no_signal)473 gst_decklink_audio_src_got_packet (GstElement * element,
474 IDeckLinkAudioInputPacket * packet, GstClockTime capture_time,
475 GstClockTime stream_time, GstClockTime stream_duration,
476 GstClockTime hardware_time, GstClockTime hardware_duration,
477 gboolean no_signal)
478 {
479 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (element);
480 GstClockTime timestamp;
481
482 GST_LOG_OBJECT (self,
483 "Got audio packet at %" GST_TIME_FORMAT " / %" GST_TIME_FORMAT
484 ", no signal %d", GST_TIME_ARGS (capture_time),
485 GST_TIME_ARGS (stream_time), no_signal);
486
487 g_mutex_lock (&self->input->lock);
488 if (self->input->videosrc) {
489 GstDecklinkVideoSrc *videosrc =
490 GST_DECKLINK_VIDEO_SRC_CAST (gst_object_ref (self->input->videosrc));
491
492 if (videosrc->drop_no_signal_frames && no_signal) {
493 g_mutex_unlock (&self->input->lock);
494 return;
495 }
496
497 if (videosrc->first_time == GST_CLOCK_TIME_NONE)
498 videosrc->first_time = stream_time;
499
500 if (videosrc->skip_first_time > 0
501 && stream_time - videosrc->first_time < videosrc->skip_first_time) {
502 GST_DEBUG_OBJECT (self,
503 "Skipping frame as requested: %" GST_TIME_FORMAT " < %"
504 GST_TIME_FORMAT, GST_TIME_ARGS (stream_time),
505 GST_TIME_ARGS (videosrc->skip_first_time + videosrc->first_time));
506 g_mutex_unlock (&self->input->lock);
507 return;
508 }
509
510 if (videosrc->output_stream_time)
511 timestamp = stream_time;
512 else
513 timestamp = gst_clock_adjust_with_calibration (NULL, stream_time,
514 videosrc->current_time_mapping.xbase,
515 videosrc->current_time_mapping.b, videosrc->current_time_mapping.num,
516 videosrc->current_time_mapping.den);
517 } else {
518 timestamp = capture_time;
519 }
520 g_mutex_unlock (&self->input->lock);
521
522 GST_LOG_OBJECT (self, "Converted times to %" GST_TIME_FORMAT,
523 GST_TIME_ARGS (timestamp));
524
525 g_mutex_lock (&self->lock);
526 if (!self->flushing) {
527 CapturePacket p;
528 guint skipped_packets = 0;
529 GstClockTime from_timestamp = GST_CLOCK_TIME_NONE;
530 GstClockTime to_timestamp = GST_CLOCK_TIME_NONE;
531
532 while (gst_queue_array_get_length (self->current_packets) >=
533 self->buffer_size) {
534 CapturePacket *tmp = (CapturePacket *)
535 gst_queue_array_pop_head_struct (self->current_packets);
536 if (skipped_packets == 0)
537 from_timestamp = tmp->timestamp;
538 skipped_packets++;
539 to_timestamp = tmp->timestamp;
540 capture_packet_clear (tmp);
541 }
542
543 if (skipped_packets > 0)
544 GST_WARNING_OBJECT (self,
545 "Dropped %u old packets from %" GST_TIME_FORMAT " to %"
546 GST_TIME_FORMAT, skipped_packets, GST_TIME_ARGS (from_timestamp),
547 GST_TIME_ARGS (to_timestamp));
548
549 memset (&p, 0, sizeof (p));
550 p.packet = packet;
551 p.timestamp = timestamp;
552 p.stream_timestamp = stream_time;
553 p.stream_duration = stream_duration;
554 p.hardware_timestamp = hardware_time;
555 p.hardware_duration = hardware_duration;
556 p.no_signal = no_signal;
557 packet->AddRef ();
558 gst_queue_array_push_tail_struct (self->current_packets, &p);
559 g_cond_signal (&self->cond);
560 }
561 g_mutex_unlock (&self->lock);
562 }
563
564 static GstFlowReturn
gst_decklink_audio_src_create(GstPushSrc * bsrc,GstBuffer ** buffer)565 gst_decklink_audio_src_create (GstPushSrc * bsrc, GstBuffer ** buffer)
566 {
567 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (bsrc);
568 GstFlowReturn flow_ret = GST_FLOW_OK;
569 const guint8 *data;
570 glong sample_count;
571 gsize data_size;
572 CapturePacket p;
573 AudioPacket *ap;
574 GstClockTime timestamp, duration;
575 GstClockTime start_time, end_time;
576 guint64 start_offset, end_offset;
577 gboolean discont = FALSE;
578 static GstStaticCaps stream_reference =
579 GST_STATIC_CAPS ("timestamp/x-decklink-stream");
580 static GstStaticCaps hardware_reference =
581 GST_STATIC_CAPS ("timestamp/x-decklink-hardware");
582
583 if (!gst_decklink_audio_src_start (self)) {
584 return GST_FLOW_NOT_NEGOTIATED;
585 }
586
587 retry:
588 g_mutex_lock (&self->lock);
589 while (gst_queue_array_is_empty (self->current_packets) && !self->flushing) {
590 g_cond_wait (&self->cond, &self->lock);
591 }
592
593 if (self->flushing) {
594 GST_DEBUG_OBJECT (self, "Flushing");
595 g_mutex_unlock (&self->lock);
596 return GST_FLOW_FLUSHING;
597 }
598
599 p = *(CapturePacket *)
600 gst_queue_array_pop_head_struct (self->current_packets);
601 g_mutex_unlock (&self->lock);
602
603 p.packet->GetBytes ((gpointer *) & data);
604 sample_count = p.packet->GetSampleFrameCount ();
605 data_size = self->info.bpf * sample_count;
606
607 if (p.timestamp == GST_CLOCK_TIME_NONE && self->next_offset == (guint64) - 1) {
608 GST_DEBUG_OBJECT (self,
609 "Got packet without timestamp before initial "
610 "timestamp after discont - dropping");
611 capture_packet_clear (&p);
612 goto retry;
613 }
614
615 ap = (AudioPacket *) g_malloc0 (sizeof (AudioPacket));
616
617 *buffer =
618 gst_buffer_new_wrapped_full ((GstMemoryFlags) GST_MEMORY_FLAG_READONLY,
619 (gpointer) data, data_size, 0, data_size, ap,
620 (GDestroyNotify) audio_packet_free);
621
622 ap->packet = p.packet;
623 p.packet->AddRef ();
624 ap->input = self->input->input;
625 ap->input->AddRef ();
626
627 timestamp = p.timestamp;
628
629 // Jitter and discontinuity handling, based on audiobasesrc
630 start_time = timestamp;
631
632 // Convert to the sample numbers
633 start_offset =
634 gst_util_uint64_scale (start_time, self->info.rate, GST_SECOND);
635
636 end_offset = start_offset + sample_count;
637 end_time = gst_util_uint64_scale_int (end_offset, GST_SECOND,
638 self->info.rate);
639
640 duration = end_time - start_time;
641
642 if (self->next_offset == (guint64) - 1) {
643 discont = TRUE;
644 } else {
645 guint64 diff, max_sample_diff;
646
647 // Check discont
648 if (start_offset <= self->next_offset)
649 diff = self->next_offset - start_offset;
650 else
651 diff = start_offset - self->next_offset;
652
653 max_sample_diff =
654 gst_util_uint64_scale_int (self->alignment_threshold, self->info.rate,
655 GST_SECOND);
656
657 // Discont!
658 if (G_UNLIKELY (diff >= max_sample_diff)) {
659 if (self->discont_wait > 0) {
660 if (self->discont_time == GST_CLOCK_TIME_NONE) {
661 self->discont_time = start_time;
662 } else if (start_time - self->discont_time >= self->discont_wait) {
663 discont = TRUE;
664 self->discont_time = GST_CLOCK_TIME_NONE;
665 }
666 } else {
667 discont = TRUE;
668 }
669 } else if (G_UNLIKELY (self->discont_time != GST_CLOCK_TIME_NONE)) {
670 // we have had a discont, but are now back on track!
671 self->discont_time = GST_CLOCK_TIME_NONE;
672 }
673 }
674
675 if (discont) {
676 // Have discont, need resync and use the capture timestamps
677 if (self->next_offset != (guint64) - 1)
678 GST_INFO_OBJECT (self, "Have discont. Expected %"
679 G_GUINT64_FORMAT ", got %" G_GUINT64_FORMAT,
680 self->next_offset, start_offset);
681 GST_BUFFER_FLAG_SET (*buffer, GST_BUFFER_FLAG_DISCONT);
682 self->next_offset = end_offset;
683 // Got a discont and adjusted, reset the discont_time marker.
684 self->discont_time = GST_CLOCK_TIME_NONE;
685 } else {
686 // No discont, just keep counting
687 timestamp =
688 gst_util_uint64_scale (self->next_offset, GST_SECOND, self->info.rate);
689 self->next_offset += sample_count;
690 duration =
691 gst_util_uint64_scale (self->next_offset, GST_SECOND,
692 self->info.rate) - timestamp;
693 }
694
695 // Detect gaps in stream time
696 self->processed += sample_count;
697 if (self->expected_stream_time != GST_CLOCK_TIME_NONE
698 && p.stream_timestamp == GST_CLOCK_TIME_NONE) {
699 /* We missed a frame. Extrapolate the timestamps */
700 p.stream_timestamp = self->expected_stream_time;
701 p.stream_duration =
702 gst_util_uint64_scale_int (sample_count, GST_SECOND, self->info.rate);
703 }
704 if (self->last_hardware_time != GST_CLOCK_TIME_NONE
705 && p.hardware_timestamp == GST_CLOCK_TIME_NONE) {
706 /* This should always happen when the previous one also does, but let's
707 * have two separate checks just in case */
708 GstClockTime start_hw_offset, end_hw_offset;
709 start_hw_offset =
710 gst_util_uint64_scale (self->last_hardware_time, self->info.rate,
711 GST_SECOND);
712 end_hw_offset = start_hw_offset + sample_count;
713 p.hardware_timestamp =
714 gst_util_uint64_scale_int (end_hw_offset, GST_SECOND, self->info.rate);
715 /* Will be the same as the stream duration - reuse it */
716 p.hardware_duration = p.stream_duration;
717 }
718
719 if (p.stream_timestamp != GST_CLOCK_TIME_NONE) {
720 GstClockTime start_stream_time, end_stream_time;
721
722 start_stream_time = p.stream_timestamp;
723
724 start_offset =
725 gst_util_uint64_scale (start_stream_time, self->info.rate, GST_SECOND);
726
727 end_offset = start_offset + sample_count;
728 end_stream_time = gst_util_uint64_scale_int (end_offset, GST_SECOND,
729 self->info.rate);
730
731 /* With drop-frame we have gaps of 1 sample every now and then (rounding
732 * errors because of the samples-per-frame pattern which is not 100%
733 * accurate), and due to rounding errors in the calculations these can be
734 * 2>x>1 */
735 if (self->expected_stream_time != GST_CLOCK_TIME_NONE &&
736 ABSDIFF (self->expected_stream_time, p.stream_timestamp) >
737 gst_util_uint64_scale (2, GST_SECOND, self->info.rate)) {
738 GstMessage *msg;
739 GstClockTime running_time;
740
741 self->dropped +=
742 gst_util_uint64_scale (ABSDIFF (self->expected_stream_time,
743 p.stream_timestamp), self->info.rate, GST_SECOND);
744 running_time =
745 gst_segment_to_running_time (&GST_BASE_SRC (self)->segment,
746 GST_FORMAT_TIME, timestamp);
747
748 msg =
749 gst_message_new_qos (GST_OBJECT (self), TRUE, running_time,
750 p.stream_timestamp, timestamp, duration);
751 gst_message_set_qos_stats (msg, GST_FORMAT_DEFAULT, self->processed,
752 self->dropped);
753 gst_element_post_message (GST_ELEMENT (self), msg);
754 }
755 self->expected_stream_time = end_stream_time;
756 }
757 self->last_hardware_time = p.hardware_timestamp;
758
759 if (p.no_signal)
760 GST_BUFFER_FLAG_SET (*buffer, GST_BUFFER_FLAG_GAP);
761 GST_BUFFER_TIMESTAMP (*buffer) = timestamp;
762 GST_BUFFER_DURATION (*buffer) = duration;
763
764 gst_buffer_add_reference_timestamp_meta (*buffer,
765 gst_static_caps_get (&stream_reference), p.stream_timestamp,
766 p.stream_duration);
767 gst_buffer_add_reference_timestamp_meta (*buffer,
768 gst_static_caps_get (&hardware_reference), p.hardware_timestamp,
769 p.hardware_duration);
770
771 GST_DEBUG_OBJECT (self,
772 "Outputting buffer %p with timestamp %" GST_TIME_FORMAT " and duration %"
773 GST_TIME_FORMAT, *buffer, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (*buffer)),
774 GST_TIME_ARGS (GST_BUFFER_DURATION (*buffer)));
775
776 capture_packet_clear (&p);
777
778 return flow_ret;
779 }
780
781 static gboolean
gst_decklink_audio_src_query(GstBaseSrc * bsrc,GstQuery * query)782 gst_decklink_audio_src_query (GstBaseSrc * bsrc, GstQuery * query)
783 {
784 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (bsrc);
785 gboolean ret = TRUE;
786
787 switch (GST_QUERY_TYPE (query)) {
788 case GST_QUERY_LATENCY:{
789 if (self->input) {
790 g_mutex_lock (&self->input->lock);
791 if (self->input->mode) {
792 GstClockTime min, max;
793
794 min =
795 gst_util_uint64_scale_ceil (GST_SECOND, self->input->mode->fps_d,
796 self->input->mode->fps_n);
797 max = self->buffer_size * min;
798
799 gst_query_set_latency (query, TRUE, min, max);
800 ret = TRUE;
801 } else {
802 ret = FALSE;
803 }
804 g_mutex_unlock (&self->input->lock);
805 } else {
806 ret = FALSE;
807 }
808
809 break;
810 }
811 default:
812 ret = GST_BASE_SRC_CLASS (parent_class)->query (bsrc, query);
813 break;
814 }
815
816 return ret;
817 }
818
819 static gboolean
gst_decklink_audio_src_unlock(GstBaseSrc * bsrc)820 gst_decklink_audio_src_unlock (GstBaseSrc * bsrc)
821 {
822 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (bsrc);
823
824 g_mutex_lock (&self->lock);
825 self->flushing = TRUE;
826 g_cond_signal (&self->cond);
827 g_mutex_unlock (&self->lock);
828
829 return TRUE;
830 }
831
832 static gboolean
gst_decklink_audio_src_unlock_stop(GstBaseSrc * bsrc)833 gst_decklink_audio_src_unlock_stop (GstBaseSrc * bsrc)
834 {
835 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (bsrc);
836
837 g_mutex_lock (&self->lock);
838 self->flushing = FALSE;
839 while (gst_queue_array_get_length (self->current_packets) > 0) {
840 CapturePacket *tmp = (CapturePacket *)
841 gst_queue_array_pop_head_struct (self->current_packets);
842 capture_packet_clear (tmp);
843 }
844 g_mutex_unlock (&self->lock);
845
846 return TRUE;
847 }
848
849 static gboolean
gst_decklink_audio_src_open(GstDecklinkAudioSrc * self)850 gst_decklink_audio_src_open (GstDecklinkAudioSrc * self)
851 {
852 GST_DEBUG_OBJECT (self, "Opening");
853
854 self->input =
855 gst_decklink_acquire_nth_input (self->device_number,
856 GST_ELEMENT_CAST (self), TRUE);
857 if (!self->input) {
858 GST_ERROR_OBJECT (self, "Failed to acquire input");
859 return FALSE;
860 }
861
862 g_object_notify (G_OBJECT (self), "hw-serial-number");
863
864 g_mutex_lock (&self->input->lock);
865 if (self->channels > 0) {
866 self->channels_found = self->channels;
867 } else {
868 if (self->input->attributes) {
869 int64_t channels_found;
870
871 HRESULT ret = self->input->attributes->GetInt
872 (BMDDeckLinkMaximumAudioChannels, &channels_found);
873 self->channels_found = channels_found;
874
875 /* Sometimes the card may report an invalid number of channels. In
876 * that case, we should (empirically) use 8. */
877 if (ret != S_OK ||
878 self->channels_found == 0 || g_enum_get_value ((GEnumClass *)
879 g_type_class_peek (GST_TYPE_DECKLINK_AUDIO_CHANNELS),
880 self->channels_found)
881 == NULL) {
882 self->channels_found = GST_DECKLINK_AUDIO_CHANNELS_8;
883 }
884 }
885 }
886 self->input->got_audio_packet = gst_decklink_audio_src_got_packet;
887 g_mutex_unlock (&self->input->lock);
888
889 return TRUE;
890 }
891
892 static gboolean
gst_decklink_audio_src_close(GstDecklinkAudioSrc * self)893 gst_decklink_audio_src_close (GstDecklinkAudioSrc * self)
894 {
895 GST_DEBUG_OBJECT (self, "Closing");
896
897 if (self->input) {
898 g_mutex_lock (&self->input->lock);
899 self->input->got_audio_packet = NULL;
900 g_mutex_unlock (&self->input->lock);
901
902 gst_decklink_release_nth_input (self->device_number,
903 GST_ELEMENT_CAST (self), TRUE);
904 self->input = NULL;
905 }
906
907 return TRUE;
908 }
909
910 static gboolean
gst_decklink_audio_src_stop(GstDecklinkAudioSrc * self)911 gst_decklink_audio_src_stop (GstDecklinkAudioSrc * self)
912 {
913 GST_DEBUG_OBJECT (self, "Stopping");
914
915 while (gst_queue_array_get_length (self->current_packets) > 0) {
916 CapturePacket *tmp = (CapturePacket *)
917 gst_queue_array_pop_head_struct (self->current_packets);
918 capture_packet_clear (tmp);
919 }
920
921 if (self->input && self->input->audio_enabled) {
922 g_mutex_lock (&self->input->lock);
923 self->input->audio_enabled = FALSE;
924 g_mutex_unlock (&self->input->lock);
925
926 self->input->input->DisableAudioInput ();
927 }
928
929 return TRUE;
930 }
931
932 #if 0
933 static gboolean
934 in_same_pipeline (GstElement * a, GstElement * b)
935 {
936 GstObject *root = NULL, *tmp;
937 gboolean ret = FALSE;
938
939 tmp = gst_object_get_parent (GST_OBJECT_CAST (a));
940 while (tmp != NULL) {
941 if (root)
942 gst_object_unref (root);
943 root = tmp;
944 tmp = gst_object_get_parent (root);
945 }
946
947 ret = root && gst_object_has_ancestor (GST_OBJECT_CAST (b), root);
948
949 if (root)
950 gst_object_unref (root);
951
952 return ret;
953 }
954 #endif
955
956 static GstStateChangeReturn
gst_decklink_audio_src_change_state(GstElement * element,GstStateChange transition)957 gst_decklink_audio_src_change_state (GstElement * element,
958 GstStateChange transition)
959 {
960 GstDecklinkAudioSrc *self = GST_DECKLINK_AUDIO_SRC_CAST (element);
961 GstStateChangeReturn ret;
962
963 switch (transition) {
964 case GST_STATE_CHANGE_NULL_TO_READY:
965 self->processed = 0;
966 self->dropped = 0;
967 self->expected_stream_time = GST_CLOCK_TIME_NONE;
968 if (!gst_decklink_audio_src_open (self)) {
969 ret = GST_STATE_CHANGE_FAILURE;
970 goto out;
971 }
972 break;
973 case GST_STATE_CHANGE_READY_TO_PAUSED:{
974 GstElement *videosrc = NULL;
975
976 // Check if there is a video src for this input too and if it
977 // is actually in the same pipeline
978 g_mutex_lock (&self->input->lock);
979 if (self->input->videosrc)
980 videosrc = GST_ELEMENT_CAST (gst_object_ref (self->input->videosrc));
981 g_mutex_unlock (&self->input->lock);
982
983 if (!videosrc) {
984 GST_ELEMENT_ERROR (self, STREAM, FAILED,
985 (NULL), ("Audio src needs a video src for its operation"));
986 ret = GST_STATE_CHANGE_FAILURE;
987 goto out;
988 }
989 // FIXME: This causes deadlocks sometimes
990 #if 0
991 else if (!in_same_pipeline (GST_ELEMENT_CAST (self), videosrc)) {
992 GST_ELEMENT_ERROR (self, STREAM, FAILED,
993 (NULL),
994 ("Audio src and video src need to be in the same pipeline"));
995 ret = GST_STATE_CHANGE_FAILURE;
996 gst_object_unref (videosrc);
997 goto out;
998 }
999 #endif
1000
1001 if (videosrc)
1002 gst_object_unref (videosrc);
1003
1004 self->flushing = FALSE;
1005 self->next_offset = -1;
1006 break;
1007 }
1008 default:
1009 break;
1010 }
1011
1012 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
1013 if (ret == GST_STATE_CHANGE_FAILURE)
1014 return ret;
1015
1016 switch (transition) {
1017 case GST_STATE_CHANGE_PAUSED_TO_READY:
1018 gst_decklink_audio_src_stop (self);
1019 break;
1020 case GST_STATE_CHANGE_READY_TO_NULL:
1021 gst_decklink_audio_src_close (self);
1022 break;
1023 default:
1024 break;
1025 }
1026 out:
1027
1028 return ret;
1029 }
1030