1 /* GStreamer
2 * Copyright (C) 2013 Alessandro Decina <alessandro.d@gmail.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
17 * Boston, MA 02110-1335, USA.
18 */
19 /**
20 * SECTION:element-atdec
21 * @title: atdec
22 *
23 * AudioToolbox based decoder.
24 *
25 * ## Example launch line
26 * |[
27 * gst-launch-1.0 -v filesrc location=file.mov ! qtdemux ! queue ! aacparse ! atdec ! autoaudiosink
28 * ]|
29 * Decode aac audio from a mov file
30 *
31 */
32
33 #ifdef HAVE_CONFIG_H
34 #include "config.h"
35 #endif
36
37 #include <gst/gst.h>
38 #include <gst/audio/gstaudiodecoder.h>
39 #include "atdec.h"
40
41 GST_DEBUG_CATEGORY_STATIC (gst_atdec_debug_category);
42 #define GST_CAT_DEFAULT gst_atdec_debug_category
43
44 static void gst_atdec_set_property (GObject * object,
45 guint property_id, const GValue * value, GParamSpec * pspec);
46 static void gst_atdec_get_property (GObject * object,
47 guint property_id, GValue * value, GParamSpec * pspec);
48 static void gst_atdec_finalize (GObject * object);
49
50 static gboolean gst_atdec_start (GstAudioDecoder * decoder);
51 static gboolean gst_atdec_stop (GstAudioDecoder * decoder);
52 static gboolean gst_atdec_set_format (GstAudioDecoder * decoder,
53 GstCaps * caps);
54 static GstFlowReturn gst_atdec_handle_frame (GstAudioDecoder * decoder,
55 GstBuffer * buffer);
56 static void gst_atdec_flush (GstAudioDecoder * decoder, gboolean hard);
57 static void gst_atdec_buffer_emptied (void *user_data,
58 AudioQueueRef queue, AudioQueueBufferRef buffer);
59
60 enum
61 {
62 PROP_0
63 };
64
65 static GstStaticPadTemplate gst_atdec_src_template =
66 GST_STATIC_PAD_TEMPLATE ("src",
67 GST_PAD_SRC,
68 GST_PAD_ALWAYS,
69 GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE ("S16LE") ", layout=interleaved;"
70 GST_AUDIO_CAPS_MAKE ("F32LE") ", layout=interleaved")
71 );
72
73 static GstStaticPadTemplate gst_atdec_sink_template =
74 GST_STATIC_PAD_TEMPLATE ("sink",
75 GST_PAD_SINK,
76 GST_PAD_ALWAYS,
77 GST_STATIC_CAPS ("audio/mpeg, mpegversion=4, framed=true, channels=[1,max];"
78 "audio/mpeg, mpegversion=1, layer=[1, 3]")
79 );
80
81 G_DEFINE_TYPE_WITH_CODE (GstATDec, gst_atdec, GST_TYPE_AUDIO_DECODER,
82 GST_DEBUG_CATEGORY_INIT (gst_atdec_debug_category, "atdec", 0,
83 "debug category for atdec element"));
84
85 static GstStaticCaps aac_caps = GST_STATIC_CAPS ("audio/mpeg, mpegversion=4");
86 static GstStaticCaps mp3_caps =
87 GST_STATIC_CAPS ("audio/mpeg, mpegversion=1, layer=[1, 3]");
88 static GstStaticCaps raw_caps = GST_STATIC_CAPS ("audio/x-raw");
89
90 static void
gst_atdec_class_init(GstATDecClass * klass)91 gst_atdec_class_init (GstATDecClass * klass)
92 {
93 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
94 GstAudioDecoderClass *audio_decoder_class = GST_AUDIO_DECODER_CLASS (klass);
95
96 gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass),
97 &gst_atdec_src_template);
98 gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass),
99 &gst_atdec_sink_template);
100
101 gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass),
102 "AudioToolbox based audio decoder",
103 "Codec/Decoder/Audio",
104 "AudioToolbox based audio decoder",
105 "Alessandro Decina <alessandro.d@gmail.com>");
106
107 gobject_class->set_property = gst_atdec_set_property;
108 gobject_class->get_property = gst_atdec_get_property;
109 gobject_class->finalize = gst_atdec_finalize;
110 audio_decoder_class->start = GST_DEBUG_FUNCPTR (gst_atdec_start);
111 audio_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_atdec_stop);
112 audio_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_atdec_set_format);
113 audio_decoder_class->handle_frame =
114 GST_DEBUG_FUNCPTR (gst_atdec_handle_frame);
115 audio_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_atdec_flush);
116 }
117
118 static void
gst_atdec_init(GstATDec * atdec)119 gst_atdec_init (GstATDec * atdec)
120 {
121 gst_audio_decoder_set_needs_format (GST_AUDIO_DECODER (atdec), TRUE);
122 atdec->queue = NULL;
123 }
124
125 void
gst_atdec_set_property(GObject * object,guint property_id,const GValue * value,GParamSpec * pspec)126 gst_atdec_set_property (GObject * object, guint property_id,
127 const GValue * value, GParamSpec * pspec)
128 {
129 GstATDec *atdec = GST_ATDEC (object);
130
131 GST_DEBUG_OBJECT (atdec, "set_property");
132
133 switch (property_id) {
134 default:
135 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
136 break;
137 }
138 }
139
140 void
gst_atdec_get_property(GObject * object,guint property_id,GValue * value,GParamSpec * pspec)141 gst_atdec_get_property (GObject * object, guint property_id,
142 GValue * value, GParamSpec * pspec)
143 {
144 GstATDec *atdec = GST_ATDEC (object);
145
146 GST_DEBUG_OBJECT (atdec, "get_property");
147
148 switch (property_id) {
149 default:
150 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
151 break;
152 }
153 }
154
155 static void
gst_atdec_destroy_queue(GstATDec * atdec,gboolean drain)156 gst_atdec_destroy_queue (GstATDec * atdec, gboolean drain)
157 {
158 AudioQueueStop (atdec->queue, drain);
159 AudioQueueDispose (atdec->queue, true);
160 atdec->queue = NULL;
161 atdec->output_position = 0;
162 atdec->input_position = 0;
163 }
164
165 void
gst_atdec_finalize(GObject * object)166 gst_atdec_finalize (GObject * object)
167 {
168 GstATDec *atdec = GST_ATDEC (object);
169
170 GST_DEBUG_OBJECT (atdec, "finalize");
171
172 if (atdec->queue)
173 gst_atdec_destroy_queue (atdec, FALSE);
174
175 G_OBJECT_CLASS (gst_atdec_parent_class)->finalize (object);
176 }
177
178 static gboolean
gst_atdec_start(GstAudioDecoder * decoder)179 gst_atdec_start (GstAudioDecoder * decoder)
180 {
181 GstATDec *atdec = GST_ATDEC (decoder);
182
183 GST_DEBUG_OBJECT (atdec, "start");
184 atdec->output_position = 0;
185 atdec->input_position = 0;
186
187 return TRUE;
188 }
189
190 static gboolean
gst_atdec_stop(GstAudioDecoder * decoder)191 gst_atdec_stop (GstAudioDecoder * decoder)
192 {
193 GstATDec *atdec = GST_ATDEC (decoder);
194
195 gst_atdec_destroy_queue (atdec, FALSE);
196
197 return TRUE;
198 }
199
200 static gboolean
can_intersect_static_caps(GstCaps * caps,GstStaticCaps * caps1)201 can_intersect_static_caps (GstCaps * caps, GstStaticCaps * caps1)
202 {
203 GstCaps *tmp;
204 gboolean ret;
205
206 tmp = gst_static_caps_get (caps1);
207 ret = gst_caps_can_intersect (caps, tmp);
208 gst_caps_unref (tmp);
209
210 return ret;
211 }
212
213 static gboolean
gst_caps_to_at_format(GstCaps * caps,AudioStreamBasicDescription * format)214 gst_caps_to_at_format (GstCaps * caps, AudioStreamBasicDescription * format)
215 {
216 int channels = 0;
217 int rate = 0;
218 GstStructure *structure;
219
220 memset (format, 0, sizeof (AudioStreamBasicDescription));
221
222 structure = gst_caps_get_structure (caps, 0);
223 gst_structure_get_int (structure, "rate", &rate);
224 gst_structure_get_int (structure, "channels", &channels);
225 format->mSampleRate = rate;
226 format->mChannelsPerFrame = channels;
227
228 if (can_intersect_static_caps (caps, &aac_caps)) {
229 format->mFormatID = kAudioFormatMPEG4AAC;
230 format->mFramesPerPacket = 1024;
231 } else if (can_intersect_static_caps (caps, &mp3_caps)) {
232 gint layer, mpegaudioversion = 1;
233
234 gst_structure_get_int (structure, "layer", &layer);
235 gst_structure_get_int (structure, "mpegaudioversion", &mpegaudioversion);
236 switch (layer) {
237 case 1:
238 format->mFormatID = kAudioFormatMPEGLayer1;
239 format->mFramesPerPacket = 384;
240 break;
241 case 2:
242 format->mFormatID = kAudioFormatMPEGLayer2;
243 format->mFramesPerPacket = 1152;
244 break;
245 case 3:
246 format->mFormatID = kAudioFormatMPEGLayer3;
247 format->mFramesPerPacket = (mpegaudioversion == 1 ? 1152 : 576);
248 break;
249 default:
250 g_warn_if_reached ();
251 format->mFormatID = kAudioFormatMPEGLayer3;
252 format->mFramesPerPacket = 1152;
253 break;
254 }
255 } else if (can_intersect_static_caps (caps, &raw_caps)) {
256 GstAudioFormat audio_format;
257 const char *audio_format_str;
258
259 format->mFormatID = kAudioFormatLinearPCM;
260 format->mFramesPerPacket = 1;
261
262 audio_format_str = gst_structure_get_string (structure, "format");
263 if (!audio_format_str)
264 audio_format_str = "S16LE";
265
266 audio_format = gst_audio_format_from_string (audio_format_str);
267 switch (audio_format) {
268 case GST_AUDIO_FORMAT_S16LE:
269 format->mFormatFlags =
270 kLinearPCMFormatFlagIsPacked | kLinearPCMFormatFlagIsSignedInteger;
271 format->mBitsPerChannel = 16;
272 format->mBytesPerPacket = format->mBytesPerFrame = 2 * channels;
273 break;
274 case GST_AUDIO_FORMAT_F32LE:
275 format->mFormatFlags =
276 kLinearPCMFormatFlagIsPacked | kLinearPCMFormatFlagIsFloat;
277 format->mBitsPerChannel = 32;
278 format->mBytesPerPacket = format->mBytesPerFrame = 4 * channels;
279 break;
280 default:
281 g_warn_if_reached ();
282 break;
283 }
284 }
285
286 return TRUE;
287 }
288
289 static gboolean
gst_atdec_set_format(GstAudioDecoder * decoder,GstCaps * caps)290 gst_atdec_set_format (GstAudioDecoder * decoder, GstCaps * caps)
291 {
292 OSStatus status;
293 AudioStreamBasicDescription input_format = { 0 };
294 AudioStreamBasicDescription output_format = { 0 };
295 GstAudioInfo output_info = { 0 };
296 AudioChannelLayout output_layout = { 0 };
297 GstCaps *output_caps;
298 AudioTimeStamp timestamp = { 0 };
299 AudioQueueBufferRef output_buffer;
300 GstATDec *atdec = GST_ATDEC (decoder);
301
302 GST_DEBUG_OBJECT (atdec, "set_format");
303
304 if (atdec->queue)
305 gst_atdec_destroy_queue (atdec, TRUE);
306
307 /* configure input_format from caps */
308 gst_caps_to_at_format (caps, &input_format);
309 /* Remember the number of samples per frame */
310 atdec->spf = input_format.mFramesPerPacket;
311
312 /* negotiate output caps */
313 output_caps = gst_pad_get_allowed_caps (GST_AUDIO_DECODER_SRC_PAD (atdec));
314 if (!output_caps)
315 output_caps =
316 gst_pad_get_pad_template_caps (GST_AUDIO_DECODER_SRC_PAD (atdec));
317 output_caps = gst_caps_fixate (output_caps);
318
319 gst_caps_set_simple (output_caps,
320 "rate", G_TYPE_INT, (int) input_format.mSampleRate,
321 "channels", G_TYPE_INT, input_format.mChannelsPerFrame, NULL);
322
323 /* configure output_format from caps */
324 gst_caps_to_at_format (output_caps, &output_format);
325
326 /* set the format we want to negotiate downstream */
327 gst_audio_info_from_caps (&output_info, output_caps);
328 gst_audio_info_set_format (&output_info,
329 output_format.mFormatFlags & kLinearPCMFormatFlagIsSignedInteger ?
330 GST_AUDIO_FORMAT_S16LE : GST_AUDIO_FORMAT_F32LE,
331 output_format.mSampleRate, output_format.mChannelsPerFrame, NULL);
332 gst_audio_decoder_set_output_format (decoder, &output_info);
333 gst_caps_unref (output_caps);
334
335 status = AudioQueueNewOutput (&input_format, gst_atdec_buffer_emptied,
336 atdec, NULL, NULL, 0, &atdec->queue);
337 if (status)
338 goto create_queue_error;
339
340 /* FIXME: figure out how to map this properly */
341 if (output_format.mChannelsPerFrame == 1)
342 output_layout.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
343 else
344 output_layout.mChannelLayoutTag = kAudioChannelLayoutTag_Stereo;
345
346 status = AudioQueueSetOfflineRenderFormat (atdec->queue,
347 &output_format, &output_layout);
348 if (status)
349 goto set_format_error;
350
351 status = AudioQueueStart (atdec->queue, NULL);
352 if (status)
353 goto start_error;
354
355 timestamp.mFlags = kAudioTimeStampSampleTimeValid;
356 timestamp.mSampleTime = 0;
357
358 status =
359 AudioQueueAllocateBuffer (atdec->queue, atdec->spf * output_info.bpf,
360 &output_buffer);
361 if (status)
362 goto allocate_output_error;
363
364 status = AudioQueueOfflineRender (atdec->queue, ×tamp, output_buffer, 0);
365 if (status)
366 goto offline_render_error;
367
368 AudioQueueFreeBuffer (atdec->queue, output_buffer);
369
370 return TRUE;
371
372 create_queue_error:
373 GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL),
374 ("AudioQueueNewOutput returned error: %d", (gint) status));
375 return FALSE;
376
377 set_format_error:
378 GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL),
379 ("AudioQueueSetOfflineRenderFormat returned error: %d", (gint) status));
380 gst_atdec_destroy_queue (atdec, FALSE);
381 return FALSE;
382
383 start_error:
384 GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL),
385 ("AudioQueueStart returned error: %d", (gint) status));
386 gst_atdec_destroy_queue (atdec, FALSE);
387 return FALSE;
388
389 allocate_output_error:
390 GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL),
391 ("AudioQueueAllocateBuffer returned error: %d", (gint) status));
392 gst_atdec_destroy_queue (atdec, FALSE);
393 return FALSE;
394
395 offline_render_error:
396 GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL),
397 ("AudioQueueOfflineRender returned error: %d", (gint) status));
398 AudioQueueFreeBuffer (atdec->queue, output_buffer);
399 gst_atdec_destroy_queue (atdec, FALSE);
400 return FALSE;
401 }
402
403 static void
gst_atdec_buffer_emptied(void * user_data,AudioQueueRef queue,AudioQueueBufferRef buffer)404 gst_atdec_buffer_emptied (void *user_data, AudioQueueRef queue,
405 AudioQueueBufferRef buffer)
406 {
407 AudioQueueFreeBuffer (queue, buffer);
408 }
409
410 static GstFlowReturn
gst_atdec_offline_render(GstATDec * atdec,GstAudioInfo * audio_info)411 gst_atdec_offline_render (GstATDec * atdec, GstAudioInfo * audio_info)
412 {
413 OSStatus status;
414 AudioTimeStamp timestamp = { 0 };
415 AudioQueueBufferRef output_buffer;
416 GstFlowReturn flow_ret = GST_FLOW_OK;
417 GstBuffer *out;
418 guint out_frames;
419
420 /* figure out how many frames we need to pull out of the queue */
421 out_frames = atdec->input_position - atdec->output_position;
422 if (out_frames > atdec->spf)
423 out_frames = atdec->spf;
424 status = AudioQueueAllocateBuffer (atdec->queue, out_frames * audio_info->bpf,
425 &output_buffer);
426 if (status)
427 goto allocate_output_failed;
428
429 /* pull the frames */
430 timestamp.mFlags = kAudioTimeStampSampleTimeValid;
431 timestamp.mSampleTime = atdec->output_position;
432 status =
433 AudioQueueOfflineRender (atdec->queue, ×tamp, output_buffer,
434 out_frames);
435 if (status)
436 goto offline_render_failed;
437
438 if (output_buffer->mAudioDataByteSize) {
439 if (output_buffer->mAudioDataByteSize % audio_info->bpf != 0)
440 goto invalid_buffer_size;
441
442 GST_DEBUG_OBJECT (atdec,
443 "Got output buffer of size %u at position %" G_GUINT64_FORMAT,
444 (guint) output_buffer->mAudioDataByteSize, atdec->output_position);
445 atdec->output_position +=
446 output_buffer->mAudioDataByteSize / audio_info->bpf;
447
448 out =
449 gst_audio_decoder_allocate_output_buffer (GST_AUDIO_DECODER (atdec),
450 output_buffer->mAudioDataByteSize);
451
452 gst_buffer_fill (out, 0, output_buffer->mAudioData,
453 output_buffer->mAudioDataByteSize);
454
455 flow_ret =
456 gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (atdec), out, 1);
457 GST_DEBUG_OBJECT (atdec, "Finished buffer: %s",
458 gst_flow_get_name (flow_ret));
459 } else {
460 GST_DEBUG_OBJECT (atdec, "Got empty output buffer");
461 flow_ret = GST_FLOW_CUSTOM_SUCCESS;
462 }
463
464 AudioQueueFreeBuffer (atdec->queue, output_buffer);
465
466 return flow_ret;
467
468 allocate_output_failed:
469 {
470 GST_ELEMENT_ERROR (atdec, STREAM, DECODE, (NULL),
471 ("AudioQueueAllocateBuffer returned error: %d", (gint) status));
472 return GST_FLOW_ERROR;
473 }
474
475 offline_render_failed:
476 {
477 AudioQueueFreeBuffer (atdec->queue, output_buffer);
478
479 GST_AUDIO_DECODER_ERROR (atdec, 1, STREAM, DECODE, (NULL),
480 ("AudioQueueOfflineRender returned error: %d", (gint) status),
481 flow_ret);
482
483 return flow_ret;
484 }
485
486 invalid_buffer_size:
487 {
488 GST_AUDIO_DECODER_ERROR (atdec, 1, STREAM, DECODE, (NULL),
489 ("AudioQueueOfflineRender returned invalid buffer size: %u (bpf %d)",
490 (guint) output_buffer->mAudioDataByteSize, audio_info->bpf),
491 flow_ret);
492
493 AudioQueueFreeBuffer (atdec->queue, output_buffer);
494
495 return flow_ret;
496 }
497 }
498
499 static GstFlowReturn
gst_atdec_handle_frame(GstAudioDecoder * decoder,GstBuffer * buffer)500 gst_atdec_handle_frame (GstAudioDecoder * decoder, GstBuffer * buffer)
501 {
502 OSStatus status;
503 AudioStreamPacketDescription packet;
504 AudioQueueBufferRef input_buffer;
505 GstAudioInfo *audio_info;
506 int size;
507 GstFlowReturn flow_ret = GST_FLOW_OK;
508 GstATDec *atdec = GST_ATDEC (decoder);
509
510 audio_info = gst_audio_decoder_get_audio_info (decoder);
511
512 if (buffer == NULL) {
513 GST_DEBUG_OBJECT (atdec, "Draining");
514 AudioQueueFlush (atdec->queue);
515
516 while (atdec->input_position > atdec->output_position
517 && flow_ret == GST_FLOW_OK) {
518 flow_ret = gst_atdec_offline_render (atdec, audio_info);
519 }
520
521 if (flow_ret == GST_FLOW_CUSTOM_SUCCESS)
522 flow_ret = GST_FLOW_OK;
523
524 return flow_ret;
525 }
526
527 /* copy the input buffer into an AudioQueueBuffer */
528 size = gst_buffer_get_size (buffer);
529 GST_DEBUG_OBJECT (atdec,
530 "Handling buffer of size %u at timestamp %" GST_TIME_FORMAT, (guint) size,
531 GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)));
532 status = AudioQueueAllocateBuffer (atdec->queue, size, &input_buffer);
533 if (status)
534 goto allocate_input_failed;
535 gst_buffer_extract (buffer, 0, input_buffer->mAudioData, size);
536 input_buffer->mAudioDataByteSize = size;
537
538 /* assume framed input */
539 packet.mStartOffset = 0;
540 packet.mVariableFramesInPacket = 1;
541 packet.mDataByteSize = size;
542
543 /* enqueue the buffer. It will get free'd once the gst_atdec_buffer_emptied
544 * callback is called
545 */
546 status = AudioQueueEnqueueBuffer (atdec->queue, input_buffer, 1, &packet);
547 if (status)
548 goto enqueue_buffer_failed;
549
550 atdec->input_position += atdec->spf;
551
552 flow_ret = gst_atdec_offline_render (atdec, audio_info);
553 if (flow_ret == GST_FLOW_CUSTOM_SUCCESS)
554 flow_ret = GST_FLOW_OK;
555
556 return flow_ret;
557
558 allocate_input_failed:
559 {
560 GST_ELEMENT_ERROR (atdec, STREAM, DECODE, (NULL),
561 ("AudioQueueAllocateBuffer returned error: %d", (gint) status));
562 return GST_FLOW_ERROR;
563 }
564
565 enqueue_buffer_failed:
566 {
567 GST_AUDIO_DECODER_ERROR (atdec, 1, STREAM, DECODE, (NULL),
568 ("AudioQueueEnqueueBuffer returned error: %d", (gint) status),
569 flow_ret);
570 return flow_ret;
571 }
572 }
573
574 static void
gst_atdec_flush(GstAudioDecoder * decoder,gboolean hard)575 gst_atdec_flush (GstAudioDecoder * decoder, gboolean hard)
576 {
577 GstATDec *atdec = GST_ATDEC (decoder);
578
579 GST_DEBUG_OBJECT (atdec, "Flushing");
580 AudioQueueReset (atdec->queue);
581 atdec->output_position = 0;
582 atdec->input_position = 0;
583 }
584