1 /* GStreamer
2  * Copyright (C) 2010, 2013 Ole André Vadla Ravnås <oleavr@soundrop.com>
3  * Copyright (C) 2012-2016 Alessandro Decina <alessandro.d@gmail.com>
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Library General Public
7  * License as published by the Free Software Foundation; either
8  * version 2 of the License, or (at your option) any later version.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * Library General Public License for more details.
14  *
15  * You should have received a copy of the GNU Library General Public
16  * License along with this library; if not, write to the
17  * Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
18  * Boston, MA 02110-1335, USA.
19  */
20 /**
21  * SECTION:element-vtdec
22  * @title: gstvtdec
23  *
24  * Apple VideoToolbox based decoder.
25  *
26  * ## Example launch line
27  * |[
28  * gst-launch-1.0 -v filesrc location=file.mov ! qtdemux ! queue ! h264parse ! vtdec ! videoconvert ! autovideosink
29  * ]|
30  * Decode h264 video from a mov file.
31  *
32  */
33 
34 #ifdef HAVE_CONFIG_H
35 #include "config.h"
36 #endif
37 
38 #include <string.h>
39 #include <gst/gst.h>
40 #include <gst/video/video.h>
41 #include <gst/video/gstvideodecoder.h>
42 #include <gst/gl/gstglcontext.h>
43 #include "vtdec.h"
44 #include "vtutil.h"
45 #include "corevideobuffer.h"
46 #include "coremediabuffer.h"
47 
48 GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
49 #define GST_CAT_DEFAULT gst_vtdec_debug_category
50 
51 enum
52 {
53   /* leave some headroom for new GstVideoCodecFrameFlags flags */
54   VTDEC_FRAME_FLAG_SKIP = (1 << 10),
55   VTDEC_FRAME_FLAG_DROP = (1 << 11),
56 };
57 
58 static void gst_vtdec_finalize (GObject * object);
59 
60 static gboolean gst_vtdec_start (GstVideoDecoder * decoder);
61 static gboolean gst_vtdec_stop (GstVideoDecoder * decoder);
62 static gboolean gst_vtdec_negotiate (GstVideoDecoder * decoder);
63 static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder,
64     GstVideoCodecState * state);
65 static gboolean gst_vtdec_flush (GstVideoDecoder * decoder);
66 static GstFlowReturn gst_vtdec_finish (GstVideoDecoder * decoder);
67 static GstFlowReturn gst_vtdec_handle_frame (GstVideoDecoder * decoder,
68     GstVideoCodecFrame * frame);
69 
70 static OSStatus gst_vtdec_create_session (GstVtdec * vtdec,
71     GstVideoFormat format, gboolean enable_hardware);
72 static void gst_vtdec_invalidate_session (GstVtdec * vtdec);
73 static CMSampleBufferRef cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec,
74     GstBuffer * buf);
75 static GstFlowReturn gst_vtdec_push_frames_if_needed (GstVtdec * vtdec,
76     gboolean drain, gboolean flush);
77 static CMFormatDescriptionRef create_format_description (GstVtdec * vtdec,
78     CMVideoCodecType cm_format);
79 static CMFormatDescriptionRef
80 create_format_description_from_codec_data (GstVtdec * vtdec,
81     CMVideoCodecType cm_format, GstBuffer * codec_data);
82 static void gst_vtdec_session_output_callback (void
83     *decompression_output_ref_con, void *source_frame_ref_con, OSStatus status,
84     VTDecodeInfoFlags info_flags, CVImageBufferRef image_buffer, CMTime pts,
85     CMTime duration);
86 static gboolean compute_h264_decode_picture_buffer_length (GstVtdec * vtdec,
87     GstBuffer * codec_data, int *length);
88 static gboolean gst_vtdec_compute_reorder_queue_length (GstVtdec * vtdec,
89     CMVideoCodecType cm_format, GstBuffer * codec_data);
90 static void gst_vtdec_set_latency (GstVtdec * vtdec);
91 static void gst_vtdec_set_context (GstElement * element, GstContext * context);
92 
93 static GstStaticPadTemplate gst_vtdec_sink_template =
94     GST_STATIC_PAD_TEMPLATE ("sink",
95     GST_PAD_SINK,
96     GST_PAD_ALWAYS,
97     GST_STATIC_CAPS ("video/x-h264, stream-format=avc, alignment=au,"
98         " width=(int)[1, MAX], height=(int)[1, MAX];"
99         "video/mpeg, mpegversion=2, systemstream=false, parsed=true;"
100         "image/jpeg")
101     );
102 
103 /* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */
104 #if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1090
105 const CFStringRef
106     kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder =
107 CFSTR ("EnableHardwareAcceleratedVideoDecoder");
108 const CFStringRef
109     kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder =
110 CFSTR ("RequireHardwareAcceleratedVideoDecoder");
111 #endif
112 
113 #define VIDEO_SRC_CAPS \
114     GST_VIDEO_CAPS_MAKE("NV12") ";"                                     \
115     GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
116         "NV12") ", "                                                    \
117     "texture-target = (string) rectangle;"
118 
119 G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
120 
121 static void
gst_vtdec_class_init(GstVtdecClass * klass)122 gst_vtdec_class_init (GstVtdecClass * klass)
123 {
124   GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
125   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
126   GstVideoDecoderClass *video_decoder_class = GST_VIDEO_DECODER_CLASS (klass);
127 
128   /* Setting up pads and setting metadata should be moved to
129      base_class_init if you intend to subclass this class. */
130   gst_element_class_add_static_pad_template (element_class,
131       &gst_vtdec_sink_template);
132   gst_element_class_add_pad_template (element_class,
133       gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
134           gst_caps_from_string (VIDEO_SRC_CAPS)));
135 
136   gst_element_class_set_static_metadata (element_class,
137       "Apple VideoToolbox decoder",
138       "Codec/Decoder/Video/Hardware",
139       "Apple VideoToolbox Decoder",
140       "Ole André Vadla Ravnås <oleavr@soundrop.com>; "
141       "Alessandro Decina <alessandro.d@gmail.com>");
142 
143   gobject_class->finalize = gst_vtdec_finalize;
144   element_class->set_context = gst_vtdec_set_context;
145   video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vtdec_start);
146   video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vtdec_stop);
147   video_decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_vtdec_negotiate);
148   video_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_vtdec_set_format);
149   video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vtdec_flush);
150   video_decoder_class->finish = GST_DEBUG_FUNCPTR (gst_vtdec_finish);
151   video_decoder_class->handle_frame =
152       GST_DEBUG_FUNCPTR (gst_vtdec_handle_frame);
153 }
154 
155 static void
gst_vtdec_init(GstVtdec * vtdec)156 gst_vtdec_init (GstVtdec * vtdec)
157 {
158   vtdec->reorder_queue = g_async_queue_new ();
159 }
160 
161 void
gst_vtdec_finalize(GObject * object)162 gst_vtdec_finalize (GObject * object)
163 {
164   GstVtdec *vtdec = GST_VTDEC (object);
165 
166   GST_DEBUG_OBJECT (vtdec, "finalize");
167 
168   g_async_queue_unref (vtdec->reorder_queue);
169 
170   G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object);
171 }
172 
173 static gboolean
gst_vtdec_start(GstVideoDecoder * decoder)174 gst_vtdec_start (GstVideoDecoder * decoder)
175 {
176   GstVtdec *vtdec = GST_VTDEC (decoder);
177 
178   GST_DEBUG_OBJECT (vtdec, "start");
179 
180   if (!vtdec->ctxh)
181     vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (decoder));
182 
183   return TRUE;
184 }
185 
186 static gboolean
gst_vtdec_stop(GstVideoDecoder * decoder)187 gst_vtdec_stop (GstVideoDecoder * decoder)
188 {
189   GstVtdec *vtdec = GST_VTDEC (decoder);
190 
191   gst_vtdec_push_frames_if_needed (vtdec, TRUE, TRUE);
192 
193   if (vtdec->input_state)
194     gst_video_codec_state_unref (vtdec->input_state);
195   vtdec->input_state = NULL;
196 
197   if (vtdec->session)
198     gst_vtdec_invalidate_session (vtdec);
199 
200   if (vtdec->texture_cache)
201     gst_video_texture_cache_free (vtdec->texture_cache);
202   vtdec->texture_cache = NULL;
203 
204   if (vtdec->ctxh)
205     gst_gl_context_helper_free (vtdec->ctxh);
206   vtdec->ctxh = NULL;
207 
208   GST_DEBUG_OBJECT (vtdec, "stop");
209 
210   return TRUE;
211 }
212 
213 static void
setup_texture_cache(GstVtdec * vtdec,GstGLContext * context)214 setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
215 {
216   GstVideoCodecState *output_state;
217 
218   g_return_if_fail (vtdec->texture_cache == NULL);
219 
220   output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
221   vtdec->texture_cache = gst_video_texture_cache_new (context);
222   gst_video_texture_cache_set_format (vtdec->texture_cache,
223       GST_VIDEO_FORMAT_NV12, output_state->caps);
224   gst_video_codec_state_unref (output_state);
225 }
226 
227 static gboolean
gst_vtdec_negotiate(GstVideoDecoder * decoder)228 gst_vtdec_negotiate (GstVideoDecoder * decoder)
229 {
230   GstVideoCodecState *output_state = NULL;
231   GstCaps *peercaps = NULL, *caps = NULL, *templcaps = NULL, *prevcaps = NULL;
232   GstVideoFormat format;
233   GstStructure *structure;
234   const gchar *s;
235   GstVtdec *vtdec;
236   OSStatus err = noErr;
237   GstCapsFeatures *features = NULL;
238   gboolean output_textures;
239 
240   vtdec = GST_VTDEC (decoder);
241   if (vtdec->session)
242     gst_vtdec_push_frames_if_needed (vtdec, TRUE, FALSE);
243 
244   output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
245   if (output_state) {
246     prevcaps = gst_caps_ref (output_state->caps);
247     gst_video_codec_state_unref (output_state);
248   }
249 
250   peercaps = gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL);
251   if (prevcaps && gst_caps_can_intersect (prevcaps, peercaps)) {
252     /* The hardware decoder can become (temporarily) unavailable across
253      * VTDecompressionSessionCreate/Destroy calls. So if the currently configured
254      * caps are still accepted by downstream we keep them so we don't have to
255      * destroy and recreate the session.
256      */
257     GST_INFO_OBJECT (vtdec,
258         "current and peer caps are compatible, keeping current caps");
259     caps = gst_caps_ref (prevcaps);
260   } else {
261     templcaps =
262         gst_pad_get_pad_template_caps (GST_VIDEO_DECODER_SRC_PAD (decoder));
263     caps =
264         gst_caps_intersect_full (peercaps, templcaps, GST_CAPS_INTERSECT_FIRST);
265     gst_caps_unref (templcaps);
266   }
267   gst_caps_unref (peercaps);
268 
269   caps = gst_caps_truncate (gst_caps_make_writable (caps));
270   structure = gst_caps_get_structure (caps, 0);
271   s = gst_structure_get_string (structure, "format");
272   format = gst_video_format_from_string (s);
273   features = gst_caps_get_features (caps, 0);
274   if (features)
275     features = gst_caps_features_copy (features);
276 
277   output_state = gst_video_decoder_set_output_state (GST_VIDEO_DECODER (vtdec),
278       format, vtdec->video_info.width, vtdec->video_info.height,
279       vtdec->input_state);
280   output_state->caps = gst_video_info_to_caps (&output_state->info);
281   if (features) {
282     gst_caps_set_features (output_state->caps, 0, features);
283     output_textures =
284         gst_caps_features_contains (features,
285         GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
286     if (output_textures)
287       gst_caps_set_simple (output_state->caps, "texture-target", G_TYPE_STRING,
288 #if !HAVE_IOS
289           GST_GL_TEXTURE_TARGET_RECTANGLE_STR,
290 #else
291           GST_GL_TEXTURE_TARGET_2D_STR,
292 #endif
293           NULL);
294   }
295   gst_caps_unref (caps);
296 
297   if (!prevcaps || !gst_caps_is_equal (prevcaps, output_state->caps)) {
298     gboolean renegotiating = vtdec->session != NULL;
299 
300     GST_INFO_OBJECT (vtdec,
301         "negotiated output format %" GST_PTR_FORMAT " previous %"
302         GST_PTR_FORMAT, output_state->caps, prevcaps);
303 
304     if (vtdec->session)
305       gst_vtdec_invalidate_session (vtdec);
306 
307     err = gst_vtdec_create_session (vtdec, format, TRUE);
308     if (err == noErr) {
309       GST_INFO_OBJECT (vtdec, "using hardware decoder");
310     } else if (err == kVTVideoDecoderNotAvailableNowErr && renegotiating) {
311       GST_WARNING_OBJECT (vtdec, "hw decoder not available anymore");
312       err = gst_vtdec_create_session (vtdec, format, FALSE);
313     }
314 
315     if (err != noErr) {
316       GST_ELEMENT_ERROR (vtdec, RESOURCE, FAILED, (NULL),
317           ("VTDecompressionSessionCreate returned %d", (int) err));
318     }
319   }
320 
321   if (vtdec->texture_cache != NULL && !output_textures) {
322     gst_video_texture_cache_free (vtdec->texture_cache);
323     vtdec->texture_cache = NULL;
324   }
325 
326   if (err == noErr && output_textures) {
327     /* call this regardless of whether caps have changed or not since a new
328      * local context could have become available
329      */
330     if (!vtdec->ctxh)
331       vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
332     gst_gl_context_helper_ensure_context (vtdec->ctxh);
333 
334     GST_INFO_OBJECT (vtdec, "pushing textures, context %p old context %p",
335         vtdec->ctxh->context,
336         vtdec->texture_cache ? vtdec->texture_cache->ctx : NULL);
337 
338     if (vtdec->texture_cache
339         && vtdec->texture_cache->ctx != vtdec->ctxh->context) {
340       gst_video_texture_cache_free (vtdec->texture_cache);
341       vtdec->texture_cache = NULL;
342     }
343     if (!vtdec->texture_cache)
344       setup_texture_cache (vtdec, vtdec->ctxh->context);
345   }
346 
347   if (prevcaps)
348     gst_caps_unref (prevcaps);
349 
350   if (err != noErr)
351     return FALSE;
352 
353   return GST_VIDEO_DECODER_CLASS (gst_vtdec_parent_class)->negotiate (decoder);
354 }
355 
356 static gboolean
gst_vtdec_set_format(GstVideoDecoder * decoder,GstVideoCodecState * state)357 gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
358 {
359   GstStructure *structure;
360   CMVideoCodecType cm_format = 0;
361   CMFormatDescriptionRef format_description = NULL;
362   const char *caps_name;
363   GstVtdec *vtdec = GST_VTDEC (decoder);
364 
365   GST_DEBUG_OBJECT (vtdec, "set_format");
366 
367   structure = gst_caps_get_structure (state->caps, 0);
368   caps_name = gst_structure_get_name (structure);
369   if (!strcmp (caps_name, "video/x-h264")) {
370     cm_format = kCMVideoCodecType_H264;
371   } else if (!strcmp (caps_name, "video/mpeg")) {
372     cm_format = kCMVideoCodecType_MPEG2Video;
373   } else if (!strcmp (caps_name, "image/jpeg")) {
374     cm_format = kCMVideoCodecType_JPEG;
375   }
376 
377   if (cm_format == kCMVideoCodecType_H264 && state->codec_data == NULL) {
378     GST_INFO_OBJECT (vtdec, "no codec data, wait for one");
379     return TRUE;
380   }
381 
382   gst_video_info_from_caps (&vtdec->video_info, state->caps);
383 
384   if (!gst_vtdec_compute_reorder_queue_length (vtdec, cm_format,
385           state->codec_data))
386     return FALSE;
387   gst_vtdec_set_latency (vtdec);
388 
389   if (state->codec_data) {
390     format_description = create_format_description_from_codec_data (vtdec,
391         cm_format, state->codec_data);
392   } else {
393     format_description = create_format_description (vtdec, cm_format);
394   }
395 
396   if (vtdec->format_description)
397     CFRelease (vtdec->format_description);
398   vtdec->format_description = format_description;
399 
400   if (vtdec->input_state)
401     gst_video_codec_state_unref (vtdec->input_state);
402   vtdec->input_state = gst_video_codec_state_ref (state);
403 
404   return gst_video_decoder_negotiate (decoder);
405 }
406 
407 static gboolean
gst_vtdec_flush(GstVideoDecoder * decoder)408 gst_vtdec_flush (GstVideoDecoder * decoder)
409 {
410   GstVtdec *vtdec = GST_VTDEC (decoder);
411 
412   GST_DEBUG_OBJECT (vtdec, "flush");
413 
414   gst_vtdec_push_frames_if_needed (vtdec, FALSE, TRUE);
415 
416   return TRUE;
417 }
418 
419 static GstFlowReturn
gst_vtdec_finish(GstVideoDecoder * decoder)420 gst_vtdec_finish (GstVideoDecoder * decoder)
421 {
422   GstVtdec *vtdec = GST_VTDEC (decoder);
423 
424   GST_DEBUG_OBJECT (vtdec, "finish");
425 
426   return gst_vtdec_push_frames_if_needed (vtdec, TRUE, FALSE);
427 }
428 
429 static GstFlowReturn
gst_vtdec_handle_frame(GstVideoDecoder * decoder,GstVideoCodecFrame * frame)430 gst_vtdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
431 {
432   OSStatus status;
433   CMSampleBufferRef cm_sample_buffer = NULL;
434   VTDecodeFrameFlags input_flags, output_flags;
435   GstVtdec *vtdec = GST_VTDEC (decoder);
436   GstFlowReturn ret = GST_FLOW_OK;
437   int decode_frame_number = frame->decode_frame_number;
438 
439   if (vtdec->format_description == NULL) {
440     ret = GST_FLOW_NOT_NEGOTIATED;
441     goto out;
442   }
443 
444   GST_LOG_OBJECT (vtdec, "got input frame %d", decode_frame_number);
445 
446   ret = gst_vtdec_push_frames_if_needed (vtdec, FALSE, FALSE);
447   if (ret != GST_FLOW_OK)
448     return ret;
449 
450   /* don't bother enabling kVTDecodeFrame_EnableTemporalProcessing at all since
451    * it's not mandatory for the underlying VT codec to respect it. KISS and do
452    * reordering ourselves.
453    */
454   input_flags = kVTDecodeFrame_EnableAsynchronousDecompression;
455   output_flags = 0;
456 
457   cm_sample_buffer =
458       cm_sample_buffer_from_gst_buffer (vtdec, frame->input_buffer);
459   status =
460       VTDecompressionSessionDecodeFrame (vtdec->session, cm_sample_buffer,
461       input_flags, frame, NULL);
462   if (status != noErr && FALSE)
463     goto error;
464 
465   GST_LOG_OBJECT (vtdec, "submitted input frame %d", decode_frame_number);
466 
467 out:
468   if (cm_sample_buffer)
469     CFRelease (cm_sample_buffer);
470   return ret;
471 
472 error:
473   GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
474       ("VTDecompressionSessionDecodeFrame returned %d", (int) status));
475   ret = GST_FLOW_ERROR;
476   goto out;
477 }
478 
479 static void
gst_vtdec_invalidate_session(GstVtdec * vtdec)480 gst_vtdec_invalidate_session (GstVtdec * vtdec)
481 {
482   g_return_if_fail (vtdec->session);
483 
484   VTDecompressionSessionInvalidate (vtdec->session);
485   CFRelease (vtdec->session);
486   vtdec->session = NULL;
487 }
488 
489 static OSStatus
gst_vtdec_create_session(GstVtdec * vtdec,GstVideoFormat format,gboolean enable_hardware)490 gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format,
491     gboolean enable_hardware)
492 {
493   CFMutableDictionaryRef output_image_buffer_attrs;
494   VTDecompressionOutputCallbackRecord callback;
495   CFMutableDictionaryRef videoDecoderSpecification;
496   OSStatus status;
497   guint32 cv_format = 0;
498 
499   g_return_val_if_fail (vtdec->session == NULL, FALSE);
500 
501   switch (format) {
502     case GST_VIDEO_FORMAT_NV12:
503       cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
504       break;
505     case GST_VIDEO_FORMAT_UYVY:
506       cv_format = kCVPixelFormatType_422YpCbCr8;
507       break;
508     case GST_VIDEO_FORMAT_RGBA:
509       cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
510       break;
511     default:
512       g_warn_if_reached ();
513       break;
514   }
515 
516   videoDecoderSpecification =
517       CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
518       &kCFTypeDictionaryValueCallBacks);
519 
520   /* This is the default on iOS and the key does not exist there */
521 #ifndef HAVE_IOS
522   gst_vtutil_dict_set_boolean (videoDecoderSpecification,
523       kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
524       enable_hardware);
525   if (enable_hardware && vtdec->require_hardware)
526     gst_vtutil_dict_set_boolean (videoDecoderSpecification,
527         kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
528         TRUE);
529 #endif
530 
531   output_image_buffer_attrs =
532       CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
533       &kCFTypeDictionaryValueCallBacks);
534   gst_vtutil_dict_set_i32 (output_image_buffer_attrs,
535       kCVPixelBufferPixelFormatTypeKey, cv_format);
536   gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferWidthKey,
537       vtdec->video_info.width);
538   gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferHeightKey,
539       vtdec->video_info.height);
540 
541   callback.decompressionOutputCallback = gst_vtdec_session_output_callback;
542   callback.decompressionOutputRefCon = vtdec;
543 
544   status = VTDecompressionSessionCreate (NULL, vtdec->format_description,
545       videoDecoderSpecification, output_image_buffer_attrs, &callback,
546       &vtdec->session);
547 
548   CFRelease (output_image_buffer_attrs);
549 
550   return status;
551 }
552 
553 static CMFormatDescriptionRef
create_format_description(GstVtdec * vtdec,CMVideoCodecType cm_format)554 create_format_description (GstVtdec * vtdec, CMVideoCodecType cm_format)
555 {
556   OSStatus status;
557   CMFormatDescriptionRef format_description;
558 
559   status = CMVideoFormatDescriptionCreate (NULL,
560       cm_format, vtdec->video_info.width, vtdec->video_info.height,
561       NULL, &format_description);
562   if (status != noErr)
563     return NULL;
564 
565   return format_description;
566 }
567 
568 static CMFormatDescriptionRef
create_format_description_from_codec_data(GstVtdec * vtdec,CMVideoCodecType cm_format,GstBuffer * codec_data)569 create_format_description_from_codec_data (GstVtdec * vtdec,
570     CMVideoCodecType cm_format, GstBuffer * codec_data)
571 {
572   CMFormatDescriptionRef fmt_desc;
573   CFMutableDictionaryRef extensions, par, atoms;
574   GstMapInfo map;
575   OSStatus status;
576 
577   /* Extensions dict */
578   extensions =
579       CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
580       &kCFTypeDictionaryValueCallBacks);
581   gst_vtutil_dict_set_string (extensions,
582       CFSTR ("CVImageBufferChromaLocationBottomField"), "left");
583   gst_vtutil_dict_set_string (extensions,
584       CFSTR ("CVImageBufferChromaLocationTopField"), "left");
585   gst_vtutil_dict_set_boolean (extensions, CFSTR ("FullRangeVideo"), FALSE);
586 
587   /* CVPixelAspectRatio dict */
588   par = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
589       &kCFTypeDictionaryValueCallBacks);
590   gst_vtutil_dict_set_i32 (par, CFSTR ("HorizontalSpacing"),
591       vtdec->video_info.par_n);
592   gst_vtutil_dict_set_i32 (par, CFSTR ("VerticalSpacing"),
593       vtdec->video_info.par_d);
594   gst_vtutil_dict_set_object (extensions, CFSTR ("CVPixelAspectRatio"),
595       (CFTypeRef *) par);
596 
597   /* SampleDescriptionExtensionAtoms dict */
598   gst_buffer_map (codec_data, &map, GST_MAP_READ);
599   atoms = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
600       &kCFTypeDictionaryValueCallBacks);
601   gst_vtutil_dict_set_data (atoms, CFSTR ("avcC"), map.data, map.size);
602   gst_vtutil_dict_set_object (extensions,
603       CFSTR ("SampleDescriptionExtensionAtoms"), (CFTypeRef *) atoms);
604   gst_buffer_unmap (codec_data, &map);
605 
606   status = CMVideoFormatDescriptionCreate (NULL,
607       cm_format, vtdec->video_info.width, vtdec->video_info.height,
608       extensions, &fmt_desc);
609 
610   if (status == noErr)
611     return fmt_desc;
612   else
613     return NULL;
614 }
615 
616 /* Custom FreeBlock function for CMBlockBuffer */
617 static void
cm_block_buffer_freeblock(void * refCon,void * doomedMemoryBlock,size_t sizeInBytes)618 cm_block_buffer_freeblock (void *refCon, void *doomedMemoryBlock,
619     size_t sizeInBytes)
620 {
621   GstMapInfo *info = (GstMapInfo *) refCon;
622 
623   gst_memory_unmap (info->memory, info);
624   gst_memory_unref (info->memory);
625   g_slice_free (GstMapInfo, info);
626 }
627 
628 static CMBlockBufferRef
cm_block_buffer_from_gst_buffer(GstBuffer * buf,GstMapFlags flags)629 cm_block_buffer_from_gst_buffer (GstBuffer * buf, GstMapFlags flags)
630 {
631   OSStatus status;
632   CMBlockBufferRef bbuf;
633   CMBlockBufferCustomBlockSource blockSource;
634   guint memcount, i;
635 
636   /* Initialize custom block source structure */
637   blockSource.version = kCMBlockBufferCustomBlockSourceVersion;
638   blockSource.AllocateBlock = NULL;
639   blockSource.FreeBlock = cm_block_buffer_freeblock;
640 
641   /* Determine number of memory blocks */
642   memcount = gst_buffer_n_memory (buf);
643   status = CMBlockBufferCreateEmpty (NULL, memcount, 0, &bbuf);
644   if (status != kCMBlockBufferNoErr) {
645     GST_ERROR ("CMBlockBufferCreateEmpty returned %d", (int) status);
646     return NULL;
647   }
648 
649   /* Go over all GstMemory objects and add them to the CMBlockBuffer */
650   for (i = 0; i < memcount; ++i) {
651     GstMemory *mem;
652     GstMapInfo *info;
653 
654     mem = gst_buffer_get_memory (buf, i);
655 
656     info = g_slice_new (GstMapInfo);
657     if (!gst_memory_map (mem, info, flags)) {
658       GST_ERROR ("failed mapping memory");
659       g_slice_free (GstMapInfo, info);
660       gst_memory_unref (mem);
661       CFRelease (bbuf);
662       return NULL;
663     }
664 
665     blockSource.refCon = info;
666     status =
667         CMBlockBufferAppendMemoryBlock (bbuf, info->data, info->size, NULL,
668         &blockSource, 0, info->size, 0);
669     if (status != kCMBlockBufferNoErr) {
670       GST_ERROR ("CMBlockBufferAppendMemoryBlock returned %d", (int) status);
671       gst_memory_unmap (mem, info);
672       g_slice_free (GstMapInfo, info);
673       gst_memory_unref (mem);
674       CFRelease (bbuf);
675       return NULL;
676     }
677   }
678 
679   return bbuf;
680 }
681 
682 static CMSampleBufferRef
cm_sample_buffer_from_gst_buffer(GstVtdec * vtdec,GstBuffer * buf)683 cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec, GstBuffer * buf)
684 {
685   OSStatus status;
686   CMBlockBufferRef bbuf = NULL;
687   CMSampleBufferRef sbuf = NULL;
688   CMSampleTimingInfo sample_timing;
689   CMSampleTimingInfo time_array[1];
690 
691   g_return_val_if_fail (vtdec->format_description, NULL);
692 
693   /* create a block buffer */
694   bbuf = cm_block_buffer_from_gst_buffer (buf, GST_MAP_READ);
695   if (bbuf == NULL) {
696     GST_ELEMENT_ERROR (vtdec, RESOURCE, FAILED, (NULL),
697         ("failed creating CMBlockBuffer"));
698     return NULL;
699   }
700 
701   /* create a sample buffer */
702   if (GST_BUFFER_DURATION_IS_VALID (buf))
703     sample_timing.duration = CMTimeMake (GST_BUFFER_DURATION (buf), GST_SECOND);
704   else
705     sample_timing.duration = kCMTimeInvalid;
706 
707   if (GST_BUFFER_PTS_IS_VALID (buf))
708     sample_timing.presentationTimeStamp =
709         CMTimeMake (GST_BUFFER_PTS (buf), GST_SECOND);
710   else
711     sample_timing.presentationTimeStamp = kCMTimeInvalid;
712 
713   if (GST_BUFFER_DTS_IS_VALID (buf))
714     sample_timing.decodeTimeStamp =
715         CMTimeMake (GST_BUFFER_DTS (buf), GST_SECOND);
716   else
717     sample_timing.decodeTimeStamp = kCMTimeInvalid;
718 
719   time_array[0] = sample_timing;
720 
721   status =
722       CMSampleBufferCreate (NULL, bbuf, TRUE, 0, 0, vtdec->format_description,
723       1, 1, time_array, 0, NULL, &sbuf);
724   CFRelease (bbuf);
725   if (status != noErr) {
726     GST_ELEMENT_ERROR (vtdec, RESOURCE, FAILED, (NULL),
727         ("CMSampleBufferCreate returned %d", (int) status));
728     return NULL;
729   }
730 
731   return sbuf;
732 }
733 
734 static gint
sort_frames_by_pts(gconstpointer f1,gconstpointer f2,gpointer user_data)735 sort_frames_by_pts (gconstpointer f1, gconstpointer f2, gpointer user_data)
736 {
737   GstVideoCodecFrame *frame1, *frame2;
738   GstClockTime pts1, pts2;
739 
740   frame1 = (GstVideoCodecFrame *) f1;
741   frame2 = (GstVideoCodecFrame *) f2;
742   pts1 = pts2 = GST_CLOCK_TIME_NONE;
743   if (frame1->output_buffer)
744     pts1 = GST_BUFFER_PTS (frame1->output_buffer);
745   if (frame2->output_buffer)
746     pts2 = GST_BUFFER_PTS (frame2->output_buffer);
747 
748   if (!GST_CLOCK_TIME_IS_VALID (pts1) || !GST_CLOCK_TIME_IS_VALID (pts2))
749     return 0;
750 
751   if (pts1 < pts2)
752     return -1;
753   else if (pts1 == pts2)
754     return 0;
755   else
756     return 1;
757 }
758 
759 static void
gst_vtdec_session_output_callback(void * decompression_output_ref_con,void * source_frame_ref_con,OSStatus status,VTDecodeInfoFlags info_flags,CVImageBufferRef image_buffer,CMTime pts,CMTime duration)760 gst_vtdec_session_output_callback (void *decompression_output_ref_con,
761     void *source_frame_ref_con, OSStatus status, VTDecodeInfoFlags info_flags,
762     CVImageBufferRef image_buffer, CMTime pts, CMTime duration)
763 {
764   GstVtdec *vtdec = (GstVtdec *) decompression_output_ref_con;
765   GstVideoCodecFrame *frame = (GstVideoCodecFrame *) source_frame_ref_con;
766   GstVideoCodecState *state;
767 
768   GST_LOG_OBJECT (vtdec, "got output frame %p %d and VT buffer %p", frame,
769       frame->decode_frame_number, image_buffer);
770 
771   frame->output_buffer = NULL;
772 
773   if (status != noErr) {
774     GST_ERROR_OBJECT (vtdec, "Error decoding frame %d", (int) status);
775   }
776 
777   if (image_buffer) {
778     GstBuffer *buf = NULL;
779 
780     /* FIXME: use gst_video_decoder_allocate_output_buffer */
781     state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
782     if (state == NULL) {
783       GST_WARNING_OBJECT (vtdec, "Output state not configured, release buffer");
784       frame->flags &= VTDEC_FRAME_FLAG_SKIP;
785     } else {
786       buf =
787           gst_core_video_buffer_new (image_buffer, &state->info,
788           vtdec->texture_cache);
789       gst_video_codec_state_unref (state);
790       GST_BUFFER_PTS (buf) = pts.value;
791       GST_BUFFER_DURATION (buf) = duration.value;
792       frame->output_buffer = buf;
793     }
794   } else {
795     if (info_flags & kVTDecodeInfo_FrameDropped) {
796       GST_DEBUG_OBJECT (vtdec, "Frame dropped by video toolbox %p %d",
797           frame, frame->decode_frame_number);
798       frame->flags |= VTDEC_FRAME_FLAG_DROP;
799     } else {
800       GST_DEBUG_OBJECT (vtdec, "Decoded frame is NULL");
801       frame->flags |= VTDEC_FRAME_FLAG_SKIP;
802     }
803   }
804 
805   g_async_queue_push_sorted (vtdec->reorder_queue, frame,
806       sort_frames_by_pts, NULL);
807 }
808 
809 static GstFlowReturn
gst_vtdec_push_frames_if_needed(GstVtdec * vtdec,gboolean drain,gboolean flush)810 gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
811     gboolean flush)
812 {
813   GstVideoCodecFrame *frame;
814   GstFlowReturn ret = GST_FLOW_OK;
815   GstVideoDecoder *decoder = GST_VIDEO_DECODER (vtdec);
816 
817   /* negotiate now so that we know whether we need to use the GL upload meta or
818    * not */
819   if (gst_pad_check_reconfigure (decoder->srcpad)) {
820     if (!gst_video_decoder_negotiate (decoder)) {
821       gst_pad_mark_reconfigure (decoder->srcpad);
822       if (GST_PAD_IS_FLUSHING (decoder->srcpad))
823         ret = GST_FLOW_FLUSHING;
824       else
825         ret = GST_FLOW_NOT_NEGOTIATED;
826       return ret;
827     }
828   }
829 
830   if (drain)
831     VTDecompressionSessionWaitForAsynchronousFrames (vtdec->session);
832 
833   /* push a buffer if there are enough frames to guarantee that we push in PTS
834    * order
835    */
836   while ((g_async_queue_length (vtdec->reorder_queue) >=
837           vtdec->reorder_queue_length) || drain || flush) {
838     frame = (GstVideoCodecFrame *) g_async_queue_try_pop (vtdec->reorder_queue);
839 
840     /* we need to check this in case reorder_queue_length=0 (jpeg for
841      * example) or we're draining/flushing
842      */
843     if (frame) {
844       if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP)
845         gst_video_decoder_release_frame (decoder, frame);
846       else if (frame->flags & VTDEC_FRAME_FLAG_DROP)
847         gst_video_decoder_drop_frame (decoder, frame);
848       else
849         ret = gst_video_decoder_finish_frame (decoder, frame);
850     }
851 
852     if (!frame || ret != GST_FLOW_OK)
853       break;
854   }
855 
856   return ret;
857 }
858 
859 static gboolean
parse_h264_profile_and_level_from_codec_data(GstVtdec * vtdec,GstBuffer * codec_data,int * profile,int * level)860 parse_h264_profile_and_level_from_codec_data (GstVtdec * vtdec,
861     GstBuffer * codec_data, int *profile, int *level)
862 {
863   GstMapInfo map;
864   guint8 *data;
865   gint size;
866   gboolean ret = TRUE;
867 
868   gst_buffer_map (codec_data, &map, GST_MAP_READ);
869   data = map.data;
870   size = map.size;
871 
872   /* parse the avcC data */
873   if (size < 7)
874     goto avcc_too_small;
875 
876   /* parse the version, this must be 1 */
877   if (data[0] != 1)
878     goto wrong_version;
879 
880   /* AVCProfileIndication */
881   /* profile_compat */
882   /* AVCLevelIndication */
883   if (profile)
884     *profile = data[1];
885 
886   if (level)
887     *level = data[3];
888 
889 out:
890   gst_buffer_unmap (codec_data, &map);
891 
892   return ret;
893 
894 avcc_too_small:
895   GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
896       ("invalid codec_data buffer length"));
897   ret = FALSE;
898   goto out;
899 
900 wrong_version:
901   GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
902       ("wrong avcC version in codec_data"));
903   ret = FALSE;
904   goto out;
905 }
906 
907 static int
get_dpb_max_mb_s_from_level(GstVtdec * vtdec,int level)908 get_dpb_max_mb_s_from_level (GstVtdec * vtdec, int level)
909 {
910   switch (level) {
911     case 10:
912       /* 1b?? */
913       return 396;
914     case 11:
915       return 900;
916     case 12:
917     case 13:
918     case 20:
919       return 2376;
920     case 21:
921       return 4752;
922     case 22:
923     case 30:
924       return 8100;
925     case 31:
926       return 18000;
927     case 32:
928       return 20480;
929     case 40:
930     case 41:
931       return 32768;
932     case 42:
933       return 34816;
934     case 50:
935       return 110400;
936     case 51:
937     case 52:
938       return 184320;
939     default:
940       GST_ERROR_OBJECT (vtdec, "unknown level %d", level);
941       return -1;
942   }
943 }
944 
945 static gboolean
gst_vtdec_compute_reorder_queue_length(GstVtdec * vtdec,CMVideoCodecType cm_format,GstBuffer * codec_data)946 gst_vtdec_compute_reorder_queue_length (GstVtdec * vtdec,
947     CMVideoCodecType cm_format, GstBuffer * codec_data)
948 {
949   if (cm_format == kCMVideoCodecType_H264) {
950     if (!compute_h264_decode_picture_buffer_length (vtdec, codec_data,
951             &vtdec->reorder_queue_length)) {
952       return FALSE;
953     }
954   } else {
955     vtdec->reorder_queue_length = 0;
956   }
957 
958   return TRUE;
959 }
960 
961 static gboolean
compute_h264_decode_picture_buffer_length(GstVtdec * vtdec,GstBuffer * codec_data,int * length)962 compute_h264_decode_picture_buffer_length (GstVtdec * vtdec,
963     GstBuffer * codec_data, int *length)
964 {
965   int profile, level;
966   int dpb_mb_size = 16;
967   int max_dpb_size_frames = 16;
968   int max_dpb_mb_s = -1;
969   int width_in_mb_s = GST_ROUND_UP_16 (vtdec->video_info.width) / dpb_mb_size;
970   int height_in_mb_s = GST_ROUND_UP_16 (vtdec->video_info.height) / dpb_mb_size;
971 
972   *length = 0;
973 
974   if (!parse_h264_profile_and_level_from_codec_data (vtdec, codec_data,
975           &profile, &level))
976     return FALSE;
977 
978   if (vtdec->video_info.width == 0 || vtdec->video_info.height == 0)
979     return FALSE;
980 
981   GST_INFO_OBJECT (vtdec, "parsed profile %d, level %d", profile, level);
982   if (profile == 66) {
983     /* baseline or constrained-baseline, we don't need to reorder */
984     return TRUE;
985   }
986 
987   max_dpb_mb_s = get_dpb_max_mb_s_from_level (vtdec, level);
988   if (max_dpb_mb_s == -1) {
989     GST_ELEMENT_ERROR (vtdec, STREAM, DECODE, (NULL),
990         ("invalid level in codec_data, could not compute max_dpb_mb_s"));
991     return FALSE;
992   }
993 
994   /* this formula is specified in sections A.3.1.h and A.3.2.f of the 2009
995    * edition of the standard */
996   *length = MIN (floor (max_dpb_mb_s / (width_in_mb_s * height_in_mb_s)),
997       max_dpb_size_frames);
998   return TRUE;
999 }
1000 
1001 static void
gst_vtdec_set_latency(GstVtdec * vtdec)1002 gst_vtdec_set_latency (GstVtdec * vtdec)
1003 {
1004   GstClockTime frame_duration;
1005   GstClockTime latency;
1006 
1007   if (vtdec->video_info.fps_n == 0) {
1008     GST_INFO_OBJECT (vtdec, "Framerate not known, can't set latency");
1009     return;
1010   }
1011 
1012   frame_duration = gst_util_uint64_scale (GST_SECOND,
1013       vtdec->video_info.fps_d, vtdec->video_info.fps_n);
1014   latency = frame_duration * vtdec->reorder_queue_length;
1015 
1016   GST_INFO_OBJECT (vtdec, "setting latency frames:%d time:%" GST_TIME_FORMAT,
1017       vtdec->reorder_queue_length, GST_TIME_ARGS (latency));
1018   gst_video_decoder_set_latency (GST_VIDEO_DECODER (vtdec), latency, latency);
1019 }
1020 
1021 static void
gst_vtdec_set_context(GstElement * element,GstContext * context)1022 gst_vtdec_set_context (GstElement * element, GstContext * context)
1023 {
1024   GstVtdec *vtdec = GST_VTDEC (element);
1025 
1026   GST_INFO_OBJECT (element, "setting context %s",
1027       gst_context_get_context_type (context));
1028   if (!vtdec->ctxh)
1029     vtdec->ctxh = gst_gl_context_helper_new (element);
1030   gst_gl_handle_set_context (element, context,
1031       &vtdec->ctxh->display, &vtdec->ctxh->other_context);
1032   GST_ELEMENT_CLASS (gst_vtdec_parent_class)->set_context (element, context);
1033 }
1034 
1035 #ifndef HAVE_IOS
1036 #define GST_TYPE_VTDEC_HW   (gst_vtdec_hw_get_type())
1037 #define GST_VTDEC_HW(obj)   (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VTDEC_HW,GstVtdecHw))
1038 #define GST_VTDEC_HW_CLASS(klass)   (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VTDEC_HW,GstVtdecHwClass))
1039 #define GST_IS_VTDEC_HW(obj)   (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VTDEC_HW))
1040 #define GST_IS_VTDEC_HW_CLASS(obj)   (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VTDEC_HW))
1041 
1042 typedef GstVtdec GstVtdecHw;
1043 typedef GstVtdecClass GstVtdecHwClass;
1044 
1045 GType gst_vtdec_hw_get_type (void);
1046 
1047 G_DEFINE_TYPE (GstVtdecHw, gst_vtdec_hw, GST_TYPE_VTDEC);
1048 
1049 static void
gst_vtdec_hw_class_init(GstVtdecHwClass * klass)1050 gst_vtdec_hw_class_init (GstVtdecHwClass * klass)
1051 {
1052   gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass),
1053       "Apple VideoToolbox decoder (hardware only)",
1054       "Codec/Decoder/Video/Hardware",
1055       "Apple VideoToolbox Decoder",
1056       "Ole André Vadla Ravnås <oleavr@soundrop.com>; "
1057       "Alessandro Decina <alessandro.d@gmail.com>");
1058 }
1059 
1060 static void
gst_vtdec_hw_init(GstVtdecHw * vtdec)1061 gst_vtdec_hw_init (GstVtdecHw * vtdec)
1062 {
1063   GST_VTDEC (vtdec)->require_hardware = TRUE;
1064 }
1065 
1066 #endif
1067 
1068 void
gst_vtdec_register_elements(GstPlugin * plugin)1069 gst_vtdec_register_elements (GstPlugin * plugin)
1070 {
1071   GST_DEBUG_CATEGORY_INIT (gst_vtdec_debug_category, "vtdec", 0,
1072       "debug category for vtdec element");
1073 
1074 #ifdef HAVE_IOS
1075   gst_element_register (plugin, "vtdec", GST_RANK_PRIMARY, GST_TYPE_VTDEC);
1076 #else
1077   gst_element_register (plugin, "vtdec_hw", GST_RANK_PRIMARY + 1,
1078       GST_TYPE_VTDEC_HW);
1079   gst_element_register (plugin, "vtdec", GST_RANK_SECONDARY, GST_TYPE_VTDEC);
1080 #endif
1081 }
1082