1 /* GStreamer Intel MSDK plugin
2 * Copyright (c) 2016, Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 *
15 * 3. Neither the name of the copyright holder nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
28 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #ifdef HAVE_CONFIG_H
33 # include <config.h>
34 #endif
35
36 #include <stdlib.h>
37
38 #include "gstmsdkdec.h"
39 #include "gstmsdkbufferpool.h"
40 #include "gstmsdkvideomemory.h"
41 #include "gstmsdksystemmemory.h"
42 #include "gstmsdkcontextutil.h"
43
44 GST_DEBUG_CATEGORY_EXTERN (gst_msdkdec_debug);
45 #define GST_CAT_DEFAULT gst_msdkdec_debug
46
47 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
48 GST_PAD_SRC,
49 GST_PAD_ALWAYS,
50 GST_STATIC_CAPS ("video/x-raw, "
51 "format = (string) { NV12 }, "
52 "framerate = (fraction) [0, MAX], "
53 "width = (int) [ 1, MAX ], height = (int) [ 1, MAX ],"
54 "interlace-mode = (string) progressive;"
55 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF,
56 "{ NV12 }") ";")
57 );
58
59 #define PROP_HARDWARE_DEFAULT TRUE
60 #define PROP_ASYNC_DEPTH_DEFAULT 1
61
62 #define IS_ALIGNED(i, n) (((i) & ((n)-1)) == 0)
63
64 #define gst_msdkdec_parent_class parent_class
65 G_DEFINE_TYPE (GstMsdkDec, gst_msdkdec, GST_TYPE_VIDEO_DECODER);
66
67 typedef struct _MsdkSurface
68 {
69 mfxFrameSurface1 *surface;
70 GstBuffer *buf;
71 GstVideoFrame data;
72 GstVideoFrame copy;
73 } MsdkSurface;
74
75 static gboolean gst_msdkdec_drain (GstVideoDecoder * decoder);
76 static gboolean gst_msdkdec_flush (GstVideoDecoder * decoder);
77 static gboolean gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset);
78
79 static GstVideoCodecFrame *
gst_msdkdec_get_oldest_frame(GstVideoDecoder * decoder)80 gst_msdkdec_get_oldest_frame (GstVideoDecoder * decoder)
81 {
82 GstVideoCodecFrame *frame = NULL, *old_frame = NULL;
83 GList *frames, *l;
84 gint count = 0;
85
86 frames = gst_video_decoder_get_frames (decoder);
87
88 for (l = frames; l != NULL; l = l->next) {
89 GstVideoCodecFrame *f = l->data;
90
91 if (!GST_CLOCK_TIME_IS_VALID (f->pts)) {
92 GST_INFO
93 ("Frame doesn't have a valid pts yet, Use gst_video_decoder_get_oldest_frame()"
94 "with out considering the PTS for selecting the frame to be finished");
95 old_frame = gst_video_decoder_get_oldest_frame (decoder);
96 break;
97 }
98
99 if (!frame || frame->pts > f->pts)
100 frame = f;
101
102 count++;
103 }
104
105 if (old_frame)
106 frame = old_frame;
107
108 if (frame) {
109 GST_LOG_OBJECT (decoder,
110 "Oldest frame is %d %" GST_TIME_FORMAT " and %d frames left",
111 frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1);
112 gst_video_codec_frame_ref (frame);
113 }
114
115 if (old_frame)
116 gst_video_codec_frame_unref (old_frame);
117
118 g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref);
119
120 return frame;
121 }
122
123 static GstFlowReturn
allocate_output_buffer(GstMsdkDec * thiz,GstBuffer ** buffer)124 allocate_output_buffer (GstMsdkDec * thiz, GstBuffer ** buffer)
125 {
126 GstFlowReturn flow;
127 GstVideoCodecFrame *frame;
128 GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
129
130 frame = gst_msdkdec_get_oldest_frame (decoder);
131 if (!frame) {
132 if (GST_PAD_IS_FLUSHING (decoder->srcpad))
133 return GST_FLOW_FLUSHING;
134 else
135 return GST_FLOW_ERROR;
136 }
137
138 if (!frame->output_buffer) {
139 flow = gst_video_decoder_allocate_output_frame (decoder, frame);
140 if (flow != GST_FLOW_OK) {
141 gst_video_codec_frame_unref (frame);
142 return flow;
143 }
144 }
145
146 *buffer = gst_buffer_ref (frame->output_buffer);
147 gst_buffer_replace (&frame->output_buffer, NULL);
148 gst_video_codec_frame_unref (frame);
149 return GST_FLOW_OK;
150 }
151
152 static void
free_surface(GstMsdkDec * thiz,MsdkSurface * s)153 free_surface (GstMsdkDec * thiz, MsdkSurface * s)
154 {
155 if (s->copy.buffer) {
156 gst_video_frame_unmap (&s->copy);
157 gst_buffer_unref (s->copy.buffer);
158 }
159
160 if (s->data.buffer)
161 gst_video_frame_unmap (&s->data);
162
163 gst_buffer_unref (s->buf);
164 thiz->decoded_msdk_surfaces = g_list_remove (thiz->decoded_msdk_surfaces, s);
165
166 g_slice_free (MsdkSurface, s);
167 }
168
169 static MsdkSurface *
get_surface(GstMsdkDec * thiz,GstBuffer * buffer)170 get_surface (GstMsdkDec * thiz, GstBuffer * buffer)
171 {
172 MsdkSurface *i;
173
174 i = g_slice_new0 (MsdkSurface);
175
176 if (gst_msdk_is_msdk_buffer (buffer)) {
177 i->surface = gst_msdk_get_surface_from_buffer (buffer);
178 i->buf = buffer;
179 } else {
180 /* Confirm to activate the side pool */
181 if (!gst_buffer_pool_is_active (thiz->pool) &&
182 !gst_buffer_pool_set_active (thiz->pool, TRUE)) {
183 g_slice_free (MsdkSurface, i);
184 return NULL;
185 }
186
187 if (!gst_video_frame_map (&i->copy, &thiz->non_msdk_pool_info, buffer,
188 GST_MAP_WRITE))
189 goto failed_unref_buffer;
190
191 if (gst_buffer_pool_acquire_buffer (thiz->pool, &buffer,
192 NULL) != GST_FLOW_OK)
193 goto failed_unmap_copy;
194
195 i->surface = gst_msdk_get_surface_from_buffer (buffer);
196 i->buf = buffer;
197
198 if (!gst_video_frame_map (&i->data, &thiz->output_info, buffer,
199 GST_MAP_READWRITE))
200 goto failed_unref_buffer2;
201 }
202
203 thiz->decoded_msdk_surfaces = g_list_append (thiz->decoded_msdk_surfaces, i);
204 return i;
205
206 failed_unref_buffer2:
207 gst_buffer_unref (buffer);
208 buffer = i->data.buffer;
209 failed_unmap_copy:
210 gst_video_frame_unmap (&i->copy);
211 failed_unref_buffer:
212 gst_buffer_unref (buffer);
213 g_slice_free (MsdkSurface, i);
214
215 GST_ERROR_OBJECT (thiz, "failed to handle buffer");
216 return NULL;
217 }
218
219 static void
gst_msdkdec_close_decoder(GstMsdkDec * thiz,gboolean reset_param)220 gst_msdkdec_close_decoder (GstMsdkDec * thiz, gboolean reset_param)
221 {
222 mfxStatus status;
223
224 if (!thiz->context || !thiz->initialized)
225 return;
226
227 GST_DEBUG_OBJECT (thiz, "Closing decoder with context %" GST_PTR_FORMAT,
228 thiz->context);
229
230 if (thiz->use_video_memory)
231 gst_msdk_frame_free (thiz->context, &thiz->alloc_resp);
232
233 status = MFXVideoDECODE_Close (gst_msdk_context_get_session (thiz->context));
234 if (status != MFX_ERR_NONE && status != MFX_ERR_NOT_INITIALIZED) {
235 GST_WARNING_OBJECT (thiz, "Decoder close failed (%s)",
236 msdk_status_to_string (status));
237 }
238
239 g_array_set_size (thiz->tasks, 0);
240
241 if (reset_param)
242 memset (&thiz->param, 0, sizeof (thiz->param));
243
244 thiz->initialized = FALSE;
245 gst_adapter_clear (thiz->adapter);
246 }
247
248 static void
gst_msdkdec_set_context(GstElement * element,GstContext * context)249 gst_msdkdec_set_context (GstElement * element, GstContext * context)
250 {
251 GstMsdkContext *msdk_context = NULL;
252 GstMsdkDec *thiz = GST_MSDKDEC (element);
253
254 if (gst_msdk_context_get_context (context, &msdk_context)) {
255 gst_object_replace ((GstObject **) & thiz->context,
256 (GstObject *) msdk_context);
257 gst_object_unref (msdk_context);
258 }
259
260 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
261 }
262
263 static gboolean
gst_msdkdec_init_decoder(GstMsdkDec * thiz)264 gst_msdkdec_init_decoder (GstMsdkDec * thiz)
265 {
266 GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
267 GstVideoInfo *info;
268 mfxSession session;
269 mfxStatus status;
270 mfxFrameAllocRequest request;
271
272 if (thiz->initialized)
273 return TRUE;
274
275 if (!thiz->context) {
276 GST_WARNING_OBJECT (thiz, "No MSDK Context");
277 return FALSE;
278 }
279
280 if (!thiz->input_state) {
281 GST_DEBUG_OBJECT (thiz, "Have no input state yet");
282 return FALSE;
283 }
284 info = &thiz->input_state->info;
285
286 GST_OBJECT_LOCK (thiz);
287
288 if (thiz->use_video_memory) {
289 gst_msdk_set_frame_allocator (thiz->context);
290 thiz->param.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY;
291 } else {
292 thiz->param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
293 }
294
295 GST_INFO_OBJECT (thiz, "This MSDK decoder uses %s memory",
296 thiz->use_video_memory ? "video" : "system");
297
298 thiz->param.AsyncDepth = thiz->async_depth;
299
300 /* We expect msdk to fill the width and height values */
301 g_return_val_if_fail (thiz->param.mfx.FrameInfo.Width
302 && thiz->param.mfx.FrameInfo.Height, FALSE);
303
304 klass->preinit_decoder (thiz);
305
306 /* Set framerate only if provided.
307 * If not, framerate will be assumed inside the driver.
308 * Also we respect the upstream provided fps values */
309 if (info->fps_n > 0 && info->fps_d > 0
310 && info->fps_n != thiz->param.mfx.FrameInfo.FrameRateExtN
311 && info->fps_d != thiz->param.mfx.FrameInfo.FrameRateExtD) {
312 thiz->param.mfx.FrameInfo.FrameRateExtN = info->fps_n;
313 thiz->param.mfx.FrameInfo.FrameRateExtD = info->fps_d;
314 }
315
316 if (info->par_n && info->par_d && !thiz->param.mfx.FrameInfo.AspectRatioW
317 && !thiz->param.mfx.FrameInfo.AspectRatioH) {
318 thiz->param.mfx.FrameInfo.AspectRatioW = info->par_n;
319 thiz->param.mfx.FrameInfo.AspectRatioH = info->par_d;
320 }
321
322 thiz->param.mfx.FrameInfo.FourCC =
323 thiz->param.mfx.FrameInfo.FourCC ? thiz->param.mfx.
324 FrameInfo.FourCC : MFX_FOURCC_NV12;
325 thiz->param.mfx.FrameInfo.ChromaFormat =
326 thiz->param.mfx.FrameInfo.ChromaFormat ? thiz->param.mfx.
327 FrameInfo.ChromaFormat : MFX_CHROMAFORMAT_YUV420;
328
329 session = gst_msdk_context_get_session (thiz->context);
330 /* validate parameters and allow the Media SDK to make adjustments */
331 status = MFXVideoDECODE_Query (session, &thiz->param, &thiz->param);
332 if (status < MFX_ERR_NONE) {
333 GST_ERROR_OBJECT (thiz, "Video Decode Query failed (%s)",
334 msdk_status_to_string (status));
335 goto failed;
336 } else if (status > MFX_ERR_NONE) {
337 GST_WARNING_OBJECT (thiz, "Video Decode Query returned: %s",
338 msdk_status_to_string (status));
339 }
340
341 klass->postinit_decoder (thiz);
342
343 status = MFXVideoDECODE_QueryIOSurf (session, &thiz->param, &request);
344 if (status < MFX_ERR_NONE) {
345 GST_ERROR_OBJECT (thiz, "Query IO surfaces failed (%s)",
346 msdk_status_to_string (status));
347 goto failed;
348 } else if (status > MFX_ERR_NONE) {
349 GST_WARNING_OBJECT (thiz, "Query IO surfaces returned: %s",
350 msdk_status_to_string (status));
351 }
352
353 if (request.NumFrameSuggested < thiz->param.AsyncDepth) {
354 GST_ERROR_OBJECT (thiz, "Required %d surfaces (%d suggested), async %d",
355 request.NumFrameMin, request.NumFrameSuggested, thiz->param.AsyncDepth);
356 goto failed;
357 }
358
359 /* account the downstream requirement */
360 if (G_LIKELY (thiz->min_prealloc_buffers))
361 request.NumFrameSuggested += thiz->min_prealloc_buffers;
362 else
363 GST_WARNING_OBJECT (thiz,
364 "Allocating resources without considering the downstream requirement"
365 "or extra scratch surface count");
366
367 if (thiz->use_video_memory) {
368 gint shared_async_depth;
369
370 shared_async_depth =
371 gst_msdk_context_get_shared_async_depth (thiz->context);
372 request.NumFrameSuggested += shared_async_depth;
373
374 request.Type |= MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
375 if (thiz->use_dmabuf)
376 request.Type |= MFX_MEMTYPE_EXPORT_FRAME;
377 gst_msdk_frame_alloc (thiz->context, &request, &thiz->alloc_resp);
378 }
379
380 /* update the prealloc_buffer count which will be used later
381 * as GstBufferPool min_buffers */
382 thiz->min_prealloc_buffers = request.NumFrameSuggested;
383
384 GST_DEBUG_OBJECT (thiz, "Required %d surfaces (%d suggested)",
385 request.NumFrameMin, request.NumFrameSuggested);
386
387 status = MFXVideoDECODE_Init (session, &thiz->param);
388 if (status < MFX_ERR_NONE) {
389 GST_ERROR_OBJECT (thiz, "Init failed (%s)", msdk_status_to_string (status));
390 goto failed;
391 } else if (status > MFX_ERR_NONE) {
392 GST_WARNING_OBJECT (thiz, "Init returned: %s",
393 msdk_status_to_string (status));
394 }
395
396 status = MFXVideoDECODE_GetVideoParam (session, &thiz->param);
397 if (status < MFX_ERR_NONE) {
398 GST_ERROR_OBJECT (thiz, "Get Video Parameters failed (%s)",
399 msdk_status_to_string (status));
400 goto failed;
401 } else if (status > MFX_ERR_NONE) {
402 GST_WARNING_OBJECT (thiz, "Get Video Parameters returned: %s",
403 msdk_status_to_string (status));
404 }
405
406 g_array_set_size (thiz->tasks, 0); /* resets array content */
407 g_array_set_size (thiz->tasks, thiz->param.AsyncDepth);
408 thiz->next_task = 0;
409
410 GST_OBJECT_UNLOCK (thiz);
411
412 thiz->initialized = TRUE;
413 return TRUE;
414
415 failed:
416 GST_OBJECT_UNLOCK (thiz);
417 return FALSE;
418 }
419
420
421 static gboolean
_gst_caps_has_feature(const GstCaps * caps,const gchar * feature)422 _gst_caps_has_feature (const GstCaps * caps, const gchar * feature)
423 {
424 guint i;
425
426 for (i = 0; i < gst_caps_get_size (caps); i++) {
427 GstCapsFeatures *const features = gst_caps_get_features (caps, i);
428 /* Skip ANY features, we need an exact match for correct evaluation */
429 if (gst_caps_features_is_any (features))
430 continue;
431 if (gst_caps_features_contains (features, feature))
432 return TRUE;
433 }
434
435 return FALSE;
436 }
437
438 static gboolean
srcpad_can_dmabuf(GstMsdkDec * thiz)439 srcpad_can_dmabuf (GstMsdkDec * thiz)
440 {
441 gboolean ret = FALSE;
442 GstCaps *caps, *out_caps;
443 GstPad *srcpad;
444
445 srcpad = GST_VIDEO_DECODER_SRC_PAD (thiz);
446 caps = gst_pad_get_pad_template_caps (srcpad);
447
448 out_caps = gst_pad_peer_query_caps (srcpad, caps);
449 if (!out_caps)
450 goto done;
451
452 if (gst_caps_is_any (out_caps) || gst_caps_is_empty (out_caps)
453 || out_caps == caps)
454 goto done;
455
456 if (_gst_caps_has_feature (out_caps, GST_CAPS_FEATURE_MEMORY_DMABUF))
457 ret = TRUE;
458
459 done:
460 if (caps)
461 gst_caps_unref (caps);
462 if (out_caps)
463 gst_caps_unref (out_caps);
464 return ret;
465 }
466
467 static gboolean
gst_msdkdec_set_src_caps(GstMsdkDec * thiz,gboolean need_allocation)468 gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
469 {
470 GstVideoCodecState *output_state;
471 GstVideoInfo *vinfo;
472 GstVideoAlignment align;
473 GstCaps *allocation_caps = NULL;
474 GstVideoFormat format;
475 guint width, height;
476 const gchar *format_str;
477
478 /* use display width and display height in output state which
479 * will be using for caps negotiation */
480 width =
481 thiz->param.mfx.FrameInfo.CropW ? thiz->param.mfx.
482 FrameInfo.CropW : GST_VIDEO_INFO_WIDTH (&thiz->input_state->info);
483 height =
484 thiz->param.mfx.FrameInfo.CropH ? thiz->param.mfx.
485 FrameInfo.CropH : GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info);
486
487 format =
488 gst_msdk_get_video_format_from_mfx_fourcc (thiz->param.mfx.
489 FrameInfo.FourCC);
490
491 if (format == GST_VIDEO_FORMAT_UNKNOWN) {
492 GST_WARNING_OBJECT (thiz, "Failed to find a valid video format\n");
493 return FALSE;
494 }
495
496 output_state =
497 gst_video_decoder_set_output_state (GST_VIDEO_DECODER (thiz),
498 format, width, height, thiz->input_state);
499 if (!output_state)
500 return FALSE;
501
502 /* Ensure output_state->caps and info has same width and height
503 * Also mandate the 32 bit alignment */
504 vinfo = &output_state->info;
505 gst_msdk_set_video_alignment (vinfo, &align);
506 gst_video_info_align (vinfo, &align);
507 output_state->caps = gst_video_info_to_caps (vinfo);
508 if (srcpad_can_dmabuf (thiz))
509 gst_caps_set_features (output_state->caps, 0,
510 gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_DMABUF, NULL));
511 thiz->output_info = output_state->info;
512
513 if (need_allocation) {
514 /* Find allocation width and height */
515 width =
516 GST_ROUND_UP_16 (thiz->param.mfx.FrameInfo.Width ? thiz->param.mfx.
517 FrameInfo.Width : GST_VIDEO_INFO_WIDTH (&output_state->info));
518 height =
519 GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height ? thiz->param.mfx.
520 FrameInfo.Height : GST_VIDEO_INFO_HEIGHT (&output_state->info));
521
522 /* set allocation width and height in allocation_caps
523 * which may or may not be similar to the output_state caps */
524 allocation_caps = gst_caps_copy (output_state->caps);
525 format_str =
526 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&thiz->output_info));
527 gst_caps_set_simple (allocation_caps, "width", G_TYPE_INT, width, "height",
528 G_TYPE_INT, height, "format", G_TYPE_STRING, format_str, NULL);
529 GST_INFO_OBJECT (thiz, "new alloc caps = %" GST_PTR_FORMAT,
530 allocation_caps);
531 gst_caps_replace (&thiz->allocation_caps, allocation_caps);
532 } else {
533 /* We keep the allocation parameters as it is to avoid pool renegotiation.
534 * For codecs like VP9, dynamic resolution change doesn't requires allocation
535 * reset if the new video frame resolution is lower than the
536 * already configured one */
537 allocation_caps = gst_caps_copy (thiz->allocation_caps);
538 }
539
540 gst_caps_replace (&output_state->allocation_caps, allocation_caps);
541 if (allocation_caps)
542 gst_caps_unref (allocation_caps);
543
544 gst_video_codec_state_unref (output_state);
545 return TRUE;
546 }
547
548 static void
gst_msdkdec_set_latency(GstMsdkDec * thiz)549 gst_msdkdec_set_latency (GstMsdkDec * thiz)
550 {
551 GstVideoInfo *info = &thiz->input_state->info;
552 gint min_delayed_frames;
553 GstClockTime latency;
554
555 min_delayed_frames = thiz->async_depth;
556
557 if (info->fps_n) {
558 latency = gst_util_uint64_scale_ceil (GST_SECOND * info->fps_d,
559 min_delayed_frames, info->fps_n);
560 } else {
561 /* FIXME: Assume 25fps. This is better than reporting no latency at
562 * all and then later failing in live pipelines
563 */
564 latency = gst_util_uint64_scale_ceil (GST_SECOND * 1,
565 min_delayed_frames, 25);
566 }
567
568 GST_INFO_OBJECT (thiz,
569 "Updating latency to %" GST_TIME_FORMAT " (%d frames)",
570 GST_TIME_ARGS (latency), min_delayed_frames);
571
572 gst_video_decoder_set_latency (GST_VIDEO_DECODER (thiz), latency, latency);
573 }
574
575 static gint
_find_msdk_surface(gconstpointer msdk_surface,gconstpointer comp_surface)576 _find_msdk_surface (gconstpointer msdk_surface, gconstpointer comp_surface)
577 {
578 MsdkSurface *cached_surface = (MsdkSurface *) msdk_surface;
579 mfxFrameSurface1 *_surface = (mfxFrameSurface1 *) comp_surface;
580
581 return cached_surface ? cached_surface->surface != _surface : -1;
582 }
583
584 static GstFlowReturn
gst_msdkdec_finish_task(GstMsdkDec * thiz,MsdkDecTask * task)585 gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task)
586 {
587 GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
588 GstFlowReturn flow;
589 GstVideoCodecFrame *frame;
590 MsdkSurface *surface;
591 mfxStatus status;
592 GList *l;
593
594 if (G_LIKELY (task->sync_point)) {
595 status =
596 MFXVideoCORE_SyncOperation (gst_msdk_context_get_session
597 (thiz->context), task->sync_point, 300000);
598 if (status != MFX_ERR_NONE) {
599 GST_ERROR_OBJECT (thiz, "failed to do sync operation");
600 return GST_FLOW_ERROR;
601 }
602 }
603
604 if (G_LIKELY (task->sync_point || (task->surface && task->decode_only))) {
605 gboolean decode_only = task->decode_only;
606
607 frame = gst_msdkdec_get_oldest_frame (decoder);
608
609 l = g_list_find_custom (thiz->decoded_msdk_surfaces, task->surface,
610 _find_msdk_surface);
611 if (l) {
612 surface = l->data;
613 } else {
614 GST_ERROR_OBJECT (thiz, "Couldn't find the cached MSDK surface");
615 return GST_FLOW_ERROR;
616 }
617
618 if (G_LIKELY (frame)) {
619 if (G_LIKELY (surface->copy.buffer == NULL)) {
620 frame->output_buffer = gst_buffer_ref (surface->buf);
621 } else {
622 gst_video_frame_copy (&surface->copy, &surface->data);
623 frame->output_buffer = gst_buffer_ref (surface->copy.buffer);
624 }
625 }
626
627 free_surface (thiz, surface);
628 task->sync_point = NULL;
629 task->surface = NULL;
630 task->decode_only = FALSE;
631
632 if (!frame)
633 return GST_FLOW_FLUSHING;
634 gst_video_codec_frame_unref (frame);
635
636 if (decode_only)
637 GST_VIDEO_CODEC_FRAME_SET_DECODE_ONLY (frame);
638 flow = gst_video_decoder_finish_frame (decoder, frame);
639 return flow;
640 }
641 return GST_FLOW_OK;
642 }
643
644 static gboolean
gst_msdkdec_start(GstVideoDecoder * decoder)645 gst_msdkdec_start (GstVideoDecoder * decoder)
646 {
647 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
648
649 if (gst_msdk_context_prepare (GST_ELEMENT_CAST (thiz), &thiz->context)) {
650 GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
651 thiz->context);
652 thiz->use_video_memory = TRUE;
653
654 if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_DECODER) {
655 GstMsdkContext *parent_context, *msdk_context;
656
657 parent_context = thiz->context;
658 msdk_context = gst_msdk_context_new_with_parent (parent_context);
659
660 if (!msdk_context) {
661 GST_ERROR_OBJECT (thiz, "Context creation failed");
662 return FALSE;
663 }
664
665 thiz->context = msdk_context;
666
667 gst_msdk_context_add_shared_async_depth (thiz->context,
668 gst_msdk_context_get_shared_async_depth (parent_context));
669 gst_object_unref (parent_context);
670
671 GST_INFO_OBJECT (thiz,
672 "Creating new context %" GST_PTR_FORMAT " with joined session",
673 thiz->context);
674 } else {
675 gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_DECODER);
676 }
677 } else {
678 if (!gst_msdk_context_ensure_context (GST_ELEMENT_CAST (thiz),
679 thiz->hardware, GST_MSDK_JOB_DECODER))
680 return FALSE;
681 GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT,
682 thiz->context);
683 }
684
685 gst_msdk_context_add_shared_async_depth (thiz->context, thiz->async_depth);
686
687 return TRUE;
688 }
689
690 static gboolean
gst_msdkdec_close(GstVideoDecoder * decoder)691 gst_msdkdec_close (GstVideoDecoder * decoder)
692 {
693 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
694
695 if (thiz->context)
696 gst_object_replace ((GstObject **) & thiz->context, NULL);
697
698 return TRUE;
699 }
700
701 static gboolean
gst_msdkdec_stop(GstVideoDecoder * decoder)702 gst_msdkdec_stop (GstVideoDecoder * decoder)
703 {
704 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
705
706 gst_msdkdec_flush (decoder);
707
708 if (thiz->input_state) {
709 gst_video_codec_state_unref (thiz->input_state);
710 thiz->input_state = NULL;
711 }
712 if (thiz->pool) {
713 gst_object_unref (thiz->pool);
714 thiz->pool = NULL;
715 }
716 gst_video_info_init (&thiz->output_info);
717 gst_video_info_init (&thiz->non_msdk_pool_info);
718
719 gst_msdkdec_close_decoder (thiz, TRUE);
720 return TRUE;
721 }
722
723 static gboolean
gst_msdkdec_set_format(GstVideoDecoder * decoder,GstVideoCodecState * state)724 gst_msdkdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
725 {
726 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
727
728 if (thiz->input_state) {
729 /* mark for re-negotiation if display resolution changes */
730 if ((GST_VIDEO_INFO_WIDTH (&thiz->input_state->info) !=
731 GST_VIDEO_INFO_WIDTH (&state->info)) ||
732 GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info) !=
733 GST_VIDEO_INFO_HEIGHT (&state->info))
734 thiz->do_renego = TRUE;
735 gst_video_codec_state_unref (thiz->input_state);
736 }
737 thiz->input_state = gst_video_codec_state_ref (state);
738
739 /* we don't set output state here to avoid caching of mismatched
740 * video information if there is dynamic resolution change in the stream.
741 * All negotiation code is consolidated in gst_msdkdec_negotiate() and
742 * this will be invoked from handle_frame() */
743
744 gst_msdkdec_set_latency (thiz);
745 return TRUE;
746 }
747
748 static void
release_msdk_surfaces(GstMsdkDec * thiz)749 release_msdk_surfaces (GstMsdkDec * thiz)
750 {
751 GList *l;
752 MsdkSurface *surface;
753
754 for (l = thiz->decoded_msdk_surfaces; l; l = l->next) {
755 surface = l->data;
756 free_surface (thiz, surface);
757 }
758 }
759
760 /* This will get invoked in the following situations:
761 * 1: begining of the stream, which requires initialization (== complete reset)
762 * 2: upstream notified a resolution change and set do_renego to TRUE.
763 * new resoulution may or may not requires full reset
764 * 3: upstream failed to notify the resoulution change but
765 * msdk detected the change (eg: vp9 stream in ivf elementary form
766 * with varying resolution frames).
767 *
768 * for any input configuration change, we deal with notification
769 * from upstream and also use msdk apis to handle the parameter initialization
770 * efficiently
771 */
772 static gboolean
gst_msdkdec_negotiate(GstMsdkDec * thiz,gboolean hard_reset)773 gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset)
774 {
775 GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
776 GST_DEBUG_OBJECT (thiz,
777 "Start Negotiating caps, pool and Init the msdk decdoer subsystem");
778
779 if (hard_reset) {
780 /* Retrieve any pending frames and push them downstream */
781 if (gst_msdkdec_drain (GST_VIDEO_DECODER (thiz)) != GST_FLOW_OK)
782 goto error_drain;
783
784 /* This will initiate the allocation query which will help to flush
785 * all the pending buffers in the pipeline so that we can stop
786 * the active bufferpool and safely invoke gst_msdk_frame_free() */
787 if (thiz->initialized) {
788 GstCaps *caps = gst_pad_get_current_caps (decoder->srcpad);
789 GstQuery *query = NULL;
790 if (caps) {
791 query = gst_query_new_allocation (caps, FALSE);
792 gst_pad_peer_query (decoder->srcpad, query);
793 gst_query_unref (query);
794 gst_caps_unref (caps);
795 }
796 }
797
798 /* De-initialize the decoder if it is already active */
799 /* Not resetting the mfxVideoParam since it already
800 * possessing the required parameters for new session decode */
801 gst_msdkdec_close_decoder (thiz, FALSE);
802
803 /* request for pool renegotiation by setting do_realloc */
804 thiz->do_realloc = TRUE;
805 }
806
807 /* At this point all pending frames(if there is any) are pushed downsteram
808 * and we are ready to negotiate the output caps */
809 if (!gst_msdkdec_set_src_caps (thiz, hard_reset))
810 return FALSE;
811
812 /* this will initiate the allocation query, we create the
813 * bufferpool in decide_allocation inorder to account
814 * the downstream min_buffer requirement
815 * Required initializations for MediaSDK operations
816 * will all be inited from decide_allocation after considering
817 * some of the downstream requirements */
818 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (thiz)))
819 goto error_negotiate;
820
821 thiz->do_renego = FALSE;
822 thiz->do_realloc = FALSE;
823
824 return TRUE;
825
826 error_drain:
827 GST_ERROR_OBJECT (thiz, "Failed to Drain the queued decoded frames");
828 return FALSE;
829
830 error_negotiate:
831 GST_ERROR_OBJECT (thiz, "Failed to renegotiation");
832 return FALSE;
833 }
834
835 static GstFlowReturn
gst_msdkdec_handle_frame(GstVideoDecoder * decoder,GstVideoCodecFrame * frame)836 gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
837 {
838 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
839 GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
840 GstFlowReturn flow;
841 GstBuffer *buffer, *input_buffer = NULL;
842 GstVideoInfo alloc_info;
843 MsdkDecTask *task = NULL;
844 mfxBitstream bitstream;
845 MsdkSurface *surface = NULL;
846 mfxSession session;
847 mfxStatus status;
848 GstMapInfo map_info;
849 guint i, retry_err_incompatible = 0;
850 gsize data_size;
851 gboolean hard_reset = FALSE;
852
853 /* configure the subclass in order to fill the CodecID field of
854 * mfxVideoParam and also to load the PluginID for some of the
855 * codecs which is mandatory to invoke the
856 * MFXVideoDECODE_DecodeHeader API.
857 *
858 * For non packetized formats (currently only vc1), there
859 * could be headers received as codec_data which are not available
860 * instream and in that case subclass implementation will
861 * push it to the internal adapter. We invoke the subclass configure
862 * well early to make sure the codec_data received has been correctly
863 * pushed to the adapter by the subclasses before doing
864 * the DecodeHeader() later on
865 */
866 if (!thiz->initialized || thiz->do_renego) {
867 /* Clear the internal adapter in renegotiation for non-packetized
868 * formats */
869 if (!thiz->is_packetized)
870 gst_adapter_clear (thiz->adapter);
871
872 if (!klass->configure || !klass->configure (thiz)) {
873 flow = GST_FLOW_OK;
874 goto error;
875 }
876 }
877
878 /* Current frame-codec could be pushed and released before this
879 * function ends -- because msdkdec pushes the oldest frame,
880 * according its PTS, and it could be this very same frame-codec
881 * among others pending frame-codecs.
882 *
883 * Instead of copying the input data into the mfxBitstream, let's
884 * keep an extra reference to frame-codec's input buffer */
885 input_buffer = gst_buffer_ref (frame->input_buffer);
886 if (!gst_buffer_map (input_buffer, &map_info, GST_MAP_READ)) {
887 gst_buffer_unref (input_buffer);
888 return GST_FLOW_ERROR;
889 }
890
891 memset (&bitstream, 0, sizeof (bitstream));
892
893 if (thiz->is_packetized) {
894 /* Packetized stream: We prefer to have a parser as connected upstream
895 * element to the decoder */
896 bitstream.Data = map_info.data;
897 bitstream.DataLength = map_info.size;
898 bitstream.MaxLength = map_info.size;
899 bitstream.DataFlag = MFX_BITSTREAM_COMPLETE_FRAME;
900 } else {
901 /* Non packetized streams: eg: vc1 advanced profile with per buffer bdu */
902 gst_adapter_push (thiz->adapter, gst_buffer_ref (input_buffer));
903 data_size = gst_adapter_available (thiz->adapter);
904
905 bitstream.Data = (mfxU8 *) gst_adapter_map (thiz->adapter, data_size);
906 bitstream.DataLength = (mfxU32) data_size;
907 bitstream.MaxLength = bitstream.DataLength;
908 }
909 GST_INFO_OBJECT (thiz,
910 "mfxBitStream=> DataLength:%d DataOffset:%d MaxLength:%d",
911 bitstream.DataLength, bitstream.DataOffset, bitstream.MaxLength);
912
913 session = gst_msdk_context_get_session (thiz->context);
914
915 if (!thiz->initialized || thiz->do_renego) {
916
917 /* gstreamer caps will not bring all the necessary parameters
918 * required for optimal decode configuration. For eg: the required numbers
919 * of surfaces to be allocated can be calculated based on H264 SEI header
920 * and this information can't be retrieved from the negotiated caps.
921 * So instead of introducing the codecparser dependency to parse the headers
922 * inside msdk plugin, we simply use the mfx apis to extract header information */
923 status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
924 if (status == MFX_ERR_MORE_DATA) {
925 flow = GST_FLOW_OK;
926 goto done;
927 }
928
929 if (!thiz->initialized)
930 hard_reset = TRUE;
931 else if (thiz->allocation_caps) {
932 gst_video_info_from_caps (&alloc_info, thiz->allocation_caps);
933
934 /* Check whether we need complete reset for dynamic resolution change */
935 if (thiz->param.mfx.FrameInfo.Width > GST_VIDEO_INFO_WIDTH (&alloc_info)
936 || thiz->param.mfx.FrameInfo.Height >
937 GST_VIDEO_INFO_HEIGHT (&alloc_info))
938 hard_reset = TRUE;
939 }
940
941 /* if subclass requested for the force reset */
942 if (thiz->force_reset_on_res_change)
943 hard_reset = TRUE;
944
945 /* Config changed dynamically and we are going to do a full reset,
946 * this will unref the input frame which has the new configuration.
947 * Keep a ref to the input_frame to keep it alive */
948 if (thiz->initialized && thiz->do_renego)
949 gst_video_codec_frame_ref (frame);
950
951 if (!gst_msdkdec_negotiate (thiz, hard_reset)) {
952 GST_ELEMENT_ERROR (thiz, CORE, NEGOTIATION,
953 ("Could not negotiate the stream"), (NULL));
954 flow = GST_FLOW_ERROR;
955 goto error;
956 }
957 }
958
959 for (;;) {
960 task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
961 flow = gst_msdkdec_finish_task (thiz, task);
962 if (flow != GST_FLOW_OK)
963 goto error;
964 if (!surface) {
965 flow = allocate_output_buffer (thiz, &buffer);
966 if (flow != GST_FLOW_OK)
967 goto error;
968 surface = get_surface (thiz, buffer);
969 if (!surface) {
970 /* Can't get a surface for some reason, finish tasks to see if
971 a surface becomes available. */
972 for (i = 0; i < thiz->tasks->len - 1; i++) {
973 thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
974 task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
975 flow = gst_msdkdec_finish_task (thiz, task);
976 if (flow != GST_FLOW_OK)
977 goto error;
978 surface = get_surface (thiz, buffer);
979 if (surface)
980 break;
981 }
982 if (!surface) {
983 GST_ERROR_OBJECT (thiz, "Couldn't get a surface");
984 flow = GST_FLOW_ERROR;
985 goto error;
986 }
987 }
988 }
989
990 status =
991 MFXVideoDECODE_DecodeFrameAsync (session, &bitstream, surface->surface,
992 &task->surface, &task->sync_point);
993
994 /* media-sdk requires complete reset since the surface is inadaquate to
995 * do further decoding */
996 if (status == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM &&
997 retry_err_incompatible++ < 1) {
998 /* MFX_ERR_INCOMPATIBLE_VIDEO_PARAM means the current mfx surface is not
999 * suitable for the current frame, call MFXVideoDECODE_DecodeHeader to get
1000 * the current frame size then do memory re-allocation, otherwise
1001 * MFXVideoDECODE_DecodeFrameAsync still will fail for next call */
1002 status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
1003 if (status == MFX_ERR_MORE_DATA) {
1004 flow = GST_FLOW_OK;
1005 goto done;
1006 }
1007
1008 /* Requires memory re-allocation, do a hard reset */
1009 if (!gst_msdkdec_negotiate (thiz, TRUE))
1010 goto error;
1011
1012 /* The current surface is freed when doing a hard reset, a new surface is
1013 * required for the new resolution */
1014 surface = NULL;
1015 continue;
1016 }
1017
1018 retry_err_incompatible = 0;
1019
1020 if (G_LIKELY (status == MFX_ERR_NONE)
1021 || (status == MFX_WRN_VIDEO_PARAM_CHANGED)) {
1022 thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
1023
1024 if (surface->surface->Data.Locked > 0)
1025 surface = NULL;
1026 else if (!thiz->use_video_memory) {
1027 /* The for loop will continue because DataLength is greater than 0 so
1028 * free the surface right away if not in use. */
1029 if (bitstream.DataLength > 0 && task->surface != surface->surface)
1030 free_surface (thiz, surface);
1031 surface = NULL;
1032 }
1033
1034 if (bitstream.DataLength == 0) {
1035 flow = GST_FLOW_OK;
1036 break;
1037 }
1038 } else if (status == MFX_ERR_MORE_DATA) {
1039 if (task->surface) {
1040 task->decode_only = TRUE;
1041 thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
1042 }
1043
1044 if (surface->surface->Data.Locked > 0)
1045 surface = NULL;
1046 flow = GST_VIDEO_DECODER_FLOW_NEED_DATA;
1047 break;
1048 } else if (status == MFX_ERR_MORE_SURFACE) {
1049 surface = NULL;
1050 continue;
1051 } else if (status == MFX_WRN_DEVICE_BUSY) {
1052 /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */
1053 g_usleep (1000);
1054
1055 if (task->surface &&
1056 task->surface == surface->surface && !task->sync_point) {
1057 free_surface (thiz, surface);
1058 surface = NULL;
1059 }
1060
1061 /* If the current surface is still busy, we should do sync oepration
1062 * then tries to decode again
1063 */
1064 thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
1065 } else if (status < MFX_ERR_NONE) {
1066 GST_ERROR_OBJECT (thiz, "DecodeFrameAsync failed (%s)",
1067 msdk_status_to_string (status));
1068 flow = GST_FLOW_ERROR;
1069 break;
1070 }
1071 }
1072
1073 if (!thiz->is_packetized) {
1074 /* flush out the data which is already consumed by msdk */
1075 gst_adapter_flush (thiz->adapter, bitstream.DataOffset);
1076 flow = GST_FLOW_OK;
1077 }
1078
1079 done:
1080 if (surface)
1081 free_surface (thiz, surface);
1082
1083 gst_buffer_unmap (input_buffer, &map_info);
1084 gst_buffer_unref (input_buffer);
1085 return flow;
1086
1087 error:
1088 if (input_buffer) {
1089 gst_buffer_unmap (input_buffer, &map_info);
1090 gst_buffer_unref (input_buffer);
1091 }
1092 gst_video_decoder_drop_frame (decoder, frame);
1093
1094 return flow;
1095 }
1096
1097
1098 static GstBufferPool *
gst_msdkdec_create_buffer_pool(GstMsdkDec * thiz,GstVideoInfo * info,guint num_buffers)1099 gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstVideoInfo * info,
1100 guint num_buffers)
1101 {
1102 GstBufferPool *pool = NULL;
1103 GstStructure *config;
1104 GstAllocator *allocator = NULL;
1105 GstVideoAlignment align;
1106 GstCaps *caps = NULL;
1107 GstAllocationParams params = { 0, 31, 0, 0, };
1108 mfxFrameAllocResponse *alloc_resp = NULL;
1109
1110 g_return_val_if_fail (info, NULL);
1111 g_return_val_if_fail (GST_VIDEO_INFO_WIDTH (info)
1112 && GST_VIDEO_INFO_HEIGHT (info), NULL);
1113
1114 alloc_resp = &thiz->alloc_resp;
1115
1116 pool = gst_msdk_buffer_pool_new (thiz->context, alloc_resp);
1117 if (!pool)
1118 goto error_no_pool;
1119
1120 if (G_UNLIKELY (!IS_ALIGNED (GST_VIDEO_INFO_WIDTH (info), 16)
1121 || !IS_ALIGNED (GST_VIDEO_INFO_HEIGHT (info), 32))) {
1122 gst_msdk_set_video_alignment (info, &align);
1123 gst_video_info_align (info, &align);
1124 }
1125
1126 caps = gst_video_info_to_caps (info);
1127
1128 /* allocators should use the same width/height/stride/height_alignment of
1129 * negotiated output caps which is what we configure in msdk_allocator */
1130 if (thiz->use_dmabuf)
1131 allocator = gst_msdk_dmabuf_allocator_new (thiz->context, info, alloc_resp);
1132 else if (thiz->use_video_memory)
1133 allocator = gst_msdk_video_allocator_new (thiz->context, info, alloc_resp);
1134 else
1135 allocator = gst_msdk_system_allocator_new (info);
1136
1137 if (!allocator)
1138 goto error_no_allocator;
1139
1140 config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));
1141 gst_buffer_pool_config_set_params (config, caps,
1142 GST_VIDEO_INFO_SIZE (info), num_buffers, 0);
1143 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
1144 gst_buffer_pool_config_add_option (config,
1145 GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
1146
1147 if (thiz->use_video_memory) {
1148 gst_buffer_pool_config_add_option (config,
1149 GST_BUFFER_POOL_OPTION_MSDK_USE_VIDEO_MEMORY);
1150 if (thiz->use_dmabuf)
1151 gst_buffer_pool_config_add_option (config,
1152 GST_BUFFER_POOL_OPTION_MSDK_USE_DMABUF);
1153 }
1154
1155 gst_buffer_pool_config_set_video_alignment (config, &align);
1156 gst_buffer_pool_config_set_allocator (config, allocator, ¶ms);
1157 gst_object_unref (allocator);
1158
1159 if (!gst_buffer_pool_set_config (pool, config))
1160 goto error_pool_config;
1161
1162 return pool;
1163
1164 error_no_pool:
1165 {
1166 GST_INFO_OBJECT (thiz, "failed to create bufferpool");
1167 return NULL;
1168 }
1169 error_no_allocator:
1170 {
1171 GST_INFO_OBJECT (thiz, "failed to create allocator");
1172 gst_object_unref (pool);
1173 return NULL;
1174 }
1175 error_pool_config:
1176 {
1177 GST_INFO_OBJECT (thiz, "failed to set config");
1178 gst_object_unref (pool);
1179 gst_object_unref (allocator);
1180 return NULL;
1181 }
1182 }
1183
1184 static gboolean
gst_msdkdec_decide_allocation(GstVideoDecoder * decoder,GstQuery * query)1185 gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
1186 {
1187 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
1188 GstBufferPool *pool = NULL;
1189 GstStructure *pool_config = NULL;
1190 GstCaps *pool_caps /*, *negotiated_caps */ ;
1191 guint size, min_buffers, max_buffers;
1192
1193 if (!GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation (decoder,
1194 query))
1195 return FALSE;
1196
1197 /* Get the buffer pool config decided by the base class. The base
1198 class ensures that there will always be at least a 0th pool in
1199 the query. */
1200 gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL);
1201 pool_config = gst_buffer_pool_get_config (pool);
1202
1203 /* Get the caps of pool and increase the min and max buffers by async_depth,
1204 * we will always have that number of decode operations in-flight */
1205 gst_buffer_pool_config_get_params (pool_config, &pool_caps, &size,
1206 &min_buffers, &max_buffers);
1207 min_buffers += thiz->async_depth;
1208 if (max_buffers)
1209 max_buffers += thiz->async_depth;
1210
1211 /* increase the min_buffers by 1 for smooth display in render pipeline */
1212 min_buffers += 1;
1213
1214 /* this will get updated with msdk requirement */
1215 thiz->min_prealloc_buffers = min_buffers;
1216
1217 if (_gst_caps_has_feature (pool_caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
1218 GST_INFO_OBJECT (decoder, "This MSDK decoder uses DMABuf memory");
1219 thiz->use_video_memory = thiz->use_dmabuf = TRUE;
1220 }
1221
1222 /* Initialize MSDK decoder before new bufferpool tries to alloc each buffer,
1223 * which requires information of frame allocation.
1224 * No effect if already initialized.
1225 */
1226 if (!gst_msdkdec_init_decoder (thiz))
1227 return FALSE;
1228
1229 /* get the updated min_buffers which account the msdk requirement too */
1230 min_buffers = thiz->min_prealloc_buffers;
1231
1232 /* Decoder always use its own pool. So we create a pool if msdk apis
1233 * previously requested for allocation (do_realloc = TRUE) */
1234 if (thiz->do_realloc || !thiz->pool) {
1235 if (thiz->pool)
1236 gst_object_replace ((GstObject **) & thiz->pool, NULL);
1237 GST_INFO_OBJECT (decoder, "create new MSDK bufferpool");
1238 thiz->pool =
1239 gst_msdkdec_create_buffer_pool (thiz, &thiz->output_info, min_buffers);
1240 if (!thiz->pool)
1241 goto failed_to_create_pool;
1242 }
1243
1244 if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)
1245 && gst_buffer_pool_has_option (pool,
1246 GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT)) {
1247 GstStructure *config;
1248 GstAllocator *allocator;
1249
1250 /* If downstream supports video meta and video alignment,
1251 * we can replace our own msdk bufferpool and use it
1252 */
1253 /* Remove downstream's pool */
1254 gst_structure_free (pool_config);
1255 gst_object_unref (pool);
1256
1257 pool = gst_object_ref (thiz->pool);
1258
1259 /* Set the allocator of new msdk bufferpool */
1260 config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));
1261
1262 if (gst_buffer_pool_config_get_allocator (config, &allocator, NULL))
1263 gst_query_set_nth_allocation_param (query, 0, allocator, NULL);
1264 gst_structure_free (config);
1265 } else {
1266 /* Unfortunately, dowstream doesn't have videometa or alignment support,
1267 * we keep msdk pool as a side-pool that will be decoded into and
1268 * then copied from.
1269 */
1270 GST_INFO_OBJECT (decoder, "Keep MSDK bufferpool as a side-pool");
1271
1272 /* Update params to downstream's pool */
1273 gst_buffer_pool_config_set_params (pool_config, pool_caps, size,
1274 min_buffers, max_buffers);
1275 if (!gst_buffer_pool_set_config (pool, pool_config))
1276 goto error_set_config;
1277 gst_video_info_from_caps (&thiz->non_msdk_pool_info, pool_caps);
1278
1279 /* update width and height with actual negotiated values */
1280 GST_VIDEO_INFO_WIDTH (&thiz->non_msdk_pool_info) =
1281 GST_VIDEO_INFO_WIDTH (&thiz->output_info);
1282 GST_VIDEO_INFO_HEIGHT (&thiz->non_msdk_pool_info) =
1283 GST_VIDEO_INFO_HEIGHT (&thiz->output_info);
1284 }
1285
1286 gst_query_set_nth_allocation_pool (query, 0, pool, size, min_buffers,
1287 max_buffers);
1288
1289 if (pool)
1290 gst_object_unref (pool);
1291
1292
1293 return TRUE;
1294
1295 failed_to_create_pool:
1296 GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
1297 if (pool)
1298 gst_object_unref (pool);
1299 return FALSE;
1300
1301 error_set_config:
1302 GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
1303 if (pool)
1304 gst_object_unref (pool);
1305 return FALSE;
1306 }
1307
1308 static GstFlowReturn
gst_msdkdec_drain(GstVideoDecoder * decoder)1309 gst_msdkdec_drain (GstVideoDecoder * decoder)
1310 {
1311 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
1312 GstFlowReturn flow;
1313 GstBuffer *buffer;
1314 MsdkDecTask *task;
1315 MsdkSurface *surface = NULL;
1316 mfxSession session;
1317 mfxStatus status;
1318 guint i;
1319
1320 if (!thiz->initialized)
1321 return GST_FLOW_OK;
1322 session = gst_msdk_context_get_session (thiz->context);
1323
1324 for (;;) {
1325 task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
1326 if ((flow = gst_msdkdec_finish_task (thiz, task)) != GST_FLOW_OK) {
1327 if (flow != GST_FLOW_FLUSHING)
1328 GST_WARNING_OBJECT (decoder,
1329 "failed to finish the task %p, but keep draining for the remaining frames",
1330 task);
1331 }
1332
1333 if (!surface) {
1334 flow = allocate_output_buffer (thiz, &buffer);
1335 if (flow != GST_FLOW_OK)
1336 return flow;
1337 surface = get_surface (thiz, buffer);
1338 if (!surface)
1339 return GST_FLOW_ERROR;
1340 }
1341
1342 status =
1343 MFXVideoDECODE_DecodeFrameAsync (session, NULL, surface->surface,
1344 &task->surface, &task->sync_point);
1345 if (G_LIKELY (status == MFX_ERR_NONE)) {
1346 thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
1347
1348 if (surface->surface->Data.Locked == 0)
1349 free_surface (thiz, surface);
1350 surface = NULL;
1351 } else if (status == MFX_WRN_VIDEO_PARAM_CHANGED) {
1352 continue;
1353 } else if (status == MFX_WRN_DEVICE_BUSY) {
1354 /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */
1355 g_usleep (1000);
1356
1357 /* If the current surface is still busy, we should do sync oepration
1358 * then tries to decode again
1359 */
1360 thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
1361 } else if (status == MFX_ERR_MORE_DATA) {
1362 break;
1363 } else if (status == MFX_ERR_MORE_SURFACE) {
1364 surface = NULL;
1365 continue;
1366 } else if (status < MFX_ERR_NONE)
1367 return GST_FLOW_ERROR;
1368 }
1369 if (surface)
1370 free_surface (thiz, surface);
1371
1372 for (i = 0; i < thiz->tasks->len; i++) {
1373 task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
1374 gst_msdkdec_finish_task (thiz, task);
1375 thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
1376 }
1377
1378 release_msdk_surfaces (thiz);
1379
1380 return GST_FLOW_OK;
1381 }
1382
1383 static gboolean
gst_msdkdec_flush(GstVideoDecoder * decoder)1384 gst_msdkdec_flush (GstVideoDecoder * decoder)
1385 {
1386 GstMsdkDec *thiz = GST_MSDKDEC (decoder);
1387
1388 return gst_msdkdec_drain (GST_VIDEO_DECODER_CAST (thiz));
1389 }
1390
1391 static GstFlowReturn
gst_msdkdec_finish(GstVideoDecoder * decoder)1392 gst_msdkdec_finish (GstVideoDecoder * decoder)
1393 {
1394 return gst_msdkdec_drain (decoder);
1395 }
1396
1397 static void
gst_msdkdec_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)1398 gst_msdkdec_set_property (GObject * object, guint prop_id, const GValue * value,
1399 GParamSpec * pspec)
1400 {
1401 GstMsdkDec *thiz = GST_MSDKDEC (object);
1402 GstState state;
1403
1404 GST_OBJECT_LOCK (thiz);
1405
1406 state = GST_STATE (thiz);
1407 if ((state != GST_STATE_READY && state != GST_STATE_NULL) &&
1408 !(pspec->flags & GST_PARAM_MUTABLE_PLAYING))
1409 goto wrong_state;
1410
1411 switch (prop_id) {
1412 case GST_MSDKDEC_PROP_HARDWARE:
1413 thiz->hardware = g_value_get_boolean (value);
1414 break;
1415 case GST_MSDKDEC_PROP_ASYNC_DEPTH:
1416 thiz->async_depth = g_value_get_uint (value);
1417 break;
1418 default:
1419 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1420 break;
1421 }
1422 GST_OBJECT_UNLOCK (thiz);
1423 return;
1424
1425 /* ERROR */
1426 wrong_state:
1427 {
1428 GST_WARNING_OBJECT (thiz, "setting property in wrong state");
1429 GST_OBJECT_UNLOCK (thiz);
1430 }
1431 }
1432
1433 static void
gst_msdkdec_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)1434 gst_msdkdec_get_property (GObject * object, guint prop_id, GValue * value,
1435 GParamSpec * pspec)
1436 {
1437 GstMsdkDec *thiz = GST_MSDKDEC (object);
1438
1439 GST_OBJECT_LOCK (thiz);
1440 switch (prop_id) {
1441 case GST_MSDKDEC_PROP_HARDWARE:
1442 g_value_set_boolean (value, thiz->hardware);
1443 break;
1444 case GST_MSDKDEC_PROP_ASYNC_DEPTH:
1445 g_value_set_uint (value, thiz->async_depth);
1446 break;
1447 default:
1448 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1449 break;
1450 }
1451 GST_OBJECT_UNLOCK (thiz);
1452 }
1453
1454 static void
gst_msdkdec_finalize(GObject * object)1455 gst_msdkdec_finalize (GObject * object)
1456 {
1457 GstMsdkDec *thiz = GST_MSDKDEC (object);
1458
1459 g_array_unref (thiz->tasks);
1460 g_object_unref (thiz->adapter);
1461 }
1462
1463 static gboolean
gst_msdkdec_preinit_decoder(GstMsdkDec * decoder)1464 gst_msdkdec_preinit_decoder (GstMsdkDec * decoder)
1465 {
1466 decoder->param.mfx.FrameInfo.Width =
1467 GST_ROUND_UP_16 (decoder->param.mfx.FrameInfo.Width);
1468 decoder->param.mfx.FrameInfo.Height =
1469 GST_ROUND_UP_32 (decoder->param.mfx.FrameInfo.Height);
1470
1471 decoder->param.mfx.FrameInfo.PicStruct =
1472 decoder->param.mfx.FrameInfo.PicStruct ? decoder->param.mfx.
1473 FrameInfo.PicStruct : MFX_PICSTRUCT_PROGRESSIVE;
1474
1475 return TRUE;
1476 }
1477
1478 static gboolean
gst_msdkdec_postinit_decoder(GstMsdkDec * decoder)1479 gst_msdkdec_postinit_decoder (GstMsdkDec * decoder)
1480 {
1481 /* Do nothing */
1482 return TRUE;
1483 }
1484
1485 static void
gst_msdkdec_class_init(GstMsdkDecClass * klass)1486 gst_msdkdec_class_init (GstMsdkDecClass * klass)
1487 {
1488 GObjectClass *gobject_class;
1489 GstElementClass *element_class;
1490 GstVideoDecoderClass *decoder_class;
1491
1492 gobject_class = G_OBJECT_CLASS (klass);
1493 element_class = GST_ELEMENT_CLASS (klass);
1494 decoder_class = GST_VIDEO_DECODER_CLASS (klass);
1495
1496 gobject_class->set_property = gst_msdkdec_set_property;
1497 gobject_class->get_property = gst_msdkdec_get_property;
1498 gobject_class->finalize = gst_msdkdec_finalize;
1499
1500 element_class->set_context = gst_msdkdec_set_context;
1501
1502 decoder_class->close = GST_DEBUG_FUNCPTR (gst_msdkdec_close);
1503 decoder_class->start = GST_DEBUG_FUNCPTR (gst_msdkdec_start);
1504 decoder_class->stop = GST_DEBUG_FUNCPTR (gst_msdkdec_stop);
1505 decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_msdkdec_set_format);
1506 decoder_class->finish = GST_DEBUG_FUNCPTR (gst_msdkdec_finish);
1507 decoder_class->handle_frame = GST_DEBUG_FUNCPTR (gst_msdkdec_handle_frame);
1508 decoder_class->decide_allocation =
1509 GST_DEBUG_FUNCPTR (gst_msdkdec_decide_allocation);
1510 decoder_class->flush = GST_DEBUG_FUNCPTR (gst_msdkdec_flush);
1511 decoder_class->drain = GST_DEBUG_FUNCPTR (gst_msdkdec_drain);
1512
1513 klass->preinit_decoder = GST_DEBUG_FUNCPTR (gst_msdkdec_preinit_decoder);
1514 klass->postinit_decoder = GST_DEBUG_FUNCPTR (gst_msdkdec_postinit_decoder);
1515
1516 g_object_class_install_property (gobject_class, GST_MSDKDEC_PROP_HARDWARE,
1517 g_param_spec_boolean ("hardware", "Hardware", "Enable hardware decoders",
1518 PROP_HARDWARE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1519
1520 g_object_class_install_property (gobject_class, GST_MSDKDEC_PROP_ASYNC_DEPTH,
1521 g_param_spec_uint ("async-depth", "Async Depth",
1522 "Depth of asynchronous pipeline",
1523 1, 20, PROP_ASYNC_DEPTH_DEFAULT,
1524 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1525
1526 gst_element_class_add_static_pad_template (element_class, &src_factory);
1527 }
1528
1529 static void
gst_msdkdec_init(GstMsdkDec * thiz)1530 gst_msdkdec_init (GstMsdkDec * thiz)
1531 {
1532 gst_video_info_init (&thiz->output_info);
1533 gst_video_info_init (&thiz->non_msdk_pool_info);
1534 thiz->tasks = g_array_new (FALSE, TRUE, sizeof (MsdkDecTask));
1535 thiz->hardware = PROP_HARDWARE_DEFAULT;
1536 thiz->async_depth = PROP_ASYNC_DEPTH_DEFAULT;
1537 thiz->is_packetized = TRUE;
1538 thiz->do_renego = TRUE;
1539 thiz->do_realloc = TRUE;
1540 thiz->force_reset_on_res_change = TRUE;
1541 thiz->adapter = gst_adapter_new ();
1542 }
1543