1 /* GStreamer Intel MSDK plugin
2 * Copyright (c) 2018, Intel Corporation
3 * Copyright (c) 2018, Igalia S.L.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * 3. Neither the name of the copyright holder nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
22 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
24 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
27 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGDECE
29 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include "gstmsdkcontext.h"
34 #ifndef _WIN32
35 #include <fcntl.h>
36 #include <unistd.h>
37 #include <va/va_drm.h>
38 #include <gudev/gudev.h>
39 #endif
40
41 GST_DEBUG_CATEGORY_STATIC (gst_debug_msdkcontext);
42 #define GST_CAT_DEFAULT gst_debug_msdkcontext
43
44 struct _GstMsdkContextPrivate
45 {
46 mfxSession session;
47 GList *cached_alloc_responses;
48 gboolean hardware;
49 gboolean is_joined;
50 gboolean has_frame_allocator;
51 GstMsdkContextJobType job_type;
52 gint shared_async_depth;
53 GMutex mutex;
54 GList *child_session_list;
55 #ifndef _WIN32
56 gint fd;
57 VADisplay dpy;
58 #endif
59 };
60
61 #define gst_msdk_context_parent_class parent_class
62 G_DEFINE_TYPE_WITH_CODE (GstMsdkContext, gst_msdk_context, GST_TYPE_OBJECT,
63 G_ADD_PRIVATE (GstMsdkContext)
64 GST_DEBUG_CATEGORY_INIT (gst_debug_msdkcontext, "msdkcontext", 0,
65 "MSDK Context"));
66
67 #ifndef _WIN32
68
69 static gint
get_device_id(void)70 get_device_id (void)
71 {
72 GUdevClient *client = NULL;
73 GUdevEnumerator *e = NULL;
74 GList *devices, *l;
75 GUdevDevice *dev, *parent;
76 const gchar *devnode_path;
77 const gchar *devnode_files[2] = { "renderD[0-9]*", "card[0-9]*" };
78 int fd = -1, i;
79
80 client = g_udev_client_new (NULL);
81 if (!client)
82 goto done;
83
84 e = g_udev_enumerator_new (client);
85 if (!e)
86 goto done;
87
88 g_udev_enumerator_add_match_subsystem (e, "drm");
89 for (i = 0; i < 2; i++) {
90 g_udev_enumerator_add_match_name (e, devnode_files[i]);
91 devices = g_udev_enumerator_execute (e);
92
93 for (l = devices; l != NULL; l = l->next) {
94 dev = (GUdevDevice *) l->data;
95
96 parent = g_udev_device_get_parent (dev);
97 if (strcmp (g_udev_device_get_subsystem (parent), "pci") != 0 ||
98 strcmp (g_udev_device_get_driver (parent), "i915") != 0) {
99 g_object_unref (parent);
100 continue;
101 }
102 g_object_unref (parent);
103
104 devnode_path = g_udev_device_get_device_file (dev);
105 fd = open (devnode_path, O_RDWR | O_CLOEXEC);
106 if (fd < 0)
107 continue;
108 GST_DEBUG ("Opened the drm device node %s", devnode_path);
109 break;
110 }
111
112 g_list_foreach (devices, (GFunc) gst_object_unref, NULL);
113 g_list_free (devices);
114 if (fd >= 0)
115 goto done;
116 }
117
118 done:
119 if (e)
120 g_object_unref (e);
121 if (client)
122 g_object_unref (client);
123
124 return fd;
125 }
126
127
128 static gboolean
gst_msdk_context_use_vaapi(GstMsdkContext * context)129 gst_msdk_context_use_vaapi (GstMsdkContext * context)
130 {
131 gint fd;
132 gint maj_ver, min_ver;
133 VADisplay va_dpy = NULL;
134 VAStatus va_status;
135 mfxStatus status;
136 GstMsdkContextPrivate *priv = context->priv;
137
138 fd = get_device_id ();
139 if (fd < 0) {
140 GST_ERROR ("Couldn't find a drm device node to open");
141 return FALSE;
142 }
143
144 va_dpy = vaGetDisplayDRM (fd);
145 if (!va_dpy) {
146 GST_ERROR ("Couldn't get a VA DRM display");
147 goto failed;
148 }
149
150 va_status = vaInitialize (va_dpy, &maj_ver, &min_ver);
151 if (va_status != VA_STATUS_SUCCESS) {
152 GST_ERROR ("Couldn't initialize VA DRM display");
153 goto failed;
154 }
155
156 status = MFXVideoCORE_SetHandle (priv->session, MFX_HANDLE_VA_DISPLAY,
157 (mfxHDL) va_dpy);
158 if (status != MFX_ERR_NONE) {
159 GST_ERROR ("Setting VAAPI handle failed (%s)",
160 msdk_status_to_string (status));
161 goto failed;
162 }
163
164 priv->fd = fd;
165 priv->dpy = va_dpy;
166
167 return TRUE;
168
169 failed:
170 if (va_dpy)
171 vaTerminate (va_dpy);
172 close (fd);
173 return FALSE;
174 }
175 #endif
176
177 static gboolean
gst_msdk_context_open(GstMsdkContext * context,gboolean hardware,GstMsdkContextJobType job_type)178 gst_msdk_context_open (GstMsdkContext * context, gboolean hardware,
179 GstMsdkContextJobType job_type)
180 {
181 GstMsdkContextPrivate *priv = context->priv;
182
183 priv->job_type = job_type;
184 priv->hardware = hardware;
185 priv->session =
186 msdk_open_session (hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE);
187 if (!priv->session)
188 goto failed;
189
190 #ifndef _WIN32
191 priv->fd = -1;
192
193 if (hardware) {
194 if (!gst_msdk_context_use_vaapi (context))
195 goto failed;
196 }
197 #endif
198
199 return TRUE;
200
201 failed:
202 msdk_close_session (priv->session);
203 return FALSE;
204 }
205
206 static void
gst_msdk_context_init(GstMsdkContext * context)207 gst_msdk_context_init (GstMsdkContext * context)
208 {
209 GstMsdkContextPrivate *priv = gst_msdk_context_get_instance_private (context);
210
211 context->priv = priv;
212
213 g_mutex_init (&priv->mutex);
214 }
215
216 static void
release_child_session(gpointer session)217 release_child_session (gpointer session)
218 {
219 mfxStatus status;
220
221 mfxSession _session = session;
222 status = MFXDisjoinSession (_session);
223 if (status != MFX_ERR_NONE)
224 GST_WARNING ("failed to disjoin (%s)", msdk_status_to_string (status));
225 msdk_close_session (_session);
226 }
227
228 static void
gst_msdk_context_finalize(GObject * obj)229 gst_msdk_context_finalize (GObject * obj)
230 {
231 GstMsdkContext *context = GST_MSDK_CONTEXT_CAST (obj);
232 GstMsdkContextPrivate *priv = context->priv;
233
234 /* child sessions will be closed when the parent session is closed */
235 if (priv->is_joined)
236 goto done;
237 else
238 g_list_free_full (priv->child_session_list, release_child_session);
239
240 msdk_close_session (priv->session);
241 g_mutex_clear (&priv->mutex);
242
243 #ifndef _WIN32
244 if (priv->dpy)
245 vaTerminate (priv->dpy);
246 if (priv->fd >= 0)
247 close (priv->fd);
248 #endif
249
250 done:
251 G_OBJECT_CLASS (parent_class)->finalize (obj);
252 }
253
254 static void
gst_msdk_context_class_init(GstMsdkContextClass * klass)255 gst_msdk_context_class_init (GstMsdkContextClass * klass)
256 {
257 GObjectClass *const g_object_class = G_OBJECT_CLASS (klass);
258
259 g_object_class->finalize = gst_msdk_context_finalize;
260 }
261
262 GstMsdkContext *
gst_msdk_context_new(gboolean hardware,GstMsdkContextJobType job_type)263 gst_msdk_context_new (gboolean hardware, GstMsdkContextJobType job_type)
264 {
265 GstMsdkContext *obj = g_object_new (GST_TYPE_MSDK_CONTEXT, NULL);
266
267 if (obj && !gst_msdk_context_open (obj, hardware, job_type)) {
268 if (obj)
269 gst_object_unref (obj);
270 return NULL;
271 }
272
273 return obj;
274 }
275
276 GstMsdkContext *
gst_msdk_context_new_with_parent(GstMsdkContext * parent)277 gst_msdk_context_new_with_parent (GstMsdkContext * parent)
278 {
279 mfxStatus status;
280 GstMsdkContext *obj = g_object_new (GST_TYPE_MSDK_CONTEXT, NULL);
281 GstMsdkContextPrivate *priv = obj->priv;
282 GstMsdkContextPrivate *parent_priv = parent->priv;
283
284 status = MFXCloneSession (parent_priv->session, &priv->session);
285 if (status != MFX_ERR_NONE) {
286 GST_ERROR ("Failed to clone mfx session");
287 g_object_unref (obj);
288 return NULL;
289 }
290
291 priv->is_joined = TRUE;
292 priv->hardware = parent_priv->hardware;
293 priv->job_type = parent_priv->job_type;
294 parent_priv->child_session_list =
295 g_list_prepend (parent_priv->child_session_list, priv->session);
296 #ifndef _WIN32
297 priv->dpy = parent_priv->dpy;
298 priv->fd = parent_priv->fd;
299
300 if (priv->hardware) {
301 status = MFXVideoCORE_SetHandle (priv->session, MFX_HANDLE_VA_DISPLAY,
302 (mfxHDL) parent_priv->dpy);
303
304 if (status != MFX_ERR_NONE) {
305 GST_ERROR ("Setting VA handle failed (%s)",
306 msdk_status_to_string (status));
307 g_object_unref (obj);
308 return NULL;
309 }
310
311 }
312 #endif
313
314 return obj;
315 }
316
317 mfxSession
gst_msdk_context_get_session(GstMsdkContext * context)318 gst_msdk_context_get_session (GstMsdkContext * context)
319 {
320 return context->priv->session;
321 }
322
323 gpointer
gst_msdk_context_get_handle(GstMsdkContext * context)324 gst_msdk_context_get_handle (GstMsdkContext * context)
325 {
326 #ifndef _WIN32
327 return context->priv->dpy;
328 #else
329 return NULL;
330 #endif
331 }
332
333 gint
gst_msdk_context_get_fd(GstMsdkContext * context)334 gst_msdk_context_get_fd (GstMsdkContext * context)
335 {
336 #ifndef _WIN32
337 return context->priv->fd;
338 #else
339 return -1;
340 #endif
341 }
342
343 static gint
_find_response(gconstpointer resp,gconstpointer comp_resp)344 _find_response (gconstpointer resp, gconstpointer comp_resp)
345 {
346 GstMsdkAllocResponse *cached_resp = (GstMsdkAllocResponse *) resp;
347 mfxFrameAllocResponse *_resp = (mfxFrameAllocResponse *) comp_resp;
348
349 return cached_resp ? cached_resp->mem_ids != _resp->mids : -1;
350 }
351
352 static gint
_find_request(gconstpointer resp,gconstpointer req)353 _find_request (gconstpointer resp, gconstpointer req)
354 {
355 GstMsdkAllocResponse *cached_resp = (GstMsdkAllocResponse *) resp;
356 mfxFrameAllocRequest *_req = (mfxFrameAllocRequest *) req;
357
358 /* Confirm if it's under the size of the cached response */
359 if (_req->Info.Width <= cached_resp->request.Info.Width &&
360 _req->Info.Height <= cached_resp->request.Info.Height) {
361 return _req->Type & cached_resp->
362 request.Type & MFX_MEMTYPE_FROM_DECODE ? 0 : -1;
363 }
364
365 return -1;
366 }
367
368 GstMsdkAllocResponse *
gst_msdk_context_get_cached_alloc_responses(GstMsdkContext * context,mfxFrameAllocResponse * resp)369 gst_msdk_context_get_cached_alloc_responses (GstMsdkContext * context,
370 mfxFrameAllocResponse * resp)
371 {
372 GstMsdkContextPrivate *priv = context->priv;
373 GList *l =
374 g_list_find_custom (priv->cached_alloc_responses, resp, _find_response);
375
376 if (l)
377 return l->data;
378 else
379 return NULL;
380 }
381
382 GstMsdkAllocResponse *
gst_msdk_context_get_cached_alloc_responses_by_request(GstMsdkContext * context,mfxFrameAllocRequest * req)383 gst_msdk_context_get_cached_alloc_responses_by_request (GstMsdkContext *
384 context, mfxFrameAllocRequest * req)
385 {
386 GstMsdkContextPrivate *priv = context->priv;
387 GList *l =
388 g_list_find_custom (priv->cached_alloc_responses, req, _find_request);
389
390 if (l)
391 return l->data;
392 else
393 return NULL;
394 }
395
396 static void
create_surfaces(GstMsdkContext * context,GstMsdkAllocResponse * resp)397 create_surfaces (GstMsdkContext * context, GstMsdkAllocResponse * resp)
398 {
399 gint i;
400 mfxMemId *mem_id;
401 mfxFrameSurface1 *surface;
402
403 for (i = 0; i < resp->response->NumFrameActual; i++) {
404 mem_id = resp->mem_ids[i];
405 surface = (mfxFrameSurface1 *) g_slice_new0 (mfxFrameSurface1);
406 if (!surface) {
407 GST_ERROR ("failed to allocate surface");
408 break;
409 }
410 surface->Data.MemId = mem_id;
411 resp->surfaces_avail = g_list_prepend (resp->surfaces_avail, surface);
412 }
413 }
414
415 static void
free_surface(gpointer surface)416 free_surface (gpointer surface)
417 {
418 g_slice_free1 (sizeof (mfxFrameSurface1), surface);
419 }
420
421 static void
remove_surfaces(GstMsdkContext * context,GstMsdkAllocResponse * resp)422 remove_surfaces (GstMsdkContext * context, GstMsdkAllocResponse * resp)
423 {
424 g_list_free_full (resp->surfaces_used, free_surface);
425 g_list_free_full (resp->surfaces_avail, free_surface);
426 g_list_free_full (resp->surfaces_locked, free_surface);
427 }
428
429 void
gst_msdk_context_add_alloc_response(GstMsdkContext * context,GstMsdkAllocResponse * resp)430 gst_msdk_context_add_alloc_response (GstMsdkContext * context,
431 GstMsdkAllocResponse * resp)
432 {
433 context->priv->cached_alloc_responses =
434 g_list_prepend (context->priv->cached_alloc_responses, resp);
435
436 create_surfaces (context, resp);
437 }
438
439 gboolean
gst_msdk_context_remove_alloc_response(GstMsdkContext * context,mfxFrameAllocResponse * resp)440 gst_msdk_context_remove_alloc_response (GstMsdkContext * context,
441 mfxFrameAllocResponse * resp)
442 {
443 GstMsdkAllocResponse *msdk_resp;
444 GstMsdkContextPrivate *priv = context->priv;
445 GList *l =
446 g_list_find_custom (priv->cached_alloc_responses, resp, _find_response);
447
448 if (!l)
449 return FALSE;
450
451 msdk_resp = l->data;
452
453 remove_surfaces (context, msdk_resp);
454
455 g_slice_free1 (sizeof (GstMsdkAllocResponse), msdk_resp);
456 priv->cached_alloc_responses =
457 g_list_delete_link (priv->cached_alloc_responses, l);
458
459 return TRUE;
460 }
461
462 static gboolean
check_surfaces_available(GstMsdkContext * context,GstMsdkAllocResponse * resp)463 check_surfaces_available (GstMsdkContext * context, GstMsdkAllocResponse * resp)
464 {
465 GList *l;
466 mfxFrameSurface1 *surface = NULL;
467 GstMsdkContextPrivate *priv = context->priv;
468 gboolean ret = FALSE;
469
470 g_mutex_lock (&priv->mutex);
471 for (l = resp->surfaces_locked; l; l = l->next) {
472 surface = l->data;
473 if (!surface->Data.Locked) {
474 resp->surfaces_locked = g_list_remove (resp->surfaces_locked, surface);
475 resp->surfaces_avail = g_list_prepend (resp->surfaces_avail, surface);
476 ret = TRUE;
477 }
478 }
479 g_mutex_unlock (&priv->mutex);
480
481 return ret;
482 }
483
484 /*
485 * There are 3 lists here in GstMsdkContext as the following:
486 * 1. surfaces_avail : surfaces which are free and unused anywhere
487 * 2. surfaces_used : surfaces coupled with a gst buffer and being used now.
488 * 3. surfaces_locked : surfaces still locked even after the gst buffer is released.
489 *
490 * Note that they need to be protected by mutex to be thread-safe.
491 */
492
493 mfxFrameSurface1 *
gst_msdk_context_get_surface_available(GstMsdkContext * context,mfxFrameAllocResponse * resp)494 gst_msdk_context_get_surface_available (GstMsdkContext * context,
495 mfxFrameAllocResponse * resp)
496 {
497 GList *l;
498 mfxFrameSurface1 *surface = NULL;
499 GstMsdkAllocResponse *msdk_resp =
500 gst_msdk_context_get_cached_alloc_responses (context, resp);
501 gint retry = 0;
502 GstMsdkContextPrivate *priv = context->priv;
503
504 retry:
505 g_mutex_lock (&priv->mutex);
506 for (l = msdk_resp->surfaces_avail; l; l = l->next) {
507 surface = l->data;
508
509 if (!surface->Data.Locked) {
510 msdk_resp->surfaces_avail =
511 g_list_remove (msdk_resp->surfaces_avail, surface);
512 msdk_resp->surfaces_used =
513 g_list_prepend (msdk_resp->surfaces_used, surface);
514 break;
515 }
516 }
517 g_mutex_unlock (&priv->mutex);
518
519 /*
520 * If a msdk context is shared by multiple msdk elements,
521 * upstream msdk element sometimes needs to wait for a gst buffer
522 * to be released in downstream.
523 *
524 * Poll the pool for a maximum of 20 milisecnds.
525 *
526 * FIXME: Is there any better way to handle this case?
527 */
528 if (!surface && retry < 20) {
529 /* If there's no surface available, find unlocked surfaces in the locked list,
530 * take it back to the available list and then search again.
531 */
532 check_surfaces_available (context, msdk_resp);
533 retry++;
534 g_usleep (1000);
535 goto retry;
536 }
537
538 return surface;
539 }
540
541 void
gst_msdk_context_put_surface_locked(GstMsdkContext * context,mfxFrameAllocResponse * resp,mfxFrameSurface1 * surface)542 gst_msdk_context_put_surface_locked (GstMsdkContext * context,
543 mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface)
544 {
545 GstMsdkContextPrivate *priv = context->priv;
546 GstMsdkAllocResponse *msdk_resp =
547 gst_msdk_context_get_cached_alloc_responses (context, resp);
548
549 g_mutex_lock (&priv->mutex);
550 if (!g_list_find (msdk_resp->surfaces_locked, surface)) {
551 msdk_resp->surfaces_used =
552 g_list_remove (msdk_resp->surfaces_used, surface);
553 msdk_resp->surfaces_locked =
554 g_list_prepend (msdk_resp->surfaces_locked, surface);
555 }
556 g_mutex_unlock (&priv->mutex);
557 }
558
559 void
gst_msdk_context_put_surface_available(GstMsdkContext * context,mfxFrameAllocResponse * resp,mfxFrameSurface1 * surface)560 gst_msdk_context_put_surface_available (GstMsdkContext * context,
561 mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface)
562 {
563 GstMsdkContextPrivate *priv = context->priv;
564 GstMsdkAllocResponse *msdk_resp =
565 gst_msdk_context_get_cached_alloc_responses (context, resp);
566
567 g_mutex_lock (&priv->mutex);
568 if (!g_list_find (msdk_resp->surfaces_avail, surface)) {
569 msdk_resp->surfaces_used =
570 g_list_remove (msdk_resp->surfaces_used, surface);
571 msdk_resp->surfaces_avail =
572 g_list_prepend (msdk_resp->surfaces_avail, surface);
573 }
574 g_mutex_unlock (&priv->mutex);
575 }
576
577 GstMsdkContextJobType
gst_msdk_context_get_job_type(GstMsdkContext * context)578 gst_msdk_context_get_job_type (GstMsdkContext * context)
579 {
580 return context->priv->job_type;
581 }
582
583 void
gst_msdk_context_add_job_type(GstMsdkContext * context,GstMsdkContextJobType job_type)584 gst_msdk_context_add_job_type (GstMsdkContext * context,
585 GstMsdkContextJobType job_type)
586 {
587 context->priv->job_type |= job_type;
588 }
589
590 gint
gst_msdk_context_get_shared_async_depth(GstMsdkContext * context)591 gst_msdk_context_get_shared_async_depth (GstMsdkContext * context)
592 {
593 return context->priv->shared_async_depth;
594 }
595
596 void
gst_msdk_context_add_shared_async_depth(GstMsdkContext * context,gint async_depth)597 gst_msdk_context_add_shared_async_depth (GstMsdkContext * context,
598 gint async_depth)
599 {
600 context->priv->shared_async_depth += async_depth;
601 }
602
603 void
gst_msdk_context_set_frame_allocator(GstMsdkContext * context,mfxFrameAllocator * allocator)604 gst_msdk_context_set_frame_allocator (GstMsdkContext * context,
605 mfxFrameAllocator * allocator)
606 {
607 GstMsdkContextPrivate *priv = context->priv;
608
609 g_mutex_lock (&priv->mutex);
610
611 if (!priv->has_frame_allocator) {
612 mfxStatus status;
613
614 status = MFXVideoCORE_SetFrameAllocator (priv->session, allocator);
615
616 if (status != MFX_ERR_NONE)
617 GST_ERROR ("Failed to set frame allocator");
618 else
619 priv->has_frame_allocator = 1;
620 }
621
622 g_mutex_unlock (&priv->mutex);
623 }
624