1 /*
2 * GStreamer
3 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
4 * Copyright (C) 2020 Rafał Dzięgiel <rafostar.github@gmail.com>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
15 *
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
20 */
21
22 #ifdef HAVE_CONFIG_H
23 #include "config.h"
24 #endif
25
26 #include <stdio.h>
27
28 #include "gtkgstglwidget.h"
29 #include "gstgtkutils.h"
30 #include <gst/gl/gstglfuncs.h>
31 #include <gst/video/video.h>
32
33 #if GST_GL_HAVE_WINDOW_X11 && defined (GDK_WINDOWING_X11)
34 #if defined(BUILD_FOR_GTK4)
35 #include <gdk/x11/gdkx.h>
36 #else
37 #include <gdk/gdkx.h>
38 #endif
39 #include <gst/gl/x11/gstgldisplay_x11.h>
40 #endif
41
42 #if GST_GL_HAVE_WINDOW_WAYLAND && defined (GDK_WINDOWING_WAYLAND)
43 #if defined(BUILD_FOR_GTK4)
44 #include <gdk/wayland/gdkwayland.h>
45 #else
46 #include <gdk/gdkwayland.h>
47 #endif
48 #include <gst/gl/wayland/gstgldisplay_wayland.h>
49 #endif
50
51 /**
52 * SECTION:gtkgstglwidget
53 * @title: GtkGstGlWidget
54 * @short_description: a #GtkGLArea that renders GStreamer video #GstBuffers
55 * @see_also: #GtkGLArea, #GstBuffer
56 *
57 * #GtkGstGLWidget is an #GtkWidget that renders GStreamer video buffers.
58 */
59
60 #define GST_CAT_DEFAULT gtk_gst_gl_widget_debug
61 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
62
63 struct _GtkGstGLWidgetPrivate
64 {
65 gboolean initted;
66 GstGLDisplay *display;
67 GdkGLContext *gdk_context;
68 GstGLContext *other_context;
69 GstGLContext *context;
70 GstGLUpload *upload;
71 GstGLShader *shader;
72 GLuint vao;
73 GLuint vertex_buffer;
74 GLint attr_position;
75 GLint attr_texture;
76 GLuint current_tex;
77 GstGLOverlayCompositor *overlay_compositor;
78 };
79
80 static const GLfloat vertices[] = {
81 1.0f, 1.0f, 0.0f, 1.0f, 0.0f,
82 -1.0f, 1.0f, 0.0f, 0.0f, 0.0f,
83 -1.0f, -1.0f, 0.0f, 0.0f, 1.0f,
84 1.0f, -1.0f, 0.0f, 1.0f, 1.0f
85 };
86
87 G_DEFINE_TYPE_WITH_CODE (GtkGstGLWidget, gtk_gst_gl_widget, GTK_TYPE_GL_AREA,
88 G_ADD_PRIVATE (GtkGstGLWidget)
89 GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "gtkgstglwidget", 0,
90 "GTK Gst GL Widget"));
91
92 static void
gtk_gst_gl_widget_bind_buffer(GtkGstGLWidget * gst_widget)93 gtk_gst_gl_widget_bind_buffer (GtkGstGLWidget * gst_widget)
94 {
95 GtkGstGLWidgetPrivate *priv = gst_widget->priv;
96 const GstGLFuncs *gl = priv->context->gl_vtable;
97
98 gl->BindBuffer (GL_ARRAY_BUFFER, priv->vertex_buffer);
99
100 /* Load the vertex position */
101 gl->VertexAttribPointer (priv->attr_position, 3, GL_FLOAT, GL_FALSE,
102 5 * sizeof (GLfloat), (void *) 0);
103
104 /* Load the texture coordinate */
105 gl->VertexAttribPointer (priv->attr_texture, 2, GL_FLOAT, GL_FALSE,
106 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
107
108 gl->EnableVertexAttribArray (priv->attr_position);
109 gl->EnableVertexAttribArray (priv->attr_texture);
110 }
111
112 static void
gtk_gst_gl_widget_unbind_buffer(GtkGstGLWidget * gst_widget)113 gtk_gst_gl_widget_unbind_buffer (GtkGstGLWidget * gst_widget)
114 {
115 GtkGstGLWidgetPrivate *priv = gst_widget->priv;
116 const GstGLFuncs *gl = priv->context->gl_vtable;
117
118 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
119
120 gl->DisableVertexAttribArray (priv->attr_position);
121 gl->DisableVertexAttribArray (priv->attr_texture);
122 }
123
124 static void
gtk_gst_gl_widget_init_redisplay(GtkGstGLWidget * gst_widget)125 gtk_gst_gl_widget_init_redisplay (GtkGstGLWidget * gst_widget)
126 {
127 GtkGstGLWidgetPrivate *priv = gst_widget->priv;
128 const GstGLFuncs *gl = priv->context->gl_vtable;
129 GError *error = NULL;
130
131 gst_gl_insert_debug_marker (priv->other_context, "initializing redisplay");
132 if (!(priv->shader = gst_gl_shader_new_default (priv->context, &error))) {
133 GST_ERROR ("Failed to initialize shader: %s", error->message);
134 return;
135 }
136
137 priv->attr_position =
138 gst_gl_shader_get_attribute_location (priv->shader, "a_position");
139 priv->attr_texture =
140 gst_gl_shader_get_attribute_location (priv->shader, "a_texcoord");
141
142 if (gl->GenVertexArrays) {
143 gl->GenVertexArrays (1, &priv->vao);
144 gl->BindVertexArray (priv->vao);
145 }
146
147 gl->GenBuffers (1, &priv->vertex_buffer);
148 gl->BindBuffer (GL_ARRAY_BUFFER, priv->vertex_buffer);
149 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
150 GL_STATIC_DRAW);
151
152 if (gl->GenVertexArrays) {
153 gtk_gst_gl_widget_bind_buffer (gst_widget);
154 gl->BindVertexArray (0);
155 }
156
157 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
158
159 priv->overlay_compositor =
160 gst_gl_overlay_compositor_new (priv->other_context);
161
162 priv->initted = TRUE;
163 }
164
165 static void
_redraw_texture(GtkGstGLWidget * gst_widget,guint tex)166 _redraw_texture (GtkGstGLWidget * gst_widget, guint tex)
167 {
168 GtkGstGLWidgetPrivate *priv = gst_widget->priv;
169 const GstGLFuncs *gl = priv->context->gl_vtable;
170 const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
171
172 if (gst_widget->base.force_aspect_ratio) {
173 GstVideoRectangle src, dst, result;
174 gint widget_width, widget_height, widget_scale;
175
176 gl->ClearColor (0.0, 0.0, 0.0, 0.0);
177 gl->Clear (GL_COLOR_BUFFER_BIT);
178
179 widget_scale = gtk_widget_get_scale_factor ((GtkWidget *) gst_widget);
180 widget_width = gtk_widget_get_allocated_width ((GtkWidget *) gst_widget);
181 widget_height = gtk_widget_get_allocated_height ((GtkWidget *) gst_widget);
182
183 src.x = 0;
184 src.y = 0;
185 src.w = gst_widget->base.display_width;
186 src.h = gst_widget->base.display_height;
187
188 dst.x = 0;
189 dst.y = 0;
190 dst.w = widget_width * widget_scale;
191 dst.h = widget_height * widget_scale;
192
193 gst_video_sink_center_rect (src, dst, &result, TRUE);
194
195 gl->Viewport (result.x, result.y, result.w, result.h);
196 }
197
198 gst_gl_shader_use (priv->shader);
199
200 if (gl->BindVertexArray)
201 gl->BindVertexArray (priv->vao);
202 gtk_gst_gl_widget_bind_buffer (gst_widget);
203
204 gl->ActiveTexture (GL_TEXTURE0);
205 gl->BindTexture (GL_TEXTURE_2D, tex);
206 gst_gl_shader_set_uniform_1i (priv->shader, "tex", 0);
207
208 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices);
209
210 if (gl->BindVertexArray)
211 gl->BindVertexArray (0);
212 else
213 gtk_gst_gl_widget_unbind_buffer (gst_widget);
214
215 gl->BindTexture (GL_TEXTURE_2D, 0);
216 }
217
218 static inline void
_draw_black(GstGLContext * context)219 _draw_black (GstGLContext * context)
220 {
221 const GstGLFuncs *gl = context->gl_vtable;
222
223 gst_gl_insert_debug_marker (context, "no buffer. rendering black");
224 gl->ClearColor (0.0, 0.0, 0.0, 0.0);
225 gl->Clear (GL_COLOR_BUFFER_BIT);
226 }
227
228 static gboolean
gtk_gst_gl_widget_render(GtkGLArea * widget,GdkGLContext * context)229 gtk_gst_gl_widget_render (GtkGLArea * widget, GdkGLContext * context)
230 {
231 GtkGstGLWidgetPrivate *priv = GTK_GST_GL_WIDGET (widget)->priv;
232 GtkGstBaseWidget *base_widget = GTK_GST_BASE_WIDGET (widget);
233
234 GTK_GST_BASE_WIDGET_LOCK (widget);
235
236 if (!priv->context || !priv->other_context)
237 goto done;
238
239 gst_gl_context_activate (priv->other_context, TRUE);
240
241 if (!priv->initted)
242 gtk_gst_gl_widget_init_redisplay (GTK_GST_GL_WIDGET (widget));
243
244 if (!priv->initted || !base_widget->negotiated) {
245 _draw_black (priv->other_context);
246 goto done;
247 }
248
249 /* Upload latest buffer */
250 if (base_widget->pending_buffer) {
251 GstBuffer *buffer = base_widget->pending_buffer;
252 GstVideoFrame gl_frame;
253 GstGLSyncMeta *sync_meta;
254
255 if (!gst_video_frame_map (&gl_frame, &base_widget->v_info, buffer,
256 GST_MAP_READ | GST_MAP_GL)) {
257 _draw_black (priv->other_context);
258 goto done;
259 }
260
261 priv->current_tex = *(guint *) gl_frame.data[0];
262 gst_gl_insert_debug_marker (priv->other_context, "redrawing texture %u",
263 priv->current_tex);
264
265 gst_gl_overlay_compositor_upload_overlays (priv->overlay_compositor,
266 buffer);
267
268 sync_meta = gst_buffer_get_gl_sync_meta (buffer);
269 if (sync_meta) {
270 /* XXX: the set_sync() seems to be needed for resizing */
271 gst_gl_sync_meta_set_sync_point (sync_meta, priv->context);
272 gst_gl_sync_meta_wait (sync_meta, priv->other_context);
273 }
274
275 gst_video_frame_unmap (&gl_frame);
276
277 if (base_widget->buffer)
278 gst_buffer_unref (base_widget->buffer);
279
280 /* Keep the buffer to ensure current_tex stay valid */
281 base_widget->buffer = buffer;
282 base_widget->pending_buffer = NULL;
283 }
284
285 GST_DEBUG ("rendering buffer %p with gdk context %p",
286 base_widget->buffer, context);
287
288 _redraw_texture (GTK_GST_GL_WIDGET (widget), priv->current_tex);
289 gst_gl_overlay_compositor_draw_overlays (priv->overlay_compositor);
290
291 gst_gl_insert_debug_marker (priv->other_context, "texture %u redrawn",
292 priv->current_tex);
293
294 done:
295 if (priv->other_context)
296 gst_gl_context_activate (priv->other_context, FALSE);
297
298 GTK_GST_BASE_WIDGET_UNLOCK (widget);
299 return FALSE;
300 }
301
302 static void
_reset_gl(GtkGstGLWidget * gst_widget)303 _reset_gl (GtkGstGLWidget * gst_widget)
304 {
305 GtkGstGLWidgetPrivate *priv = gst_widget->priv;
306 const GstGLFuncs *gl = priv->other_context->gl_vtable;
307
308 if (!priv->gdk_context)
309 priv->gdk_context = gtk_gl_area_get_context (GTK_GL_AREA (gst_widget));
310
311 if (priv->gdk_context == NULL)
312 return;
313
314 gdk_gl_context_make_current (priv->gdk_context);
315 gst_gl_context_activate (priv->other_context, TRUE);
316
317 if (priv->vao) {
318 gl->DeleteVertexArrays (1, &priv->vao);
319 priv->vao = 0;
320 }
321
322 if (priv->vertex_buffer) {
323 gl->DeleteBuffers (1, &priv->vertex_buffer);
324 priv->vertex_buffer = 0;
325 }
326
327 if (priv->upload) {
328 gst_object_unref (priv->upload);
329 priv->upload = NULL;
330 }
331
332 if (priv->shader) {
333 gst_object_unref (priv->shader);
334 priv->shader = NULL;
335 }
336
337 if (priv->overlay_compositor)
338 gst_object_unref (priv->overlay_compositor);
339
340 gst_gl_context_activate (priv->other_context, FALSE);
341
342 gst_object_unref (priv->other_context);
343 priv->other_context = NULL;
344
345 gdk_gl_context_clear_current ();
346
347 g_object_unref (priv->gdk_context);
348 priv->gdk_context = NULL;
349 }
350
351 static void
gtk_gst_gl_widget_finalize(GObject * object)352 gtk_gst_gl_widget_finalize (GObject * object)
353 {
354 GtkGstGLWidgetPrivate *priv = GTK_GST_GL_WIDGET (object)->priv;
355 GtkGstBaseWidget *base_widget = GTK_GST_BASE_WIDGET (object);
356
357 if (priv->other_context)
358 gst_gtk_invoke_on_main ((GThreadFunc) _reset_gl, base_widget);
359
360 if (priv->context)
361 gst_object_unref (priv->context);
362
363 if (priv->display)
364 gst_object_unref (priv->display);
365
366 gtk_gst_base_widget_finalize (object);
367 G_OBJECT_CLASS (gtk_gst_gl_widget_parent_class)->finalize (object);
368 }
369
370 static void
gtk_gst_gl_widget_class_init(GtkGstGLWidgetClass * klass)371 gtk_gst_gl_widget_class_init (GtkGstGLWidgetClass * klass)
372 {
373 GObjectClass *gobject_klass = (GObjectClass *) klass;
374 GtkGLAreaClass *gl_widget_klass = (GtkGLAreaClass *) klass;
375
376 gtk_gst_base_widget_class_init (GTK_GST_BASE_WIDGET_CLASS (klass));
377
378 gobject_klass->finalize = gtk_gst_gl_widget_finalize;
379 gl_widget_klass->render = gtk_gst_gl_widget_render;
380 }
381
382 static void
gtk_gst_gl_widget_init(GtkGstGLWidget * gst_widget)383 gtk_gst_gl_widget_init (GtkGstGLWidget * gst_widget)
384 {
385 GtkGstBaseWidget *base_widget = GTK_GST_BASE_WIDGET (gst_widget);
386 GdkDisplay *display;
387 GtkGstGLWidgetPrivate *priv;
388
389 gtk_gst_base_widget_init (base_widget);
390
391 gst_widget->priv = priv = gtk_gst_gl_widget_get_instance_private (gst_widget);
392
393 display = gdk_display_get_default ();
394
395 #if GST_GL_HAVE_WINDOW_X11 && defined (GDK_WINDOWING_X11)
396 if (GDK_IS_X11_DISPLAY (display)) {
397 priv->display = (GstGLDisplay *)
398 gst_gl_display_x11_new_with_display (gdk_x11_display_get_xdisplay
399 (display));
400 }
401 #endif
402 #if GST_GL_HAVE_WINDOW_WAYLAND && defined (GDK_WINDOWING_WAYLAND)
403 if (GDK_IS_WAYLAND_DISPLAY (display)) {
404 struct wl_display *wayland_display =
405 gdk_wayland_display_get_wl_display (display);
406 priv->display = (GstGLDisplay *)
407 gst_gl_display_wayland_new_with_display (wayland_display);
408 }
409 #endif
410
411 (void) display;
412
413 if (!priv->display)
414 priv->display = gst_gl_display_new ();
415
416 GST_INFO ("Created %" GST_PTR_FORMAT, priv->display);
417
418 /* GTK4 always has alpha */
419 #if !defined(BUILD_FOR_GTK4)
420 gtk_gl_area_set_has_alpha (GTK_GL_AREA (gst_widget),
421 !base_widget->ignore_alpha);
422 #endif
423 }
424
425 static void
_get_gl_context(GtkGstGLWidget * gst_widget)426 _get_gl_context (GtkGstGLWidget * gst_widget)
427 {
428 GtkGstGLWidgetPrivate *priv = gst_widget->priv;
429 GstGLPlatform platform = GST_GL_PLATFORM_NONE;
430 GstGLAPI gl_api = GST_GL_API_NONE;
431 guintptr gl_handle = 0;
432
433 gtk_widget_realize (GTK_WIDGET (gst_widget));
434
435 if (priv->other_context)
436 gst_object_unref (priv->other_context);
437 priv->other_context = NULL;
438
439 if (priv->gdk_context)
440 g_object_unref (priv->gdk_context);
441
442 priv->gdk_context = gtk_gl_area_get_context (GTK_GL_AREA (gst_widget));
443 if (priv->gdk_context == NULL) {
444 GError *error = gtk_gl_area_get_error (GTK_GL_AREA (gst_widget));
445
446 GST_ERROR_OBJECT (gst_widget, "Error creating GdkGLContext : %s",
447 error ? error->message : "No error set by Gdk");
448 g_clear_error (&error);
449 return;
450 }
451
452 g_object_ref (priv->gdk_context);
453
454 gdk_gl_context_make_current (priv->gdk_context);
455
456 #if GST_GL_HAVE_WINDOW_X11 && defined (GDK_WINDOWING_X11)
457 if (GST_IS_GL_DISPLAY_X11 (priv->display)) {
458 #if GST_GL_HAVE_PLATFORM_GLX
459 if (!gl_handle) {
460 platform = GST_GL_PLATFORM_GLX;
461 gl_handle = gst_gl_context_get_current_gl_context (platform);
462 }
463 #endif
464
465 #if GST_GL_HAVE_PLATFORM_EGL
466 if (!gl_handle) {
467 platform = GST_GL_PLATFORM_EGL;
468 gl_handle = gst_gl_context_get_current_gl_context (platform);
469 }
470 #endif
471
472 if (gl_handle) {
473 gl_api = gst_gl_context_get_current_gl_api (platform, NULL, NULL);
474 priv->other_context =
475 gst_gl_context_new_wrapped (priv->display, gl_handle,
476 platform, gl_api);
477 }
478 }
479 #endif
480 #if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (GDK_WINDOWING_WAYLAND)
481 if (GST_IS_GL_DISPLAY_WAYLAND (priv->display)) {
482 platform = GST_GL_PLATFORM_EGL;
483 gl_api = gst_gl_context_get_current_gl_api (platform, NULL, NULL);
484 gl_handle = gst_gl_context_get_current_gl_context (platform);
485 if (gl_handle)
486 priv->other_context =
487 gst_gl_context_new_wrapped (priv->display, gl_handle,
488 platform, gl_api);
489 }
490 #endif
491
492 (void) platform;
493 (void) gl_api;
494 (void) gl_handle;
495
496 if (priv->other_context) {
497 GError *error = NULL;
498
499 GST_INFO ("Retrieved Gdk OpenGL context %" GST_PTR_FORMAT,
500 priv->other_context);
501 gst_gl_context_activate (priv->other_context, TRUE);
502 if (!gst_gl_context_fill_info (priv->other_context, &error)) {
503 GST_ERROR ("failed to retrieve gdk context info: %s", error->message);
504 g_clear_error (&error);
505 g_object_unref (priv->other_context);
506 priv->other_context = NULL;
507 } else {
508 gst_gl_context_activate (priv->other_context, FALSE);
509 }
510 } else {
511 GST_WARNING ("Could not retrieve Gdk OpenGL context");
512 }
513 }
514
515 GtkWidget *
gtk_gst_gl_widget_new(void)516 gtk_gst_gl_widget_new (void)
517 {
518 return (GtkWidget *) g_object_new (GTK_TYPE_GST_GL_WIDGET, NULL);
519 }
520
521 gboolean
gtk_gst_gl_widget_init_winsys(GtkGstGLWidget * gst_widget)522 gtk_gst_gl_widget_init_winsys (GtkGstGLWidget * gst_widget)
523 {
524 GtkGstGLWidgetPrivate *priv = gst_widget->priv;
525 GError *error = NULL;
526
527 g_return_val_if_fail (GTK_IS_GST_GL_WIDGET (gst_widget), FALSE);
528 g_return_val_if_fail (priv->display != NULL, FALSE);
529
530 GTK_GST_BASE_WIDGET_LOCK (gst_widget);
531
532 if (priv->display && priv->gdk_context && priv->other_context) {
533 GST_TRACE ("have already initialized contexts");
534 GTK_GST_BASE_WIDGET_UNLOCK (gst_widget);
535 return TRUE;
536 }
537
538 if (!priv->other_context) {
539 GTK_GST_BASE_WIDGET_UNLOCK (gst_widget);
540 gst_gtk_invoke_on_main ((GThreadFunc) _get_gl_context, gst_widget);
541 GTK_GST_BASE_WIDGET_LOCK (gst_widget);
542 }
543
544 if (!GST_IS_GL_CONTEXT (priv->other_context)) {
545 GST_FIXME ("Could not retrieve Gdk OpenGL context");
546 GTK_GST_BASE_WIDGET_UNLOCK (gst_widget);
547 return FALSE;
548 }
549
550 GST_OBJECT_LOCK (priv->display);
551 if (!gst_gl_display_create_context (priv->display, priv->other_context,
552 &priv->context, &error)) {
553 GST_WARNING ("Could not create OpenGL context: %s",
554 error ? error->message : "Unknown");
555 g_clear_error (&error);
556 GST_OBJECT_UNLOCK (priv->display);
557 GTK_GST_BASE_WIDGET_UNLOCK (gst_widget);
558 return FALSE;
559 }
560 gst_gl_display_add_context (priv->display, priv->context);
561 GST_OBJECT_UNLOCK (priv->display);
562
563 GTK_GST_BASE_WIDGET_UNLOCK (gst_widget);
564 return TRUE;
565 }
566
567 GstGLContext *
gtk_gst_gl_widget_get_gtk_context(GtkGstGLWidget * gst_widget)568 gtk_gst_gl_widget_get_gtk_context (GtkGstGLWidget * gst_widget)
569 {
570 if (!gst_widget->priv->other_context)
571 return NULL;
572
573 return gst_object_ref (gst_widget->priv->other_context);
574 }
575
576 GstGLContext *
gtk_gst_gl_widget_get_context(GtkGstGLWidget * gst_widget)577 gtk_gst_gl_widget_get_context (GtkGstGLWidget * gst_widget)
578 {
579 if (!gst_widget->priv->context)
580 return NULL;
581
582 return gst_object_ref (gst_widget->priv->context);
583 }
584
585 GstGLDisplay *
gtk_gst_gl_widget_get_display(GtkGstGLWidget * gst_widget)586 gtk_gst_gl_widget_get_display (GtkGstGLWidget * gst_widget)
587 {
588 if (!gst_widget->priv->display)
589 return NULL;
590
591 return gst_object_ref (gst_widget->priv->display);
592 }
593