1 /*
2 * GStreamer
3 * Copyright (C) 2009 Julien Isorce <julien.isorce@gmail.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 /**
22 * SECTION:element-glvideomixer
23 * @title: glvideomixer
24 *
25 * Composites a number of streams into a single output scene using OpenGL in
26 * a similar fashion to compositor and videomixer. See the compositor plugin
27 * for documentation about the #GstGLVideoMixerPad properties.
28 *
29 * ## Examples
30 * |[
31 * gst-launch-1.0 glvideomixer name=m ! glimagesink \
32 * videotestsrc ! video/x-raw, format=YUY2 ! glupload ! glcolorconvert ! m. \
33 * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! queue ! \
34 * glupload ! glcolorconvert ! m. \
35 * videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. \
36 * videotestsrc ! glupload ! glfiltercube ! queue ! m. \
37 * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m.
38 * ]|
39 *
40 */
41
42 #ifdef HAVE_CONFIG_H
43 #include "config.h"
44 #endif
45
46 #include <string.h>
47 #include <gst/controller/gstproxycontrolbinding.h>
48 #include <gst/gl/gstglfuncs.h>
49 #include <gst/video/gstvideoaffinetransformationmeta.h>
50
51 #include "gstglvideomixer.h"
52
53 #include "gstglmixerbin.h"
54 #include "gstglutils.h"
55
56 #define GST_CAT_DEFAULT gst_gl_video_mixer_debug
57 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
58
59 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
60 GST_PAD_SINK,
61 GST_PAD_REQUEST,
62 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
63 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
64 "RGBA"))
65 );
66
67 #define GST_TYPE_GL_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type())
68 static GType
gst_gl_video_mixer_background_get_type(void)69 gst_gl_video_mixer_background_get_type (void)
70 {
71 static GType mixer_background_type = 0;
72
73 static const GEnumValue mixer_background[] = {
74 {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"},
75 {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"},
76 {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"},
77 {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT,
78 "Transparent Background to enable further compositing", "transparent"},
79 {0, NULL, NULL},
80 };
81
82 if (!mixer_background_type) {
83 mixer_background_type =
84 g_enum_register_static ("GstGLVideoMixerBackground", mixer_background);
85 }
86 return mixer_background_type;
87 }
88
89 #define GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION (gst_gl_video_mixer_blend_equation_get_type())
90 static GType
gst_gl_video_mixer_blend_equation_get_type(void)91 gst_gl_video_mixer_blend_equation_get_type (void)
92 {
93 static GType mixer_blend_equation_type = 0;
94
95 static const GEnumValue mixer_blend_equations[] = {
96 {GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD, "Add", "add"},
97 {GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT, "Subtract", "subtract"},
98 {GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT, "Reverse Subtract",
99 "reverse-subtract"},
100 {0, NULL, NULL},
101 };
102
103 if (!mixer_blend_equation_type) {
104 mixer_blend_equation_type =
105 g_enum_register_static ("GstGLVideoMixerBlendEquation",
106 mixer_blend_equations);
107 }
108 return mixer_blend_equation_type;
109 }
110
111 #define GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION (gst_gl_video_mixer_blend_function_get_type())
112 static GType
gst_gl_video_mixer_blend_function_get_type(void)113 gst_gl_video_mixer_blend_function_get_type (void)
114 {
115 static GType mixer_blend_function_type = 0;
116
117 static const GEnumValue mixer_blend_funcs[] = {
118 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO, "Zero", "zero"},
119 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE, "One", "one"},
120 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR, "Source Color", "src-color"},
121 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR,
122 "One Minus Source Color", "one-minus-src-color"},
123 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR, "Destination Color",
124 "dst-color"},
125 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR,
126 "One Minus Destination Color", "one-minus-dst-color"},
127 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA, "Source Alpha", "src-alpha"},
128 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA,
129 "One Minus Source Alpha", "one-minus-src-alpha"},
130 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA, "Destination Alpha",
131 "dst-alpha"},
132 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA,
133 "One Minus Destination Alpha", "one-minus-dst-alpha"},
134 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR, "Constant Color",
135 "constant-color"},
136 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR,
137 "One Minus Constant Color", "one-minus-contant-color"},
138 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA, "Constant Alpha",
139 "constant-alpha"},
140 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_ALPHA,
141 "One Minus Constant Alpha", "one-minus-contant-alpha"},
142 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE,
143 "Source Alpha Saturate", "src-alpha-saturate"},
144 {0, NULL, NULL},
145 };
146
147 if (!mixer_blend_function_type) {
148 mixer_blend_function_type =
149 g_enum_register_static ("GstGLVideoMixerBlendFunction",
150 mixer_blend_funcs);
151 }
152 return mixer_blend_function_type;
153 }
154
155 #define DEFAULT_PAD_XPOS 0
156 #define DEFAULT_PAD_YPOS 0
157 #define DEFAULT_PAD_WIDTH 0
158 #define DEFAULT_PAD_HEIGHT 0
159 #define DEFAULT_PAD_ALPHA 1.0
160 #define DEFAULT_PAD_ZORDER 0
161 #define DEFAULT_PAD_REPEAT_AFTER_EOS FALSE
162 #define DEFAULT_PAD_BLEND_EQUATION_RGB GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD
163 #define DEFAULT_PAD_BLEND_EQUATION_ALPHA GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD
164 #define DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA
165 #define DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE
166 #define DEFAULT_PAD_BLEND_FUNCTION_DST_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA
167 #define DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA
168
169 enum
170 {
171 PROP_INPUT_0,
172 PROP_INPUT_XPOS,
173 PROP_INPUT_YPOS,
174 PROP_INPUT_WIDTH,
175 PROP_INPUT_HEIGHT,
176 PROP_INPUT_ALPHA,
177 PROP_INPUT_BLEND_EQUATION_RGB,
178 PROP_INPUT_BLEND_EQUATION_ALPHA,
179 PROP_INPUT_BLEND_FUNCTION_SRC_RGB,
180 PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA,
181 PROP_INPUT_BLEND_FUNCTION_DST_RGB,
182 PROP_INPUT_BLEND_FUNCTION_DST_ALPHA,
183 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_RED,
184 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
185 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
186 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
187 PROP_INPUT_ZORDER,
188 PROP_INPUT_REPEAT_AFTER_EOS,
189 };
190
191 static void gst_gl_video_mixer_input_get_property (GObject * object,
192 guint prop_id, GValue * value, GParamSpec * pspec);
193 static void gst_gl_video_mixer_input_set_property (GObject * object,
194 guint prop_id, const GValue * value, GParamSpec * pspec);
195
196 typedef struct _GstGLVideoMixerInput GstGLVideoMixerInput;
197 typedef GstGhostPadClass GstGLVideoMixerInputClass;
198
199 struct _GstGLVideoMixerInput
200 {
201 GstGhostPad parent;
202
203 GstSegment segment;
204
205 GstPad *mixer_pad;
206 };
207
208 GType gst_gl_video_mixer_input_get_type (void);
209 G_DEFINE_TYPE (GstGLVideoMixerInput, gst_gl_video_mixer_input,
210 GST_TYPE_GHOST_PAD);
211
212 static void
gst_gl_video_mixer_input_init(GstGLVideoMixerInput * self)213 gst_gl_video_mixer_input_init (GstGLVideoMixerInput * self)
214 {
215 }
216
217 static void
gst_gl_video_mixer_input_class_init(GstGLVideoMixerInputClass * klass)218 gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass)
219 {
220 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
221
222 gobject_class->set_property = gst_gl_video_mixer_input_set_property;
223 gobject_class->get_property = gst_gl_video_mixer_input_get_property;
224
225 g_object_class_install_property (gobject_class, PROP_INPUT_ZORDER,
226 g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture",
227 0, 10000, DEFAULT_PAD_ZORDER,
228 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
229 g_object_class_install_property (gobject_class, PROP_INPUT_REPEAT_AFTER_EOS,
230 g_param_spec_boolean ("repeat-after-eos", "Repeat After EOS",
231 "Aggregate the last "
232 "frame on pads that are EOS till they are released",
233 DEFAULT_PAD_REPEAT_AFTER_EOS,
234 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
235 g_object_class_install_property (gobject_class, PROP_INPUT_XPOS,
236 g_param_spec_int ("xpos", "X Position", "X Position of the picture",
237 G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
238 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
239 g_object_class_install_property (gobject_class, PROP_INPUT_YPOS,
240 g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
241 G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
242 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
243 g_object_class_install_property (gobject_class, PROP_INPUT_WIDTH,
244 g_param_spec_int ("width", "Width", "Width of the picture", G_MININT,
245 G_MAXINT, DEFAULT_PAD_WIDTH,
246 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
247 g_object_class_install_property (gobject_class, PROP_INPUT_HEIGHT,
248 g_param_spec_int ("height", "Height", "Height of the picture", G_MININT,
249 G_MAXINT, DEFAULT_PAD_HEIGHT,
250 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
251 g_object_class_install_property (gobject_class, PROP_INPUT_ALPHA,
252 g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
253 DEFAULT_PAD_ALPHA,
254 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
255 g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_EQUATION_RGB,
256 g_param_spec_enum ("blend-equation-rgb", "Blend Equation RGB",
257 "Blend Equation for RGB", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
258 DEFAULT_PAD_BLEND_EQUATION_RGB,
259 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
260 g_object_class_install_property (gobject_class,
261 PROP_INPUT_BLEND_EQUATION_ALPHA,
262 g_param_spec_enum ("blend-equation-alpha", "Blend Equation Alpha",
263 "Blend Equation for Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
264 DEFAULT_PAD_BLEND_EQUATION_ALPHA,
265 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
266 g_object_class_install_property (gobject_class,
267 PROP_INPUT_BLEND_FUNCTION_SRC_RGB,
268 g_param_spec_enum ("blend-function-src-rgb", "Blend Function Source RGB",
269 "Blend Function for Source RGB",
270 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
271 DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB,
272 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
273 g_object_class_install_property (gobject_class,
274 PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA,
275 g_param_spec_enum ("blend-function-src-alpha",
276 "Blend Function Source Alpha", "Blend Function for Source Alpha",
277 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
278 DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA,
279 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
280 g_object_class_install_property (gobject_class,
281 PROP_INPUT_BLEND_FUNCTION_DST_RGB,
282 g_param_spec_enum ("blend-function-dst-rgb",
283 "Blend Function Destination RGB",
284 "Blend Function for Destination RGB",
285 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
286 DEFAULT_PAD_BLEND_FUNCTION_DST_RGB,
287 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
288 g_object_class_install_property (gobject_class,
289 PROP_INPUT_BLEND_FUNCTION_DST_ALPHA,
290 g_param_spec_enum ("blend-function-dst-alpha",
291 "Blend Function Destination Alpha",
292 "Blend Function for Destination Alpha",
293 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
294 DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA,
295 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
296 g_object_class_install_property (gobject_class,
297 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_RED,
298 g_param_spec_double ("blend-constant-color-red",
299 "Blend Constant Color Red", "Blend Constant Color Red", 0.0, 1.0, 0.0,
300 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
301 g_object_class_install_property (gobject_class,
302 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
303 g_param_spec_double ("blend-constant-color-green",
304 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
305 0.0,
306 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
307 g_object_class_install_property (gobject_class,
308 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
309 g_param_spec_double ("blend-constant-color-blue",
310 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
311 0.0,
312 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
313 g_object_class_install_property (gobject_class,
314 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
315 g_param_spec_double ("blend-constant-color-alpha",
316 "Blend Constant Color Alpha", "Blend Constant Color Alpha", 0.0, 1.0,
317 0.0,
318 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
319 }
320
321 static void
gst_gl_video_mixer_input_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)322 gst_gl_video_mixer_input_get_property (GObject * object, guint prop_id,
323 GValue * value, GParamSpec * pspec)
324 {
325 GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object;
326
327 if (self->mixer_pad)
328 g_object_get_property (G_OBJECT (self->mixer_pad), pspec->name, value);
329 }
330
331 static void
gst_gl_video_mixer_input_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)332 gst_gl_video_mixer_input_set_property (GObject * object, guint prop_id,
333 const GValue * value, GParamSpec * pspec)
334 {
335 GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object;
336
337 if (self->mixer_pad)
338 g_object_set_property (G_OBJECT (self->mixer_pad), pspec->name, value);
339 }
340
341 static GstGhostPad *
_create_video_mixer_input(GstGLMixerBin * self,GstPad * mixer_pad)342 _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad)
343 {
344 GstGLVideoMixerInput *input =
345 g_object_new (gst_gl_video_mixer_input_get_type (), "name",
346 GST_OBJECT_NAME (mixer_pad), "direction", GST_PAD_DIRECTION (mixer_pad),
347 NULL);
348
349 if (!gst_ghost_pad_construct (GST_GHOST_PAD (input))) {
350 gst_object_unref (input);
351 return NULL;
352 }
353 #define ADD_BINDING(obj,ref,prop) \
354 gst_object_add_control_binding (GST_OBJECT (obj), \
355 gst_proxy_control_binding_new (GST_OBJECT (obj), prop, \
356 GST_OBJECT (ref), prop));
357 ADD_BINDING (mixer_pad, input, "zorder");
358 ADD_BINDING (mixer_pad, input, "xpos");
359 ADD_BINDING (mixer_pad, input, "ypos");
360 ADD_BINDING (mixer_pad, input, "width");
361 ADD_BINDING (mixer_pad, input, "height");
362 ADD_BINDING (mixer_pad, input, "alpha");
363 ADD_BINDING (mixer_pad, input, "blend-equation-rgb");
364 ADD_BINDING (mixer_pad, input, "blend-equation-alpha");
365 ADD_BINDING (mixer_pad, input, "blend-function-src-rgb");
366 ADD_BINDING (mixer_pad, input, "blend-function-src-alpha");
367 ADD_BINDING (mixer_pad, input, "blend-function-dst-rgb");
368 ADD_BINDING (mixer_pad, input, "blend-function-dst-alpha");
369 ADD_BINDING (mixer_pad, input, "blend-constant-color-red");
370 ADD_BINDING (mixer_pad, input, "blend-constant-color-green");
371 ADD_BINDING (mixer_pad, input, "blend-constant-color-blue");
372 ADD_BINDING (mixer_pad, input, "blend-constant-color-alpha");
373 #undef ADD_BINDING
374
375 input->mixer_pad = mixer_pad;
376
377 return GST_GHOST_PAD (input);
378 }
379
380 enum
381 {
382 PROP_BIN_0,
383 PROP_BIN_BACKGROUND,
384 };
385 #define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER
386
387 static void gst_gl_video_mixer_bin_get_property (GObject * object,
388 guint prop_id, GValue * value, GParamSpec * pspec);
389 static void gst_gl_video_mixer_bin_set_property (GObject * object,
390 guint prop_id, const GValue * value, GParamSpec * pspec);
391
392 typedef GstGLMixerBin GstGLVideoMixerBin;
393 typedef GstGLMixerBinClass GstGLVideoMixerBinClass;
394
395 G_DEFINE_TYPE (GstGLVideoMixerBin, gst_gl_video_mixer_bin,
396 GST_TYPE_GL_MIXER_BIN);
397
398 static void
gst_gl_video_mixer_bin_init(GstGLVideoMixerBin * self)399 gst_gl_video_mixer_bin_init (GstGLVideoMixerBin * self)
400 {
401 GstGLMixerBin *mix_bin = GST_GL_MIXER_BIN (self);
402
403 gst_gl_mixer_bin_finish_init_with_element (mix_bin,
404 g_object_new (GST_TYPE_GL_VIDEO_MIXER, NULL));
405 }
406
407 static void
gst_gl_video_mixer_bin_class_init(GstGLVideoMixerBinClass * klass)408 gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass)
409 {
410 GstGLMixerBinClass *mixer_class = GST_GL_MIXER_BIN_CLASS (klass);
411 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
412 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
413
414 mixer_class->create_input_pad = _create_video_mixer_input;
415
416 gobject_class->set_property = gst_gl_video_mixer_bin_set_property;
417 gobject_class->get_property = gst_gl_video_mixer_bin_get_property;
418
419 g_object_class_install_property (gobject_class, PROP_BIN_BACKGROUND,
420 g_param_spec_enum ("background", "Background", "Background type",
421 GST_TYPE_GL_VIDEO_MIXER_BACKGROUND,
422 DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
423
424 gst_element_class_set_metadata (element_class, "OpenGL video_mixer bin",
425 "Bin/Filter/Effect/Video/Compositor", "OpenGL video_mixer bin",
426 "Matthew Waters <matthew@centricular.com>");
427 }
428
429 static void
gst_gl_video_mixer_bin_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)430 gst_gl_video_mixer_bin_get_property (GObject * object, guint prop_id,
431 GValue * value, GParamSpec * pspec)
432 {
433 GstGLMixerBin *self = GST_GL_MIXER_BIN (object);
434
435 if (self->mixer)
436 g_object_get_property (G_OBJECT (self->mixer), pspec->name, value);
437 }
438
439 static void
gst_gl_video_mixer_bin_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)440 gst_gl_video_mixer_bin_set_property (GObject * object, guint prop_id,
441 const GValue * value, GParamSpec * pspec)
442 {
443 GstGLMixerBin *self = GST_GL_MIXER_BIN (object);
444
445 if (self->mixer)
446 g_object_set_property (G_OBJECT (self->mixer), pspec->name, value);
447 }
448
449 enum
450 {
451 PROP_0,
452 PROP_BACKGROUND,
453 };
454
455 static void gst_gl_video_mixer_child_proxy_init (gpointer g_iface,
456 gpointer iface_data);
457
458 #define DEBUG_INIT \
459 GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element");
460
461 #define gst_gl_video_mixer_parent_class parent_class
462 G_DEFINE_TYPE_WITH_CODE (GstGLVideoMixer, gst_gl_video_mixer, GST_TYPE_GL_MIXER,
463 G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
464 gst_gl_video_mixer_child_proxy_init); DEBUG_INIT);
465
466 static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id,
467 const GValue * value, GParamSpec * pspec);
468 static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id,
469 GValue * value, GParamSpec * pspec);
470
471 static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps);
472 static GstCaps *_fixate_caps (GstAggregator * agg, GstCaps * caps);
473 static gboolean gst_gl_video_mixer_propose_allocation (GstAggregator *
474 agg, GstAggregatorPad * agg_pad, GstQuery * decide_query, GstQuery * query);
475 static void gst_gl_video_mixer_reset (GstGLMixer * mixer);
476 static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer,
477 GstCaps * outcaps);
478
479 static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer,
480 GstGLMemory * out_tex);
481 static gboolean gst_gl_video_mixer_callback (gpointer stuff);
482
483 /* *INDENT-OFF* */
484
485 /* fragment source */
486 static const gchar *video_mixer_f_src =
487 "uniform sampler2D texture; \n"
488 "uniform float alpha;\n"
489 "varying vec2 v_texcoord; \n"
490 "void main() \n"
491 "{ \n"
492 " vec4 rgba = texture2D(texture, v_texcoord);\n"
493 " gl_FragColor = vec4(rgba.rgb, rgba.a * alpha);\n"
494 "} \n";
495
496 /* checker vertex source */
497 static const gchar *checker_v_src =
498 "attribute vec4 a_position;\n"
499 "void main()\n"
500 "{\n"
501 " gl_Position = a_position;\n"
502 "}\n";
503
504 /* checker fragment source */
505 static const gchar *checker_f_src =
506 "const float blocksize = 8.0;\n"
507 "void main ()\n"
508 "{\n"
509 " vec4 high = vec4(0.667, 0.667, 0.667, 1.0);\n"
510 " vec4 low = vec4(0.333, 0.333, 0.333, 1.0);\n"
511 " if (mod(gl_FragCoord.x, blocksize * 2.0) >= blocksize) {\n"
512 " if (mod(gl_FragCoord.y, blocksize * 2.0) >= blocksize)\n"
513 " gl_FragColor = low;\n"
514 " else\n"
515 " gl_FragColor = high;\n"
516 " } else {\n"
517 " if (mod(gl_FragCoord.y, blocksize * 2.0) < blocksize)\n"
518 " gl_FragColor = low;\n"
519 " else\n"
520 " gl_FragColor = high;\n"
521 " }\n"
522 "}\n";
523 /* *INDENT-ON* */
524
525 #define GST_TYPE_GL_VIDEO_MIXER_PAD (gst_gl_video_mixer_pad_get_type())
526 #define GST_GL_VIDEO_MIXER_PAD(obj) \
527 (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_VIDEO_MIXER_PAD, GstGLVideoMixerPad))
528 #define GST_GL_VIDEO_MIXER_PAD_CLASS(klass) \
529 (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_VIDEO_MIXER_PAD, GstGLVideoMixerPadClass))
530 #define GST_IS_GL_VIDEO_MIXER_PAD(obj) \
531 (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_VIDEO_MIXER_PAD))
532 #define GST_IS_GL_VIDEO_MIXER_PAD_CLASS(klass) \
533 (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_VIDEO_MIXER_PAD))
534
535 typedef struct _GstGLVideoMixerPad GstGLVideoMixerPad;
536 typedef struct _GstGLVideoMixerPadClass GstGLVideoMixerPadClass;
537 typedef struct _GstGLVideoMixerCollect GstGLVideoMixerCollect;
538
539 /**
540 * GstGLVideoMixerPad:
541 *
542 * The opaque #GstGLVideoMixerPad structure.
543 */
544 struct _GstGLVideoMixerPad
545 {
546 GstGLMixerPad parent;
547
548 /* < private > */
549 /* properties */
550 gint xpos, ypos;
551 gint width, height;
552 gdouble alpha;
553
554 GstGLVideoMixerBlendEquation blend_equation_rgb;
555 GstGLVideoMixerBlendEquation blend_equation_alpha;
556 GstGLVideoMixerBlendFunction blend_function_src_rgb;
557 GstGLVideoMixerBlendFunction blend_function_src_alpha;
558 GstGLVideoMixerBlendFunction blend_function_dst_rgb;
559 GstGLVideoMixerBlendFunction blend_function_dst_alpha;
560 gdouble blend_constant_color_red;
561 gdouble blend_constant_color_green;
562 gdouble blend_constant_color_blue;
563 gdouble blend_constant_color_alpha;
564
565 gboolean geometry_change;
566 GLuint vertex_buffer;
567 gfloat m_matrix[16];
568 };
569
570 struct _GstGLVideoMixerPadClass
571 {
572 GstGLMixerPadClass parent_class;
573 };
574
575 GType gst_gl_video_mixer_pad_get_type (void);
576 G_DEFINE_TYPE (GstGLVideoMixerPad, gst_gl_video_mixer_pad,
577 GST_TYPE_GL_MIXER_PAD);
578
579 static void gst_gl_video_mixer_pad_set_property (GObject * object,
580 guint prop_id, const GValue * value, GParamSpec * pspec);
581 static void gst_gl_video_mixer_pad_get_property (GObject * object,
582 guint prop_id, GValue * value, GParamSpec * pspec);
583
584 enum
585 {
586 PROP_PAD_0,
587 PROP_PAD_XPOS,
588 PROP_PAD_YPOS,
589 PROP_PAD_WIDTH,
590 PROP_PAD_HEIGHT,
591 PROP_PAD_ALPHA,
592 PROP_PAD_BLEND_EQUATION_RGB,
593 PROP_PAD_BLEND_EQUATION_ALPHA,
594 PROP_PAD_BLEND_FUNCTION_SRC_RGB,
595 PROP_PAD_BLEND_FUNCTION_SRC_ALPHA,
596 PROP_PAD_BLEND_FUNCTION_DST_RGB,
597 PROP_PAD_BLEND_FUNCTION_DST_ALPHA,
598 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED,
599 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
600 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
601 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
602 };
603
604 static void
gst_gl_video_mixer_pad_init(GstGLVideoMixerPad * pad)605 gst_gl_video_mixer_pad_init (GstGLVideoMixerPad * pad)
606 {
607 pad->alpha = DEFAULT_PAD_ALPHA;
608 pad->blend_equation_rgb = DEFAULT_PAD_BLEND_EQUATION_RGB;
609 pad->blend_equation_alpha = DEFAULT_PAD_BLEND_EQUATION_ALPHA;
610 pad->blend_function_src_rgb = DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB;
611 pad->blend_function_src_alpha = DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA;
612 pad->blend_function_dst_rgb = DEFAULT_PAD_BLEND_FUNCTION_DST_RGB;
613 pad->blend_function_dst_alpha = DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA;
614 memset (pad->m_matrix, 0, sizeof (gfloat) * 4 * 4);
615 pad->m_matrix[0] = 1.0;
616 pad->m_matrix[5] = 1.0;
617 pad->m_matrix[10] = 1.0;
618 pad->m_matrix[15] = 1.0;
619 }
620
621 static void
gst_gl_video_mixer_pad_class_init(GstGLVideoMixerPadClass * klass)622 gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass)
623 {
624 GObjectClass *gobject_class = (GObjectClass *) klass;
625
626 gobject_class->set_property = gst_gl_video_mixer_pad_set_property;
627 gobject_class->get_property = gst_gl_video_mixer_pad_get_property;
628
629 g_object_class_install_property (gobject_class, PROP_PAD_XPOS,
630 g_param_spec_int ("xpos", "X Position", "X Position of the picture",
631 G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
632 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
633 g_object_class_install_property (gobject_class, PROP_PAD_YPOS,
634 g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
635 G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
636 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
637 g_object_class_install_property (gobject_class, PROP_PAD_WIDTH,
638 g_param_spec_int ("width", "Width", "Width of the picture",
639 G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH,
640 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
641 g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT,
642 g_param_spec_int ("height", "Height", "Height of the picture",
643 G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT,
644 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
645 g_object_class_install_property (gobject_class, PROP_PAD_ALPHA,
646 g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
647 DEFAULT_PAD_ALPHA,
648 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
649 g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_EQUATION_RGB,
650 g_param_spec_enum ("blend-equation-rgb", "Blend Equation RGB",
651 "Blend Equation for RGB",
652 GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
653 DEFAULT_PAD_BLEND_EQUATION_RGB,
654 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
655 g_object_class_install_property (gobject_class,
656 PROP_INPUT_BLEND_EQUATION_ALPHA,
657 g_param_spec_enum ("blend-equation-alpha", "Blend Equation Alpha",
658 "Blend Equation for Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
659 DEFAULT_PAD_BLEND_EQUATION_ALPHA,
660 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
661 g_object_class_install_property (gobject_class,
662 PROP_INPUT_BLEND_FUNCTION_SRC_RGB,
663 g_param_spec_enum ("blend-function-src-rgb", "Blend Function Source RGB",
664 "Blend Function for Source RGB",
665 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
666 DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB,
667 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
668 g_object_class_install_property (gobject_class,
669 PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA,
670 g_param_spec_enum ("blend-function-src-alpha",
671 "Blend Function Source Alpha", "Blend Function for Source Alpha",
672 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
673 DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA,
674 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
675 g_object_class_install_property (gobject_class,
676 PROP_INPUT_BLEND_FUNCTION_DST_RGB,
677 g_param_spec_enum ("blend-function-dst-rgb",
678 "Blend Function Destination RGB",
679 "Blend Function for Destination RGB",
680 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
681 DEFAULT_PAD_BLEND_FUNCTION_DST_RGB,
682 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
683 g_object_class_install_property (gobject_class,
684 PROP_INPUT_BLEND_FUNCTION_DST_ALPHA,
685 g_param_spec_enum ("blend-function-dst-alpha",
686 "Blend Function Destination Alpha",
687 "Blend Function for Destination Alpha",
688 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
689 DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA,
690 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
691 g_object_class_install_property (gobject_class,
692 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED,
693 g_param_spec_double ("blend-constant-color-red",
694 "Blend Constant Color Red", "Blend Constant Color Red", 0.0, 1.0, 0.0,
695 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
696 g_object_class_install_property (gobject_class,
697 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
698 g_param_spec_double ("blend-constant-color-green",
699 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
700 0.0,
701 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
702 g_object_class_install_property (gobject_class,
703 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
704 g_param_spec_double ("blend-constant-color-blue",
705 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
706 0.0,
707 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
708 g_object_class_install_property (gobject_class,
709 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
710 g_param_spec_double ("blend-constant-color-alpha",
711 "Blend Constant Color Alpha", "Blend Constant Color Alpha", 0.0, 1.0,
712 0.0,
713 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
714 }
715
716 static void
gst_gl_video_mixer_pad_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)717 gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id,
718 GValue * value, GParamSpec * pspec)
719 {
720 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (object);
721
722 switch (prop_id) {
723 case PROP_PAD_XPOS:
724 g_value_set_int (value, pad->xpos);
725 break;
726 case PROP_PAD_YPOS:
727 g_value_set_int (value, pad->ypos);
728 break;
729 case PROP_PAD_WIDTH:
730 g_value_set_int (value, pad->width);
731 break;
732 case PROP_PAD_HEIGHT:
733 g_value_set_int (value, pad->height);
734 break;
735 case PROP_PAD_ALPHA:
736 g_value_set_double (value, pad->alpha);
737 break;
738 case PROP_PAD_BLEND_EQUATION_RGB:
739 g_value_set_enum (value, pad->blend_equation_rgb);
740 break;
741 case PROP_PAD_BLEND_EQUATION_ALPHA:
742 g_value_set_enum (value, pad->blend_equation_alpha);
743 break;
744 case PROP_PAD_BLEND_FUNCTION_SRC_RGB:
745 g_value_set_enum (value, pad->blend_function_src_rgb);
746 break;
747 case PROP_PAD_BLEND_FUNCTION_SRC_ALPHA:
748 g_value_set_enum (value, pad->blend_function_src_alpha);
749 break;
750 case PROP_PAD_BLEND_FUNCTION_DST_RGB:
751 g_value_set_enum (value, pad->blend_function_dst_rgb);
752 break;
753 case PROP_PAD_BLEND_FUNCTION_DST_ALPHA:
754 g_value_set_enum (value, pad->blend_function_dst_alpha);
755 break;
756 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED:
757 g_value_set_double (value, pad->blend_constant_color_red);
758 break;
759 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN:
760 g_value_set_double (value, pad->blend_constant_color_green);
761 break;
762 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE:
763 g_value_set_double (value, pad->blend_constant_color_blue);
764 break;
765 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA:
766 g_value_set_double (value, pad->blend_constant_color_alpha);
767 break;
768 default:
769 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
770 break;
771 }
772 }
773
774 static void
gst_gl_video_mixer_pad_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)775 gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id,
776 const GValue * value, GParamSpec * pspec)
777 {
778 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (object);
779 GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (GST_PAD (pad)));
780
781 switch (prop_id) {
782 case PROP_PAD_XPOS:
783 pad->xpos = g_value_get_int (value);
784 pad->geometry_change = TRUE;
785 break;
786 case PROP_PAD_YPOS:
787 pad->ypos = g_value_get_int (value);
788 pad->geometry_change = TRUE;
789 break;
790 case PROP_PAD_WIDTH:
791 pad->width = g_value_get_int (value);
792 pad->geometry_change = TRUE;
793 break;
794 case PROP_PAD_HEIGHT:
795 pad->height = g_value_get_int (value);
796 pad->geometry_change = TRUE;
797 break;
798 case PROP_PAD_ALPHA:
799 pad->alpha = g_value_get_double (value);
800 break;
801 case PROP_PAD_BLEND_EQUATION_RGB:
802 pad->blend_equation_rgb = g_value_get_enum (value);
803 break;
804 case PROP_PAD_BLEND_EQUATION_ALPHA:
805 pad->blend_equation_alpha = g_value_get_enum (value);
806 break;
807 case PROP_PAD_BLEND_FUNCTION_SRC_RGB:
808 pad->blend_function_src_rgb = g_value_get_enum (value);
809 break;
810 case PROP_PAD_BLEND_FUNCTION_SRC_ALPHA:
811 pad->blend_function_src_alpha = g_value_get_enum (value);
812 break;
813 case PROP_PAD_BLEND_FUNCTION_DST_RGB:
814 pad->blend_function_dst_rgb = g_value_get_enum (value);
815 break;
816 case PROP_PAD_BLEND_FUNCTION_DST_ALPHA:
817 pad->blend_function_dst_alpha = g_value_get_enum (value);
818 break;
819 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED:
820 pad->blend_constant_color_red = g_value_get_double (value);
821 break;
822 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN:
823 pad->blend_constant_color_green = g_value_get_double (value);
824 break;
825 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE:
826 pad->blend_constant_color_blue = g_value_get_double (value);
827 break;
828 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA:
829 pad->blend_constant_color_alpha = g_value_get_double (value);
830 break;
831 default:
832 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
833 break;
834 }
835
836 gst_object_unref (mix);
837 }
838
839 static void
_del_buffer(GstGLContext * context,GLuint * pBuffer)840 _del_buffer (GstGLContext * context, GLuint * pBuffer)
841 {
842 context->gl_vtable->DeleteBuffers (1, pBuffer);
843 }
844
845 static GstPad *
gst_gl_video_mixer_request_new_pad(GstElement * element,GstPadTemplate * templ,const gchar * req_name,const GstCaps * caps)846 gst_gl_video_mixer_request_new_pad (GstElement * element,
847 GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps)
848 {
849 GstPad *newpad;
850
851 newpad = (GstPad *)
852 GST_ELEMENT_CLASS (parent_class)->request_new_pad (element,
853 templ, req_name, caps);
854
855 if (newpad == NULL)
856 goto could_not_create;
857
858 gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad),
859 GST_OBJECT_NAME (newpad));
860
861 return newpad;
862
863 could_not_create:
864 {
865 GST_DEBUG_OBJECT (element, "could not create/add pad");
866 return NULL;
867 }
868 }
869
870 static void
gst_gl_video_mixer_release_pad(GstElement * element,GstPad * p)871 gst_gl_video_mixer_release_pad (GstElement * element, GstPad * p)
872 {
873 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (p);
874
875 gst_child_proxy_child_removed (GST_CHILD_PROXY (element), G_OBJECT (pad),
876 GST_OBJECT_NAME (pad));
877
878 /* we call the base class first as this will remove the pad from
879 * the aggregator, thus stopping misc callbacks from being called,
880 * one of which (process_textures) will recreate the vertex_buffer
881 * if it is destroyed */
882 GST_ELEMENT_CLASS (g_type_class_peek_parent (G_OBJECT_GET_CLASS (element)))
883 ->release_pad (element, p);
884
885 if (pad->vertex_buffer) {
886 GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element);
887 gst_gl_context_thread_add (mix->context, (GstGLContextThreadFunc)
888 _del_buffer, &pad->vertex_buffer);
889 pad->vertex_buffer = 0;
890 }
891 }
892
893 static void
gst_gl_video_mixer_class_init(GstGLVideoMixerClass * klass)894 gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass)
895 {
896 GObjectClass *gobject_class;
897 GstElementClass *element_class;
898 GstAggregatorClass *agg_class = (GstAggregatorClass *) klass;
899 GstVideoAggregatorClass *vagg_class = (GstVideoAggregatorClass *) klass;
900
901 gobject_class = (GObjectClass *) klass;
902 element_class = GST_ELEMENT_CLASS (klass);
903 element_class->request_new_pad = gst_gl_video_mixer_request_new_pad;
904 element_class->release_pad = gst_gl_video_mixer_release_pad;
905
906 gobject_class->set_property = gst_gl_video_mixer_set_property;
907 gobject_class->get_property = gst_gl_video_mixer_get_property;
908
909 gst_element_class_set_metadata (element_class, "OpenGL video_mixer",
910 "Filter/Effect/Video/Compositor", "OpenGL video_mixer",
911 "Matthew Waters <matthew@centricular.com>");
912
913 gst_element_class_add_static_pad_template_with_gtype (element_class,
914 &sink_factory, GST_TYPE_GL_VIDEO_MIXER_PAD);
915
916 g_object_class_install_property (gobject_class, PROP_BACKGROUND,
917 g_param_spec_enum ("background", "Background", "Background type",
918 GST_TYPE_GL_VIDEO_MIXER_BACKGROUND,
919 DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
920
921 GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_video_mixer_init_shader;
922 GST_GL_MIXER_CLASS (klass)->reset = gst_gl_video_mixer_reset;
923 GST_GL_MIXER_CLASS (klass)->process_textures =
924 gst_gl_video_mixer_process_textures;
925
926
927 vagg_class->update_caps = _update_caps;
928
929 agg_class->fixate_src_caps = _fixate_caps;
930 agg_class->propose_allocation = gst_gl_video_mixer_propose_allocation;
931
932 GST_GL_BASE_MIXER_CLASS (klass)->supported_gl_api =
933 GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
934 }
935
936 static void
gst_gl_video_mixer_init(GstGLVideoMixer * video_mixer)937 gst_gl_video_mixer_init (GstGLVideoMixer * video_mixer)
938 {
939 video_mixer->background = DEFAULT_BACKGROUND;
940 video_mixer->shader = NULL;
941 }
942
943 static void
gst_gl_video_mixer_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)944 gst_gl_video_mixer_set_property (GObject * object, guint prop_id,
945 const GValue * value, GParamSpec * pspec)
946 {
947 GstGLVideoMixer *mixer = GST_GL_VIDEO_MIXER (object);
948
949 switch (prop_id) {
950 case PROP_BACKGROUND:
951 mixer->background = g_value_get_enum (value);
952 break;
953 default:
954 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
955 break;
956 }
957 }
958
959 static void
gst_gl_video_mixer_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)960 gst_gl_video_mixer_get_property (GObject * object, guint prop_id,
961 GValue * value, GParamSpec * pspec)
962 {
963 GstGLVideoMixer *mixer = GST_GL_VIDEO_MIXER (object);
964
965 switch (prop_id) {
966 case PROP_BACKGROUND:
967 g_value_set_enum (value, mixer->background);
968 break;
969 default:
970 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
971 break;
972 }
973 }
974
975 static gboolean
gst_gl_video_mixer_propose_allocation(GstAggregator * agg,GstAggregatorPad * agg_pad,GstQuery * decide_query,GstQuery * query)976 gst_gl_video_mixer_propose_allocation (GstAggregator * agg,
977 GstAggregatorPad * agg_pad, GstQuery * decide_query, GstQuery * query)
978 {
979 if (!GST_AGGREGATOR_CLASS (parent_class)->propose_allocation (agg,
980 agg_pad, decide_query, query))
981 return FALSE;
982
983 gst_query_add_allocation_meta (query,
984 GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, 0);
985
986 return TRUE;
987 }
988
989 static void
_mixer_pad_get_output_size(GstGLVideoMixer * mix,GstGLVideoMixerPad * mix_pad,gint out_par_n,gint out_par_d,gint * width,gint * height)990 _mixer_pad_get_output_size (GstGLVideoMixer * mix,
991 GstGLVideoMixerPad * mix_pad, gint out_par_n, gint out_par_d, gint * width,
992 gint * height)
993 {
994 GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (mix_pad);
995 gint pad_width, pad_height;
996 guint dar_n, dar_d;
997
998 /* FIXME: Anything better we can do here? */
999 if (!vagg_pad->info.finfo
1000 || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) {
1001 GST_DEBUG_OBJECT (mix_pad, "Have no caps yet");
1002 *width = 0;
1003 *height = 0;
1004 return;
1005 }
1006
1007 pad_width =
1008 mix_pad->width <=
1009 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : mix_pad->width;
1010 pad_height =
1011 mix_pad->height <=
1012 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : mix_pad->height;
1013
1014 if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height,
1015 GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
1016 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) {
1017 GST_WARNING_OBJECT (mix_pad, "Cannot calculate display aspect ratio");
1018 *width = *height = 0;
1019 return;
1020 }
1021 GST_LOG_OBJECT (mix_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width,
1022 pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
1023 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d);
1024
1025 if (pad_height % dar_n == 0) {
1026 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
1027 } else if (pad_width % dar_d == 0) {
1028 pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n);
1029 } else {
1030 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
1031 }
1032
1033 *width = pad_width;
1034 *height = pad_height;
1035 }
1036
1037 static GstCaps *
_update_caps(GstVideoAggregator * vagg,GstCaps * caps)1038 _update_caps (GstVideoAggregator * vagg, GstCaps * caps)
1039 {
1040 GstCaps *ret;
1041 GList *l;
1042
1043 GST_OBJECT_LOCK (vagg);
1044 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
1045 GstVideoAggregatorPad *vaggpad = l->data;
1046
1047 if (!vaggpad->info.finfo)
1048 continue;
1049
1050 if (GST_VIDEO_INFO_FORMAT (&vaggpad->info) == GST_VIDEO_FORMAT_UNKNOWN)
1051 continue;
1052
1053 if (GST_VIDEO_INFO_MULTIVIEW_MODE (&vaggpad->info) !=
1054 GST_VIDEO_MULTIVIEW_MODE_NONE
1055 && GST_VIDEO_INFO_MULTIVIEW_MODE (&vaggpad->info) !=
1056 GST_VIDEO_MULTIVIEW_MODE_MONO) {
1057 GST_FIXME_OBJECT (vaggpad, "Multiview support is not implemented yet");
1058 GST_OBJECT_UNLOCK (vagg);
1059 return NULL;
1060 }
1061
1062 }
1063
1064 GST_OBJECT_UNLOCK (vagg);
1065
1066 ret = gst_caps_ref (caps);
1067
1068 return ret;
1069 }
1070
1071 static GstCaps *
_fixate_caps(GstAggregator * agg,GstCaps * caps)1072 _fixate_caps (GstAggregator * agg, GstCaps * caps)
1073 {
1074 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1075 GstGLVideoMixer *mix = GST_GL_VIDEO_MIXER (vagg);
1076 gint best_width = 0, best_height = 0;
1077 gint best_fps_n = 0, best_fps_d = 0;
1078 gint par_n, par_d;
1079 gdouble best_fps = 0.;
1080 GstCaps *ret = NULL;
1081 GstStructure *s;
1082 GList *l;
1083
1084 ret = gst_caps_make_writable (caps);
1085
1086 /* we need this to calculate how large to make the output frame */
1087 s = gst_caps_get_structure (ret, 0);
1088 if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
1089 gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL);
1090 }
1091 gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1);
1092 gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d);
1093
1094 GST_OBJECT_LOCK (vagg);
1095 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
1096 GstVideoAggregatorPad *vaggpad = l->data;
1097 GstGLVideoMixerPad *mixer_pad = GST_GL_VIDEO_MIXER_PAD (vaggpad);
1098 gint this_width, this_height;
1099 gint width, height;
1100 gint fps_n, fps_d;
1101 gdouble cur_fps;
1102
1103 fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info);
1104 fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info);
1105 _mixer_pad_get_output_size (mix, mixer_pad, par_n, par_d, &width, &height);
1106
1107 if (width == 0 || height == 0)
1108 continue;
1109
1110 this_width = width + MAX (mixer_pad->xpos, 0);
1111 this_height = height + MAX (mixer_pad->ypos, 0);
1112
1113 if (best_width < this_width)
1114 best_width = this_width;
1115 if (best_height < this_height)
1116 best_height = this_height;
1117
1118 if (fps_d == 0)
1119 cur_fps = 0.0;
1120 else
1121 gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
1122
1123 if (best_fps < cur_fps) {
1124 best_fps = cur_fps;
1125 best_fps_n = fps_n;
1126 best_fps_d = fps_d;
1127 }
1128 }
1129 GST_OBJECT_UNLOCK (vagg);
1130
1131 if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) {
1132 best_fps_n = 25;
1133 best_fps_d = 1;
1134 best_fps = 25.0;
1135 }
1136
1137 s = gst_caps_get_structure (ret, 0);
1138 gst_structure_fixate_field_nearest_int (s, "width", best_width);
1139 gst_structure_fixate_field_nearest_int (s, "height", best_height);
1140 gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
1141 best_fps_d);
1142 ret = gst_caps_fixate (ret);
1143
1144 return ret;
1145 }
1146
1147 static gboolean
_reset_pad_gl(GstElement * agg,GstPad * aggpad,gpointer udata)1148 _reset_pad_gl (GstElement * agg, GstPad * aggpad, gpointer udata)
1149 {
1150 const GstGLFuncs *gl = GST_GL_BASE_MIXER (agg)->context->gl_vtable;
1151 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (aggpad);
1152
1153 if (pad->vertex_buffer) {
1154 gl->DeleteBuffers (1, &pad->vertex_buffer);
1155 pad->vertex_buffer = 0;
1156 }
1157
1158 return TRUE;
1159 }
1160
1161 static void
_reset_gl(GstGLContext * context,GstGLVideoMixer * video_mixer)1162 _reset_gl (GstGLContext * context, GstGLVideoMixer * video_mixer)
1163 {
1164 const GstGLFuncs *gl = GST_GL_BASE_MIXER (video_mixer)->context->gl_vtable;
1165
1166 if (video_mixer->vao) {
1167 gl->DeleteVertexArrays (1, &video_mixer->vao);
1168 video_mixer->vao = 0;
1169 }
1170
1171 if (video_mixer->vbo_indices) {
1172 gl->DeleteBuffers (1, &video_mixer->vbo_indices);
1173 video_mixer->vbo_indices = 0;
1174 }
1175
1176 if (video_mixer->checker_vbo) {
1177 gl->DeleteBuffers (1, &video_mixer->checker_vbo);
1178 video_mixer->checker_vbo = 0;
1179 }
1180
1181 gst_element_foreach_sink_pad (GST_ELEMENT (video_mixer), _reset_pad_gl, NULL);
1182 }
1183
1184 static void
gst_gl_video_mixer_reset(GstGLMixer * mixer)1185 gst_gl_video_mixer_reset (GstGLMixer * mixer)
1186 {
1187 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer);
1188 GstGLContext *context = GST_GL_BASE_MIXER (mixer)->context;
1189
1190 GST_DEBUG_OBJECT (mixer, "context:%p", context);
1191
1192 if (video_mixer->shader)
1193 gst_object_unref (video_mixer->shader);
1194 video_mixer->shader = NULL;
1195
1196 if (video_mixer->checker)
1197 gst_object_unref (video_mixer->checker);
1198 video_mixer->checker = NULL;
1199
1200 if (GST_GL_BASE_MIXER (mixer)->context)
1201 gst_gl_context_thread_add (context, (GstGLContextThreadFunc) _reset_gl,
1202 mixer);
1203 }
1204
1205 static gboolean
gst_gl_video_mixer_init_shader(GstGLMixer * mixer,GstCaps * outcaps)1206 gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps)
1207 {
1208 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer);
1209 gchar *frag_str;
1210 gboolean ret;
1211
1212 if (video_mixer->shader)
1213 gst_object_unref (video_mixer->shader);
1214
1215 /* need reconfigure output geometry */
1216 video_mixer->output_geo_change = TRUE;
1217
1218 frag_str =
1219 g_strdup_printf ("%s%s",
1220 gst_gl_shader_string_get_highest_precision (GST_GL_BASE_MIXER
1221 (mixer)->context, GST_GLSL_VERSION_NONE,
1222 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY),
1223 video_mixer_f_src);
1224
1225 ret = gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context,
1226 gst_gl_shader_string_vertex_mat4_vertex_transform,
1227 frag_str, &video_mixer->shader);
1228 g_free (frag_str);
1229 return ret;
1230 }
1231
1232 static void
_video_mixer_process_gl(GstGLContext * context,GstGLVideoMixer * video_mixer)1233 _video_mixer_process_gl (GstGLContext * context, GstGLVideoMixer * video_mixer)
1234 {
1235 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1236
1237 gst_gl_framebuffer_draw_to_texture (mixer->fbo, video_mixer->out_tex,
1238 gst_gl_video_mixer_callback, video_mixer);
1239 }
1240
1241 static gboolean
gst_gl_video_mixer_process_textures(GstGLMixer * mix,GstGLMemory * out_tex)1242 gst_gl_video_mixer_process_textures (GstGLMixer * mix, GstGLMemory * out_tex)
1243 {
1244 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mix);
1245 GstGLContext *context = GST_GL_BASE_MIXER (mix)->context;
1246
1247 video_mixer->out_tex = out_tex;
1248
1249 gst_gl_context_thread_add (context,
1250 (GstGLContextThreadFunc) _video_mixer_process_gl, video_mixer);
1251
1252 return TRUE;
1253 }
1254
1255 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
1256
1257 static void
_init_vbo_indices(GstGLVideoMixer * mixer)1258 _init_vbo_indices (GstGLVideoMixer * mixer)
1259 {
1260 const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1261
1262 if (!mixer->vbo_indices) {
1263 gl->GenBuffers (1, &mixer->vbo_indices);
1264 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, mixer->vbo_indices);
1265 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
1266 GL_STATIC_DRAW);
1267 }
1268 }
1269
1270 static gboolean
_draw_checker_background(GstGLVideoMixer * video_mixer)1271 _draw_checker_background (GstGLVideoMixer * video_mixer)
1272 {
1273 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1274 const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1275 gint attr_position_loc = 0;
1276
1277 /* *INDENT-OFF* */
1278 gfloat v_vertices[] = {
1279 -1.0,-1.0, 0.0f,
1280 1.0,-1.0, 0.0f,
1281 1.0, 1.0, 0.0f,
1282 -1.0, 1.0, 0.0f,
1283 };
1284 /* *INDENT-ON* */
1285
1286 if (!video_mixer->checker) {
1287 gchar *frag_str;
1288
1289 frag_str =
1290 g_strdup_printf ("%s%s",
1291 gst_gl_shader_string_get_highest_precision (GST_GL_BASE_MIXER
1292 (mixer)->context, GST_GLSL_VERSION_NONE,
1293 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY),
1294 checker_f_src);
1295
1296 if (!gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context,
1297 checker_v_src, frag_str, &video_mixer->checker)) {
1298 g_free (frag_str);
1299 return FALSE;
1300 }
1301 g_free (frag_str);
1302 }
1303
1304 gst_gl_shader_use (video_mixer->checker);
1305 attr_position_loc =
1306 gst_gl_shader_get_attribute_location (video_mixer->checker, "a_position");
1307
1308 _init_vbo_indices (video_mixer);
1309
1310 if (!video_mixer->checker_vbo) {
1311 gl->GenBuffers (1, &video_mixer->checker_vbo);
1312 gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo);
1313 gl->BufferData (GL_ARRAY_BUFFER, 4 * 3 * sizeof (GLfloat), v_vertices,
1314 GL_STATIC_DRAW);
1315 } else {
1316 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, video_mixer->vbo_indices);
1317 gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo);
1318 }
1319
1320 gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT,
1321 GL_FALSE, 3 * sizeof (GLfloat), (void *) 0);
1322
1323 gl->EnableVertexAttribArray (attr_position_loc);
1324
1325 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
1326
1327 gl->DisableVertexAttribArray (attr_position_loc);
1328 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1329 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1330
1331 return TRUE;
1332 }
1333
1334 static gboolean
_draw_background(GstGLVideoMixer * video_mixer)1335 _draw_background (GstGLVideoMixer * video_mixer)
1336 {
1337 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1338 const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1339
1340 switch (video_mixer->background) {
1341 case GST_GL_VIDEO_MIXER_BACKGROUND_BLACK:
1342 gl->ClearColor (0.0, 0.0, 0.0, 1.0);
1343 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
1344 break;
1345 case GST_GL_VIDEO_MIXER_BACKGROUND_WHITE:
1346 gl->ClearColor (1.0, 1.0, 1.0, 1.0);
1347 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
1348 break;
1349 case GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT:
1350 gl->ClearColor (0.0, 0.0, 0.0, 0.0);
1351 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
1352 break;
1353 case GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER:
1354 return _draw_checker_background (video_mixer);
1355 break;
1356 default:
1357 break;
1358 }
1359
1360 return TRUE;
1361 }
1362
1363 static guint
_blend_equation_to_gl(GstGLVideoMixerBlendEquation equation)1364 _blend_equation_to_gl (GstGLVideoMixerBlendEquation equation)
1365 {
1366 switch (equation) {
1367 case GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD:
1368 return GL_FUNC_ADD;
1369 case GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT:
1370 return GL_FUNC_SUBTRACT;
1371 case GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT:
1372 return GL_FUNC_REVERSE_SUBTRACT;
1373 default:
1374 g_assert_not_reached ();
1375 return 0;
1376 }
1377 }
1378
1379 static guint
_blend_function_to_gl(GstGLVideoMixerBlendFunction equation)1380 _blend_function_to_gl (GstGLVideoMixerBlendFunction equation)
1381 {
1382 switch (equation) {
1383 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO:
1384 return GL_ZERO;
1385 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE:
1386 return GL_ONE;
1387 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR:
1388 return GL_SRC_COLOR;
1389 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR:
1390 return GL_ONE_MINUS_SRC_COLOR;
1391 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR:
1392 return GL_DST_COLOR;
1393 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR:
1394 return GL_ONE_MINUS_DST_COLOR;
1395 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA:
1396 return GL_SRC_ALPHA;
1397 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA:
1398 return GL_ONE_MINUS_SRC_ALPHA;
1399 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA:
1400 return GL_DST_ALPHA;
1401 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA:
1402 return GL_ONE_MINUS_DST_ALPHA;
1403 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR:
1404 return GL_CONSTANT_COLOR;
1405 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR:
1406 return GL_ONE_MINUS_CONSTANT_COLOR;
1407 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA:
1408 return GL_CONSTANT_ALPHA;
1409 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_ALPHA:
1410 return GL_ONE_MINUS_CONSTANT_ALPHA;
1411 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE:
1412 return GL_SRC_ALPHA_SATURATE;
1413 default:
1414 g_assert_not_reached ();
1415 return 0;
1416 }
1417 }
1418
1419 static gboolean
_set_blend_state(GstGLVideoMixer * video_mixer,GstGLVideoMixerPad * mix_pad)1420 _set_blend_state (GstGLVideoMixer * video_mixer, GstGLVideoMixerPad * mix_pad)
1421 {
1422 const GstGLFuncs *gl = GST_GL_BASE_MIXER (video_mixer)->context->gl_vtable;
1423 gboolean require_separate = FALSE;
1424 guint gl_func_src_rgb, gl_func_src_alpha, gl_func_dst_rgb, gl_func_dst_alpha;
1425 guint gl_equation_rgb, gl_equation_alpha;
1426
1427 require_separate =
1428 mix_pad->blend_equation_rgb != mix_pad->blend_equation_alpha
1429 || mix_pad->blend_function_src_rgb != mix_pad->blend_function_src_alpha
1430 || mix_pad->blend_function_dst_rgb != mix_pad->blend_function_dst_alpha;
1431
1432 if (require_separate && (!gl->BlendFuncSeparate
1433 || !gl->BlendEquationSeparate)) {
1434 GST_ERROR_OBJECT (mix_pad,
1435 "separated blend equations/functions requested however "
1436 "glBlendFuncSeparate or glBlendEquationSeparate not available");
1437 return FALSE;
1438 }
1439
1440 if (mix_pad->blend_function_dst_rgb ==
1441 GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE) {
1442 GST_ERROR_OBJECT (mix_pad,
1443 "Destination RGB blend function cannot be \'SRC_ALPHA_SATURATE\'");
1444 return FALSE;
1445 }
1446
1447 if (mix_pad->blend_function_dst_alpha ==
1448 GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE) {
1449 GST_ERROR_OBJECT (mix_pad,
1450 "Destination alpha blend function cannot be \'SRC_ALPHA_SATURATE\'");
1451 return FALSE;
1452 }
1453
1454 gl_equation_rgb = _blend_equation_to_gl (mix_pad->blend_equation_rgb);
1455 gl_equation_alpha = _blend_equation_to_gl (mix_pad->blend_equation_alpha);
1456
1457 gl_func_src_rgb = _blend_function_to_gl (mix_pad->blend_function_src_rgb);
1458 gl_func_src_alpha = _blend_function_to_gl (mix_pad->blend_function_src_alpha);
1459 gl_func_dst_rgb = _blend_function_to_gl (mix_pad->blend_function_dst_rgb);
1460 gl_func_dst_alpha = _blend_function_to_gl (mix_pad->blend_function_dst_alpha);
1461
1462 if (gl->BlendEquationSeparate)
1463 gl->BlendEquationSeparate (gl_equation_rgb, gl_equation_alpha);
1464 else
1465 gl->BlendEquation (gl_equation_rgb);
1466
1467 if (gl->BlendFuncSeparate)
1468 gl->BlendFuncSeparate (gl_func_src_rgb, gl_func_dst_rgb, gl_func_src_alpha,
1469 gl_func_dst_alpha);
1470 else
1471 gl->BlendFunc (gl_func_src_rgb, gl_func_dst_rgb);
1472
1473 gl->BlendColor (mix_pad->blend_constant_color_red,
1474 mix_pad->blend_constant_color_green, mix_pad->blend_constant_color_blue,
1475 mix_pad->blend_constant_color_alpha);
1476
1477 return TRUE;
1478 }
1479
1480 /* opengl scene, params: input texture (not the output mixer->texture) */
1481 static gboolean
gst_gl_video_mixer_callback(gpointer stuff)1482 gst_gl_video_mixer_callback (gpointer stuff)
1483 {
1484 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (stuff);
1485 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (stuff);
1486 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1487 GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1488 GLint attr_position_loc = 0;
1489 GLint attr_texture_loc = 0;
1490 guint out_width, out_height;
1491 GList *walk;
1492
1493 out_width = GST_VIDEO_INFO_WIDTH (&vagg->info);
1494 out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info);
1495
1496 gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context);
1497 gl->BindTexture (GL_TEXTURE_2D, 0);
1498
1499 gl->Disable (GL_DEPTH_TEST);
1500 gl->Disable (GL_CULL_FACE);
1501
1502 if (gl->GenVertexArrays) {
1503 if (!video_mixer->vao)
1504 gl->GenVertexArrays (1, &video_mixer->vao);
1505 gl->BindVertexArray (video_mixer->vao);
1506 }
1507
1508 if (!_draw_background (video_mixer))
1509 return FALSE;
1510
1511 gst_gl_shader_use (video_mixer->shader);
1512
1513 attr_position_loc =
1514 gst_gl_shader_get_attribute_location (video_mixer->shader, "a_position");
1515 attr_texture_loc =
1516 gst_gl_shader_get_attribute_location (video_mixer->shader, "a_texcoord");
1517
1518 gl->Enable (GL_BLEND);
1519
1520 GST_OBJECT_LOCK (video_mixer);
1521 walk = GST_ELEMENT (video_mixer)->sinkpads;
1522 while (walk) {
1523 GstGLMixerPad *mix_pad = walk->data;
1524 GstGLVideoMixerPad *pad = walk->data;
1525 GstVideoAggregatorPad *vagg_pad = walk->data;
1526 GstVideoInfo *v_info;
1527 guint in_tex;
1528 guint in_width, in_height;
1529
1530 /* *INDENT-OFF* */
1531 gfloat v_vertices[] = {
1532 -1.0,-1.0, 0.0f, 0.0f, 0.0f,
1533 1.0,-1.0, 0.0f, 1.0f, 0.0f,
1534 1.0, 1.0, 0.0f, 1.0f, 1.0f,
1535 -1.0, 1.0, 0.0f, 0.0f, 1.0f,
1536 };
1537 /* *INDENT-ON* */
1538
1539 v_info = &GST_VIDEO_AGGREGATOR_PAD (pad)->info;
1540 in_width = GST_VIDEO_INFO_WIDTH (v_info);
1541 in_height = GST_VIDEO_INFO_HEIGHT (v_info);
1542
1543 if (!mix_pad->current_texture || in_width <= 0 || in_height <= 0
1544 || pad->alpha == 0.0f) {
1545 GST_DEBUG ("skipping texture:%u pad:%p width:%u height:%u alpha:%f",
1546 mix_pad->current_texture, pad, in_width, in_height, pad->alpha);
1547 walk = g_list_next (walk);
1548 continue;
1549 }
1550
1551 if (!_set_blend_state (video_mixer, pad)) {
1552 GST_FIXME_OBJECT (pad, "skipping due to incorrect blend parameters");
1553 walk = g_list_next (walk);
1554 continue;
1555 }
1556
1557 in_tex = mix_pad->current_texture;
1558
1559 _init_vbo_indices (video_mixer);
1560
1561 if (video_mixer->output_geo_change
1562 || pad->geometry_change || !pad->vertex_buffer) {
1563 gint pad_width, pad_height;
1564 gfloat w, h;
1565
1566 _mixer_pad_get_output_size (video_mixer, pad,
1567 GST_VIDEO_INFO_PAR_N (&vagg->info),
1568 GST_VIDEO_INFO_PAR_D (&vagg->info), &pad_width, &pad_height);
1569
1570 w = ((gfloat) pad_width / (gfloat) out_width);
1571 h = ((gfloat) pad_height / (gfloat) out_height);
1572
1573 pad->m_matrix[0] = w;
1574 pad->m_matrix[5] = h;
1575 pad->m_matrix[12] =
1576 2. * (gfloat) pad->xpos / (gfloat) out_width - (1. - w);
1577 pad->m_matrix[13] =
1578 2. * (gfloat) pad->ypos / (gfloat) out_height - (1. - h);
1579
1580 GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with "
1581 "alpha:%f", in_tex, in_width, in_height, pad->m_matrix[12],
1582 pad->m_matrix[13], pad->m_matrix[0], pad->m_matrix[5], pad->alpha);
1583
1584 if (!pad->vertex_buffer)
1585 gl->GenBuffers (1, &pad->vertex_buffer);
1586
1587 gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer);
1588 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), v_vertices,
1589 GL_STATIC_DRAW);
1590
1591 pad->geometry_change = FALSE;
1592 } else {
1593 gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer);
1594 }
1595 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, video_mixer->vbo_indices);
1596
1597 gl->ActiveTexture (GL_TEXTURE0);
1598 gl->BindTexture (GL_TEXTURE_2D, in_tex);
1599 gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0);
1600 gst_gl_shader_set_uniform_1f (video_mixer->shader, "alpha", pad->alpha);
1601
1602 {
1603 GstVideoAffineTransformationMeta *af_meta;
1604 gfloat matrix[16];
1605 gfloat af_matrix[16];
1606 GstBuffer *buffer =
1607 gst_video_aggregator_pad_get_current_buffer (vagg_pad);
1608
1609 af_meta = gst_buffer_get_video_affine_transformation_meta (buffer);
1610 gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, af_matrix);
1611 gst_gl_multiply_matrix4 (af_matrix, pad->m_matrix, matrix);
1612 gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader,
1613 "u_transformation", 1, FALSE, matrix);
1614 }
1615
1616 gl->EnableVertexAttribArray (attr_position_loc);
1617 gl->EnableVertexAttribArray (attr_texture_loc);
1618
1619 gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT,
1620 GL_FALSE, 5 * sizeof (GLfloat), (void *) 0);
1621
1622 gl->VertexAttribPointer (attr_texture_loc, 2, GL_FLOAT,
1623 GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
1624
1625 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
1626
1627 walk = g_list_next (walk);
1628 }
1629
1630 video_mixer->output_geo_change = FALSE;
1631 GST_OBJECT_UNLOCK (video_mixer);
1632
1633 if (gl->GenVertexArrays) {
1634 gl->BindVertexArray (0);
1635 } else {
1636 gl->DisableVertexAttribArray (attr_position_loc);
1637 gl->DisableVertexAttribArray (attr_texture_loc);
1638
1639 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1640 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1641 gl->BindTexture (GL_TEXTURE_2D, 0);
1642 }
1643
1644 gl->Disable (GL_BLEND);
1645
1646 gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context);
1647
1648 return TRUE;
1649 }
1650
1651 /* GstChildProxy implementation */
1652 static GObject *
gst_gl_video_mixer_child_proxy_get_child_by_index(GstChildProxy * child_proxy,guint index)1653 gst_gl_video_mixer_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
1654 guint index)
1655 {
1656 GstGLVideoMixer *gl_video_mixer = GST_GL_VIDEO_MIXER (child_proxy);
1657 GObject *obj = NULL;
1658
1659 GST_OBJECT_LOCK (gl_video_mixer);
1660 obj = g_list_nth_data (GST_ELEMENT_CAST (gl_video_mixer)->sinkpads, index);
1661 if (obj)
1662 gst_object_ref (obj);
1663 GST_OBJECT_UNLOCK (gl_video_mixer);
1664
1665 return obj;
1666 }
1667
1668 static guint
gst_gl_video_mixer_child_proxy_get_children_count(GstChildProxy * child_proxy)1669 gst_gl_video_mixer_child_proxy_get_children_count (GstChildProxy * child_proxy)
1670 {
1671 guint count = 0;
1672 GstGLVideoMixer *gl_video_mixer = GST_GL_VIDEO_MIXER (child_proxy);
1673
1674 GST_OBJECT_LOCK (gl_video_mixer);
1675 count = GST_ELEMENT_CAST (gl_video_mixer)->numsinkpads;
1676 GST_OBJECT_UNLOCK (gl_video_mixer);
1677 GST_INFO_OBJECT (gl_video_mixer, "Children Count: %d", count);
1678
1679 return count;
1680 }
1681
1682 static void
gst_gl_video_mixer_child_proxy_init(gpointer g_iface,gpointer iface_data)1683 gst_gl_video_mixer_child_proxy_init (gpointer g_iface, gpointer iface_data)
1684 {
1685 GstChildProxyInterface *iface = g_iface;
1686
1687 iface->get_child_by_index = gst_gl_video_mixer_child_proxy_get_child_by_index;
1688 iface->get_children_count = gst_gl_video_mixer_child_proxy_get_children_count;
1689 }
1690