1 /*
2 * glshader gstreamer plugin
3 * Copyrithg (C) 2008 Filippo Argiolas <filippo.argiolas@gmail.com>
4 * Copyright (C) 2009 Luc Deschenaux <luc.deschenaux@freesurf.ch>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
15 *
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
20 */
21
22 /**
23 * SECTION:element-glshader
24 * @title: glshader
25 *
26 * OpenGL fragment shader filter
27 *
28 * ## Examples
29 * |[
30 * gst-launch-1.0 videotestsrc ! glupload ! glshader fragment="\"`cat myshader.frag`\"" ! glimagesink
31 * ]|
32 * FBO (Frame Buffer Object) and GLSL (OpenGL Shading Language) are required.
33 * Depending on the exact OpenGL version chosen and the exact requirements of
34 * the OpenGL implementation, a #version header may be required.
35 *
36 * The following is a simple OpenGL ES (also usable with OpenGL 3 core contexts)
37 * passthrough shader with the required inputs.
38 * |[
39 * #version 100
40 * #ifdef GL_ES
41 * precision mediump float;
42 * #endif
43 * varying vec2 v_texcoord;
44 * uniform sampler2D tex;
45 * uniform float time;
46 * uniform float width;
47 * uniform float height;
48 *
49 * void main () {
50 * gl_FragColor = texture2D( tex, v_texcoord );
51 * }
52 * ]|
53 *
54 */
55 #ifdef HAVE_CONFIG_H
56 #include "config.h"
57 #endif
58
59 #include <gst/gl/gstglfuncs.h>
60
61 #include "gstglfiltershader.h"
62 #ifdef HAVE_GRAPHENE
63 #include <graphene-gobject.h>
64 #endif
65
66 enum
67 {
68 PROP_0,
69 PROP_SHADER,
70 PROP_VERTEX,
71 PROP_FRAGMENT,
72 PROP_UNIFORMS,
73 PROP_UPDATE_SHADER,
74 PROP_LAST,
75 };
76
77 enum
78 {
79 SIGNAL_0,
80 SIGNAL_CREATE_SHADER,
81 SIGNAL_LAST,
82 };
83
84 static guint gst_gl_shader_signals[SIGNAL_LAST] = { 0 };
85
86 #define GST_CAT_DEFAULT gst_gl_filtershader_debug
87 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
88
89 #define DEBUG_INIT \
90 GST_DEBUG_CATEGORY_INIT (gst_gl_filtershader_debug, "glshader", 0, "glshader element");
91 #define gst_gl_filtershader_parent_class parent_class
92 G_DEFINE_TYPE_WITH_CODE (GstGLFilterShader, gst_gl_filtershader,
93 GST_TYPE_GL_FILTER, DEBUG_INIT);
94
95 static void gst_gl_filtershader_finalize (GObject * object);
96 static void gst_gl_filtershader_set_property (GObject * object, guint prop_id,
97 const GValue * value, GParamSpec * pspec);
98 static void gst_gl_filtershader_get_property (GObject * object, guint prop_id,
99 GValue * value, GParamSpec * pspec);
100 static gboolean gst_gl_filtershader_gl_start (GstGLBaseFilter * base);
101 static void gst_gl_filtershader_gl_stop (GstGLBaseFilter * base);
102 static gboolean gst_gl_filtershader_filter (GstGLFilter * filter,
103 GstBuffer * inbuf, GstBuffer * outbuf);
104 static gboolean gst_gl_filtershader_filter_texture (GstGLFilter * filter,
105 GstGLMemory * in_tex, GstGLMemory * out_tex);
106 static gboolean gst_gl_filtershader_hcallback (GstGLFilter * filter,
107 GstGLMemory * in_tex, gpointer stuff);
108
109 static void
gst_gl_filtershader_class_init(GstGLFilterShaderClass * klass)110 gst_gl_filtershader_class_init (GstGLFilterShaderClass * klass)
111 {
112 GObjectClass *gobject_class;
113 GstElementClass *element_class;
114
115 gobject_class = (GObjectClass *) klass;
116 element_class = GST_ELEMENT_CLASS (klass);
117
118 gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
119
120 gobject_class->finalize = gst_gl_filtershader_finalize;
121 gobject_class->set_property = gst_gl_filtershader_set_property;
122 gobject_class->get_property = gst_gl_filtershader_get_property;
123
124 g_object_class_install_property (gobject_class, PROP_SHADER,
125 g_param_spec_object ("shader", "Shader object",
126 "GstGLShader to use", GST_TYPE_GL_SHADER,
127 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
128
129 g_object_class_install_property (gobject_class, PROP_VERTEX,
130 g_param_spec_string ("vertex", "Vertex Source",
131 "GLSL vertex source", NULL,
132 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
133
134 g_object_class_install_property (gobject_class, PROP_FRAGMENT,
135 g_param_spec_string ("fragment", "Fragment Source",
136 "GLSL fragment source", NULL,
137 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
138 /* FIXME: add other stages */
139
140 g_object_class_install_property (gobject_class, PROP_UNIFORMS,
141 g_param_spec_boxed ("uniforms", "GLSL Uniforms",
142 "GLSL Uniforms", GST_TYPE_STRUCTURE,
143 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
144
145 g_object_class_install_property (gobject_class, PROP_UPDATE_SHADER,
146 g_param_spec_boolean ("update-shader", "Update Shader",
147 "Emit the \'create-shader\' signal for the next frame",
148 FALSE, G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS));
149
150 /*
151 * GstGLFilterShader::create-shader:
152 * @object: the #GstGLFilterShader
153 *
154 * Ask's the application for a shader to render with as a result of
155 * inititialization or setting the 'update-shader' property.
156 *
157 * Returns: a new #GstGLShader for use in the rendering pipeline
158 */
159 gst_gl_shader_signals[SIGNAL_CREATE_SHADER] =
160 g_signal_new ("create-shader", G_TYPE_FROM_CLASS (klass),
161 G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
162 GST_TYPE_GL_SHADER, 0);
163
164 gst_element_class_set_metadata (element_class,
165 "OpenGL fragment shader filter", "Filter/Effect",
166 "Perform operations with a GLSL shader", "<matthew@centricular.com>");
167
168 GST_GL_FILTER_CLASS (klass)->filter = gst_gl_filtershader_filter;
169 GST_GL_FILTER_CLASS (klass)->filter_texture =
170 gst_gl_filtershader_filter_texture;
171
172 GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_filtershader_gl_start;
173 GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_filtershader_gl_stop;
174 GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
175 GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3;
176 }
177
178 static void
gst_gl_filtershader_init(GstGLFilterShader * filtershader)179 gst_gl_filtershader_init (GstGLFilterShader * filtershader)
180 {
181 filtershader->new_source = TRUE;
182 }
183
184 static void
gst_gl_filtershader_finalize(GObject * object)185 gst_gl_filtershader_finalize (GObject * object)
186 {
187 GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);
188
189 g_free (filtershader->vertex);
190 filtershader->vertex = NULL;
191
192 g_free (filtershader->fragment);
193 filtershader->fragment = NULL;
194
195 if (filtershader->uniforms)
196 gst_structure_free (filtershader->uniforms);
197 filtershader->uniforms = NULL;
198
199 G_OBJECT_CLASS (gst_gl_filtershader_parent_class)->finalize (object);
200 }
201
202 static void
gst_gl_filtershader_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)203 gst_gl_filtershader_set_property (GObject * object, guint prop_id,
204 const GValue * value, GParamSpec * pspec)
205 {
206 GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);
207
208 switch (prop_id) {
209 case PROP_SHADER:
210 GST_OBJECT_LOCK (filtershader);
211 gst_object_replace ((GstObject **) & filtershader->shader,
212 g_value_dup_object (value));
213 filtershader->new_source = FALSE;
214 GST_OBJECT_UNLOCK (filtershader);
215 break;
216 case PROP_VERTEX:
217 GST_OBJECT_LOCK (filtershader);
218 g_free (filtershader->vertex);
219 filtershader->vertex = g_value_dup_string (value);
220 filtershader->new_source = TRUE;
221 GST_OBJECT_UNLOCK (filtershader);
222 break;
223 case PROP_FRAGMENT:
224 GST_OBJECT_LOCK (filtershader);
225 g_free (filtershader->fragment);
226 filtershader->fragment = g_value_dup_string (value);
227 filtershader->new_source = TRUE;
228 GST_OBJECT_UNLOCK (filtershader);
229 break;
230 case PROP_UNIFORMS:
231 GST_OBJECT_LOCK (filtershader);
232 if (filtershader->uniforms)
233 gst_structure_free (filtershader->uniforms);
234 filtershader->uniforms = g_value_dup_boxed (value);
235 filtershader->new_uniforms = TRUE;
236 GST_OBJECT_UNLOCK (filtershader);
237 break;
238 case PROP_UPDATE_SHADER:
239 GST_OBJECT_LOCK (filtershader);
240 filtershader->update_shader = g_value_get_boolean (value);
241 GST_OBJECT_UNLOCK (filtershader);
242 break;
243 default:
244 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
245 break;
246 }
247 }
248
249 static void
gst_gl_filtershader_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)250 gst_gl_filtershader_get_property (GObject * object, guint prop_id,
251 GValue * value, GParamSpec * pspec)
252 {
253 GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);
254
255 switch (prop_id) {
256 case PROP_SHADER:
257 GST_OBJECT_LOCK (filtershader);
258 g_value_set_object (value, filtershader->shader);
259 GST_OBJECT_UNLOCK (filtershader);
260 break;
261 case PROP_VERTEX:
262 GST_OBJECT_LOCK (filtershader);
263 g_value_set_string (value, filtershader->vertex);
264 GST_OBJECT_UNLOCK (filtershader);
265 break;
266 case PROP_FRAGMENT:
267 GST_OBJECT_LOCK (filtershader);
268 g_value_set_string (value, filtershader->fragment);
269 GST_OBJECT_UNLOCK (filtershader);
270 break;
271 case PROP_UNIFORMS:
272 GST_OBJECT_LOCK (filtershader);
273 g_value_set_boxed (value, filtershader->uniforms);
274 GST_OBJECT_UNLOCK (filtershader);
275 break;
276 default:
277 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
278 break;
279 }
280 }
281
282 static void
gst_gl_filtershader_gl_stop(GstGLBaseFilter * base)283 gst_gl_filtershader_gl_stop (GstGLBaseFilter * base)
284 {
285 GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (base);
286
287 if (filtershader->shader)
288 gst_object_unref (filtershader->shader);
289 filtershader->shader = NULL;
290
291 GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base);
292 }
293
294 static gboolean
gst_gl_filtershader_gl_start(GstGLBaseFilter * base)295 gst_gl_filtershader_gl_start (GstGLBaseFilter * base)
296 {
297 return GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base);
298 }
299
300 static inline gboolean
_gst_clock_time_to_double(GstClockTime time,gdouble * result)301 _gst_clock_time_to_double (GstClockTime time, gdouble * result)
302 {
303 if (!GST_CLOCK_TIME_IS_VALID (time))
304 return FALSE;
305
306 *result = (gdouble) time / GST_SECOND;
307
308 return TRUE;
309 }
310
311 static inline gboolean
_gint64_time_val_to_double(gint64 time,gdouble * result)312 _gint64_time_val_to_double (gint64 time, gdouble * result)
313 {
314 if (time == -1)
315 return FALSE;
316
317 *result = (gdouble) time / GST_USECOND;
318
319 return TRUE;
320 }
321
322 static gboolean
gst_gl_filtershader_filter(GstGLFilter * filter,GstBuffer * inbuf,GstBuffer * outbuf)323 gst_gl_filtershader_filter (GstGLFilter * filter, GstBuffer * inbuf,
324 GstBuffer * outbuf)
325 {
326 GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);
327
328 if (!_gst_clock_time_to_double (GST_BUFFER_PTS (inbuf), &filtershader->time)) {
329 if (!_gst_clock_time_to_double (GST_BUFFER_DTS (inbuf),
330 &filtershader->time))
331 _gint64_time_val_to_double (g_get_monotonic_time (), &filtershader->time);
332 }
333
334 return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
335 }
336
337 static gboolean
gst_gl_filtershader_filter_texture(GstGLFilter * filter,GstGLMemory * in_tex,GstGLMemory * out_tex)338 gst_gl_filtershader_filter_texture (GstGLFilter * filter, GstGLMemory * in_tex,
339 GstGLMemory * out_tex)
340 {
341 GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);
342
343 gst_gl_filter_render_to_target (filter, in_tex, out_tex,
344 gst_gl_filtershader_hcallback, NULL);
345
346 if (!filtershader->shader)
347 return FALSE;
348
349 return TRUE;
350 }
351
352 static gboolean
_set_uniform(GQuark field_id,const GValue * value,gpointer user_data)353 _set_uniform (GQuark field_id, const GValue * value, gpointer user_data)
354 {
355 GstGLShader *shader = user_data;
356 const gchar *field_name = g_quark_to_string (field_id);
357
358 if (G_TYPE_CHECK_VALUE_TYPE ((value), G_TYPE_INT)) {
359 gst_gl_shader_set_uniform_1i (shader, field_name, g_value_get_int (value));
360 } else if (G_TYPE_CHECK_VALUE_TYPE ((value), G_TYPE_FLOAT)) {
361 gst_gl_shader_set_uniform_1f (shader, field_name,
362 g_value_get_float (value));
363 #ifdef HAVE_GRAPHENE
364 } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC2)) {
365 graphene_vec2_t *vec2 = g_value_get_boxed (value);
366 float x = graphene_vec2_get_x (vec2);
367 float y = graphene_vec2_get_y (vec2);
368 gst_gl_shader_set_uniform_2f (shader, field_name, x, y);
369 } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC3)) {
370 graphene_vec3_t *vec3 = g_value_get_boxed (value);
371 float x = graphene_vec3_get_x (vec3);
372 float y = graphene_vec3_get_y (vec3);
373 float z = graphene_vec3_get_z (vec3);
374 gst_gl_shader_set_uniform_3f (shader, field_name, x, y, z);
375 } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC4)) {
376 graphene_vec4_t *vec4 = g_value_get_boxed (value);
377 float x = graphene_vec4_get_x (vec4);
378 float y = graphene_vec4_get_y (vec4);
379 float z = graphene_vec4_get_z (vec4);
380 float w = graphene_vec4_get_w (vec4);
381 gst_gl_shader_set_uniform_4f (shader, field_name, x, y, z, w);
382 } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_MATRIX)) {
383 graphene_matrix_t *matrix = g_value_get_boxed (value);
384 float matrix_f[16];
385 graphene_matrix_to_float (matrix, matrix_f);
386 gst_gl_shader_set_uniform_matrix_4fv (shader, field_name, 1, FALSE,
387 matrix_f);
388 #endif
389 } else {
390 /* FIXME: Add support for unsigned ints, non 4x4 matrices, etc */
391 GST_FIXME ("Don't know how to set the \'%s\' paramater. Unknown type",
392 field_name);
393 return TRUE;
394 }
395
396 return TRUE;
397 }
398
399 static void
_update_uniforms(GstGLFilterShader * filtershader)400 _update_uniforms (GstGLFilterShader * filtershader)
401 {
402 if (filtershader->new_uniforms && filtershader->uniforms) {
403 gst_gl_shader_use (filtershader->shader);
404
405 gst_structure_foreach (filtershader->uniforms,
406 (GstStructureForeachFunc) _set_uniform, filtershader->shader);
407 filtershader->new_uniforms = FALSE;
408 }
409 }
410
411 static GstGLShader *
_maybe_recompile_shader(GstGLFilterShader * filtershader)412 _maybe_recompile_shader (GstGLFilterShader * filtershader)
413 {
414 GstGLContext *context = GST_GL_BASE_FILTER (filtershader)->context;
415 GstGLShader *shader;
416 GError *error = NULL;
417
418 GST_OBJECT_LOCK (filtershader);
419
420 if (!filtershader->shader || filtershader->update_shader) {
421 filtershader->update_shader = FALSE;
422 GST_OBJECT_UNLOCK (filtershader);
423 g_signal_emit (filtershader, gst_gl_shader_signals[SIGNAL_CREATE_SHADER], 0,
424 &shader);
425 GST_OBJECT_LOCK (filtershader);
426
427 if (shader) {
428 if (filtershader->shader)
429 gst_object_unref (filtershader->shader);
430 filtershader->new_source = FALSE;
431 filtershader->shader = gst_object_ref (shader);
432 filtershader->new_uniforms = TRUE;
433 _update_uniforms (filtershader);
434 GST_OBJECT_UNLOCK (filtershader);
435 return shader;
436 }
437 }
438
439 if (filtershader->shader) {
440 shader = gst_object_ref (filtershader->shader);
441 _update_uniforms (filtershader);
442 GST_OBJECT_UNLOCK (filtershader);
443 return shader;
444 }
445
446 if (filtershader->new_source) {
447 GstGLSLStage *stage;
448
449 shader = gst_gl_shader_new (context);
450
451 if (filtershader->vertex) {
452 if (!(stage = gst_glsl_stage_new_with_string (context, GL_VERTEX_SHADER,
453 GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_NONE,
454 filtershader->vertex))) {
455 g_set_error (&error, GST_GLSL_ERROR, GST_GLSL_ERROR_COMPILE,
456 "Failed to create shader vertex stage");
457 goto print_error;
458 }
459 } else {
460 stage = gst_glsl_stage_new_default_vertex (context);
461 }
462
463 if (!gst_gl_shader_compile_attach_stage (shader, stage, &error)) {
464 gst_object_unref (stage);
465 goto print_error;
466 }
467
468 if (filtershader->fragment) {
469 if (!(stage = gst_glsl_stage_new_with_string (context, GL_FRAGMENT_SHADER,
470 GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_NONE,
471 filtershader->fragment))) {
472 g_set_error (&error, GST_GLSL_ERROR, GST_GLSL_ERROR_COMPILE,
473 "Failed to create shader fragment stage");
474 goto print_error;
475 }
476 } else {
477 stage = gst_glsl_stage_new_default_fragment (context);
478 }
479
480 if (!gst_gl_shader_compile_attach_stage (shader, stage, &error)) {
481 gst_object_unref (stage);
482 goto print_error;
483 }
484
485 if (!gst_gl_shader_link (shader, &error)) {
486 goto print_error;
487 }
488 if (filtershader->shader)
489 gst_object_unref (filtershader->shader);
490 filtershader->shader = gst_object_ref (shader);
491 filtershader->new_source = FALSE;
492 filtershader->new_uniforms = TRUE;
493 _update_uniforms (filtershader);
494
495 GST_OBJECT_UNLOCK (filtershader);
496 return shader;
497 } else if (filtershader->shader) {
498 _update_uniforms (filtershader);
499 shader = gst_object_ref (filtershader->shader);
500 GST_OBJECT_UNLOCK (filtershader);
501 return shader;
502 }
503
504 return NULL;
505
506 print_error:
507 if (shader) {
508 gst_object_unref (shader);
509 shader = NULL;
510 }
511
512 GST_OBJECT_UNLOCK (filtershader);
513 GST_ELEMENT_ERROR (filtershader, RESOURCE, NOT_FOUND,
514 ("%s", error->message), (NULL));
515 return NULL;
516 }
517
518 static gboolean
gst_gl_filtershader_hcallback(GstGLFilter * filter,GstGLMemory * in_tex,gpointer stuff)519 gst_gl_filtershader_hcallback (GstGLFilter * filter, GstGLMemory * in_tex,
520 gpointer stuff)
521 {
522 GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);
523 GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
524 GstGLShader *shader;
525
526 if (!(shader = _maybe_recompile_shader (filtershader)))
527 return FALSE;
528
529 gl->ClearColor (0.0, 0.0, 0.0, 1.0);
530 gl->Clear (GL_COLOR_BUFFER_BIT);
531
532 gst_gl_shader_use (shader);
533
534 /* FIXME: propertise these */
535 gst_gl_shader_set_uniform_1i (shader, "tex", 0);
536 gst_gl_shader_set_uniform_1f (shader, "width",
537 GST_VIDEO_INFO_WIDTH (&filter->out_info));
538 gst_gl_shader_set_uniform_1f (shader, "height",
539 GST_VIDEO_INFO_HEIGHT (&filter->out_info));
540 gst_gl_shader_set_uniform_1f (shader, "time", filtershader->time);
541
542 /* FIXME: propertise these */
543 filter->draw_attr_position_loc =
544 gst_gl_shader_get_attribute_location (shader, "a_position");
545 filter->draw_attr_texture_loc =
546 gst_gl_shader_get_attribute_location (shader, "a_texcoord");
547
548 gl->ActiveTexture (GL_TEXTURE0);
549 gl->BindTexture (GL_TEXTURE_2D, gst_gl_memory_get_texture_id (in_tex));
550
551 gst_gl_filter_draw_fullscreen_quad (filter);
552
553 gst_object_unref (shader);
554
555 return TRUE;
556 }
557