1 /*
2 * GStreamer
3 * Copyright (C) 2014 Lubosz Sarnecki <lubosz@gmail.com>
4 * Copyright (C) 2016 Matthew Waters <matthew@centricular.com>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
15 *
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
20 */
21
22 /**
23 * SECTION:element-gltransformation
24 * @title: gltransformation
25 *
26 * Transforms video on the GPU.
27 *
28 * ## Examples
29 * |[
30 * gst-launch-1.0 gltestsrc ! gltransformation rotation-z=45 ! glimagesink
31 * ]| A pipeline to rotate by 45 degrees
32 * |[
33 * gst-launch-1.0 gltestsrc ! gltransformation translation-x=0.5 ! glimagesink
34 * ]| Translate the video by 0.5
35 * |[
36 * gst-launch-1.0 gltestsrc ! gltransformation scale-y=0.5 scale-x=0.5 ! glimagesink
37 * ]| Resize the video by 0.5
38 * |[
39 * gst-launch-1.0 gltestsrc ! gltransformation rotation-x=-45 ortho=True ! glimagesink
40 * ]| Rotate the video around the X-Axis by -45° with an orthographic projection
41 *
42 */
43
44 #ifdef HAVE_CONFIG_H
45 #include "config.h"
46 #endif
47
48 #include "gstgltransformation.h"
49
50 #include <gst/gl/gstglapi.h>
51 #include <graphene-gobject.h>
52 #include "gstglutils.h"
53
54 #define GST_CAT_DEFAULT gst_gl_transformation_debug
55 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
56
57 #define gst_gl_transformation_parent_class parent_class
58
59 #define VEC4_FORMAT "%f,%f,%f,%f"
60 #define VEC4_ARGS(v) graphene_vec4_get_x (v), graphene_vec4_get_y (v), graphene_vec4_get_z (v), graphene_vec4_get_w (v)
61 #define VEC3_FORMAT "%f,%f,%f"
62 #define VEC3_ARGS(v) graphene_vec3_get_x (v), graphene_vec3_get_y (v), graphene_vec3_get_z (v)
63 #define VEC2_FORMAT "%f,%f"
64 #define VEC2_ARGS(v) graphene_vec2_get_x (v), graphene_vec2_get_y (v)
65 #define POINT3D_FORMAT "%f,%f,%f"
66 #define POINT3D_ARGS(p) (p)->x, (p)->y, (p)->z
67
68 enum
69 {
70 PROP_0,
71 PROP_FOV,
72 PROP_ORTHO,
73 PROP_TRANSLATION_X,
74 PROP_TRANSLATION_Y,
75 PROP_TRANSLATION_Z,
76 PROP_ROTATION_X,
77 PROP_ROTATION_Y,
78 PROP_ROTATION_Z,
79 PROP_SCALE_X,
80 PROP_SCALE_Y,
81 PROP_MVP,
82 PROP_PIVOT_X,
83 PROP_PIVOT_Y,
84 PROP_PIVOT_Z,
85 };
86
87 #define DEBUG_INIT \
88 GST_DEBUG_CATEGORY_INIT (gst_gl_transformation_debug, "gltransformation", 0, "gltransformation element");
89
90 G_DEFINE_TYPE_WITH_CODE (GstGLTransformation, gst_gl_transformation,
91 GST_TYPE_GL_FILTER, DEBUG_INIT);
92
93 static void gst_gl_transformation_set_property (GObject * object, guint prop_id,
94 const GValue * value, GParamSpec * pspec);
95 static void gst_gl_transformation_get_property (GObject * object, guint prop_id,
96 GValue * value, GParamSpec * pspec);
97
98 static gboolean gst_gl_transformation_set_caps (GstGLFilter * filter,
99 GstCaps * incaps, GstCaps * outcaps);
100 static gboolean gst_gl_transformation_src_event (GstBaseTransform * trans,
101 GstEvent * event);
102 static gboolean gst_gl_transformation_filter_meta (GstBaseTransform * trans,
103 GstQuery * query, GType api, const GstStructure * params);
104 static gboolean gst_gl_transformation_decide_allocation (GstBaseTransform *
105 trans, GstQuery * query);
106
107 static void gst_gl_transformation_gl_stop (GstGLBaseFilter * filter);
108 static gboolean gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter);
109 static gboolean gst_gl_transformation_callback (gpointer stuff);
110 static void gst_gl_transformation_build_mvp (GstGLTransformation *
111 transformation);
112
113 static GstFlowReturn
114 gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
115 GstBuffer * inbuf, GstBuffer ** outbuf);
116 static gboolean gst_gl_transformation_filter (GstGLFilter * filter,
117 GstBuffer * inbuf, GstBuffer * outbuf);
118 static gboolean gst_gl_transformation_filter_texture (GstGLFilter * filter,
119 GstGLMemory * in_tex, GstGLMemory * out_tex);
120
121 static void
gst_gl_transformation_class_init(GstGLTransformationClass * klass)122 gst_gl_transformation_class_init (GstGLTransformationClass * klass)
123 {
124 GObjectClass *gobject_class;
125 GstElementClass *element_class;
126 GstBaseTransformClass *base_transform_class;
127
128 gobject_class = (GObjectClass *) klass;
129 element_class = GST_ELEMENT_CLASS (klass);
130 base_transform_class = GST_BASE_TRANSFORM_CLASS (klass);
131
132 gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
133
134 gobject_class->set_property = gst_gl_transformation_set_property;
135 gobject_class->get_property = gst_gl_transformation_get_property;
136
137 base_transform_class->src_event = gst_gl_transformation_src_event;
138 base_transform_class->decide_allocation =
139 gst_gl_transformation_decide_allocation;
140 base_transform_class->filter_meta = gst_gl_transformation_filter_meta;
141
142 GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_transformation_gl_start;
143 GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_transformation_gl_stop;
144
145 GST_GL_FILTER_CLASS (klass)->set_caps = gst_gl_transformation_set_caps;
146 GST_GL_FILTER_CLASS (klass)->filter = gst_gl_transformation_filter;
147 GST_GL_FILTER_CLASS (klass)->filter_texture =
148 gst_gl_transformation_filter_texture;
149 GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
150 gst_gl_transformation_prepare_output_buffer;
151
152 g_object_class_install_property (gobject_class, PROP_FOV,
153 g_param_spec_float ("fov", "Fov", "Field of view angle in degrees",
154 0.0, G_MAXFLOAT, 90.0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
155
156 g_object_class_install_property (gobject_class, PROP_ORTHO,
157 g_param_spec_boolean ("ortho", "Orthographic",
158 "Use orthographic projection", FALSE,
159 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
160
161 /* Rotation */
162 g_object_class_install_property (gobject_class, PROP_ROTATION_X,
163 g_param_spec_float ("rotation-x", "X Rotation",
164 "Rotates the video around the X-Axis in degrees.",
165 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
166 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
167
168 g_object_class_install_property (gobject_class, PROP_ROTATION_Y,
169 g_param_spec_float ("rotation-y", "Y Rotation",
170 "Rotates the video around the Y-Axis in degrees.",
171 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
172 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
173
174 g_object_class_install_property (gobject_class, PROP_ROTATION_Z,
175 g_param_spec_float ("rotation-z", "Z Rotation",
176 "Rotates the video around the Z-Axis in degrees.",
177 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
178 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
179
180 /* Translation */
181 g_object_class_install_property (gobject_class, PROP_TRANSLATION_X,
182 g_param_spec_float ("translation-x", "X Translation",
183 "Translates the video at the X-Axis, in universal [0-1] coordinate.",
184 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
185 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
186
187 g_object_class_install_property (gobject_class, PROP_TRANSLATION_Y,
188 g_param_spec_float ("translation-y", "Y Translation",
189 "Translates the video at the Y-Axis, in universal [0-1] coordinate.",
190 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
191 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
192
193 g_object_class_install_property (gobject_class, PROP_TRANSLATION_Z,
194 g_param_spec_float ("translation-z", "Z Translation",
195 "Translates the video at the Z-Axis, in universal [0-1] coordinate.",
196 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
197 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
198
199 /* Scale */
200 g_object_class_install_property (gobject_class, PROP_SCALE_X,
201 g_param_spec_float ("scale-x", "X Scale",
202 "Scale multiplier for the X-Axis.",
203 -G_MAXFLOAT, G_MAXFLOAT, 1.0,
204 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
205
206 g_object_class_install_property (gobject_class, PROP_SCALE_Y,
207 g_param_spec_float ("scale-y", "Y Scale",
208 "Scale multiplier for the Y-Axis.",
209 -G_MAXFLOAT, G_MAXFLOAT, 1.0,
210 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
211
212 /* Pivot */
213 g_object_class_install_property (gobject_class, PROP_PIVOT_X,
214 g_param_spec_float ("pivot-x", "X Pivot",
215 "Rotation pivot point X coordinate, where 0 is the center,"
216 " -1 the left border, +1 the right border and <-1, >1 outside.",
217 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
218 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
219
220 g_object_class_install_property (gobject_class, PROP_PIVOT_Y,
221 g_param_spec_float ("pivot-y", "Y Pivot",
222 "Rotation pivot point X coordinate, where 0 is the center,"
223 " -1 the left border, +1 the right border and <-1, >1 outside.",
224 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
225 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
226
227 g_object_class_install_property (gobject_class, PROP_PIVOT_Z,
228 g_param_spec_float ("pivot-z", "Z Pivot",
229 "Relevant for rotation in 3D space. You look into the negative Z axis direction",
230 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
231 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
232
233 /* MVP */
234 g_object_class_install_property (gobject_class, PROP_MVP,
235 g_param_spec_boxed ("mvp-matrix",
236 "Modelview Projection Matrix",
237 "The final Graphene 4x4 Matrix for transformation",
238 GRAPHENE_TYPE_MATRIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
239
240 gst_element_class_set_metadata (element_class, "OpenGL transformation filter",
241 "Filter/Effect/Video", "Transform video on the GPU",
242 "Lubosz Sarnecki <lubosz@gmail.com>\n"
243 "Matthew Waters <matthew@centricular.com>");
244
245 GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
246 GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
247 }
248
249 static void
gst_gl_transformation_init(GstGLTransformation * filter)250 gst_gl_transformation_init (GstGLTransformation * filter)
251 {
252 filter->shader = NULL;
253 filter->fov = 90;
254 filter->aspect = 1.0;
255 filter->znear = 0.1f;
256 filter->zfar = 100.0;
257
258 filter->xscale = 1.0;
259 filter->yscale = 1.0;
260
261 filter->in_tex = 0;
262
263 gst_gl_transformation_build_mvp (filter);
264 }
265
266 static void
gst_gl_transformation_build_mvp(GstGLTransformation * transformation)267 gst_gl_transformation_build_mvp (GstGLTransformation * transformation)
268 {
269 GstGLFilter *filter = GST_GL_FILTER (transformation);
270 graphene_matrix_t modelview_matrix;
271
272 if (!filter->out_info.finfo) {
273 graphene_matrix_init_identity (&transformation->model_matrix);
274 graphene_matrix_init_identity (&transformation->view_matrix);
275 graphene_matrix_init_identity (&transformation->projection_matrix);
276 } else {
277 graphene_point3d_t translation_vector =
278 GRAPHENE_POINT3D_INIT (transformation->xtranslation * 2.0 *
279 transformation->aspect,
280 transformation->ytranslation * 2.0,
281 transformation->ztranslation * 2.0);
282
283 graphene_point3d_t pivot_vector =
284 GRAPHENE_POINT3D_INIT (-transformation->xpivot * transformation->aspect,
285 transformation->ypivot,
286 -transformation->zpivot);
287
288 graphene_point3d_t negative_pivot_vector;
289
290 graphene_vec3_t center;
291 graphene_vec3_t up;
292
293 gboolean current_passthrough;
294 gboolean passthrough;
295
296 graphene_vec3_init (&transformation->camera_position, 0.f, 0.f, 1.f);
297 graphene_vec3_init (¢er, 0.f, 0.f, 0.f);
298 graphene_vec3_init (&up, 0.f, 1.f, 0.f);
299
300 /* Translate into pivot origin */
301 graphene_matrix_init_translate (&transformation->model_matrix,
302 &pivot_vector);
303
304 /* Scale */
305 graphene_matrix_scale (&transformation->model_matrix,
306 transformation->xscale, transformation->yscale, 1.0f);
307
308 /* Rotation */
309 graphene_matrix_rotate (&transformation->model_matrix,
310 transformation->xrotation, graphene_vec3_x_axis ());
311 graphene_matrix_rotate (&transformation->model_matrix,
312 transformation->yrotation, graphene_vec3_y_axis ());
313 graphene_matrix_rotate (&transformation->model_matrix,
314 transformation->zrotation, graphene_vec3_z_axis ());
315
316 /* Translate back from pivot origin */
317 graphene_point3d_scale (&pivot_vector, -1.0, &negative_pivot_vector);
318 graphene_matrix_translate (&transformation->model_matrix,
319 &negative_pivot_vector);
320
321 /* Translation */
322 graphene_matrix_translate (&transformation->model_matrix,
323 &translation_vector);
324
325 if (transformation->ortho) {
326 graphene_matrix_init_ortho (&transformation->projection_matrix,
327 -transformation->aspect, transformation->aspect,
328 -1, 1, transformation->znear, transformation->zfar);
329 } else {
330 graphene_matrix_init_perspective (&transformation->projection_matrix,
331 transformation->fov,
332 transformation->aspect, transformation->znear, transformation->zfar);
333 }
334
335 graphene_matrix_init_look_at (&transformation->view_matrix,
336 &transformation->camera_position, ¢er, &up);
337
338 current_passthrough =
339 gst_base_transform_is_passthrough (GST_BASE_TRANSFORM (transformation));
340 passthrough = transformation->xtranslation == 0.
341 && transformation->ytranslation == 0.
342 && transformation->ztranslation == 0. && transformation->xrotation == 0.
343 && transformation->yrotation == 0. && transformation->zrotation == 0.
344 && transformation->xscale == 1. && transformation->yscale == 1.
345 && gst_video_info_is_equal (&filter->in_info, &filter->out_info);
346 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (transformation),
347 passthrough);
348 if (current_passthrough != passthrough) {
349 gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (transformation));
350 }
351 }
352
353 graphene_matrix_multiply (&transformation->model_matrix,
354 &transformation->view_matrix, &modelview_matrix);
355 graphene_matrix_multiply (&modelview_matrix,
356 &transformation->projection_matrix, &transformation->mvp_matrix);
357
358 graphene_matrix_inverse (&transformation->model_matrix,
359 &transformation->inv_model_matrix);
360 graphene_matrix_inverse (&transformation->view_matrix,
361 &transformation->inv_view_matrix);
362 graphene_matrix_inverse (&transformation->projection_matrix,
363 &transformation->inv_projection_matrix);
364 }
365
366 static void
gst_gl_transformation_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)367 gst_gl_transformation_set_property (GObject * object, guint prop_id,
368 const GValue * value, GParamSpec * pspec)
369 {
370 GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
371
372 switch (prop_id) {
373 case PROP_FOV:
374 filter->fov = g_value_get_float (value);
375 break;
376 case PROP_ORTHO:
377 filter->ortho = g_value_get_boolean (value);
378 break;
379 case PROP_TRANSLATION_X:
380 filter->xtranslation = g_value_get_float (value);
381 break;
382 case PROP_TRANSLATION_Y:
383 filter->ytranslation = g_value_get_float (value);
384 break;
385 case PROP_TRANSLATION_Z:
386 filter->ztranslation = g_value_get_float (value);
387 break;
388 case PROP_ROTATION_X:
389 filter->xrotation = g_value_get_float (value);
390 break;
391 case PROP_ROTATION_Y:
392 filter->yrotation = g_value_get_float (value);
393 break;
394 case PROP_ROTATION_Z:
395 filter->zrotation = g_value_get_float (value);
396 break;
397 case PROP_SCALE_X:
398 filter->xscale = g_value_get_float (value);
399 break;
400 case PROP_SCALE_Y:
401 filter->yscale = g_value_get_float (value);
402 break;
403 case PROP_PIVOT_X:
404 filter->xpivot = g_value_get_float (value);
405 break;
406 case PROP_PIVOT_Y:
407 filter->ypivot = g_value_get_float (value);
408 break;
409 case PROP_PIVOT_Z:
410 filter->zpivot = g_value_get_float (value);
411 break;
412 default:
413 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
414 break;
415 }
416 gst_gl_transformation_build_mvp (filter);
417 }
418
419 static void
gst_gl_transformation_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)420 gst_gl_transformation_get_property (GObject * object, guint prop_id,
421 GValue * value, GParamSpec * pspec)
422 {
423 GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
424
425 switch (prop_id) {
426 case PROP_FOV:
427 g_value_set_float (value, filter->fov);
428 break;
429 case PROP_ORTHO:
430 g_value_set_boolean (value, filter->ortho);
431 break;
432 case PROP_TRANSLATION_X:
433 g_value_set_float (value, filter->xtranslation);
434 break;
435 case PROP_TRANSLATION_Y:
436 g_value_set_float (value, filter->ytranslation);
437 break;
438 case PROP_TRANSLATION_Z:
439 g_value_set_float (value, filter->ztranslation);
440 break;
441 case PROP_ROTATION_X:
442 g_value_set_float (value, filter->xrotation);
443 break;
444 case PROP_ROTATION_Y:
445 g_value_set_float (value, filter->yrotation);
446 break;
447 case PROP_ROTATION_Z:
448 g_value_set_float (value, filter->zrotation);
449 break;
450 case PROP_SCALE_X:
451 g_value_set_float (value, filter->xscale);
452 break;
453 case PROP_SCALE_Y:
454 g_value_set_float (value, filter->yscale);
455 break;
456 case PROP_PIVOT_X:
457 g_value_set_float (value, filter->xpivot);
458 break;
459 case PROP_PIVOT_Y:
460 g_value_set_float (value, filter->ypivot);
461 break;
462 case PROP_PIVOT_Z:
463 g_value_set_float (value, filter->zpivot);
464 break;
465 case PROP_MVP:
466 /* FIXME: need to decompose this to support navigation events */
467 g_value_set_boxed (value, (gconstpointer) & filter->mvp_matrix);
468 break;
469 default:
470 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
471 break;
472 }
473 }
474
475 static gboolean
gst_gl_transformation_set_caps(GstGLFilter * filter,GstCaps * incaps,GstCaps * outcaps)476 gst_gl_transformation_set_caps (GstGLFilter * filter, GstCaps * incaps,
477 GstCaps * outcaps)
478 {
479 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
480
481 transformation->aspect =
482 (gdouble) GST_VIDEO_INFO_WIDTH (&filter->out_info) /
483 (gdouble) GST_VIDEO_INFO_HEIGHT (&filter->out_info);
484
485 transformation->caps_change = TRUE;
486
487 gst_gl_transformation_build_mvp (transformation);
488
489 return TRUE;
490 }
491
492 static void
_intersect_plane_and_ray(graphene_plane_t * video_plane,graphene_ray_t * ray,graphene_point3d_t * result)493 _intersect_plane_and_ray (graphene_plane_t * video_plane, graphene_ray_t * ray,
494 graphene_point3d_t * result)
495 {
496 float t = graphene_ray_get_distance_to_plane (ray, video_plane);
497 GST_TRACE ("Calculated a distance of %f to the plane", t);
498 graphene_ray_get_position_at (ray, t, result);
499 }
500
501 static void
_screen_coord_to_world_ray(GstGLTransformation * transformation,float x,float y,graphene_ray_t * ray)502 _screen_coord_to_world_ray (GstGLTransformation * transformation, float x,
503 float y, graphene_ray_t * ray)
504 {
505 GstGLFilter *filter = GST_GL_FILTER (transformation);
506 gfloat w = (gfloat) GST_VIDEO_INFO_WIDTH (&filter->in_info);
507 gfloat h = (gfloat) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
508 graphene_vec3_t ray_eye_vec3, ray_world_dir, *ray_origin, *ray_direction;
509 graphene_vec3_t ray_ortho_dir;
510 graphene_point3d_t ray_clip, ray_eye;
511 graphene_vec2_t screen_coord;
512
513 /* GL is y-flipped. i.e. 0, 0 is the bottom left corner in screen space */
514 graphene_vec2_init (&screen_coord, (2. * x / w - 1.) / transformation->aspect,
515 1. - 2. * y / h);
516
517 graphene_point3d_init (&ray_clip, graphene_vec2_get_x (&screen_coord),
518 graphene_vec2_get_y (&screen_coord), -1.);
519 graphene_matrix_transform_point3d (&transformation->inv_projection_matrix,
520 &ray_clip, &ray_eye);
521
522 graphene_vec3_init (&ray_eye_vec3, ray_eye.x, ray_eye.y, -1.);
523
524 if (transformation->ortho) {
525 graphene_vec3_init (&ray_ortho_dir, 0., 0., 1.);
526
527 ray_origin = &ray_eye_vec3;
528 ray_direction = &ray_ortho_dir;
529 } else {
530 graphene_matrix_transform_vec3 (&transformation->inv_view_matrix,
531 &ray_eye_vec3, &ray_world_dir);
532 graphene_vec3_normalize (&ray_world_dir, &ray_world_dir);
533
534 ray_origin = &transformation->camera_position;
535 ray_direction = &ray_world_dir;
536 }
537
538 graphene_ray_init_from_vec3 (ray, ray_origin, ray_direction);
539
540 GST_TRACE_OBJECT (transformation, "Calculated ray origin: " VEC3_FORMAT
541 " direction: " VEC3_FORMAT " from screen coordinates: " VEC2_FORMAT
542 " with %s projection",
543 VEC3_ARGS (ray_origin), VEC3_ARGS (ray_direction),
544 VEC2_ARGS (&screen_coord),
545 transformation->ortho ? "ortho" : "perspection");
546 }
547
548 static void
_init_world_video_plane(GstGLTransformation * transformation,graphene_plane_t * video_plane)549 _init_world_video_plane (GstGLTransformation * transformation,
550 graphene_plane_t * video_plane)
551 {
552 graphene_point3d_t bottom_left, bottom_right, top_left, top_right;
553 graphene_point3d_t world_bottom_left, world_bottom_right;
554 graphene_point3d_t world_top_left, world_top_right;
555
556 graphene_point3d_init (&top_left, -transformation->aspect, 1., 0.);
557 graphene_point3d_init (&top_right, transformation->aspect, 1., 0.);
558 graphene_point3d_init (&bottom_left, -transformation->aspect, -1., 0.);
559 graphene_point3d_init (&bottom_right, transformation->aspect, -1., 0.);
560
561 graphene_matrix_transform_point3d (&transformation->model_matrix,
562 &bottom_left, &world_bottom_left);
563 graphene_matrix_transform_point3d (&transformation->model_matrix,
564 &bottom_right, &world_bottom_right);
565 graphene_matrix_transform_point3d (&transformation->model_matrix,
566 &top_left, &world_top_left);
567 graphene_matrix_transform_point3d (&transformation->model_matrix,
568 &top_right, &world_top_right);
569
570 graphene_plane_init_from_points (video_plane, &world_bottom_left,
571 &world_top_right, &world_top_left);
572 }
573
574 static gboolean
_screen_coord_to_model_coord(GstGLTransformation * transformation,double x,double y,double * res_x,double * res_y)575 _screen_coord_to_model_coord (GstGLTransformation * transformation,
576 double x, double y, double *res_x, double *res_y)
577 {
578 GstGLFilter *filter = GST_GL_FILTER (transformation);
579 double w = (double) GST_VIDEO_INFO_WIDTH (&filter->in_info);
580 double h = (double) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
581 graphene_point3d_t world_point, model_coord;
582 graphene_plane_t video_plane;
583 graphene_ray_t ray;
584 double new_x, new_y;
585
586 _init_world_video_plane (transformation, &video_plane);
587 _screen_coord_to_world_ray (transformation, x, y, &ray);
588 _intersect_plane_and_ray (&video_plane, &ray, &world_point);
589 graphene_matrix_transform_point3d (&transformation->inv_model_matrix,
590 &world_point, &model_coord);
591
592 /* ndc to pixels. We render the frame Y-flipped so need to unflip the
593 * y coordinate */
594 new_x = (model_coord.x + 1.) * w / 2;
595 new_y = (1. - model_coord.y) * h / 2;
596
597 if (new_x < 0. || new_x > w || new_y < 0. || new_y > h)
598 /* coords off video surface */
599 return FALSE;
600
601 GST_DEBUG_OBJECT (transformation, "converted %f,%f to %f,%f", x, y, new_x,
602 new_y);
603
604 if (res_x)
605 *res_x = new_x;
606 if (res_y)
607 *res_y = new_y;
608
609 return TRUE;
610 }
611
612 #if 0
613 /* debugging facilities for transforming vertices from model space to screen
614 * space */
615 static void
616 _ndc_to_viewport (GstGLTransformation * transformation, graphene_vec3_t * ndc,
617 int x, int y, int w, int h, float near, float far, graphene_vec3_t * result)
618 {
619 GstGLFilter *filter = GST_GL_FILTER (transformation);
620 /* center of the viewport */
621 int o_x = x + w / 2;
622 int o_y = y + h / 2;
623
624 graphene_vec3_init (result, graphene_vec3_get_x (ndc) * w / 2 + o_x,
625 graphene_vec3_get_y (ndc) * h / 2 + o_y,
626 (far - near) * graphene_vec3_get_z (ndc) / 2 + (far + near) / 2);
627 }
628
629 static void
630 _perspective_division (graphene_vec4_t * clip, graphene_vec3_t * result)
631 {
632 float w = graphene_vec4_get_w (clip);
633
634 graphene_vec3_init (result, graphene_vec4_get_x (clip) / w,
635 graphene_vec4_get_y (clip) / w, graphene_vec4_get_z (clip) / w);
636 }
637
638 static void
639 _vertex_to_screen_coord (GstGLTransformation * transformation,
640 graphene_vec4_t * vertex, graphene_vec3_t * view)
641 {
642 GstGLFilter *filter = GST_GL_FILTER (transformation);
643 gint w = GST_VIDEO_INFO_WIDTH (&filter->in_info);
644 gint h = GST_VIDEO_INFO_HEIGHT (&filter->in_info);
645 graphene_vec4_t clip;
646 graphene_vec3_t ndc;
647
648 graphene_matrix_transform_vec4 (&transformation->mvp_matrix, vertex, &clip);
649 _perspective_division (&clip, &ndc);
650 _ndc_to_viewport (transformation, &ndc, 0, 0, w, h, 0., 1., view);
651 }
652 #endif
653
654 static gboolean
gst_gl_transformation_src_event(GstBaseTransform * trans,GstEvent * event)655 gst_gl_transformation_src_event (GstBaseTransform * trans, GstEvent * event)
656 {
657 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
658 GstStructure *structure;
659 gboolean ret;
660
661 GST_DEBUG_OBJECT (trans, "handling %s event", GST_EVENT_TYPE_NAME (event));
662
663 switch (GST_EVENT_TYPE (event)) {
664 case GST_EVENT_NAVIGATION:{
665 gdouble x, y;
666 event =
667 GST_EVENT (gst_mini_object_make_writable (GST_MINI_OBJECT (event)));
668
669 structure = (GstStructure *) gst_event_get_structure (event);
670 if (gst_structure_get_double (structure, "pointer_x", &x) &&
671 gst_structure_get_double (structure, "pointer_y", &y)) {
672 gdouble new_x, new_y;
673
674 if (!_screen_coord_to_model_coord (transformation, x, y, &new_x,
675 &new_y)) {
676 gst_event_unref (event);
677 return TRUE;
678 }
679
680 gst_structure_set (structure, "pointer_x", G_TYPE_DOUBLE, new_x,
681 "pointer_y", G_TYPE_DOUBLE, new_y, NULL);
682 }
683 break;
684 }
685 default:
686 break;
687 }
688
689 ret = GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (trans, event);
690
691 return ret;
692 }
693
694 static gboolean
gst_gl_transformation_filter_meta(GstBaseTransform * trans,GstQuery * query,GType api,const GstStructure * params)695 gst_gl_transformation_filter_meta (GstBaseTransform * trans, GstQuery * query,
696 GType api, const GstStructure * params)
697 {
698 if (api == GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE)
699 return TRUE;
700
701 if (api == GST_GL_SYNC_META_API_TYPE)
702 return TRUE;
703
704 return FALSE;
705 }
706
707 static gboolean
gst_gl_transformation_decide_allocation(GstBaseTransform * trans,GstQuery * query)708 gst_gl_transformation_decide_allocation (GstBaseTransform * trans,
709 GstQuery * query)
710 {
711 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
712
713 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
714 query))
715 return FALSE;
716
717 if (gst_query_find_allocation_meta (query,
718 GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) {
719 transformation->downstream_supports_affine_meta = TRUE;
720 } else {
721 transformation->downstream_supports_affine_meta = FALSE;
722 }
723
724 return TRUE;
725 }
726
727 static void
gst_gl_transformation_gl_stop(GstGLBaseFilter * base_filter)728 gst_gl_transformation_gl_stop (GstGLBaseFilter * base_filter)
729 {
730 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
731 const GstGLFuncs *gl = base_filter->context->gl_vtable;
732
733 if (transformation->vao) {
734 gl->DeleteVertexArrays (1, &transformation->vao);
735 transformation->vao = 0;
736 }
737
738 if (transformation->vertex_buffer) {
739 gl->DeleteBuffers (1, &transformation->vertex_buffer);
740 transformation->vertex_buffer = 0;
741 }
742
743 if (transformation->vbo_indices) {
744 gl->DeleteBuffers (1, &transformation->vbo_indices);
745 transformation->vbo_indices = 0;
746 }
747
748 if (transformation->shader) {
749 gst_object_unref (transformation->shader);
750 transformation->shader = NULL;
751 }
752
753 GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base_filter);
754 }
755
756 static gboolean
gst_gl_transformation_gl_start(GstGLBaseFilter * base_filter)757 gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter)
758 {
759 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
760
761 if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base_filter))
762 return FALSE;
763
764 if (gst_gl_context_get_gl_api (base_filter->context)) {
765 gchar *frag_str;
766 gboolean ret;
767
768 frag_str =
769 gst_gl_shader_string_fragment_get_default (base_filter->context,
770 GST_GLSL_VERSION_NONE,
771 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
772
773 /* blocking call, wait until the opengl thread has compiled the shader */
774 ret = gst_gl_context_gen_shader (base_filter->context,
775 gst_gl_shader_string_vertex_mat4_vertex_transform,
776 frag_str, &transformation->shader);
777
778 g_free (frag_str);
779
780 return ret;
781 }
782 return TRUE;
783 }
784
785 static GstFlowReturn
gst_gl_transformation_prepare_output_buffer(GstBaseTransform * trans,GstBuffer * inbuf,GstBuffer ** outbuf)786 gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
787 GstBuffer * inbuf, GstBuffer ** outbuf)
788 {
789 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
790 GstGLFilter *filter = GST_GL_FILTER (trans);
791
792 if (transformation->downstream_supports_affine_meta &&
793 gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
794 GstVideoAffineTransformationMeta *af_meta;
795 graphene_matrix_t upstream_matrix, tmp, tmp2, inv_aspect, yflip;
796 float upstream[16], downstream[16];
797
798 *outbuf = gst_buffer_make_writable (inbuf);
799
800 af_meta = gst_buffer_get_video_affine_transformation_meta (inbuf);
801 if (!af_meta)
802 af_meta = gst_buffer_add_video_affine_transformation_meta (*outbuf);
803
804 GST_LOG_OBJECT (trans, "applying transformation to existing affine "
805 "transformation meta");
806
807 gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, upstream);
808
809 /* apply the transformation to the existing affine meta */
810 graphene_matrix_init_from_float (&upstream_matrix, upstream);
811 graphene_matrix_init_scale (&inv_aspect, transformation->aspect, -1., 1.);
812 graphene_matrix_init_scale (&yflip, 1., -1., 1.);
813
814 /* invert the aspect effects */
815 graphene_matrix_multiply (&upstream_matrix, &inv_aspect, &tmp2);
816 /* apply the transformation */
817 graphene_matrix_multiply (&tmp2, &transformation->mvp_matrix, &tmp);
818 /* and undo yflip */
819 graphene_matrix_multiply (&tmp, &yflip, &tmp2);
820
821 graphene_matrix_to_float (&tmp2, downstream);
822 gst_gl_set_affine_transformation_meta_from_ndc_ext (af_meta, downstream);
823
824 return GST_FLOW_OK;
825 }
826
827 return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (trans,
828 inbuf, outbuf);
829 }
830
831 static gboolean
gst_gl_transformation_filter(GstGLFilter * filter,GstBuffer * inbuf,GstBuffer * outbuf)832 gst_gl_transformation_filter (GstGLFilter * filter,
833 GstBuffer * inbuf, GstBuffer * outbuf)
834 {
835 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
836
837 if (transformation->downstream_supports_affine_meta &&
838 gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
839 return TRUE;
840 } else {
841 return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
842 }
843 }
844
845 static gboolean
gst_gl_transformation_filter_texture(GstGLFilter * filter,GstGLMemory * in_tex,GstGLMemory * out_tex)846 gst_gl_transformation_filter_texture (GstGLFilter * filter,
847 GstGLMemory * in_tex, GstGLMemory * out_tex)
848 {
849 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
850
851 transformation->in_tex = in_tex;
852
853 gst_gl_framebuffer_draw_to_texture (filter->fbo, out_tex,
854 (GstGLFramebufferFunc) gst_gl_transformation_callback, transformation);
855
856 return TRUE;
857 }
858
859 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
860
861 static void
_upload_vertices(GstGLTransformation * transformation)862 _upload_vertices (GstGLTransformation * transformation)
863 {
864 const GstGLFuncs *gl =
865 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
866
867 /* *INDENT-OFF* */
868 GLfloat vertices[] = {
869 -transformation->aspect, -1.0, 0.0, 1.0, 0.0, 0.0,
870 transformation->aspect, -1.0, 0.0, 1.0, 1.0, 0.0,
871 transformation->aspect, 1.0, 0.0, 1.0, 1.0, 1.0,
872 -transformation->aspect, 1.0, 0.0, 1.0, 0.0, 1.0,
873 };
874 /* *INDENT-ON* */
875
876 gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
877
878 gl->BufferData (GL_ARRAY_BUFFER, 4 * 6 * sizeof (GLfloat), vertices,
879 GL_STATIC_DRAW);
880 }
881
882 static void
_bind_buffer(GstGLTransformation * transformation)883 _bind_buffer (GstGLTransformation * transformation)
884 {
885 const GstGLFuncs *gl =
886 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
887
888 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
889 gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
890
891 /* Load the vertex position */
892 gl->VertexAttribPointer (transformation->attr_position, 4, GL_FLOAT,
893 GL_FALSE, 6 * sizeof (GLfloat), (void *) 0);
894
895 /* Load the texture coordinate */
896 gl->VertexAttribPointer (transformation->attr_texture, 2, GL_FLOAT, GL_FALSE,
897 6 * sizeof (GLfloat), (void *) (4 * sizeof (GLfloat)));
898
899 gl->EnableVertexAttribArray (transformation->attr_position);
900 gl->EnableVertexAttribArray (transformation->attr_texture);
901 }
902
903 static void
_unbind_buffer(GstGLTransformation * transformation)904 _unbind_buffer (GstGLTransformation * transformation)
905 {
906 const GstGLFuncs *gl =
907 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
908
909 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
910 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
911
912 gl->DisableVertexAttribArray (transformation->attr_position);
913 gl->DisableVertexAttribArray (transformation->attr_texture);
914 }
915
916 static gboolean
gst_gl_transformation_callback(gpointer stuff)917 gst_gl_transformation_callback (gpointer stuff)
918 {
919 GstGLFilter *filter = GST_GL_FILTER (stuff);
920 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
921 GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
922
923 GLfloat temp_matrix[16];
924
925 gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
926 gl->BindTexture (GL_TEXTURE_2D, 0);
927
928 gl->ClearColor (0.f, 0.f, 0.f, 0.f);
929 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
930
931 gst_gl_shader_use (transformation->shader);
932
933 gl->ActiveTexture (GL_TEXTURE0);
934 gl->BindTexture (GL_TEXTURE_2D, transformation->in_tex->tex_id);
935 gst_gl_shader_set_uniform_1i (transformation->shader, "texture", 0);
936
937 graphene_matrix_to_float (&transformation->mvp_matrix, temp_matrix);
938 gst_gl_shader_set_uniform_matrix_4fv (transformation->shader,
939 "u_transformation", 1, GL_FALSE, temp_matrix);
940
941 if (!transformation->vertex_buffer) {
942 transformation->attr_position =
943 gst_gl_shader_get_attribute_location (transformation->shader,
944 "a_position");
945
946 transformation->attr_texture =
947 gst_gl_shader_get_attribute_location (transformation->shader,
948 "a_texcoord");
949
950 if (gl->GenVertexArrays) {
951 gl->GenVertexArrays (1, &transformation->vao);
952 gl->BindVertexArray (transformation->vao);
953 }
954
955 gl->GenBuffers (1, &transformation->vertex_buffer);
956
957 gl->GenBuffers (1, &transformation->vbo_indices);
958 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
959 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
960 GL_STATIC_DRAW);
961
962 transformation->caps_change = TRUE;
963 }
964
965 if (gl->GenVertexArrays)
966 gl->BindVertexArray (transformation->vao);
967
968 if (transformation->caps_change)
969 _upload_vertices (transformation);
970 _bind_buffer (transformation);
971
972 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
973
974 if (gl->GenVertexArrays)
975 gl->BindVertexArray (0);
976 else
977 _unbind_buffer (transformation);
978
979 gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
980 transformation->caps_change = FALSE;
981
982 return TRUE;
983 }
984