1 /*
2 * GStreamer
3 * Copyright (C) 2016 Matthew Waters <matthew@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 /**
22 * SECTION:element-glvideo_flip
23 * @title: glvideo_flip
24 *
25 * Transforms video on the GPU.
26 *
27 * ## Examples
28 * |[
29 * gst-launch-1.0 videotestsrc ! glupload ! glvideoflip method=clockwise ! glimagesinkelement
30 * ]| This pipeline flips the test image 90 degrees clockwise.
31 *
32 */
33
34 #ifdef HAVE_CONFIG_H
35 #include "config.h"
36 #endif
37
38 #include "gstglvideoflip.h"
39
40 #define GST_CAT_DEFAULT gst_gl_video_flip_debug
41 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
42
43 #define DEFAULT_METHOD GST_GL_VIDEO_FLIP_METHOD_IDENTITY
44
45 enum
46 {
47 PROP_0,
48 PROP_METHOD,
49 PROP_VIDEO_DIRECTION
50 };
51
52 static GstStaticPadTemplate _sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
53 GST_PAD_SINK,
54 GST_PAD_ALWAYS,
55 GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
56 "format = (string) RGBA, "
57 "width = " GST_VIDEO_SIZE_RANGE ", "
58 "height = " GST_VIDEO_SIZE_RANGE ", "
59 "framerate = " GST_VIDEO_FPS_RANGE ", "
60 "texture-target = (string) 2D"));
61
62 static GstStaticPadTemplate _src_template = GST_STATIC_PAD_TEMPLATE ("src",
63 GST_PAD_SRC,
64 GST_PAD_ALWAYS,
65 GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
66 "format = (string) RGBA, "
67 "width = " GST_VIDEO_SIZE_RANGE ", "
68 "height = " GST_VIDEO_SIZE_RANGE ", "
69 "framerate = " GST_VIDEO_FPS_RANGE ", "
70 "texture-target = (string) 2D"));
71
72 #define GST_TYPE_GL_VIDEO_FLIP_METHOD (gst_video_flip_method_get_type())
73 static const GEnumValue video_flip_methods[] = {
74 {GST_GL_VIDEO_FLIP_METHOD_IDENTITY, "Identity (no rotation)", "none"},
75 {GST_GL_VIDEO_FLIP_METHOD_90R, "Rotate clockwise 90 degrees", "clockwise"},
76 {GST_GL_VIDEO_FLIP_METHOD_180, "Rotate 180 degrees", "rotate-180"},
77 {GST_GL_VIDEO_FLIP_METHOD_90L, "Rotate counter-clockwise 90 degrees",
78 "counterclockwise"},
79 {GST_GL_VIDEO_FLIP_METHOD_FLIP_HORIZ, "Flip horizontally", "horizontal-flip"},
80 {GST_GL_VIDEO_FLIP_METHOD_FLIP_VERT, "Flip vertically", "vertical-flip"},
81 {GST_GL_VIDEO_FLIP_METHOD_FLIP_UL_LR,
82 "Flip across upper left/lower right diagonal", "upper-left-diagonal"},
83 {GST_GL_VIDEO_FLIP_METHOD_FLIP_UR_LL,
84 "Flip across upper right/lower left diagonal", "upper-right-diagonal"},
85 {GST_GL_VIDEO_FLIP_METHOD_AUTO,
86 "Select flip method based on image-orientation tag", "automatic"},
87 {0, NULL, NULL},
88 };
89
90 static GType
gst_video_flip_method_get_type(void)91 gst_video_flip_method_get_type (void)
92 {
93 static GType video_flip_method_type = 0;
94
95 if (!video_flip_method_type) {
96 video_flip_method_type = g_enum_register_static ("GstGLVideoFlipMethod",
97 video_flip_methods);
98 }
99 return video_flip_method_type;
100 }
101
102 static void gst_gl_video_flip_finalize (GObject * object);
103 static void gst_gl_video_flip_set_property (GObject * object, guint prop_id,
104 const GValue * value, GParamSpec * pspec);
105 static void gst_gl_video_flip_get_property (GObject * object, guint prop_id,
106 GValue * value, GParamSpec * pspec);
107
108 static GstPadProbeReturn _input_sink_probe (GstPad * pad,
109 GstPadProbeInfo * info, gpointer user_data);
110 static GstPadProbeReturn _trans_src_probe (GstPad * pad, GstPadProbeInfo * info,
111 gpointer user_data);
112
113 static void
114 gst_gl_video_flip_video_direction_interface_init (GstVideoDirectionInterface
115 * iface);
116
117 #define gst_gl_video_flip_parent_class parent_class
118 G_DEFINE_TYPE_WITH_CODE (GstGLVideoFlip, gst_gl_video_flip,
119 GST_TYPE_BIN, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT,
120 "glvideoflip", 0, "glvideoflip element");
121 G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_DIRECTION,
122 gst_gl_video_flip_video_direction_interface_init););
123
124 static void
gst_gl_video_flip_video_direction_interface_init(GstVideoDirectionInterface * iface)125 gst_gl_video_flip_video_direction_interface_init (GstVideoDirectionInterface
126 * iface)
127 {
128 /* We implement the video-direction property */
129 }
130
131 static void
gst_gl_video_flip_class_init(GstGLVideoFlipClass * klass)132 gst_gl_video_flip_class_init (GstGLVideoFlipClass * klass)
133 {
134 GObjectClass *gobject_class;
135 GstElementClass *element_class;
136
137 gobject_class = (GObjectClass *) klass;
138 element_class = GST_ELEMENT_CLASS (klass);
139
140 gobject_class->finalize = gst_gl_video_flip_finalize;
141 gobject_class->set_property = gst_gl_video_flip_set_property;
142 gobject_class->get_property = gst_gl_video_flip_get_property;
143
144 g_object_class_install_property (gobject_class, PROP_METHOD,
145 g_param_spec_enum ("method", "method",
146 "method (deprecated, use video-direction instead)",
147 GST_TYPE_GL_VIDEO_FLIP_METHOD, DEFAULT_METHOD,
148 GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | G_PARAM_CONSTRUCT |
149 G_PARAM_STATIC_STRINGS));
150 g_object_class_override_property (gobject_class, PROP_VIDEO_DIRECTION,
151 "video-direction");
152
153 gst_element_class_add_static_pad_template (element_class, &_src_template);
154 gst_element_class_add_static_pad_template (element_class, &_sink_template);
155
156 gst_element_class_set_metadata (element_class, "OpenGL video flip filter",
157 "Filter/Effect/Video", "Flip video on the GPU",
158 "Matthew Waters <matthew@centricular.com>");
159 }
160
161 static void
gst_gl_video_flip_init(GstGLVideoFlip * flip)162 gst_gl_video_flip_init (GstGLVideoFlip * flip)
163 {
164 gboolean res = TRUE;
165 GstPad *pad;
166
167 flip->aspect = 1.0;
168
169 flip->input_capsfilter = gst_element_factory_make ("capsfilter", NULL);
170 res &= gst_bin_add (GST_BIN (flip), flip->input_capsfilter);
171
172 flip->transformation = gst_element_factory_make ("gltransformation", NULL);
173 g_object_set (flip->transformation, "ortho", TRUE, NULL);
174 res &= gst_bin_add (GST_BIN (flip), flip->transformation);
175
176 flip->output_capsfilter = gst_element_factory_make ("capsfilter", NULL);
177 res &= gst_bin_add (GST_BIN (flip), flip->output_capsfilter);
178
179 res &=
180 gst_element_link_pads (flip->input_capsfilter, "src",
181 flip->transformation, "sink");
182 res &=
183 gst_element_link_pads (flip->transformation, "src",
184 flip->output_capsfilter, "sink");
185
186 pad = gst_element_get_static_pad (flip->input_capsfilter, "sink");
187 if (!pad) {
188 res = FALSE;
189 } else {
190 GST_DEBUG_OBJECT (flip, "setting target sink pad %" GST_PTR_FORMAT, pad);
191 flip->sinkpad = gst_ghost_pad_new ("sink", pad);
192 flip->sink_probe = gst_pad_add_probe (flip->sinkpad,
193 GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM |
194 GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM,
195 (GstPadProbeCallback) _input_sink_probe, flip, NULL);
196 gst_element_add_pad (GST_ELEMENT_CAST (flip), flip->sinkpad);
197 gst_object_unref (pad);
198 }
199
200 pad = gst_element_get_static_pad (flip->transformation, "src");
201 flip->src_probe = gst_pad_add_probe (pad,
202 GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM,
203 (GstPadProbeCallback) _trans_src_probe, flip, NULL);
204 gst_object_unref (pad);
205
206 pad = gst_element_get_static_pad (flip->output_capsfilter, "src");
207 if (!pad) {
208 res = FALSE;
209 } else {
210 GST_DEBUG_OBJECT (flip, "setting target sink pad %" GST_PTR_FORMAT, pad);
211 flip->srcpad = gst_ghost_pad_new ("src", pad);
212 gst_element_add_pad (GST_ELEMENT_CAST (flip), flip->srcpad);
213 gst_object_unref (pad);
214 }
215
216 if (!res) {
217 GST_WARNING_OBJECT (flip, "Failed to add/connect the necessary machinery");
218 }
219 }
220
221 static void
gst_gl_video_flip_finalize(GObject * object)222 gst_gl_video_flip_finalize (GObject * object)
223 {
224 GstGLVideoFlip *flip = GST_GL_VIDEO_FLIP (object);
225
226 gst_caps_replace (&flip->input_caps, NULL);
227
228 G_OBJECT_CLASS (parent_class)->finalize (object);
229 }
230
231 /* Caps negotiation happens like this:
232 *
233 * 1. caps/accept-caps queries bypass the capsfilters on either side of the
234 * transformation element so the fixed caps don't get in the way.
235 * 2. Receiving a caps event on the sink pad will set fixed caps on either side
236 * of the transformation element.
237 */
238 static GstCaps *
_transform_caps(GstGLVideoFlip * vf,GstPadDirection direction,GstCaps * caps)239 _transform_caps (GstGLVideoFlip * vf, GstPadDirection direction, GstCaps * caps)
240 {
241 GstCaps *output = gst_caps_copy (caps);
242 gint i;
243
244 for (i = 0; i < gst_caps_get_size (output); i++) {
245 GstStructure *structure = gst_caps_get_structure (output, i);
246 gint width, height;
247 gint par_n, par_d;
248
249 if (gst_structure_get_int (structure, "width", &width) &&
250 gst_structure_get_int (structure, "height", &height)) {
251
252 switch (vf->active_method) {
253 case GST_VIDEO_ORIENTATION_90R:
254 case GST_VIDEO_ORIENTATION_90L:
255 case GST_VIDEO_ORIENTATION_UL_LR:
256 case GST_VIDEO_ORIENTATION_UR_LL:
257 gst_structure_set (structure, "width", G_TYPE_INT, height,
258 "height", G_TYPE_INT, width, NULL);
259 if (gst_structure_get_fraction (structure, "pixel-aspect-ratio",
260 &par_n, &par_d)) {
261 if (par_n != 1 || par_d != 1) {
262 GValue val = { 0, };
263
264 g_value_init (&val, GST_TYPE_FRACTION);
265 gst_value_set_fraction (&val, par_d, par_n);
266 gst_structure_set_value (structure, "pixel-aspect-ratio", &val);
267 g_value_unset (&val);
268 }
269 }
270 break;
271 case GST_VIDEO_ORIENTATION_IDENTITY:
272 case GST_VIDEO_ORIENTATION_180:
273 case GST_VIDEO_ORIENTATION_HORIZ:
274 case GST_VIDEO_ORIENTATION_VERT:
275 break;
276 default:
277 g_assert_not_reached ();
278 break;
279 }
280 }
281 }
282
283 return output;
284 }
285
286 /* with object lock */
287 static void
_set_active_method(GstGLVideoFlip * vf,GstVideoOrientationMethod method,GstCaps * caps)288 _set_active_method (GstGLVideoFlip * vf, GstVideoOrientationMethod method,
289 GstCaps * caps)
290 {
291 gfloat rot_z = 0., scale_x = 1.0, scale_y = 1.0;
292 GstCaps *output_caps, *templ;
293 GstPad *srcpad;
294
295 switch (method) {
296 case GST_VIDEO_ORIENTATION_IDENTITY:
297 break;
298 case GST_VIDEO_ORIENTATION_90R:
299 scale_x *= vf->aspect;
300 scale_y *= 1. / vf->aspect;
301 rot_z = 90.;
302 break;
303 case GST_VIDEO_ORIENTATION_180:
304 rot_z = 180.;
305 break;
306 case GST_VIDEO_ORIENTATION_90L:
307 scale_x *= vf->aspect;
308 scale_y *= 1. / vf->aspect;
309 rot_z = 270.;
310 break;
311 case GST_VIDEO_ORIENTATION_HORIZ:
312 scale_x *= -1.;
313 break;
314 case GST_VIDEO_ORIENTATION_UR_LL:
315 scale_x *= -vf->aspect;
316 scale_y *= 1. / vf->aspect;
317 rot_z = 90.;
318 break;
319 case GST_VIDEO_ORIENTATION_VERT:
320 scale_x *= -1.;
321 rot_z = 180.;
322 break;
323 case GST_VIDEO_ORIENTATION_UL_LR:
324 scale_x *= -vf->aspect;
325 scale_y *= 1. / vf->aspect;
326 rot_z = 270.;
327 break;
328 default:
329 break;
330 }
331 vf->active_method = method;
332
333 output_caps = _transform_caps (vf, GST_PAD_SINK, caps);
334 gst_caps_replace (&vf->input_caps, caps);
335
336 srcpad = gst_element_get_static_pad (vf->transformation, "src");
337 templ = gst_pad_get_pad_template_caps (srcpad);
338 gst_object_unref (srcpad);
339
340 gst_caps_append (output_caps, gst_caps_ref (templ));
341 GST_OBJECT_UNLOCK (vf);
342
343 g_object_set (vf->input_capsfilter, "caps", gst_caps_ref (caps), NULL);
344 g_object_set (vf->output_capsfilter, "caps", output_caps, NULL);
345 g_object_set (vf->transformation, "rotation-z", rot_z, "scale-x", scale_x,
346 "scale-y", scale_y, NULL);
347 GST_OBJECT_LOCK (vf);
348 }
349
350 static void
gst_gl_video_flip_set_method(GstGLVideoFlip * vf,GstVideoOrientationMethod method,gboolean from_tag)351 gst_gl_video_flip_set_method (GstGLVideoFlip * vf,
352 GstVideoOrientationMethod method, gboolean from_tag)
353 {
354 GST_OBJECT_LOCK (vf);
355
356 if (method == GST_VIDEO_ORIENTATION_CUSTOM) {
357 GST_WARNING_OBJECT (vf, "unsupported custom orientation");
358 GST_OBJECT_UNLOCK (vf);
359 return;
360 }
361
362 /* Store updated method */
363 if (from_tag)
364 vf->tag_method = method;
365 else
366 vf->method = method;
367
368 /* Get the new method */
369 if (vf->method == GST_VIDEO_ORIENTATION_AUTO)
370 method = vf->tag_method;
371 else
372 method = vf->method;
373
374 if (vf->input_caps)
375 _set_active_method (vf, method, vf->input_caps);
376 else {
377 /* just store the configured method here. The actual transform configuration
378 * will be done once caps are configured. See caps handling in
379 * _input_sink_probe. */
380 vf->active_method = method;
381 }
382
383 GST_OBJECT_UNLOCK (vf);
384 }
385
386 static void
gst_gl_video_flip_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)387 gst_gl_video_flip_set_property (GObject * object, guint prop_id,
388 const GValue * value, GParamSpec * pspec)
389 {
390 GstGLVideoFlip *vf = GST_GL_VIDEO_FLIP (object);
391
392 switch (prop_id) {
393 case PROP_METHOD:
394 case PROP_VIDEO_DIRECTION:
395 gst_gl_video_flip_set_method (vf, g_value_get_enum (value), FALSE);
396 break;
397 default:
398 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
399 break;
400 }
401 }
402
403 static void
gst_gl_video_flip_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)404 gst_gl_video_flip_get_property (GObject * object, guint prop_id,
405 GValue * value, GParamSpec * pspec)
406 {
407 GstGLVideoFlip *vf = GST_GL_VIDEO_FLIP (object);
408
409 switch (prop_id) {
410 case PROP_METHOD:
411 case PROP_VIDEO_DIRECTION:
412 g_value_set_enum (value, vf->method);
413 break;
414 default:
415 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
416 break;
417 }
418 }
419
420 static GstPadProbeReturn
_input_sink_probe(GstPad * pad,GstPadProbeInfo * info,gpointer user_data)421 _input_sink_probe (GstPad * pad, GstPadProbeInfo * info, gpointer user_data)
422 {
423 GstGLVideoFlip *vf = GST_GL_VIDEO_FLIP (user_data);
424
425 if (GST_PAD_PROBE_INFO_TYPE (info) & GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) {
426 GstEvent *event = GST_PAD_PROBE_INFO_EVENT (info);
427
428 switch (GST_EVENT_TYPE (event)) {
429 case GST_EVENT_TAG:{
430 GstTagList *taglist;
431 gchar *orientation;
432
433 gst_event_parse_tag (event, &taglist);
434
435 if (gst_tag_list_get_string (taglist, "image-orientation",
436 &orientation)) {
437 if (!g_strcmp0 ("rotate-0", orientation))
438 gst_gl_video_flip_set_method (vf, GST_VIDEO_ORIENTATION_IDENTITY,
439 TRUE);
440 else if (!g_strcmp0 ("rotate-90", orientation))
441 gst_gl_video_flip_set_method (vf, GST_VIDEO_ORIENTATION_90R, TRUE);
442 else if (!g_strcmp0 ("rotate-180", orientation))
443 gst_gl_video_flip_set_method (vf, GST_VIDEO_ORIENTATION_180, TRUE);
444 else if (!g_strcmp0 ("rotate-270", orientation))
445 gst_gl_video_flip_set_method (vf, GST_VIDEO_ORIENTATION_90L, TRUE);
446 else if (!g_strcmp0 ("flip-rotate-0", orientation))
447 gst_gl_video_flip_set_method (vf,
448 GST_VIDEO_ORIENTATION_HORIZ, TRUE);
449 else if (!g_strcmp0 ("flip-rotate-90", orientation))
450 gst_gl_video_flip_set_method (vf,
451 GST_VIDEO_ORIENTATION_UR_LL, TRUE);
452 else if (!g_strcmp0 ("flip-rotate-180", orientation))
453 gst_gl_video_flip_set_method (vf, GST_VIDEO_ORIENTATION_VERT, TRUE);
454 else if (!g_strcmp0 ("flip-rotate-270", orientation))
455 gst_gl_video_flip_set_method (vf,
456 GST_VIDEO_ORIENTATION_UL_LR, TRUE);
457
458 g_free (orientation);
459 }
460 break;
461 }
462 case GST_EVENT_CAPS:{
463 GstCaps *caps;
464 GstVideoInfo v_info;
465
466 gst_event_parse_caps (event, &caps);
467 GST_OBJECT_LOCK (vf);
468 if (gst_video_info_from_caps (&v_info, caps))
469 vf->aspect =
470 (gfloat) GST_VIDEO_INFO_WIDTH (&v_info) /
471 (gfloat) GST_VIDEO_INFO_HEIGHT (&v_info);
472 else
473 vf->aspect = 1.0;
474 _set_active_method (vf, vf->active_method, caps);
475 GST_OBJECT_UNLOCK (vf);
476 break;
477 }
478 default:
479 break;
480 }
481 } else if (GST_PAD_PROBE_INFO_TYPE (info) &
482 GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM) {
483 GstQuery *query = GST_PAD_PROBE_INFO_QUERY (info);
484
485 switch (GST_QUERY_TYPE (query)) {
486 /* bypass the capsfilter */
487 case GST_QUERY_CAPS:
488 case GST_QUERY_ACCEPT_CAPS:{
489 GstPad *pad = gst_element_get_static_pad (vf->transformation, "sink");
490 if (gst_pad_query (pad, query)) {
491 gst_object_unref (pad);
492 return GST_PAD_PROBE_HANDLED;
493 } else {
494 gst_object_unref (pad);
495 return GST_PAD_PROBE_DROP;
496 }
497 }
498 default:
499 break;
500 }
501 }
502
503 return GST_PAD_PROBE_OK;
504 }
505
506 static GstPadProbeReturn
_trans_src_probe(GstPad * pad,GstPadProbeInfo * info,gpointer user_data)507 _trans_src_probe (GstPad * pad, GstPadProbeInfo * info, gpointer user_data)
508 {
509 GstGLVideoFlip *vf = GST_GL_VIDEO_FLIP (user_data);
510
511 if (GST_PAD_PROBE_INFO_TYPE (info) & GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM) {
512 GstQuery *query = GST_PAD_PROBE_INFO_QUERY (info);
513
514 switch (GST_QUERY_TYPE (query)) {
515 /* bypass the capsfilter */
516 case GST_QUERY_CAPS:
517 case GST_QUERY_ACCEPT_CAPS:{
518 if (gst_pad_peer_query (vf->srcpad, query))
519 return GST_PAD_PROBE_HANDLED;
520 else
521 return GST_PAD_PROBE_DROP;
522 }
523 default:
524 break;
525 }
526 }
527
528 return GST_PAD_PROBE_OK;
529 }
530