1 /*
2 * GStreamer
3 * Copyright (C) 2009 Julien Isorce <julien.isorce@mail.com>
4 * Copyright (C) 2014 Jan Schmidt <jan@centricular.com>
5 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
21 */
22
23 /**
24 * SECTION:gstglviewconvert
25 * @title: GstGLViewConvert
26 * @short_description: convert between steroscopic/multiview video formats
27 * @see_also: #GstGLColorConvert, #GstGLContext
28 *
29 * Convert stereoscopic/multiview video using fragment shaders.
30 */
31
32 #ifdef HAVE_CONFIG_H
33 #include "config.h"
34 #endif
35
36 #include "gstglviewconvert.h"
37
38 #include <gst/video/gstvideoaffinetransformationmeta.h>
39
40 #include "gl.h"
41 #include "gstglsl_private.h"
42 #include "gstglutils_private.h"
43
44 #define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
45 #define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
46 #define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
47 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
48 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
49
50 static GstStaticCaps caps_template =
51 GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
52 "format = (string) RGBA, "
53 "width = " GST_VIDEO_SIZE_RANGE ", "
54 "height = " GST_VIDEO_SIZE_RANGE ", "
55 "framerate = " GST_VIDEO_FPS_RANGE ", "
56 "texture-target = (string) { 2D, rectangle, external-oes } ");
57
58 #define GST_CAT_DEFAULT gst_gl_view_convert_debug
59 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
60
61 enum
62 {
63 PROP_0,
64 PROP_INPUT_LAYOUT,
65 PROP_INPUT_FLAGS,
66 PROP_OUTPUT_LAYOUT,
67 PROP_OUTPUT_FLAGS,
68 PROP_OUTPUT_DOWNMIX_MODE
69 };
70
71 #define DEFAULT_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS
72
73 struct _GstGLViewConvertPrivate
74 {
75 gboolean result;
76
77 GstVideoMultiviewMode input_mode;
78 GstVideoMultiviewFlags input_flags;
79 GstVideoMultiviewMode output_mode;
80 GstVideoMultiviewFlags output_flags;
81
82 GstBuffer *primary_in;
83 GstBuffer *auxilliary_in;
84
85 GstBuffer *primary_out;
86 GstBuffer *auxilliary_out;
87
88 GstGLMemory *in_tex[GST_VIDEO_MAX_PLANES];
89 GstGLMemory *out_tex[GST_VIDEO_MAX_PLANES];
90 guint n_out_tex;
91
92 GLuint vao;
93 GLuint vertex_buffer;
94 GLuint vbo_indices;
95 GLuint attr_position;
96 GLuint attr_texture;
97 };
98
99 #define DEBUG_INIT \
100 GST_DEBUG_CATEGORY_INIT (gst_gl_view_convert_debug, "glviewconvert", 0, "glviewconvert object");
101
102 G_DEFINE_TYPE_WITH_CODE (GstGLViewConvert, gst_gl_view_convert,
103 GST_TYPE_OBJECT, G_ADD_PRIVATE (GstGLViewConvert) DEBUG_INIT);
104
105 static void gst_gl_view_convert_set_property (GObject * object,
106 guint prop_id, const GValue * value, GParamSpec * pspec);
107 static void gst_gl_view_convert_get_property (GObject * object,
108 guint prop_id, GValue * value, GParamSpec * pspec);
109 static void gst_gl_view_convert_finalize (GObject * object);
110
111 static void _do_view_convert (GstGLContext * context,
112 GstGLViewConvert * viewconvert);
113
114 /* *INDENT-OFF* */
115 /* These match the order and number of DOWNMIX_ANAGLYPH_* modes */
116 static GLfloat downmix_matrices[][2][9] = {
117 { /* Green-Magenta Dubois */
118 {-0.062f, 0.284f, -0.015f, -0.158f, 0.668f, -0.027f, -0.039f, 0.143f, 0.021f},
119 {0.529f, -0.016f, 0.009f, 0.705f, -0.015f, 0.075f, 0.024f, -0.065f, 0.937f}
120 },
121 { /* Red-Cyan Dubois */
122 /* Source of this matrix: http://www.site.uottawa.ca/~edubois/anaglyph/LeastSquaresHowToPhotoshop.pdf */
123 {0.437f, -0.062f, -0.048f, 0.449f, -0.062f, -0.050f, 0.164f, -0.024f, -0.017f},
124 {-0.011f, 0.377f, -0.026f, -0.032f, 0.761f, -0.093f, -0.007f, 0.009f, 1.234f}
125 },
126 { /* Amber-blue Dubois */
127 {1.062f, -0.026f, -0.038f, -0.205f, 0.908f, -0.173f, 0.299f, 0.068f, 0.022f},
128 {-0.016f, 0.006f, 0.094f, -0.123f, 0.062f, 0.185f, -0.017f, -0.017f, 0.911f}
129 }
130 };
131
132 static gfloat identity_matrix[] = {
133 1.0f, 0.0f, 0.0f, 0.0f,
134 0.0f, 1.0f, 0.0f, 0.0f,
135 0.0f, 0.0f, 1.0f, 0.0f,
136 0.0f, 0.0f, 0.0f, 1.0f,
137 };
138 /* *INDENT-ON* */
139
140 #define glsl_OES_extension_string "#extension GL_OES_EGL_image_external : require \n"
141
142 /* *INDENT-OFF* */
143 static const gchar *fragment_header =
144 "uniform sampler2D tex_l;\n"
145 "uniform sampler2D tex_r;\n"
146 "uniform float width;\n"
147 "uniform float height;\n"
148 "uniform mat3 downmix[2];\n"
149 "uniform vec2 tex_scale[2];\n"
150 "uniform vec2 offsets[2];\n";
151
152 static const gchar *frag_input =
153 " vec2 l_tex = v_texcoord * tex_scale[0] + offsets[0];\n"
154 " vec2 r_tex = v_texcoord * tex_scale[1] + offsets[1];\n"
155 " l = texture2D(tex_l, l_tex).rgba;\n"
156 " r = texture2D(tex_r, r_tex).rgba;\n";
157
158 static const gchar *frag_output_downmix =
159 " vec3 lcol = l.rgb * l.a + vec3(1.0-l.a);\n"
160 " vec3 rcol = r.rgb * r.a + vec3(1.0-r.a);\n"
161 " if (l.a + r.a > 0.0) {\n"
162 " lcol = clamp (downmix[0] * lcol, 0.0, 1.0);\n"
163 " rcol = clamp (downmix[1] * rcol, 0.0, 1.0);\n"
164 " gl_FragColor = vec4 (lcol + rcol, 1.0);\n"
165 " } else {\n"
166 " gl_FragColor = vec4 (0.0);\n"
167 " }\n";
168
169 static const gchar *frag_output_left =
170 " gl_FragColor = l;\n";
171
172 static const gchar *frag_output_right =
173 " gl_FragColor = r;\n";
174
175 static const gchar *frag_output_side_by_side =
176 " if (v_texcoord.x < 0.5) {\n"
177 " gl_FragColor = l;\n"
178 " } else {\n"
179 " gl_FragColor = r;\n"
180 " };\n";
181
182 static const gchar *frag_output_top_bottom =
183 "if (v_texcoord.y < 0.5) {\n"
184 " gl_FragColor = l;\n"
185 "} else {\n"
186 " gl_FragColor = r;\n"
187 "};\n";
188
189 static const gchar *frag_output_column_interleaved =
190 "if (int(mod(l_tex.x * width, 2.0)) == 0) {\n"
191 " gl_FragColor = l;\n"
192 "} else {\n"
193 " gl_FragColor = r;\n"
194 "};\n";
195
196 static const gchar *frag_output_row_interleaved =
197 "if (int(mod(l_tex.y * height, 2.0)) == 0) {\n"
198 " gl_FragColor = l;\n"
199 "} else {\n"
200 " gl_FragColor = r;\n"
201 "};\n";
202
203 static const gchar *frag_output_checkerboard =
204 "if (int(mod(l_tex.x * width, 2.0)) == \n"
205 " int(mod(l_tex.y * height, 2.0))) {\n"
206 " gl_FragColor = l;\n"
207 "} else {\n"
208 " gl_FragColor = r;\n"
209 "};\n";
210
211 static const gchar *frag_output_separated =
212 "gl_FragData[0] = l;\n"
213 "gl_FragData[1] = r;\n";
214 /* *INDENT-ON* */
215
216 static const GLfloat vertices[] = {
217 1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
218 -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
219 -1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
220 1.0f, 1.0f, 0.0f, 1.0f, 1.0f
221 };
222
223 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
224
225 static void
gst_gl_view_convert_class_init(GstGLViewConvertClass * klass)226 gst_gl_view_convert_class_init (GstGLViewConvertClass * klass)
227 {
228 GObjectClass *gobject_class = (GObjectClass *) klass;
229
230 gobject_class->set_property = gst_gl_view_convert_set_property;
231 gobject_class->get_property = gst_gl_view_convert_get_property;
232 gobject_class->finalize = gst_gl_view_convert_finalize;
233
234 g_object_class_install_property (gobject_class, PROP_INPUT_LAYOUT,
235 g_param_spec_enum ("input-mode-override",
236 "Input Multiview Mode Override",
237 "Override any input information about multiview layout",
238 GST_TYPE_VIDEO_MULTIVIEW_MODE,
239 GST_VIDEO_MULTIVIEW_MODE_NONE,
240 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
241 g_object_class_install_property (gobject_class, PROP_INPUT_FLAGS,
242 g_param_spec_flags ("input-flags-override",
243 "Input Multiview Flags Override",
244 "Override any input information about multiview layout flags",
245 GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
246 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
247 g_object_class_install_property (gobject_class, PROP_OUTPUT_LAYOUT,
248 g_param_spec_enum ("output-mode-override",
249 "Output Multiview Mode Override",
250 "Override automatic output mode selection for multiview layout",
251 GST_TYPE_VIDEO_MULTIVIEW_MODE, GST_VIDEO_MULTIVIEW_MODE_NONE,
252 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
253 g_object_class_install_property (gobject_class, PROP_OUTPUT_FLAGS,
254 g_param_spec_flags ("output-flags-override",
255 "Output Multiview Flags Override",
256 "Override automatic negotiation for output multiview layout flags",
257 GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
258 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
259 g_object_class_install_property (gobject_class, PROP_OUTPUT_DOWNMIX_MODE,
260 g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output",
261 "Output anaglyph type to generate when downmixing to mono",
262 GST_TYPE_GL_STEREO_DOWNMIX, DEFAULT_DOWNMIX,
263 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
264 }
265
266 static void
gst_gl_view_convert_init(GstGLViewConvert * convert)267 gst_gl_view_convert_init (GstGLViewConvert * convert)
268 {
269 convert->priv = gst_gl_view_convert_get_instance_private (convert);
270
271 convert->shader = NULL;
272 convert->downmix_mode = DEFAULT_DOWNMIX;
273 convert->priv->input_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
274 convert->priv->input_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
275 convert->priv->output_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
276 convert->priv->output_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
277
278 convert->input_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
279 convert->input_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
280 convert->output_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
281 convert->output_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
282
283 gst_video_info_init (&convert->in_info);
284 gst_video_info_init (&convert->out_info);
285 }
286
287 static void
gst_gl_view_convert_finalize(GObject * object)288 gst_gl_view_convert_finalize (GObject * object)
289 {
290 GstGLViewConvert *viewconvert;
291
292 viewconvert = GST_GL_VIEW_CONVERT (object);
293
294 gst_gl_view_convert_reset (viewconvert);
295
296 gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
297 gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
298 gst_buffer_replace (&viewconvert->priv->primary_out, NULL);
299 gst_buffer_replace (&viewconvert->priv->auxilliary_out, NULL);
300
301 if (viewconvert->context) {
302 gst_object_unref (viewconvert->context);
303 viewconvert->context = NULL;
304 }
305
306 G_OBJECT_CLASS (gst_gl_view_convert_parent_class)->finalize (object);
307 }
308
309 /**
310 * gst_gl_view_convert_new:
311 *
312 * Returns: (transfer full): a new #GstGLViewConvert
313 *
314 * Since: 1.6
315 */
316 GstGLViewConvert *
gst_gl_view_convert_new(void)317 gst_gl_view_convert_new (void)
318 {
319 GstGLViewConvert *convert;
320
321 convert = g_object_new (GST_TYPE_GL_VIEW_CONVERT, NULL);
322 gst_object_ref_sink (convert);
323
324 return convert;
325 }
326
327 /**
328 * gst_gl_view_convert_set_context:
329 * @viewconvert: a #GstGLViewConvert
330 * @context: the #GstGLContext to set
331 *
332 * Set @context on @viewconvert
333 *
334 * Since: 1.6
335 */
336 void
gst_gl_view_convert_set_context(GstGLViewConvert * viewconvert,GstGLContext * context)337 gst_gl_view_convert_set_context (GstGLViewConvert * viewconvert,
338 GstGLContext * context)
339 {
340 g_return_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert));
341
342 if (gst_object_replace ((GstObject **) & viewconvert->context,
343 GST_OBJECT (context)))
344 gst_gl_view_convert_reset (viewconvert);
345 }
346
347 static gboolean
_view_convert_set_format(GstGLViewConvert * viewconvert,GstVideoInfo * in_info,GstGLTextureTarget from_target,GstVideoInfo * out_info,GstGLTextureTarget to_target)348 _view_convert_set_format (GstGLViewConvert * viewconvert,
349 GstVideoInfo * in_info, GstGLTextureTarget from_target,
350 GstVideoInfo * out_info, GstGLTextureTarget to_target)
351 {
352 gboolean passthrough;
353 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), FALSE);
354
355 if (gst_video_info_is_equal (in_info, &viewconvert->in_info) &&
356 gst_video_info_is_equal (out_info, &viewconvert->out_info) &&
357 viewconvert->from_texture_target == from_target &&
358 viewconvert->to_texture_target == to_target)
359 return TRUE;
360
361 if (GST_VIDEO_INFO_FORMAT (in_info) != GST_VIDEO_FORMAT_RGBA ||
362 GST_VIDEO_INFO_FORMAT (out_info) != GST_VIDEO_FORMAT_RGBA) {
363 GST_ERROR_OBJECT (viewconvert,
364 "Multiview conversion can currently only be performed on RGBA textures");
365 return FALSE;
366 }
367
368 passthrough = gst_video_info_is_equal (in_info, out_info) &&
369 from_target == to_target;
370
371 if (!passthrough && to_target != GST_GL_TEXTURE_TARGET_2D
372 && to_target != GST_GL_TEXTURE_TARGET_RECTANGLE)
373 return FALSE;
374
375 /* FIXME: Compare what changed and decide if we need a full reset or not */
376 GST_OBJECT_LOCK (viewconvert);
377 gst_gl_view_convert_reset (viewconvert);
378
379 viewconvert->in_info = *in_info;
380 viewconvert->out_info = *out_info;
381 viewconvert->from_texture_target = from_target;
382 viewconvert->to_texture_target = to_target;
383 viewconvert->caps_passthrough = passthrough;
384
385 gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
386 gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
387 gst_buffer_replace (&viewconvert->priv->primary_out, NULL);
388 gst_buffer_replace (&viewconvert->priv->auxilliary_out, NULL);
389 GST_OBJECT_UNLOCK (viewconvert);
390
391 return TRUE;
392 }
393
394 /**
395 * gst_gl_view_convert_set_caps:
396 * @viewconvert: a #GstGLViewConvert
397 * @in_caps: input #GstCaps
398 * @out_caps: output #GstCaps
399 *
400 * Initializes @viewconvert with the information required for conversion.
401 *
402 * Since: 1.6
403 */
404 gboolean
gst_gl_view_convert_set_caps(GstGLViewConvert * viewconvert,GstCaps * in_caps,GstCaps * out_caps)405 gst_gl_view_convert_set_caps (GstGLViewConvert * viewconvert,
406 GstCaps * in_caps, GstCaps * out_caps)
407 {
408 GstVideoInfo in_info, out_info;
409 GstCapsFeatures *in_features, *out_features;
410 GstGLTextureTarget from_target = GST_GL_TEXTURE_TARGET_2D;
411 GstGLTextureTarget to_target = GST_GL_TEXTURE_TARGET_2D;
412
413 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), FALSE);
414 g_return_val_if_fail (GST_IS_CAPS (in_caps), FALSE);
415 g_return_val_if_fail (GST_IS_CAPS (out_caps), FALSE);
416
417 GST_INFO_OBJECT (viewconvert,
418 "Configuring multiview conversion from caps %" GST_PTR_FORMAT
419 " to %" GST_PTR_FORMAT, in_caps, out_caps);
420
421 in_features = gst_caps_get_features (in_caps, 0);
422 out_features = gst_caps_get_features (out_caps, 0);
423
424 if (!gst_caps_features_contains (in_features,
425 GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
426 return FALSE;
427 if (!gst_caps_features_contains (out_features,
428 GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
429 return FALSE;
430
431 if (!gst_video_info_from_caps (&in_info, in_caps))
432 return FALSE;
433 if (!gst_video_info_from_caps (&out_info, out_caps))
434 return FALSE;
435
436 {
437 GstStructure *in_s = gst_caps_get_structure (in_caps, 0);
438 GstStructure *out_s = gst_caps_get_structure (out_caps, 0);
439
440 if (gst_structure_has_field_typed (in_s, "texture-target", G_TYPE_STRING)) {
441 from_target =
442 gst_gl_texture_target_from_string (gst_structure_get_string (in_s,
443 "texture-target"));
444 }
445
446 if (gst_structure_has_field_typed (out_s, "texture-target", G_TYPE_STRING)) {
447 to_target =
448 gst_gl_texture_target_from_string (gst_structure_get_string (out_s,
449 "texture-target"));
450 }
451
452 if (to_target == GST_GL_TEXTURE_TARGET_NONE
453 || from_target == GST_GL_TEXTURE_TARGET_NONE)
454 /* invalid caps */
455 return FALSE;
456 }
457
458 return _view_convert_set_format (viewconvert, &in_info, from_target,
459 &out_info, to_target);
460 }
461
462 /* Function that can halve the value
463 * of ints, fractions, int/fraction ranges and lists of ints/fractions */
464 static gboolean
_halve_value(GValue * out,const GValue * in_value)465 _halve_value (GValue * out, const GValue * in_value)
466 {
467 /* Fundamental fixed types first */
468 if (G_VALUE_HOLDS_INT (in_value)) {
469 g_value_init (out, G_TYPE_INT);
470 g_value_set_int (out, MAX (g_value_get_int (in_value) / 2, 1));
471 } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
472 gint num, den;
473 num = gst_value_get_fraction_numerator (in_value);
474 den = gst_value_get_fraction_denominator (in_value);
475 g_value_init (out, GST_TYPE_FRACTION);
476 /* Don't adjust 'infinite' fractions */
477 if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
478 /* FIXME - could do better approximation when den > G_MAXINT/2? */
479 den = den > G_MAXINT / 2 ? G_MAXINT : den * 2;
480 }
481 gst_value_set_fraction (out, num, den);
482 } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
483 gint range_min = gst_value_get_int_range_min (in_value);
484 gint range_max = gst_value_get_int_range_max (in_value);
485 gint range_step = gst_value_get_int_range_step (in_value);
486 g_value_init (out, GST_TYPE_INT_RANGE);
487 if (range_min != 1)
488 range_min = MAX (1, range_min / 2);
489 if (range_max != G_MAXINT)
490 range_max = MAX (1, range_max / 2);
491 gst_value_set_int_range_step (out, range_min,
492 range_max, MAX (1, range_step / 2));
493 } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
494 GValue min_out = G_VALUE_INIT;
495 GValue max_out = G_VALUE_INIT;
496 const GValue *range_min = gst_value_get_fraction_range_min (in_value);
497 const GValue *range_max = gst_value_get_fraction_range_max (in_value);
498 _halve_value (&min_out, range_min);
499 _halve_value (&max_out, range_max);
500 g_value_init (out, GST_TYPE_FRACTION_RANGE);
501 gst_value_set_fraction_range (out, &min_out, &max_out);
502 g_value_unset (&min_out);
503 g_value_unset (&max_out);
504 } else if (GST_VALUE_HOLDS_LIST (in_value)) {
505 gint i;
506 g_value_init (out, GST_TYPE_LIST);
507 for (i = 0; i < gst_value_list_get_size (in_value); i++) {
508 const GValue *entry;
509 GValue tmp = G_VALUE_INIT;
510
511 entry = gst_value_list_get_value (in_value, i);
512 /* Random list values might not be the right type */
513 if (!_halve_value (&tmp, entry))
514 goto fail;
515 gst_value_list_append_and_take_value (out, &tmp);
516 }
517 } else {
518 return FALSE;
519 }
520
521 return TRUE;
522 fail:
523 g_value_unset (out);
524 return FALSE;
525 }
526
527 static GstStructure *
_halve_structure_field(const GstStructure * in,const gchar * field_name)528 _halve_structure_field (const GstStructure * in, const gchar * field_name)
529 {
530 GstStructure *out;
531 const GValue *in_value = gst_structure_get_value (in, field_name);
532 GValue tmp = G_VALUE_INIT;
533
534 if (G_UNLIKELY (in_value == NULL))
535 return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
536
537 if (!_halve_value (&tmp, in_value))
538 return NULL;
539
540 out = gst_structure_copy (in);
541 gst_structure_set_value (out, field_name, &tmp);
542 g_value_unset (&tmp);
543
544 return out;
545 }
546
547 /* Function that can double the value
548 * of ints, fractions, int/fraction ranges and lists of ints/fractions */
549 static gboolean
_double_value(GValue * out,const GValue * in_value)550 _double_value (GValue * out, const GValue * in_value)
551 {
552 /* Fundamental fixed types first */
553 if (G_VALUE_HOLDS_INT (in_value)) {
554 gint n = g_value_get_int (in_value);
555 g_value_init (out, G_TYPE_INT);
556 if (n <= G_MAXINT / 2)
557 g_value_set_int (out, n * 2);
558 else
559 g_value_set_int (out, G_MAXINT);
560 } else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
561 gint num, den;
562 num = gst_value_get_fraction_numerator (in_value);
563 den = gst_value_get_fraction_denominator (in_value);
564 g_value_init (out, GST_TYPE_FRACTION);
565 /* Don't adjust 'infinite' fractions */
566 if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
567 /* FIXME - could do better approximation when num > G_MAXINT/2? */
568 num = num > G_MAXINT / 2 ? G_MAXINT : num * 2;
569 }
570 gst_value_set_fraction (out, num, den);
571 } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
572 gint range_min = gst_value_get_int_range_min (in_value);
573 gint range_max = gst_value_get_int_range_max (in_value);
574 gint range_step = gst_value_get_int_range_step (in_value);
575 if (range_min != 1) {
576 range_min = MIN (G_MAXINT / 2, range_min);
577 range_min *= 2;
578 }
579 if (range_max != G_MAXINT) {
580 range_max = MIN (G_MAXINT / 2, range_max);
581 range_max *= 2;
582 }
583 range_step = MIN (G_MAXINT / 2, range_step);
584 g_value_init (out, GST_TYPE_INT_RANGE);
585 gst_value_set_int_range_step (out, range_min, range_max, range_step);
586 } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
587 GValue min_out = G_VALUE_INIT;
588 GValue max_out = G_VALUE_INIT;
589 const GValue *range_min = gst_value_get_fraction_range_min (in_value);
590 const GValue *range_max = gst_value_get_fraction_range_max (in_value);
591 _double_value (&min_out, range_min);
592 _double_value (&max_out, range_max);
593 g_value_init (out, GST_TYPE_FRACTION_RANGE);
594 gst_value_set_fraction_range (out, &min_out, &max_out);
595 g_value_unset (&min_out);
596 g_value_unset (&max_out);
597 } else if (GST_VALUE_HOLDS_LIST (in_value)) {
598 gint i;
599 g_value_init (out, GST_TYPE_LIST);
600 for (i = 0; i < gst_value_list_get_size (in_value); i++) {
601 const GValue *entry;
602 GValue tmp = G_VALUE_INIT;
603
604 entry = gst_value_list_get_value (in_value, i);
605 /* Random list values might not be the right type */
606 if (!_double_value (&tmp, entry))
607 goto fail;
608 gst_value_list_append_and_take_value (out, &tmp);
609 }
610 } else {
611 return FALSE;
612 }
613
614 return TRUE;
615 fail:
616 g_value_unset (out);
617 return FALSE;
618 }
619
620 static GstStructure *
_double_structure_field(const GstStructure * in,const gchar * field_name)621 _double_structure_field (const GstStructure * in, const gchar * field_name)
622 {
623 GstStructure *out;
624 const GValue *in_value = gst_structure_get_value (in, field_name);
625 GValue tmp = G_VALUE_INIT;
626
627 if (G_UNLIKELY (in_value == NULL))
628 return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
629
630 if (!_double_value (&tmp, in_value))
631 return NULL;
632
633 out = gst_structure_copy (in);
634 gst_structure_set_value (out, field_name, &tmp);
635 g_value_unset (&tmp);
636
637 return out;
638 }
639
640 /* Return a copy of the caps with the requested field halved in value/range */
641 #if 0
642 static GstCaps *
643 _halve_caps_field (const GstCaps * in, const gchar * field_name)
644 {
645 gint i;
646 GstCaps *out = gst_caps_new_empty ();
647
648 for (i = 0; i < gst_caps_get_size (in); i++) {
649 const GstStructure *cur = gst_caps_get_structure (in, i);
650 GstCapsFeatures *f = gst_caps_get_features (in, i);
651
652 GstStructure *res = _halve_structure_field (cur, field_name);
653 out =
654 gst_caps_merge_structure_full (out, res,
655 f ? gst_caps_features_copy (f) : NULL);
656 }
657
658 return out;
659 }
660 #endif
661
662 /* Return a copy of the caps with the requested field doubled in value/range */
663 static GstCaps *
_double_caps_field(const GstCaps * in,const gchar * field_name)664 _double_caps_field (const GstCaps * in, const gchar * field_name)
665 {
666 gint i;
667 GstCaps *out = gst_caps_new_empty ();
668
669 for (i = 0; i < gst_caps_get_size (in); i++) {
670 const GstStructure *cur = gst_caps_get_structure (in, i);
671 GstCapsFeatures *f = gst_caps_get_features (in, i);
672
673 GstStructure *res = _double_structure_field (cur, field_name);
674 out =
675 gst_caps_merge_structure_full (out, res,
676 f ? gst_caps_features_copy (f) : NULL);
677 }
678
679 return out;
680 }
681
682 /* Takes ownership of the input caps */
683 static GstCaps *
_expand_par_for_half_aspect(GstCaps * in,gboolean vertical_half_aspect)684 _expand_par_for_half_aspect (GstCaps * in, gboolean vertical_half_aspect)
685 {
686
687 guint mview_flags, mview_flags_mask;
688 GstCaps *out;
689 GstStructure *tmp;
690
691 out = gst_caps_new_empty ();
692
693 while (gst_caps_get_size (in) > 0) {
694 GstStructure *s;
695 GstCapsFeatures *features;
696
697 features = gst_caps_get_features (in, 0);
698 if (features)
699 features = gst_caps_features_copy (features);
700
701 s = gst_caps_steal_structure (in, 0);
702
703 if (!gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
704 &mview_flags_mask)) {
705 gst_caps_append_structure_full (out, s, features);
706 continue;
707 }
708 /* If the input doesn't care about the half-aspect flag, allow current PAR in either variant */
709 if ((mview_flags_mask & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) == 0) {
710 gst_caps_append_structure_full (out, s, features);
711 continue;
712 }
713 if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
714 /* No par field, dont-care the half-aspect flag */
715 gst_structure_set (s, "multiview-flags",
716 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
717 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
718 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
719 gst_caps_append_structure_full (out, s, features);
720 continue;
721 }
722
723 /* Halve or double PAR base on inputs input specified. */
724
725 /* Append a copy with the half-aspect flag as-is */
726 tmp = gst_structure_copy (s);
727 out = gst_caps_merge_structure_full (out, tmp,
728 features ? gst_caps_features_copy (features) : NULL);
729
730 /* and then a copy inverted */
731 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
732 /* Input is half-aspect. Double/halve the PAR, clear the flag */
733 if (vertical_half_aspect)
734 tmp = _halve_structure_field (s, "pixel-aspect-ratio");
735 else
736 tmp = _double_structure_field (s, "pixel-aspect-ratio");
737 /* Clear the flag */
738 gst_structure_set (tmp, "multiview-flags",
739 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
740 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
741 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
742 } else {
743 if (vertical_half_aspect)
744 tmp = _double_structure_field (s, "pixel-aspect-ratio");
745 else
746 tmp = _halve_structure_field (s, "pixel-aspect-ratio");
747 /* Set the flag */
748 gst_structure_set (tmp, "multiview-flags",
749 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
750 mview_flags | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
751 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
752 }
753
754 out = gst_caps_merge_structure_full (out, tmp,
755 features ? gst_caps_features_copy (features) : NULL);
756
757 gst_structure_free (s);
758 if (features)
759 gst_caps_features_free (features);
760 }
761
762 gst_caps_unref (in);
763
764 return out;
765 }
766
767 /* If input supports top-bottom or row-interleaved, we may halve height to mono frames.
768 * If input supports left-right, checkerboard, quincunx or column-interleaved,
769 * we may halve width to mono frames.
770 * For output of top-bottom or row-interleaved, we may double the mono height
771 * For output of left-right, checkerboard, quincunx or column-interleaved,
772 * we may double the mono width.
773 * In all cases, if input has half-aspect and output does not, we may double the PAR
774 * And if input does *not* have half-aspect flag and output does not, we may halve the PAR
775 */
776 static GstCaps *
_expand_structure(GstGLViewConvert * viewconvert,GstCaps * out_caps,GstStructure * structure,GstCapsFeatures * features)777 _expand_structure (GstGLViewConvert * viewconvert,
778 GstCaps * out_caps, GstStructure * structure, GstCapsFeatures * features)
779 {
780 GstCaps *expanded_caps, *tmp;
781 GstCaps *mono_caps;
782 const gchar *default_mview_mode_str = NULL;
783 guint mview_flags, mview_flags_mask;
784 const GValue *in_modes;
785 gint i;
786
787 /* Empty caps to accumulate into */
788 expanded_caps = gst_caps_new_empty ();
789
790 /* First, set defaults if multiview flags are missing */
791 default_mview_mode_str =
792 gst_video_multiview_mode_to_caps_string (GST_VIDEO_MULTIVIEW_MODE_MONO);
793
794 mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
795 mview_flags_mask = GST_FLAG_SET_MASK_EXACT;
796
797 if (!gst_structure_has_field (structure, "multiview-mode")) {
798 gst_structure_set (structure,
799 "multiview-mode", G_TYPE_STRING, default_mview_mode_str, NULL);
800 }
801 if (!gst_structure_has_field (structure, "multiview-flags")) {
802 gst_structure_set (structure,
803 "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
804 mview_flags_mask, NULL);
805 } else {
806 gst_structure_get_flagset (structure, "multiview-flags",
807 &mview_flags, &mview_flags_mask);
808 }
809
810 in_modes = gst_structure_get_value (structure, "multiview-mode");
811 mono_caps = gst_caps_new_empty ();
812 if (gst_value_intersect (NULL, in_modes,
813 gst_video_multiview_get_mono_modes ())) {
814 GstStructure *new_struct = gst_structure_copy (structure);
815 gst_structure_set_value (new_struct, "multiview-mode",
816 gst_video_multiview_get_mono_modes ());
817 /* Half-aspect makes no sense for mono or unpacked, get rid of it */
818 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
819 gst_structure_set (new_struct, "multiview-flags",
820 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
821 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
822 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
823 }
824 gst_caps_append_structure_full (mono_caps, new_struct,
825 features ? gst_caps_features_copy (features) : NULL);
826 }
827 if (gst_value_intersect (NULL, in_modes,
828 gst_video_multiview_get_unpacked_modes ())) {
829 GstStructure *new_struct = gst_structure_copy (structure);
830
831 gst_structure_set_value (new_struct, "multiview-mode",
832 gst_video_multiview_get_mono_modes ());
833
834 /* Half-aspect makes no sense for mono or unpacked, get rid of it */
835 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
836 gst_structure_set (new_struct, "multiview-flags",
837 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
838 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
839 mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
840 }
841 gst_caps_append_structure_full (mono_caps, new_struct,
842 features ? gst_caps_features_copy (features) : NULL);
843 }
844
845 if (gst_value_intersect (NULL, in_modes,
846 gst_video_multiview_get_doubled_height_modes ())) {
847 /* Append mono formats with height halved */
848 GstStructure *new_struct = _halve_structure_field (structure, "height");
849 gst_structure_set_value (new_struct, "multiview-mode",
850 gst_video_multiview_get_mono_modes ());
851 /* Normalise the half-aspect flag away */
852 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
853 GstStructure *s =
854 _halve_structure_field (new_struct, "pixel-aspect-ratio");
855 gst_structure_set (structure, "multiview-flags",
856 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
857 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
858 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
859 gst_structure_free (new_struct);
860 new_struct = s;
861 }
862 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
863 features ? gst_caps_features_copy (features) : NULL);
864 }
865 if (gst_value_intersect (NULL, in_modes,
866 gst_video_multiview_get_doubled_width_modes ())) {
867 /* Append mono formats with width halved */
868 GstStructure *new_struct = _halve_structure_field (structure, "width");
869 gst_structure_set_value (new_struct, "multiview-mode",
870 gst_video_multiview_get_mono_modes ());
871 /* Normalise the half-aspect flag away */
872 if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
873 GstStructure *s =
874 _double_structure_field (new_struct, "pixel-aspect-ratio");
875 gst_structure_set (structure, "multiview-flags",
876 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
877 mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
878 mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
879 gst_structure_free (new_struct);
880 new_struct = s;
881 }
882 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
883 features ? gst_caps_features_copy (features) : NULL);
884 }
885 if (gst_value_intersect (NULL, in_modes,
886 gst_video_multiview_get_doubled_size_modes ())) {
887 /* Append checkerboard/doubled size formats with width & height halved */
888 GstStructure *new_struct_w = _halve_structure_field (structure, "width");
889 GstStructure *new_struct_wh =
890 _halve_structure_field (new_struct_w, "height");
891 gst_structure_free (new_struct_w);
892 gst_structure_set_value (new_struct_wh, "multiview-mode",
893 gst_video_multiview_get_mono_modes ());
894 mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct_wh,
895 features ? gst_caps_features_copy (features) : NULL);
896 }
897
898 /* Everything is normalised now, unset the flags we can change */
899 /* Remove the views field, as these are all 'mono' modes
900 * Need to do this before we expand caps back out to frame packed modes */
901 for (i = 0; i < gst_caps_get_size (mono_caps); i++) {
902 GstStructure *s = gst_caps_get_structure (mono_caps, i);
903 gst_structure_remove_fields (s, "views", NULL);
904 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
905 &mview_flags_mask)) {
906 /* Preserve only the half-aspect and mixed-mono flags, for now.
907 * The rest we can change */
908 mview_flags_mask &=
909 (GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT |
910 GST_VIDEO_MULTIVIEW_FLAGS_MIXED_MONO);
911 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
912 mview_flags, mview_flags_mask, NULL);
913 }
914 }
915
916 GST_TRACE_OBJECT (viewconvert,
917 "Collected single-view caps %" GST_PTR_FORMAT, mono_caps);
918 /* Put unpacked and mono modes first. We don't care about flags. Clear them */
919 tmp = gst_caps_copy (mono_caps);
920 for (i = 0; i < gst_caps_get_size (tmp); i++) {
921 GstStructure *s = gst_caps_get_structure (tmp, i);
922 gst_structure_remove_fields (s, "views", NULL);
923 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
924 &mview_flags_mask)) {
925 /* We can change any flags for mono modes - half-aspect and mixed-mono have no meaning */
926 mview_flags_mask = 0;
927 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
928 mview_flags, mview_flags_mask, NULL);
929 }
930 }
931 expanded_caps = gst_caps_merge (expanded_caps, tmp);
932
933 /* Unpacked output modes have 2 views, for now */
934 tmp = gst_caps_copy (mono_caps);
935 gst_caps_set_value (tmp, "multiview-mode",
936 gst_video_multiview_get_unpacked_modes ());
937 for (i = 0; i < gst_caps_get_size (tmp); i++) {
938 GstStructure *s = gst_caps_get_structure (tmp, i);
939 gst_structure_set (s, "views", G_TYPE_INT, 2, NULL);
940 if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
941 &mview_flags_mask)) {
942 /* We can change any flags for unpacked modes - half-aspect and mixed-mono have no meaning */
943 mview_flags_mask = 0;
944 gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
945 mview_flags, mview_flags_mask, NULL);
946 }
947 }
948 expanded_caps = gst_caps_merge (expanded_caps, tmp);
949
950 /* Double height output modes */
951 tmp = _double_caps_field (mono_caps, "height");
952 gst_caps_set_value (tmp, "multiview-mode",
953 gst_video_multiview_get_doubled_height_modes ());
954 tmp = _expand_par_for_half_aspect (tmp, TRUE);
955
956 expanded_caps = gst_caps_merge (expanded_caps, tmp);
957
958 /* Double width output modes */
959 tmp = _double_caps_field (mono_caps, "width");
960 gst_caps_set_value (tmp, "multiview-mode",
961 gst_video_multiview_get_doubled_width_modes ());
962 tmp = _expand_par_for_half_aspect (tmp, FALSE);
963
964 expanded_caps = gst_caps_merge (expanded_caps, tmp);
965
966 /* Double size output modes */
967 {
968 GstCaps *tmp_w = _double_caps_field (mono_caps, "width");
969 tmp = _double_caps_field (tmp_w, "height");
970 gst_caps_unref (tmp_w);
971 gst_caps_set_value (tmp, "multiview-mode",
972 gst_video_multiview_get_doubled_size_modes ());
973 expanded_caps = gst_caps_merge (expanded_caps, tmp);
974 }
975
976 /* We're done with the mono caps now */
977 gst_caps_unref (mono_caps);
978
979 GST_TRACE_OBJECT (viewconvert,
980 "expanded transform caps now %" GST_PTR_FORMAT, expanded_caps);
981
982 if (gst_caps_is_empty (expanded_caps)) {
983 gst_caps_unref (expanded_caps);
984 return out_caps;
985 }
986 /* Really, we can rescale - so at this point we can append full-range
987 * height/width/PAR as an unpreferred final option. */
988 tmp = gst_caps_copy (expanded_caps);
989 gst_caps_set_simple (tmp, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
990 "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL);
991
992 out_caps = gst_caps_merge (out_caps, expanded_caps);
993 out_caps = gst_caps_merge (out_caps, tmp);
994 return out_caps;
995 }
996
997 static GstCaps *
_intersect_with_mview_mode(GstCaps * caps,GstVideoMultiviewMode mode,GstVideoMultiviewFlags flags)998 _intersect_with_mview_mode (GstCaps * caps,
999 GstVideoMultiviewMode mode, GstVideoMultiviewFlags flags)
1000 {
1001 GstCaps *filter, *result;
1002 const gchar *caps_str;
1003
1004 caps_str = gst_video_multiview_mode_to_caps_string (mode);
1005
1006 filter = gst_caps_new_simple ("video/x-raw",
1007 "multiview-mode", G_TYPE_STRING,
1008 caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1009 GST_FLAG_SET_MASK_EXACT, NULL);
1010
1011 if (mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1012 mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
1013 gst_caps_set_simple (filter, "views", G_TYPE_INT, 2, NULL);
1014
1015 gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1016
1017 GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1018 " with caps %" GST_PTR_FORMAT, caps, filter);
1019
1020 result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1021 gst_caps_unref (filter);
1022 return result;
1023 }
1024
1025 static GstCaps *
_intersect_with_mview_modes(GstCaps * caps,const GValue * modes)1026 _intersect_with_mview_modes (GstCaps * caps, const GValue * modes)
1027 {
1028 GstCaps *filter, *result;
1029
1030 filter = gst_caps_new_empty_simple ("video/x-raw");
1031
1032 gst_caps_set_value (filter, "multiview-mode", modes);
1033 gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
1034
1035 GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
1036 " with caps %" GST_PTR_FORMAT, caps, filter);
1037
1038 result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
1039 gst_caps_unref (filter);
1040 return result;
1041 }
1042
1043 /**
1044 * gst_gl_view_convert_transform_caps:
1045 * @viewconvert: a #GstGLViewConvert
1046 * @direction: a #GstPadDirection
1047 * @caps: (transfer none): the #GstCaps to transform
1048 * @filter: (transfer none): a set of filter #GstCaps
1049 *
1050 * Provides an implementation of #GstBaseTransformClass.transform_caps()
1051 *
1052 * Returns: (transfer full): the converted #GstCaps
1053 *
1054 * Since: 1.6
1055 */
1056 GstCaps *
gst_gl_view_convert_transform_caps(GstGLViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * filter)1057 gst_gl_view_convert_transform_caps (GstGLViewConvert * viewconvert,
1058 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
1059 {
1060 gint i, n;
1061 GstCaps *base_caps = gst_static_caps_get (&caps_template);
1062 GstCaps *out_caps, *tmp_caps;
1063
1064 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), NULL);
1065
1066 GST_DEBUG_OBJECT (viewconvert, "Direction %s "
1067 "input caps %" GST_PTR_FORMAT " filter %" GST_PTR_FORMAT,
1068 direction == GST_PAD_SINK ? "sink" : "src", caps, filter);
1069
1070 /* We can only process GLmemory RGBA caps, start from that */
1071 caps = gst_caps_intersect (caps, base_caps);
1072 gst_caps_unref (base_caps);
1073
1074 /* Change input/output to the formats we can convert to/from,
1075 * but keep the original caps at the start - we will always prefer
1076 * passthrough */
1077 if (direction == GST_PAD_SINK) {
1078 out_caps = gst_caps_copy (caps);
1079 if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1080 GstVideoMultiviewMode mode = viewconvert->input_mode_override;
1081 GstVideoMultiviewFlags flags = viewconvert->input_flags_override;
1082
1083 const gchar *caps_str = gst_video_multiview_mode_to_caps_string (mode);
1084 /* Coerce the input caps before transforming, so the sizes come out right */
1085 gst_caps_set_simple (out_caps, "multiview-mode", G_TYPE_STRING,
1086 caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
1087 GST_FLAG_SET_MASK_EXACT, NULL);
1088 }
1089 } else {
1090 out_caps = gst_caps_new_empty ();
1091 }
1092
1093 for (i = 0; i < gst_caps_get_size (caps); i++) {
1094 GstStructure *structure = gst_caps_get_structure (caps, i);
1095 GstCapsFeatures *features = gst_caps_get_features (caps, i);
1096 out_caps = _expand_structure (viewconvert, out_caps, structure, features);
1097 }
1098
1099 if (gst_caps_is_empty (out_caps))
1100 goto out;
1101
1102 /* If we have an output mode override, limit things to that */
1103 if (direction == GST_PAD_SINK &&
1104 viewconvert->output_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1105
1106 tmp_caps = _intersect_with_mview_mode (out_caps,
1107 viewconvert->output_mode_override, viewconvert->output_flags_override);
1108
1109 gst_caps_unref (out_caps);
1110 out_caps = tmp_caps;
1111 } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1112 /* Prepend a copy of our preferred input caps in case the peer
1113 * can handle them */
1114 tmp_caps = _intersect_with_mview_mode (out_caps,
1115 viewconvert->input_mode_override, viewconvert->input_flags_override);
1116 out_caps = gst_caps_merge (out_caps, tmp_caps);
1117 }
1118 if (direction == GST_PAD_SRC) {
1119 GstStructure *s;
1120 /* When generating input caps, we also need a copy of the mono caps
1121 * without multiview-mode or flags for backwards compat, at the end */
1122 tmp_caps = _intersect_with_mview_mode (caps,
1123 GST_VIDEO_MULTIVIEW_MODE_MONO, GST_VIDEO_MULTIVIEW_FLAGS_NONE);
1124 if (!gst_caps_is_empty (tmp_caps)) {
1125 s = gst_caps_get_structure (tmp_caps, 0);
1126 gst_structure_remove_fields (s, "multiview-mode", "multiview-flags",
1127 NULL);
1128 out_caps = gst_caps_merge (out_caps, tmp_caps);
1129 } else
1130 gst_caps_unref (tmp_caps);
1131 }
1132 out:
1133 gst_caps_unref (caps);
1134
1135 n = gst_caps_get_size (out_caps);
1136 for (i = 0; i < n; i++) {
1137 GstStructure *s = gst_caps_get_structure (out_caps, i);
1138
1139 gst_structure_remove_fields (s, "texture-target", NULL);
1140 }
1141
1142 GST_DEBUG_OBJECT (viewconvert, "Returning caps %" GST_PTR_FORMAT, out_caps);
1143 return out_caps;
1144 }
1145
1146 static guint
_get_target_bitmask_from_g_value(const GValue * targets)1147 _get_target_bitmask_from_g_value (const GValue * targets)
1148 {
1149 guint new_targets = 0;
1150
1151 if (targets == NULL) {
1152 new_targets = 1 << GST_GL_TEXTURE_TARGET_2D;
1153 } else if (G_TYPE_CHECK_VALUE_TYPE (targets, G_TYPE_STRING)) {
1154 GstGLTextureTarget target;
1155 const gchar *str;
1156
1157 str = g_value_get_string (targets);
1158 target = gst_gl_texture_target_from_string (str);
1159
1160 if (target)
1161 new_targets |= 1 << target;
1162 } else if (G_TYPE_CHECK_VALUE_TYPE (targets, GST_TYPE_LIST)) {
1163 gint j, m;
1164
1165 m = gst_value_list_get_size (targets);
1166 for (j = 0; j < m; j++) {
1167 const GValue *val = gst_value_list_get_value (targets, j);
1168 GstGLTextureTarget target;
1169 const gchar *str;
1170
1171 str = g_value_get_string (val);
1172 target = gst_gl_texture_target_from_string (str);
1173 if (target)
1174 new_targets |= 1 << target;
1175 }
1176 }
1177
1178 return new_targets;
1179 }
1180
1181 static GstCaps *
_fixate_texture_target(GstGLViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * other)1182 _fixate_texture_target (GstGLViewConvert * viewconvert,
1183 GstPadDirection direction, GstCaps * caps, GstCaps * other)
1184 {
1185 GValue item = G_VALUE_INIT;
1186 const GValue *targets, *other_targets;
1187 guint targets_mask = 0, other_targets_mask = 0, result_mask;
1188 GstStructure *s, *s_other;
1189
1190 other = gst_caps_make_writable (other);
1191 s = gst_caps_get_structure (caps, 0);
1192 s_other = gst_caps_get_structure (other, 0);
1193
1194 other_targets = gst_structure_get_value (s_other, "texture-target");
1195 targets = gst_structure_get_value (s, "texture-target");
1196
1197 targets_mask = _get_target_bitmask_from_g_value (targets);
1198 other_targets_mask = _get_target_bitmask_from_g_value (other_targets);
1199
1200 result_mask = targets_mask & other_targets_mask;
1201 if (result_mask == 0) {
1202 /* nothing we can do here */
1203 return gst_caps_fixate (other);
1204 }
1205
1206 if (direction == GST_PAD_SINK) {
1207 result_mask &=
1208 (1 << GST_GL_TEXTURE_TARGET_2D | 1 << GST_GL_TEXTURE_TARGET_RECTANGLE);
1209 } else {
1210 /* if the src caps has 2D support we can 'convert' to anything */
1211 if (targets_mask & (1 << GST_GL_TEXTURE_TARGET_2D))
1212 result_mask = -1;
1213 else
1214 result_mask = other_targets_mask;
1215 }
1216
1217 g_value_init (&item, G_TYPE_STRING);
1218 if (result_mask & (1 << GST_GL_TEXTURE_TARGET_2D)) {
1219 g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_2D_STR);
1220 } else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_RECTANGLE)) {
1221 g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_RECTANGLE_STR);
1222 } else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES)) {
1223 g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_EXTERNAL_OES_STR);
1224 }
1225
1226 gst_structure_set_value (s_other, "texture-target", &item);
1227
1228 g_value_unset (&item);
1229
1230 return gst_caps_fixate (other);
1231 }
1232
1233 /**
1234 * gst_gl_view_convert_fixate_caps:
1235 * @viewconvert: a #GstGLViewConvert
1236 * @direction: a #GstPadDirection
1237 * @caps: (transfer none): the #GstCaps of @direction
1238 * @othercaps: (transfer full): the #GstCaps to fixate
1239 *
1240 * Provides an implementation of #GstBaseTransformClass.fixate_caps()
1241 *
1242 * Returns: (transfer full): the fixated #GstCaps
1243 *
1244 * Since: 1.6
1245 */
1246 GstCaps *
gst_gl_view_convert_fixate_caps(GstGLViewConvert * viewconvert,GstPadDirection direction,GstCaps * caps,GstCaps * othercaps)1247 gst_gl_view_convert_fixate_caps (GstGLViewConvert * viewconvert,
1248 GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
1249 {
1250 GstVideoMultiviewMode mode = viewconvert->output_mode_override;
1251 GstVideoMultiviewFlags flags = viewconvert->output_flags_override;
1252 GstCaps *tmp;
1253
1254 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), NULL);
1255
1256 othercaps = gst_caps_make_writable (othercaps);
1257 GST_LOG_OBJECT (viewconvert, "dir %s fixating %" GST_PTR_FORMAT
1258 " against caps %" GST_PTR_FORMAT,
1259 direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1260
1261 if (direction == GST_PAD_SINK) {
1262 if (mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1263 /* We have a requested output mode and are fixating source caps, try and enforce it */
1264 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1265 gst_caps_unref (othercaps);
1266 othercaps = tmp;
1267 } else {
1268 /* See if we can do passthrough */
1269 GstVideoInfo info;
1270
1271 if (gst_video_info_from_caps (&info, caps)) {
1272 GstVideoMultiviewMode mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&info);
1273 GstVideoMultiviewFlags flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&info);
1274
1275 if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1276 mode = viewconvert->input_mode_override;
1277 flags = viewconvert->input_flags_override;
1278 }
1279
1280 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1281 if (gst_caps_is_empty (tmp)) {
1282 /* Nope, we can't pass our input caps downstream */
1283 gst_caps_unref (tmp);
1284 } else {
1285 gst_caps_unref (othercaps);
1286 othercaps = tmp;
1287 goto done;
1288 }
1289 }
1290
1291 /* Prefer an unpacked mode for output */
1292 tmp =
1293 _intersect_with_mview_modes (othercaps,
1294 gst_video_multiview_get_unpacked_modes ());
1295 if (!gst_caps_is_empty (tmp)) {
1296 gst_caps_unref (othercaps);
1297 othercaps = tmp;
1298 } else {
1299 gst_caps_unref (tmp);
1300 }
1301 }
1302 } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1303 /* See if we can coerce the caps into matching input mode/flags,
1304 * in case it doesn't care at all, but allow it not to too */
1305 mode = viewconvert->input_mode_override;
1306 flags = viewconvert->input_flags_override;
1307 tmp = _intersect_with_mview_mode (othercaps, mode, flags);
1308 if (gst_caps_is_empty (tmp)) {
1309 /* Nope, we can pass our input caps downstream */
1310 gst_caps_unref (tmp);
1311 } else {
1312 gst_caps_unref (othercaps);
1313 othercaps = tmp;
1314 }
1315 }
1316
1317 othercaps = _fixate_texture_target (viewconvert, direction, caps, othercaps);
1318
1319 done:
1320 GST_DEBUG_OBJECT (viewconvert, "dir %s fixated to %" GST_PTR_FORMAT
1321 " against caps %" GST_PTR_FORMAT,
1322 direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
1323 return othercaps;
1324 }
1325
1326 /**
1327 * gst_gl_view_convert_reset:
1328 * @viewconvert: a #GstGLViewConvert
1329 *
1330 * Reset @viewconvert to the default state. Further operation will require
1331 * setting the caps with gst_gl_view_convert_set_caps().
1332 *
1333 * Since: 1.6
1334 */
1335 void
gst_gl_view_convert_reset(GstGLViewConvert * viewconvert)1336 gst_gl_view_convert_reset (GstGLViewConvert * viewconvert)
1337 {
1338 g_return_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert));
1339 if (viewconvert->shader)
1340 gst_object_unref (viewconvert->shader);
1341 viewconvert->shader = NULL;
1342
1343 if (viewconvert->fbo)
1344 gst_object_unref (viewconvert->fbo);
1345 viewconvert->fbo = NULL;
1346
1347 viewconvert->initted = FALSE;
1348 viewconvert->reconfigure = FALSE;
1349 }
1350
1351 static void
gst_gl_view_convert_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)1352 gst_gl_view_convert_set_property (GObject * object, guint prop_id,
1353 const GValue * value, GParamSpec * pspec)
1354 {
1355 GstGLViewConvert *convert = GST_GL_VIEW_CONVERT (object);
1356 switch (prop_id) {
1357 case PROP_INPUT_LAYOUT:
1358 convert->input_mode_override = g_value_get_enum (value);
1359 break;
1360 case PROP_INPUT_FLAGS:
1361 convert->input_flags_override = g_value_get_flags (value);
1362 break;
1363 case PROP_OUTPUT_LAYOUT:
1364 convert->output_mode_override = g_value_get_enum (value);
1365 break;
1366 case PROP_OUTPUT_FLAGS:
1367 convert->output_flags_override = g_value_get_flags (value);
1368 break;
1369 case PROP_OUTPUT_DOWNMIX_MODE:
1370 convert->downmix_mode = g_value_get_enum (value);
1371 break;
1372 default:
1373 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1374 break;
1375 }
1376 GST_OBJECT_LOCK (convert);
1377 convert->reconfigure = TRUE;
1378 GST_OBJECT_UNLOCK (convert);
1379 }
1380
1381 static void
gst_gl_view_convert_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)1382 gst_gl_view_convert_get_property (GObject * object, guint prop_id,
1383 GValue * value, GParamSpec * pspec)
1384 {
1385 GstGLViewConvert *convert = GST_GL_VIEW_CONVERT (object);
1386 switch (prop_id) {
1387 case PROP_INPUT_LAYOUT:
1388 g_value_set_enum (value, convert->input_mode_override);
1389 break;
1390 case PROP_INPUT_FLAGS:
1391 g_value_set_flags (value, convert->input_flags_override);
1392 break;
1393 case PROP_OUTPUT_LAYOUT:
1394 g_value_set_enum (value, convert->output_mode_override);
1395 break;
1396 case PROP_OUTPUT_FLAGS:
1397 g_value_set_flags (value, convert->output_flags_override);
1398 break;
1399 case PROP_OUTPUT_DOWNMIX_MODE:
1400 g_value_set_enum (value, convert->downmix_mode);
1401 break;
1402 default:
1403 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1404 break;
1405 }
1406 }
1407
1408 /**
1409 * gst_gl_view_convert_perform:
1410 * @viewconvert: a #GstGLViewConvert
1411 * @inbuf: (transfer none): the #GstGLMemory filled #GstBuffer to convert
1412 *
1413 * Converts the data contained by @inbuf using the formats specified by the
1414 * #GstCaps passed to gst_gl_view_convert_set_caps()
1415 *
1416 * Returns: (transfer full): a converted #GstBuffer or %NULL
1417 *
1418 * Since: 1.6
1419 */
1420 GstBuffer *
gst_gl_view_convert_perform(GstGLViewConvert * viewconvert,GstBuffer * inbuf)1421 gst_gl_view_convert_perform (GstGLViewConvert * viewconvert, GstBuffer * inbuf)
1422 {
1423 GstBuffer *out;
1424
1425 if (gst_gl_view_convert_submit_input_buffer (viewconvert,
1426 GST_BUFFER_IS_DISCONT (inbuf), gst_buffer_ref (inbuf)) != GST_FLOW_OK)
1427 return NULL;
1428 if (gst_gl_view_convert_get_output (viewconvert, &out) != GST_FLOW_OK)
1429 return NULL;
1430
1431 return out;
1432 }
1433
1434 /* called by _init_convert (in the gl thread) */
1435 static gboolean
_init_view_convert_fbo(GstGLViewConvert * viewconvert)1436 _init_view_convert_fbo (GstGLViewConvert * viewconvert)
1437 {
1438 guint out_width, out_height;
1439
1440 out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info);
1441 out_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info);
1442
1443 viewconvert->fbo =
1444 gst_gl_framebuffer_new_with_default_depth (viewconvert->context,
1445 out_width, out_height);
1446
1447 return viewconvert->fbo != NULL;
1448 }
1449
1450 /* free after use */
1451 static gchar *
_get_shader_string(GstGLViewConvert * viewconvert,GstGLShader * shader,GstVideoMultiviewMode in_mode,GstVideoMultiviewMode out_mode,GstGLSLVersion version,GstGLSLProfile profile)1452 _get_shader_string (GstGLViewConvert * viewconvert, GstGLShader * shader,
1453 GstVideoMultiviewMode in_mode, GstVideoMultiviewMode out_mode,
1454 GstGLSLVersion version, GstGLSLProfile profile)
1455 {
1456 const gchar *input_str, *output_str;
1457 gboolean mono_input = FALSE;
1458 gchar *tmp, *tmp2;
1459 GString *str = g_string_new (NULL);
1460 guint n_outputs = 1;
1461
1462 switch (in_mode) {
1463 case GST_VIDEO_MULTIVIEW_MODE_NONE:
1464 case GST_VIDEO_MULTIVIEW_MODE_MONO:
1465 case GST_VIDEO_MULTIVIEW_MODE_LEFT:
1466 case GST_VIDEO_MULTIVIEW_MODE_RIGHT:
1467 mono_input = TRUE;
1468 /* Fall through */
1469 default:
1470 input_str = frag_input;
1471 break;
1472 }
1473
1474 switch (out_mode) {
1475 case GST_VIDEO_MULTIVIEW_MODE_LEFT:
1476 output_str = frag_output_left;
1477 break;
1478 case GST_VIDEO_MULTIVIEW_MODE_RIGHT:
1479 output_str = frag_output_right;
1480 break;
1481 case GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX:
1482 /* FIXME: implement properly with sub-sampling */
1483 case GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE:
1484 output_str = frag_output_side_by_side;
1485 break;
1486 case GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM:
1487 output_str = frag_output_top_bottom;
1488 break;
1489 case GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED:
1490 output_str = frag_output_column_interleaved;
1491 break;
1492 case GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED:
1493 output_str = frag_output_row_interleaved;
1494 break;
1495 case GST_VIDEO_MULTIVIEW_MODE_SEPARATED:
1496 case GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME:
1497 output_str = frag_output_separated;
1498 n_outputs = 2;
1499 break;
1500 case GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD:
1501 output_str = frag_output_checkerboard;
1502 break;
1503 case GST_VIDEO_MULTIVIEW_MODE_NONE:
1504 case GST_VIDEO_MULTIVIEW_MODE_MONO:
1505 default:
1506 if (mono_input)
1507 output_str = frag_output_left;
1508 else
1509 output_str = frag_output_downmix;
1510 break;
1511 }
1512
1513 if (viewconvert->from_texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
1514 g_string_append (str, glsl_OES_extension_string);
1515
1516 g_string_append (str,
1517 gst_gl_shader_string_get_highest_precision (viewconvert->context, version,
1518 profile));
1519 g_string_append (str, fragment_header);
1520
1521 /* GL 3.3+ and GL ES 3.x */
1522 if ((profile == GST_GLSL_PROFILE_CORE && version >= GST_GLSL_VERSION_330)
1523 || (profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)) {
1524 if (n_outputs > 1) {
1525 gint i;
1526
1527 for (i = 0; i < n_outputs; i++) {
1528 g_string_append_printf (str,
1529 "layout(location = %d) out vec4 fragColor_%d;\n", i, i);
1530 }
1531 } else {
1532 g_string_append (str, "layout (location = 0) out vec4 fragColor;\n");
1533 }
1534 } else if (profile == GST_GLSL_PROFILE_CORE
1535 && version >= GST_GLSL_VERSION_150) {
1536 /* no layout specifiers, use glBindFragDataLocation instead */
1537 if (n_outputs > 1) {
1538 gint i;
1539
1540 for (i = 0; i < n_outputs; i++) {
1541 gchar *var_name = g_strdup_printf ("fragColor_%d", i);
1542 g_string_append_printf (str, "out vec4 %s;\n", var_name);
1543 gst_gl_shader_bind_frag_data_location (shader, i, var_name);
1544 g_free (var_name);
1545 }
1546 } else {
1547 g_string_append (str, "out vec4 fragColor;\n");
1548 gst_gl_shader_bind_frag_data_location (shader, 0, "fragColor");
1549 }
1550 }
1551
1552 {
1553 const gchar *varying = NULL;
1554
1555 if ((profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)
1556 || (profile == GST_GLSL_PROFILE_CORE
1557 && version >= GST_GLSL_VERSION_150)) {
1558 varying = "in";
1559 } else {
1560 varying = "varying";
1561 }
1562 g_string_append_printf (str,
1563 "\n%s vec2 v_texcoord;\nvoid main() {\nvec4 l, r;\n", varying);
1564 }
1565
1566 g_string_append (str, input_str);
1567 g_string_append (str, output_str);
1568 g_string_append (str, "\n}");
1569 tmp = g_string_free (str, FALSE);
1570
1571 tmp2 =
1572 _gst_glsl_mangle_shader (tmp, GL_FRAGMENT_SHADER,
1573 GST_GL_TEXTURE_TARGET_2D, viewconvert->from_texture_target,
1574 viewconvert->context, &version, &profile);
1575
1576 return tmp2;
1577 }
1578
1579 static void
_bind_buffer(GstGLViewConvert * viewconvert)1580 _bind_buffer (GstGLViewConvert * viewconvert)
1581 {
1582 const GstGLFuncs *gl = viewconvert->context->gl_vtable;
1583 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, viewconvert->priv->vbo_indices);
1584 gl->BindBuffer (GL_ARRAY_BUFFER, viewconvert->priv->vertex_buffer);
1585 /* Load the vertex position */
1586 gl->VertexAttribPointer (viewconvert->priv->attr_position, 3, GL_FLOAT,
1587 GL_FALSE, 5 * sizeof (GLfloat), (void *) 0);
1588 /* Load the texture coordinate */
1589 gl->VertexAttribPointer (viewconvert->priv->attr_texture, 2, GL_FLOAT,
1590 GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
1591 gl->EnableVertexAttribArray (viewconvert->priv->attr_position);
1592 gl->EnableVertexAttribArray (viewconvert->priv->attr_texture);
1593 }
1594
1595 static void
_unbind_buffer(GstGLViewConvert * viewconvert)1596 _unbind_buffer (GstGLViewConvert * viewconvert)
1597 {
1598 const GstGLFuncs *gl = viewconvert->context->gl_vtable;
1599 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1600 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1601 gl->DisableVertexAttribArray (viewconvert->priv->attr_position);
1602 gl->DisableVertexAttribArray (viewconvert->priv->attr_texture);
1603 }
1604
1605 /* Called in the gl thread */
1606 static gboolean
_init_view_convert(GstGLViewConvert * viewconvert)1607 _init_view_convert (GstGLViewConvert * viewconvert)
1608 {
1609 GstGLViewConvertPrivate *priv = viewconvert->priv;
1610 GstVideoMultiviewMode in_mode = priv->input_mode;
1611 GstVideoMultiviewMode out_mode = priv->output_mode;
1612 GstVideoMultiviewFlags in_flags = priv->input_flags;
1613 GstVideoMultiviewFlags out_flags = priv->output_flags;
1614 gfloat tex_scale[2][2] = {
1615 {1., 1.},
1616 {1., 1.}
1617 };
1618 gfloat offsets[2][2] = {
1619 {0., 0.},
1620 {0., 0.}
1621 };
1622 gchar *fragment_source_str;
1623 GstGLFuncs *gl;
1624 gint l_index, r_index;
1625
1626 gl = viewconvert->context->gl_vtable;
1627 if (viewconvert->reconfigure)
1628 gst_gl_view_convert_reset (viewconvert);
1629 if (viewconvert->initted)
1630 return TRUE;
1631
1632 GST_LOG_OBJECT (viewconvert,
1633 "Initializing multiview conversion from %s mode %d flags 0x%x w %u h %u to "
1634 "%s mode %d flags 0x%x w %u h %u",
1635 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
1636 (&viewconvert->in_info)), in_mode, in_flags,
1637 viewconvert->in_info.width, viewconvert->in_info.height,
1638 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
1639 (&viewconvert->out_info)), out_mode, out_flags,
1640 viewconvert->out_info.width, viewconvert->out_info.height);
1641
1642 if (!gl->CreateProgramObject && !gl->CreateProgram) {
1643 GST_ERROR_OBJECT (viewconvert, "Cannot perform multiview conversion "
1644 "without OpenGL shaders");
1645 goto error;
1646 }
1647
1648 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED
1649 || out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1650 if (!gl->DrawBuffers) {
1651 GST_ERROR_OBJECT (viewconvert,
1652 "Separate texture output mode requested however the current "
1653 "OpenGL API does not support drawing to multiple buffers");
1654 goto error;
1655 }
1656 }
1657
1658 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
1659 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
1660 l_index = 0;
1661 r_index = 1;
1662 } else {
1663 GST_LOG_OBJECT (viewconvert, "Switching left/right views");
1664 /* Swap the views */
1665 l_index = 1;
1666 r_index = 0;
1667 }
1668
1669 if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */
1670 } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
1671 in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
1672 /* Side-by-side input */
1673 offsets[r_index][0] += 0.5 * tex_scale[r_index][0];
1674 tex_scale[0][0] *= 0.5f; /* Half horizontal scale */
1675 tex_scale[1][0] *= 0.5f;
1676 } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) { /* top-bottom */
1677 offsets[r_index][1] += 0.5 * tex_scale[r_index][1];
1678 tex_scale[0][1] *= 0.5f; /* Half vertical scale */
1679 tex_scale[1][1] *= 0.5f;
1680 }
1681
1682 /* Flipped is vertical, flopped is horizontal.
1683 * Adjust and offset per-view scaling. This needs to be done
1684 * after the input scaling already splits the views, before
1685 * adding any output scaling. */
1686 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED) !=
1687 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED)) {
1688 offsets[l_index][1] += tex_scale[l_index][1];
1689 tex_scale[l_index][1] *= -1.0;
1690 }
1691 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED) !=
1692 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED)) {
1693 offsets[l_index][0] += tex_scale[l_index][0];
1694 tex_scale[l_index][0] *= -1.0;
1695 }
1696 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED) !=
1697 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED)) {
1698 offsets[r_index][1] += tex_scale[r_index][1];
1699 tex_scale[r_index][1] *= -1.0;
1700 }
1701 if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED) !=
1702 (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED)) {
1703 offsets[r_index][0] += tex_scale[r_index][0];
1704 tex_scale[r_index][0] *= -1.0;
1705 }
1706
1707 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
1708 out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
1709 /* Side-by-Side */
1710 offsets[1][0] -= tex_scale[1][0];
1711 tex_scale[0][0] *= 2.0f;
1712 tex_scale[1][0] *= 2.0f;
1713 } else if (out_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) {
1714 offsets[1][1] -= tex_scale[1][1];
1715 tex_scale[0][1] *= 2.0f;
1716 tex_scale[1][1] *= 2.0f;
1717 }
1718
1719 GST_DEBUG_OBJECT (viewconvert,
1720 "Scaling matrix [ %f, %f ] [ %f %f]. Offsets [ %f, %f ] [ %f, %f ]",
1721 tex_scale[0][0], tex_scale[0][1],
1722 tex_scale[1][0], tex_scale[1][1],
1723 offsets[0][0], offsets[0][1], offsets[1][0], offsets[1][1]);
1724
1725 viewconvert->shader = gst_gl_shader_new (viewconvert->context);
1726 {
1727 GstGLSLVersion version;
1728 GstGLSLProfile profile;
1729 GstGLSLStage *vert, *frag;
1730 gchar *tmp, *tmp1, *version_str;
1731 const gchar *strings[2];
1732 GError *error = NULL;
1733
1734 tmp =
1735 _gst_glsl_mangle_shader
1736 (gst_gl_shader_string_vertex_mat4_vertex_transform, GL_VERTEX_SHADER,
1737 GST_GL_TEXTURE_TARGET_2D, viewconvert->from_texture_target,
1738 viewconvert->context, &version, &profile);
1739
1740 tmp1 = gst_glsl_version_profile_to_string (version, profile);
1741 version_str = g_strdup_printf ("#version %s\n", tmp1);
1742 g_free (tmp1);
1743 strings[0] = version_str;
1744
1745 strings[1] = tmp;
1746 vert =
1747 gst_glsl_stage_new_with_strings (viewconvert->context,
1748 GL_VERTEX_SHADER, version, profile, 2, strings);
1749 g_free (tmp);
1750
1751 if (!gst_gl_shader_compile_attach_stage (viewconvert->shader, vert, &error)) {
1752 GST_ERROR_OBJECT (viewconvert, "Failed to compile vertex stage %s",
1753 error->message);
1754 gst_object_unref (viewconvert->shader);
1755 viewconvert->shader = NULL;
1756 g_free (version_str);
1757 goto error;
1758 }
1759
1760 fragment_source_str = _get_shader_string (viewconvert, viewconvert->shader,
1761 in_mode, out_mode, version, profile);
1762 strings[1] = fragment_source_str;
1763
1764 frag =
1765 gst_glsl_stage_new_with_strings (viewconvert->context,
1766 GL_FRAGMENT_SHADER, version, profile, 2, strings);
1767 g_free (version_str);
1768
1769 if (!gst_gl_shader_compile_attach_stage (viewconvert->shader, frag, &error)) {
1770 GST_ERROR_OBJECT (viewconvert, "Failed to compile fragment stage %s",
1771 error->message);
1772 g_free (fragment_source_str);
1773 gst_object_unref (viewconvert->shader);
1774 viewconvert->shader = NULL;
1775 goto error;
1776 }
1777 g_free (fragment_source_str);
1778
1779 if (!gst_gl_shader_link (viewconvert->shader, &error)) {
1780 GST_ERROR_OBJECT (viewconvert, "Failed to link conversion shader %s",
1781 error->message);
1782 gst_object_unref (viewconvert->shader);
1783 viewconvert->shader = NULL;
1784 goto error;
1785 }
1786 }
1787
1788 viewconvert->priv->attr_position =
1789 gst_gl_shader_get_attribute_location (viewconvert->shader, "a_position");
1790 viewconvert->priv->attr_texture =
1791 gst_gl_shader_get_attribute_location (viewconvert->shader, "a_texcoord");
1792 gst_gl_shader_use (viewconvert->shader);
1793 gst_gl_shader_set_uniform_2fv (viewconvert->shader, "tex_scale",
1794 2, tex_scale[0]);
1795 gst_gl_shader_set_uniform_2fv (viewconvert->shader, "offsets", 2, offsets[0]);
1796 gst_gl_shader_set_uniform_1f (viewconvert->shader, "width",
1797 GST_VIDEO_INFO_WIDTH (&viewconvert->out_info));
1798 gst_gl_shader_set_uniform_1f (viewconvert->shader, "height",
1799 GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info));
1800 gst_gl_shader_set_uniform_matrix_3fv (viewconvert->shader, "downmix",
1801 2, FALSE, &downmix_matrices[viewconvert->downmix_mode][0][0]);
1802 gst_gl_shader_set_uniform_matrix_4fv (viewconvert->shader, "u_transformation",
1803 1, FALSE, identity_matrix);
1804 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1805 in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1806 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_l", l_index);
1807 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_r", r_index);
1808 } else {
1809 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_l", 0);
1810 gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_r", 0);
1811 }
1812 gst_gl_context_clear_shader (viewconvert->context);
1813 if (!_init_view_convert_fbo (viewconvert)) {
1814 goto error;
1815 }
1816
1817 if (!viewconvert->priv->vertex_buffer) {
1818 if (gl->GenVertexArrays) {
1819 gl->GenVertexArrays (1, &viewconvert->priv->vao);
1820 gl->BindVertexArray (viewconvert->priv->vao);
1821 }
1822
1823 gl->GenBuffers (1, &viewconvert->priv->vertex_buffer);
1824 gl->BindBuffer (GL_ARRAY_BUFFER, viewconvert->priv->vertex_buffer);
1825 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
1826 GL_STATIC_DRAW);
1827 gl->GenBuffers (1, &viewconvert->priv->vbo_indices);
1828 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, viewconvert->priv->vbo_indices);
1829 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
1830 GL_STATIC_DRAW);
1831 if (gl->GenVertexArrays) {
1832 _bind_buffer (viewconvert);
1833 gl->BindVertexArray (0);
1834 }
1835
1836 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1837 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1838 }
1839
1840 viewconvert->initted = TRUE;
1841 return TRUE;
1842 error:
1843 return FALSE;
1844 }
1845
1846 static gboolean
_do_view_convert_draw(GstGLContext * context,GstGLViewConvert * viewconvert)1847 _do_view_convert_draw (GstGLContext * context, GstGLViewConvert * viewconvert)
1848 {
1849 GstGLViewConvertPrivate *priv = viewconvert->priv;
1850 GstGLFuncs *gl;
1851 guint out_width, out_height;
1852 gint out_views, i;
1853 GLenum multipleRT[] = {
1854 GL_COLOR_ATTACHMENT0,
1855 GL_COLOR_ATTACHMENT1,
1856 GL_COLOR_ATTACHMENT2
1857 };
1858 GstVideoMultiviewMode in_mode = priv->input_mode;
1859 GstVideoMultiviewMode out_mode = priv->output_mode;
1860 guint from_gl_target =
1861 gst_gl_texture_target_to_gl (viewconvert->from_texture_target);
1862
1863 gl = context->gl_vtable;
1864
1865 gst_gl_framebuffer_bind (viewconvert->fbo);
1866
1867 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1868 out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1869 out_views = viewconvert->out_info.views;
1870 } else {
1871 out_views = 1;
1872 }
1873
1874 /* attach the texture to the FBO to renderer to */
1875 for (i = 0; i < out_views; i++) {
1876 GstGLBaseMemory *tex = (GstGLBaseMemory *) priv->out_tex[i];
1877
1878 gst_gl_framebuffer_attach (viewconvert->fbo, GL_COLOR_ATTACHMENT0 + i, tex);
1879 }
1880
1881 if (gl->DrawBuffers)
1882 gl->DrawBuffers (out_views, multipleRT);
1883 else if (gl->DrawBuffer)
1884 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
1885
1886 gst_gl_framebuffer_get_effective_dimensions (viewconvert->fbo, &out_width,
1887 &out_height);
1888 gl->Viewport (0, 0, out_width, out_height);
1889
1890 gst_gl_shader_use (viewconvert->shader);
1891
1892 /* FIXME: the auxillary buffer could have a different transform matrix */
1893 {
1894 GstVideoAffineTransformationMeta *af_meta;
1895 gfloat matrix[16];
1896
1897 af_meta =
1898 gst_buffer_get_video_affine_transformation_meta (priv->primary_in);
1899 gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix);
1900 gst_gl_shader_set_uniform_matrix_4fv (viewconvert->shader,
1901 "u_transformation", 1, FALSE, matrix);
1902 }
1903
1904 if (gl->BindVertexArray)
1905 gl->BindVertexArray (priv->vao);
1906 _bind_buffer (viewconvert);
1907
1908 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
1909 in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
1910 if (priv->in_tex[1] == NULL) {
1911 GST_ERROR_OBJECT (viewconvert,
1912 "No 2nd view available during conversion!");
1913 return FALSE;
1914 }
1915 gl->ActiveTexture (GL_TEXTURE1);
1916 gl->BindTexture (from_gl_target, priv->in_tex[1]->tex_id);
1917 }
1918
1919 gl->ActiveTexture (GL_TEXTURE0);
1920 gl->BindTexture (from_gl_target, priv->in_tex[0]->tex_id);
1921
1922 gl->ClearColor (0.0, 0.0, 0.0, 1.0);
1923 gl->Clear (GL_COLOR_BUFFER_BIT);
1924
1925 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, NULL);
1926
1927 if (gl->BindVertexArray)
1928 gl->BindVertexArray (0);
1929 else
1930 _unbind_buffer (viewconvert);
1931 if (gl->DrawBuffer)
1932 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
1933 /* we are done with the shader */
1934 gst_gl_context_clear_shader (context);
1935 gst_gl_context_clear_framebuffer (context);
1936
1937 return TRUE;
1938 }
1939
1940 static gboolean
_gen_buffer(GstGLViewConvert * viewconvert,GstBuffer ** target)1941 _gen_buffer (GstGLViewConvert * viewconvert, GstBuffer ** target)
1942 {
1943 GstGLVideoAllocationParams *params;
1944 GstGLMemoryAllocator *mem_allocator;
1945 GstAllocator *allocator;
1946
1947 *target = gst_buffer_new ();
1948
1949 allocator =
1950 GST_ALLOCATOR (gst_gl_memory_allocator_get_default
1951 (viewconvert->context));
1952 mem_allocator = GST_GL_MEMORY_ALLOCATOR (allocator);
1953 params = gst_gl_video_allocation_params_new (viewconvert->context, NULL,
1954 &viewconvert->out_info, 0, NULL, viewconvert->to_texture_target, 0);
1955
1956 if (!gst_gl_memory_setup_buffer (mem_allocator, *target, params, NULL, NULL,
1957 0)) {
1958 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
1959 gst_object_unref (allocator);
1960 return FALSE;
1961 }
1962 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
1963 gst_object_unref (allocator);
1964
1965 gst_buffer_add_video_meta_full (*target, 0,
1966 GST_VIDEO_INFO_FORMAT (&viewconvert->out_info),
1967 GST_VIDEO_INFO_WIDTH (&viewconvert->out_info),
1968 GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info),
1969 GST_VIDEO_INFO_N_PLANES (&viewconvert->out_info),
1970 viewconvert->out_info.offset, viewconvert->out_info.stride);
1971
1972 return TRUE;
1973 }
1974
1975 static void
_do_view_convert(GstGLContext * context,GstGLViewConvert * viewconvert)1976 _do_view_convert (GstGLContext * context, GstGLViewConvert * viewconvert)
1977 {
1978 GstGLViewConvertPrivate *priv = viewconvert->priv;
1979 guint in_width, in_height, out_width, out_height;
1980 GstMapInfo out_info[GST_VIDEO_MAX_PLANES], in_info[GST_VIDEO_MAX_PLANES];
1981 GstGLMemory *dest_tex[GST_VIDEO_MAX_PLANES];
1982 gboolean res = TRUE;
1983 gint i = 0, j = 0;
1984 gint in_views, out_views;
1985 GstVideoMultiviewMode in_mode;
1986 GstVideoMultiviewMode out_mode;
1987 GstGLSyncMeta *sync_meta;
1988
1989 out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info);
1990 out_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info);
1991 in_width = GST_VIDEO_INFO_WIDTH (&viewconvert->in_info);
1992 in_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->in_info);
1993
1994 g_return_if_fail (priv->primary_out == NULL);
1995 g_return_if_fail (priv->auxilliary_out == NULL);
1996
1997 in_mode = priv->input_mode;
1998 out_mode = priv->output_mode;
1999
2000 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
2001 in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
2002 in_views = viewconvert->in_info.views;
2003 else
2004 in_views = 1;
2005
2006 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
2007 out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
2008 out_views = viewconvert->out_info.views;
2009 else
2010 out_views = 1;
2011
2012 if (!_init_view_convert (viewconvert)) {
2013 priv->result = FALSE;
2014 return;
2015 }
2016
2017 if (!_gen_buffer (viewconvert, &priv->primary_out)) {
2018 GST_ERROR_OBJECT (viewconvert,
2019 "Failed to setup memory for primary output buffer");
2020 priv->result = FALSE;
2021 return;
2022 }
2023
2024 if (out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2025 if (!_gen_buffer (viewconvert, &priv->auxilliary_out)) {
2026 GST_ERROR_OBJECT (viewconvert,
2027 "Failed to setup memory for second view output buffer");
2028 priv->result = FALSE;
2029 return;
2030 }
2031 }
2032
2033 for (i = 0; i < in_views; i++) {
2034 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME && i > 0) {
2035 priv->in_tex[i] =
2036 (GstGLMemory *) gst_buffer_peek_memory (priv->auxilliary_in, 0);
2037 } else {
2038 priv->in_tex[i] =
2039 (GstGLMemory *) gst_buffer_peek_memory (priv->primary_in, i);
2040 }
2041 if (!gst_is_gl_memory ((GstMemory *) priv->in_tex[i])) {
2042 GST_ERROR_OBJECT (viewconvert, "input must be GstGLMemory");
2043 res = FALSE;
2044 goto out;
2045 }
2046 if (!gst_memory_map ((GstMemory *) priv->in_tex[i],
2047 &in_info[i], GST_MAP_READ | GST_MAP_GL)) {
2048 GST_ERROR_OBJECT (viewconvert, "failed to map input memory %p",
2049 priv->in_tex[i]);
2050 res = FALSE;
2051 goto out;
2052 }
2053 }
2054
2055 for (j = 0; j < out_views; j++) {
2056 GstGLMemory *out_tex;
2057 guint width, height;
2058 GstVideoInfo temp_info;
2059
2060 if (j > 0 && out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2061 dest_tex[j] = out_tex =
2062 (GstGLMemory *) gst_buffer_peek_memory (priv->auxilliary_out, 0);
2063 } else {
2064 dest_tex[j] = out_tex =
2065 (GstGLMemory *) gst_buffer_peek_memory (priv->primary_out, j);
2066 }
2067
2068 if (!gst_is_gl_memory ((GstMemory *) out_tex)) {
2069 GST_ERROR_OBJECT (viewconvert, "output must be GstGLMemory");
2070 res = FALSE;
2071 goto out;
2072 }
2073
2074 width = gst_gl_memory_get_texture_width (out_tex);
2075 height = gst_gl_memory_get_texture_height (out_tex);
2076 gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, width,
2077 height);
2078 if (out_tex->tex_format == GST_GL_LUMINANCE
2079 || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
2080 || out_width != width || out_height != height) {
2081 /* Luminance formats are not color renderable */
2082 /* renderering to a framebuffer only renders the intersection of all
2083 * the attachments i.e. the smallest attachment size */
2084 if (!priv->out_tex[j]) {
2085 GstGLVideoAllocationParams *params;
2086 GstGLBaseMemoryAllocator *base_mem_allocator;
2087 GstAllocator *allocator;
2088 GstVideoInfo temp_info;
2089
2090 gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, out_width,
2091 out_height);
2092
2093 allocator =
2094 GST_ALLOCATOR (gst_gl_memory_allocator_get_default (context));
2095 base_mem_allocator = GST_GL_BASE_MEMORY_ALLOCATOR (allocator);
2096 params = gst_gl_video_allocation_params_new (context, NULL, &temp_info,
2097 0, NULL, viewconvert->to_texture_target, GST_GL_RGBA);
2098
2099 priv->out_tex[j] =
2100 (GstGLMemory *) gst_gl_base_memory_alloc (base_mem_allocator,
2101 (GstGLAllocationParams *) params);
2102
2103 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
2104 gst_object_unref (allocator);
2105 }
2106 } else {
2107 priv->out_tex[j] = out_tex;
2108 }
2109
2110 if (!gst_memory_map ((GstMemory *) priv->out_tex[j],
2111 &out_info[j], GST_MAP_WRITE | GST_MAP_GL)) {
2112 GST_ERROR_OBJECT (viewconvert, "failed to map output memory %p",
2113 priv->out_tex[i]);
2114 res = FALSE;
2115 goto out;
2116 }
2117 }
2118 priv->n_out_tex = out_views;
2119
2120 if (priv->primary_in) {
2121 if ((sync_meta = gst_buffer_get_gl_sync_meta (priv->primary_in))) {
2122 gst_gl_sync_meta_wait (sync_meta, context);
2123 }
2124 }
2125
2126 if (priv->auxilliary_in) {
2127 if ((sync_meta = gst_buffer_get_gl_sync_meta (priv->auxilliary_in))) {
2128 gst_gl_sync_meta_wait (sync_meta, context);
2129 }
2130 }
2131
2132 GST_LOG_OBJECT (viewconvert, "multiview splitting to textures:%p,%p,%p,%p "
2133 "dimensions:%ux%u, from textures:%p,%p,%p,%p dimensions:%ux%u",
2134 priv->out_tex[0], priv->out_tex[1],
2135 priv->out_tex[2], priv->out_tex[3],
2136 out_width, out_height, priv->in_tex[0],
2137 priv->in_tex[1], priv->in_tex[2], priv->in_tex[3], in_width, in_height);
2138
2139 if (!_do_view_convert_draw (context, viewconvert))
2140 res = FALSE;
2141 out:
2142 for (j--; j >= 0; j--) {
2143 GstGLMemory *out_tex;
2144 guint width, height;
2145
2146 out_tex = dest_tex[j];
2147
2148 width = gst_gl_memory_get_texture_width (out_tex);
2149 height = gst_gl_memory_get_texture_height (out_tex);
2150
2151 gst_memory_unmap ((GstMemory *) priv->out_tex[j], &out_info[j]);
2152 if (out_tex != priv->out_tex[j]) {
2153 GstMapInfo to_info, from_info;
2154 if (!gst_memory_map ((GstMemory *) priv->out_tex[j],
2155 &from_info, GST_MAP_READ | GST_MAP_GL)) {
2156 GST_ERROR_OBJECT (viewconvert, "Failed to map intermediate memory");
2157 res = FALSE;
2158 continue;
2159 }
2160 if (!gst_memory_map ((GstMemory *) out_tex, &to_info,
2161 GST_MAP_WRITE | GST_MAP_GL)) {
2162 GST_ERROR_OBJECT (viewconvert, "Failed to map intermediate memory");
2163 res = FALSE;
2164 continue;
2165 }
2166 gst_gl_memory_copy_into (priv->out_tex[j], out_tex->tex_id,
2167 viewconvert->to_texture_target, out_tex->tex_format, width, height);
2168 gst_memory_unmap ((GstMemory *) out_tex, &to_info);
2169 }
2170
2171 priv->out_tex[j] = NULL;
2172 }
2173
2174 for (i--; i >= 0; i--) {
2175 gst_memory_unmap ((GstMemory *) priv->in_tex[i], &in_info[i]);
2176 }
2177
2178 if (!res) {
2179 gst_buffer_replace (&priv->primary_out, NULL);
2180 gst_buffer_replace (&priv->auxilliary_out, NULL);
2181 }
2182
2183 if (priv->primary_out) {
2184 if ((sync_meta = gst_buffer_add_gl_sync_meta (context, priv->primary_out)))
2185 gst_gl_sync_meta_set_sync_point (sync_meta, context);
2186 }
2187
2188 if (priv->auxilliary_out) {
2189 if ((sync_meta =
2190 gst_buffer_add_gl_sync_meta (context, priv->auxilliary_out)))
2191 gst_gl_sync_meta_set_sync_point (sync_meta, context);
2192 }
2193
2194 priv->result = res;
2195 return;
2196 }
2197
2198 /**
2199 * gst_gl_view_convert_submit_input_buffer:
2200 * @viewconvert: a #GstGLViewConvert
2201 * @is_discont: true if we have a discontinuity
2202 * @input: (transfer full): a #GstBuffer
2203 *
2204 * Submit @input to be processed by @viewconvert
2205 *
2206 * Returns: a #GstFlowReturn
2207 *
2208 * Since: 1.6
2209 */
2210 GstFlowReturn
gst_gl_view_convert_submit_input_buffer(GstGLViewConvert * viewconvert,gboolean is_discont,GstBuffer * input)2211 gst_gl_view_convert_submit_input_buffer (GstGLViewConvert * viewconvert,
2212 gboolean is_discont, GstBuffer * input)
2213 {
2214 GstFlowReturn ret = GST_FLOW_OK;
2215 GstVideoMultiviewMode mode;
2216 GstBuffer **target;
2217
2218 if (is_discont) {
2219 gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
2220 gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
2221 }
2222
2223 mode = viewconvert->input_mode_override;
2224 if (mode == GST_VIDEO_MULTIVIEW_MODE_NONE)
2225 mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->in_info);
2226
2227 target = &viewconvert->priv->primary_in;
2228
2229 /* For frame-by-frame mode, we need to collect the 2nd eye into
2230 * our auxilliary buffer */
2231 if (mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2232 if (!GST_BUFFER_FLAG_IS_SET (input, GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE))
2233 target = &viewconvert->priv->auxilliary_in;
2234 }
2235
2236 if (*target)
2237 gst_buffer_unref (*target);
2238 *target = input;
2239
2240 return ret;
2241 }
2242
2243 /**
2244 * gst_gl_view_convert_get_output:
2245 * @viewconvert: a #GstGLViewConvert
2246 * @outbuf_ptr: (out): a #GstBuffer
2247 *
2248 * Retrieve the processed output buffer placing the output in @outbuf_ptr.
2249 *
2250 * Returns: a #GstFlowReturn
2251 *
2252 * Since: 1.6
2253 */
2254 GstFlowReturn
gst_gl_view_convert_get_output(GstGLViewConvert * viewconvert,GstBuffer ** outbuf_ptr)2255 gst_gl_view_convert_get_output (GstGLViewConvert * viewconvert,
2256 GstBuffer ** outbuf_ptr)
2257 {
2258 GstGLViewConvertPrivate *priv = viewconvert->priv;
2259 GstBuffer *outbuf = NULL;
2260 GstFlowReturn ret = GST_FLOW_OK;
2261 GstVideoMultiviewMode in_mode, out_mode;
2262 GstVideoMultiviewFlags in_flags, out_flags;
2263
2264 g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), GST_FLOW_ERROR);
2265 g_return_val_if_fail (GST_IS_GL_CONTEXT (viewconvert->context),
2266 GST_FLOW_ERROR);
2267
2268 GST_OBJECT_LOCK (viewconvert);
2269
2270 /* See if a buffer is available already */
2271 if (priv->primary_out) {
2272 outbuf = viewconvert->priv->primary_out;
2273 priv->primary_out = NULL;
2274 goto done;
2275 }
2276 if (viewconvert->priv->auxilliary_out) {
2277 outbuf = priv->auxilliary_out;
2278 priv->auxilliary_out = NULL;
2279 goto done;
2280 }
2281
2282 /* Check prereqs before processing a new input buffer */
2283 if (priv->primary_in == NULL)
2284 goto done;
2285
2286 in_mode = viewconvert->input_mode_override;
2287 in_flags = viewconvert->input_flags_override;
2288 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
2289 in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->in_info);
2290 in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&viewconvert->in_info);
2291 }
2292
2293 /* Configured output mode already takes any override
2294 * into account */
2295 out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->out_info);
2296 out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&viewconvert->out_info);
2297
2298 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
2299 /* For frame-by-frame, we need 2 input buffers */
2300 if (priv->auxilliary_in == NULL) {
2301 GST_LOG_OBJECT (viewconvert,
2302 "Can't generate output yet - frame-by-frame mode");
2303 goto done;
2304 }
2305 }
2306
2307 /* Store the current conversion in the priv vars */
2308 priv->input_mode = in_mode;
2309 priv->input_flags = in_flags;
2310 priv->output_mode = out_mode;
2311 priv->output_flags = out_flags;
2312
2313 if (priv->input_mode == priv->output_mode &&
2314 priv->input_flags == priv->output_flags &&
2315 viewconvert->in_info.width == viewconvert->out_info.width &&
2316 viewconvert->in_info.height == viewconvert->out_info.height &&
2317 viewconvert->from_texture_target == viewconvert->to_texture_target) {
2318 /* passthrough - just pass input buffers */
2319 outbuf = gst_buffer_ref (priv->primary_in);
2320 if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
2321 priv->auxilliary_out = gst_buffer_ref (priv->auxilliary_in);
2322 goto done_clear_input;
2323 }
2324
2325 /* We can't output to OES textures, they're only supported for passthrough */
2326 if (viewconvert->to_texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) {
2327 ret = GST_FLOW_ERROR;
2328 goto done_clear_input;
2329 }
2330
2331 /* Generate new output buffer(s) */
2332 gst_gl_context_thread_add (viewconvert->context,
2333 (GstGLContextThreadFunc) _do_view_convert, viewconvert);
2334
2335 if (!priv->result) {
2336 if (priv->primary_out)
2337 gst_object_unref (priv->primary_out);
2338 if (priv->auxilliary_out)
2339 gst_object_unref (priv->auxilliary_out);
2340 priv->primary_out = NULL;
2341 priv->auxilliary_out = NULL;
2342 ret = GST_FLOW_ERROR;
2343 goto done_clear_input;
2344 }
2345
2346 outbuf = priv->primary_out;
2347 if (outbuf) {
2348 GstVideoOverlayCompositionMeta *composition_meta;
2349
2350 gst_buffer_copy_into (outbuf, priv->primary_in,
2351 GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1);
2352 GST_BUFFER_FLAG_SET (outbuf,
2353 GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE |
2354 GST_VIDEO_BUFFER_FLAG_MULTIPLE_VIEW);
2355
2356 composition_meta =
2357 gst_buffer_get_video_overlay_composition_meta (priv->primary_in);
2358 if (composition_meta) {
2359 GST_DEBUG ("found video overlay composition meta, applying on output.");
2360 gst_buffer_add_video_overlay_composition_meta
2361 (outbuf, composition_meta->overlay);
2362 }
2363 }
2364
2365 if (priv->auxilliary_out) {
2366 GstVideoOverlayCompositionMeta *composition_meta;
2367
2368 gst_buffer_copy_into (priv->auxilliary_out,
2369 priv->primary_out, GST_BUFFER_COPY_FLAGS, 0, -1);
2370 GST_BUFFER_FLAG_UNSET (priv->auxilliary_out,
2371 GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE);
2372
2373 composition_meta =
2374 gst_buffer_get_video_overlay_composition_meta (priv->primary_out);
2375 if (composition_meta) {
2376 GST_DEBUG ("found video overlay composition meta, applying on output.");
2377 gst_buffer_add_video_overlay_composition_meta
2378 (priv->auxilliary_out, composition_meta->overlay);
2379 }
2380 }
2381 priv->primary_out = NULL;
2382
2383 done_clear_input:
2384 /* Invalidate input buffers now they've been used */
2385 gst_buffer_replace (&priv->primary_in, NULL);
2386 gst_buffer_replace (&priv->auxilliary_in, NULL);
2387
2388 done:
2389 GST_OBJECT_UNLOCK (viewconvert);
2390 *outbuf_ptr = outbuf;
2391 return ret;
2392 }
2393
2394 #ifndef GST_REMOVE_DEPRECATED
2395 #ifdef GST_DISABLE_DEPRECATED
2396 GST_GL_API GType gst_gl_stereo_downmix_mode_get_type (void);
2397 #endif
2398
2399 GType
gst_gl_stereo_downmix_mode_get_type(void)2400 gst_gl_stereo_downmix_mode_get_type (void)
2401 {
2402 return gst_gl_stereo_downmix_get_type ();
2403 }
2404 #endif
2405