1 /*
2 * GStreamer
3 * Copyright (C) 2012-2014 Matthew Waters <ystree00@gmail.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 #ifdef HAVE_CONFIG_H
22 #include "config.h"
23 #endif
24
25 #include <string.h>
26 #include <stdio.h>
27
28 #include "gstglcolorconvert.h"
29
30 #include "gl.h"
31 #include "gstglfuncs.h"
32 #include "gstglsl_private.h"
33
34 /**
35 * SECTION:gstglcolorconvert
36 * @title: GstGLColorConvert
37 * @short_description: convert between video color spaces and formats
38 * @see_also: #GstGLUpload, #GstGLMemory, #GstGLBaseMemory
39 *
40 * #GstGLColorConvert is an object that converts between color spaces and/or
41 * formats using OpenGL Shaders.
42 *
43 * A #GstGLColorConvert can be created with gst_gl_color_convert_new(), the
44 * configuration negotiated with gst_gl_color_convert_transform_caps() and the
45 * conversion performed with gst_gl_color_convert_perform().
46 *
47 * The glcolorconvertelement provides a GStreamer element that uses
48 * #GstGLColorConvert to convert between video formats and color spaces.
49 */
50
51 #define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
52 #define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
53 #define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
54 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
55 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
56
57 static void _do_convert (GstGLContext * context, GstGLColorConvert * convert);
58 static gboolean _init_convert (GstGLColorConvert * convert);
59 static gboolean _init_convert_fbo (GstGLColorConvert * convert);
60 static GstBuffer *_gst_gl_color_convert_perform_unlocked (GstGLColorConvert *
61 convert, GstBuffer * inbuf);
62
63 static gboolean _do_convert_draw (GstGLContext * context,
64 GstGLColorConvert * convert);
65
66 /* *INDENT-OFF* */
67
68 #define YUV_TO_RGB_COEFFICIENTS \
69 "uniform vec3 offset;\n" \
70 "uniform vec3 coeff1;\n" \
71 "uniform vec3 coeff2;\n" \
72 "uniform vec3 coeff3;\n"
73
74 /* FIXME: use the colormatrix support from videoconvert */
75
76 /* BT. 601 standard with the following ranges:
77 * Y = [16..235] (of 255)
78 * Cb/Cr = [16..240] (of 255)
79 */
80 static const gfloat from_yuv_bt601_offset[] = {-0.0625f, -0.5f, -0.5f};
81 static const gfloat from_yuv_bt601_rcoeff[] = {1.164f, 0.000f, 1.596f};
82 static const gfloat from_yuv_bt601_gcoeff[] = {1.164f,-0.391f,-0.813f};
83 static const gfloat from_yuv_bt601_bcoeff[] = {1.164f, 2.018f, 0.000f};
84
85 /* BT. 709 standard with the following ranges:
86 * Y = [16..235] (of 255)
87 * Cb/Cr = [16..240] (of 255)
88 */
89 static const gfloat from_yuv_bt709_offset[] = {-0.0625f, -0.5f, -0.5f};
90 static const gfloat from_yuv_bt709_rcoeff[] = {1.164f, 0.000f, 1.787f};
91 static const gfloat from_yuv_bt709_gcoeff[] = {1.164f,-0.213f,-0.531f};
92 static const gfloat from_yuv_bt709_bcoeff[] = {1.164f,2.112f, 0.000f};
93
94 #define RGB_TO_YUV_COEFFICIENTS \
95 "uniform vec3 offset;\n" \
96 "uniform vec3 coeff1;\n" \
97 "uniform vec3 coeff2;\n" \
98 "uniform vec3 coeff3;\n"
99
100 /* Matrix inverses of the color matrices found above */
101 /* BT. 601 standard with the following ranges:
102 * Y = [16..235] (of 255)
103 * Cb/Cr = [16..240] (of 255)
104 */
105 static const gfloat from_rgb_bt601_offset[] = {0.0625f, 0.5f, 0.5f};
106 static const gfloat from_rgb_bt601_ycoeff[] = {0.256816f, 0.504154f, 0.0979137f};
107 static const gfloat from_rgb_bt601_ucoeff[] = {-0.148246f, -0.29102f, 0.439266f};
108 static const gfloat from_rgb_bt601_vcoeff[] = {0.439271f, -0.367833f, -0.071438f};
109
110 /* BT. 709 standard with the following ranges:
111 * Y = [16..235] (of 255)
112 * Cb/Cr = [16..240] (of 255)
113 */
114 static const gfloat from_rgb_bt709_offset[] = {0.0625f, 0.5f, 0.5f};
115 static const gfloat from_rgb_bt709_ycoeff[] = {0.182604f, 0.614526f, 0.061976f};
116 static const gfloat from_rgb_bt709_ucoeff[] = {-0.100640f, -0.338688f, 0.439327f};
117 static const gfloat from_rgb_bt709_vcoeff[] = {0.440654f, -0.400285f, -0.040370f};
118
119 /* GRAY16 to RGB conversion
120 * data transfered as GL_LUMINANCE_ALPHA then convert back to GRAY16
121 * high byte weight as : 255*256/65535
122 * ([0~1] denormalize to [0~255],shift to high byte,normalize to [0~1])
123 * low byte weight as : 255/65535 (similar)
124 * */
125 #define COMPOSE_WEIGHT \
126 "const vec2 compose_weight = vec2(0.996109, 0.003891);\n"
127
128 #define DEFAULT_UNIFORMS \
129 "uniform vec2 tex_scale0;\n" \
130 "uniform vec2 tex_scale1;\n" \
131 "uniform vec2 tex_scale2;\n" \
132 "uniform float width;\n" \
133 "uniform float height;\n" \
134 "uniform float poffset_x;\n" \
135 "uniform float poffset_y;\n"
136
137 #define MAX_FUNCTIONS 4
138
139 #define glsl_OES_extension_string "#extension GL_OES_EGL_image_external : require \n"
140
141 struct shader_templ
142 {
143 const gchar *extensions;
144 const gchar *uniforms;
145 const gchar *functions[MAX_FUNCTIONS];
146
147 GstGLTextureTarget target;
148 };
149
150 #define glsl_func_yuv_to_rgb \
151 "vec3 yuv_to_rgb (vec3 val, vec3 offset, vec3 ycoeff, vec3 ucoeff, vec3 vcoeff) {\n" \
152 " vec3 rgb;\n" \
153 " val += offset;\n" \
154 " rgb.r = dot(val, ycoeff);\n" \
155 " rgb.g = dot(val, ucoeff);\n" \
156 " rgb.b = dot(val, vcoeff);\n" \
157 " return rgb;\n" \
158 "}\n"
159
160 #define glsl_func_rgb_to_yuv \
161 "vec3 rgb_to_yuv (vec3 val, vec3 offset, vec3 rcoeff, vec3 gcoeff, vec3 bcoeff) {\n" \
162 " vec3 yuv;\n" \
163 " yuv.r = dot(val.rgb, rcoeff);\n" \
164 " yuv.g = dot(val.rgb, gcoeff);\n" \
165 " yuv.b = dot(val.rgb, bcoeff);\n" \
166 " yuv += offset;\n" \
167 " return yuv;\n" \
168 "}\n"
169
170 /* Channel reordering for XYZ <-> ZYX conversion */
171 static const gchar templ_REORDER_BODY[] =
172 "vec4 t = texture2D(tex, texcoord * tex_scale0);\n"
173 "%s\n" /* clobber alpha channel? */
174 "gl_FragColor = vec4(t.%c, t.%c, t.%c, t.%c);\n";
175
176 static const struct shader_templ templ_REORDER =
177 { NULL,
178 DEFAULT_UNIFORMS "uniform sampler2D tex;\n",
179 { NULL, },
180 GST_GL_TEXTURE_TARGET_2D
181 };
182
183 /* GRAY16 to RGB conversion
184 * data transfered as GL_LUMINANCE_ALPHA then convert back to GRAY16
185 * high byte weight as : 255*256/65535
186 * ([0~1] denormalize to [0~255],shift to high byte,normalize to [0~1])
187 * low byte weight as : 255/65535 (similar)
188 * */
189 static const gchar templ_COMPOSE_BODY[] =
190 "vec4 rgba;\n"
191 "vec4 t = texture2D(tex, texcoord * tex_scale0);\n"
192 "rgba.rgb = vec3 (dot(t.%c%c, compose_weight));"
193 "rgba.a = 1.0;\n"
194 "gl_FragColor = vec4(rgba.%c, rgba.%c, rgba.%c, rgba.%c);\n";
195
196 static const struct shader_templ templ_COMPOSE =
197 { NULL,
198 DEFAULT_UNIFORMS COMPOSE_WEIGHT "uniform sampler2D tex;\n",
199 { NULL, },
200 GST_GL_TEXTURE_TARGET_2D
201 };
202
203 static const gchar templ_AYUV_to_RGB_BODY[] =
204 "vec4 texel, rgba;\n"
205 "texel = texture2D(tex, texcoord * tex_scale0);\n"
206 "rgba.rgb = yuv_to_rgb (texel.yzw, offset, coeff1, coeff2, coeff3);\n"
207 "rgba.a = texel.r;\n"
208 "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
209
210 static const struct shader_templ templ_AYUV_to_RGB =
211 { NULL,
212 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D tex;\n",
213 { glsl_func_yuv_to_rgb, NULL, },
214 GST_GL_TEXTURE_TARGET_2D
215 };
216
217 static const gchar templ_RGB_to_AYUV_BODY[] =
218 "vec4 texel, ayuv;\n"
219 "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
220 "ayuv.yzw = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3);\n"
221 "ayuv.x = %s;\n"
222 "gl_FragColor = ayuv;\n";
223
224 static const struct shader_templ templ_RGB_to_AYUV =
225 { NULL,
226 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n",
227 { glsl_func_rgb_to_yuv, NULL, },
228 GST_GL_TEXTURE_TARGET_2D
229 };
230
231 static const gchar templ_VUYA_to_RGB_BODY[] =
232 "vec4 texel, rgba;\n"
233 "texel = texture2D(tex, texcoord * tex_scale0);\n"
234 "rgba.rgb = yuv_to_rgb (texel.zyx, offset, coeff1, coeff2, coeff3);\n"
235 "rgba.a = texel.w;\n"
236 "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
237
238 static const struct shader_templ templ_VUYA_to_RGB =
239 { NULL,
240 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D tex;\n",
241 { glsl_func_yuv_to_rgb, NULL, },
242 GST_GL_TEXTURE_TARGET_2D
243 };
244
245 static const gchar templ_RGB_to_VUYA_BODY[] =
246 "vec4 texel, ayuv;\n"
247 "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
248 "ayuv.zyx = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3);\n"
249 "ayuv.w = %s;\n"
250 "gl_FragColor = ayuv;\n";
251
252 static const struct shader_templ templ_RGB_to_VUYA =
253 { NULL,
254 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n",
255 { glsl_func_rgb_to_yuv, NULL, },
256 GST_GL_TEXTURE_TARGET_2D
257 };
258
259 /* YUV to RGB conversion */
260 static const gchar templ_PLANAR_YUV_to_RGB_BODY[] =
261 "vec4 texel, rgba;\n"
262 /* FIXME: should get the sampling right... */
263 "texel.x = texture2D(Ytex, texcoord * tex_scale0).r;\n"
264 "texel.y = texture2D(Utex, texcoord * tex_scale1).r;\n"
265 "texel.z = texture2D(Vtex, texcoord * tex_scale2).r;\n"
266 "rgba.rgb = yuv_to_rgb (texel.xyz, offset, coeff1, coeff2, coeff3);\n"
267 "rgba.a = 1.0;\n"
268 "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
269
270 static const struct shader_templ templ_PLANAR_YUV_to_RGB =
271 { NULL,
272 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex;\n",
273 { glsl_func_yuv_to_rgb, NULL, },
274 GST_GL_TEXTURE_TARGET_2D
275 };
276
277 static const gchar templ_RGB_to_PLANAR_YUV_BODY[] =
278 "vec4 texel;\n"
279 "vec3 yuv;\n"
280 "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
281 /* FIXME: this is not quite correct yet */
282 "vec4 uv_texel = vec4(0.0);\n"
283 /* One u and v sample can be generated by a nxm sized block given by */
284 /* @chroma_sampling. The result is the average of all the values in the */
285 /* block computed with a rolling average. */
286 "vec2 unnormalization;\n"
287 "if (texcoord.x == v_texcoord.x) {\n"
288 " unnormalization = vec2(width, height);\n"
289 "} else {\n"
290 " unnormalization = vec2 (1.0);\n"
291 "}\n"
292 /* scale for chroma size */
293 "vec2 chroma_pos = texcoord * chroma_sampling * unnormalization;\n"
294 /* offset chroma to the center of the first texel in the block */
295 "chroma_pos -= clamp(chroma_sampling * 0.5 - 0.5, vec2(0.0), chroma_sampling);\n"
296 "if (chroma_pos.x < width && chroma_pos.y < height) {\n"
297 " for (int i = 0; i < int(chroma_sampling.x); i++) {\n"
298 " vec2 delta = vec2 (float(i), 0.0);\n"
299 " for (int j = 0; j < int(chroma_sampling.y); j++) {\n"
300 " int n = (i+1)*(j+1);\n"
301 " delta.y = float(j);\n"
302 " vec4 s = texture2D(tex, (chroma_pos + delta) / unnormalization).%c%c%c%c;\n"
303 /* rolling average */
304 " uv_texel = (float(n-1) * uv_texel + s) / float(n);\n"
305 " }\n"
306 " }\n"
307 "}\n"
308 "yuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n"
309 "yuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n"
310 "gl_FragData[0] = vec4(yuv.x, 0.0, 0.0, 1.0);\n"
311 "gl_FragData[1] = vec4(yuv.y, 0.0, 0.0, 1.0);\n"
312 "gl_FragData[2] = vec4(yuv.z, 0.0, 0.0, 1.0);\n";
313
314 static const struct shader_templ templ_RGB_to_PLANAR_YUV =
315 { NULL,
316 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n"
317 "uniform vec2 chroma_sampling;\n",
318 { glsl_func_rgb_to_yuv, NULL, },
319 GST_GL_TEXTURE_TARGET_2D
320 };
321
322 /* NV12/NV21 to RGB conversion */
323 static const gchar templ_NV12_NV21_to_RGB_BODY[] =
324 "vec4 rgba;\n"
325 "vec3 yuv;\n"
326 /* FIXME: should get the sampling right... */
327 "yuv.x=texture2D(Ytex, texcoord * tex_scale0).r;\n"
328 "yuv.yz=texture2D(UVtex, texcoord * tex_scale1).%c%c;\n"
329 "rgba.rgb = yuv_to_rgb (yuv, offset, coeff1, coeff2, coeff3);\n"
330 "rgba.a = 1.0;\n"
331 "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
332
333 static const struct shader_templ templ_NV12_NV21_to_RGB =
334 { NULL,
335 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, UVtex;\n",
336 { glsl_func_yuv_to_rgb, NULL, },
337 GST_GL_TEXTURE_TARGET_2D
338 };
339
340 /* RGB to NV12/NV21 conversion */
341 /* NV12: u, v
342 NV21: v, u */
343 static const gchar templ_RGB_to_NV12_NV21_BODY[] =
344 "vec4 texel, uv_texel;\n"
345 "vec3 yuv;\n"
346 "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
347 "uv_texel = texture2D(tex, texcoord * tex_scale0 * 2.0).%c%c%c%c;\n"
348 "yuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n"
349 "yuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n"
350 "gl_FragData[0] = vec4(yuv.x, 0.0, 0.0, 1.0);\n"
351 "gl_FragData[1] = vec4(yuv.%c, yuv.%c, 0.0, 1.0);\n";
352
353 static const struct shader_templ templ_RGB_to_NV12_NV21 =
354 { NULL,
355 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n",
356 { glsl_func_rgb_to_yuv, NULL, },
357 GST_GL_TEXTURE_TARGET_2D
358 };
359
360 /* YUY2:r,g,a
361 UYVY:a,b,r */
362 static const gchar templ_YUY2_UYVY_to_RGB_BODY[] =
363 "vec4 rgba, uv_texel;\n"
364 "vec3 yuv;\n"
365 /* FIXME: should get the sampling right... */
366 "float dx1 = -poffset_x;\n"
367 "float dx2 = 0.0;\n"
368 "yuv.x = texture2D(Ytex, texcoord * tex_scale0).%c;\n"
369 /* v_texcoord are normalized, texcoord may not be e.g. rectangle textures */
370 "float inorder = mod (v_texcoord.x * width, 2.0);\n"
371 "if (inorder < 1.0) {\n"
372 " dx2 = -dx1;\n"
373 " dx1 = 0.0;\n"
374 "}\n"
375 "uv_texel.rg = texture2D(Ytex, texcoord * tex_scale0 + vec2(dx1, 0.0)).r%c;\n"
376 "uv_texel.ba = texture2D(Ytex, texcoord * tex_scale0 + vec2(dx2, 0.0)).r%c;\n"
377 "yuv.yz = uv_texel.%c%c;\n"
378 "rgba.rgb = yuv_to_rgb (yuv, offset, coeff1, coeff2, coeff3);\n"
379 "rgba.a = 1.0;\n"
380 "gl_FragColor = vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
381
382 static const struct shader_templ templ_YUY2_UYVY_to_RGB =
383 { NULL,
384 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex;\n",
385 { glsl_func_yuv_to_rgb, NULL, },
386 GST_GL_TEXTURE_TARGET_2D
387 };
388
389 static const gchar templ_RGB_to_YUY2_UYVY_BODY[] =
390 "vec4 texel1, texel2;\n"
391 "vec3 yuv, yuv1, yuv2;\n"
392 "float fx, dx, fy;\n"
393 /* v_texcoord are normalized, texcoord may not be e.g. rectangle textures */
394 "float inorder = mod (v_texcoord.x * width, 2.0);\n"
395 "fx = texcoord.x;\n"
396 "dx = poffset_x;\n"
397 "if (inorder > 1.0) {\n"
398 " dx = -dx;\n"
399 "}\n"
400 "fy = texcoord.y;\n"
401 "texel1 = texture2D(tex, vec2(fx, fy)).%c%c%c%c;\n"
402 "texel2 = texture2D(tex, vec2(fx + dx, fy)).%c%c%c%c;\n"
403 "yuv1 = rgb_to_yuv (texel1.rgb, offset, coeff1, coeff2, coeff3);\n"
404 "yuv2 = rgb_to_yuv (texel2.rgb, offset, coeff1, coeff2, coeff3);\n"
405 "yuv.x = yuv1.x;\n"
406 "yuv.yz = (yuv1.yz + yuv2.yz) * 0.5;\n"
407 "if (inorder < 1.0) {\n"
408 " gl_FragColor = vec4(yuv.%c, yuv.%c, 0.0, 0.0);\n"
409 "} else {\n"
410 " gl_FragColor = vec4(yuv.%c, yuv.%c, 0.0, 0.0);\n"
411 "}\n";
412
413 static const struct shader_templ templ_RGB_to_YUY2_UYVY =
414 { NULL,
415 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n",
416 { glsl_func_rgb_to_yuv, NULL, },
417 GST_GL_TEXTURE_TARGET_2D
418 };
419
420 static const gchar text_vertex_shader[] =
421 "attribute vec4 a_position; \n"
422 "attribute vec2 a_texcoord; \n"
423 "varying vec2 v_texcoord; \n"
424 "void main() \n"
425 "{ \n"
426 " gl_Position = a_position; \n"
427 " v_texcoord = a_texcoord; \n"
428 "} \n";
429
430 static const GLfloat vertices[] = {
431 1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
432 -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
433 -1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
434 1.0f, 1.0f, 0.0f, 1.0f, 1.0f
435 };
436
437 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
438
439 /* *INDENT-ON* */
440
441 struct ConvertInfo
442 {
443 gint in_n_textures;
444 gint out_n_textures;
445 const struct shader_templ *templ;
446 gchar *frag_body;
447 gchar *frag_prog;
448 const gchar *shader_tex_names[GST_VIDEO_MAX_PLANES];
449 gfloat *cms_offset;
450 gfloat *cms_coeff1; /* r,y */
451 gfloat *cms_coeff2; /* g,u */
452 gfloat *cms_coeff3; /* b,v */
453 gfloat chroma_sampling[2];
454 };
455
456 struct _GstGLColorConvertPrivate
457 {
458 gboolean result;
459
460 struct ConvertInfo convert_info;
461
462 GstGLTextureTarget from_texture_target;
463 GstGLTextureTarget to_texture_target;
464
465 GstGLMemory *in_tex[GST_VIDEO_MAX_PLANES];
466 GstGLMemory *out_tex[GST_VIDEO_MAX_PLANES];
467
468 GstGLFormat in_tex_formats[GST_VIDEO_MAX_PLANES];
469
470 GLuint vao;
471 GLuint vertex_buffer;
472 GLuint vbo_indices;
473 GLuint attr_position;
474 GLuint attr_texture;
475
476 GstCaps *in_caps;
477 GstCaps *out_caps;
478
479 GstBufferPool *pool;
480 gboolean pool_started;
481 };
482
483 GST_DEBUG_CATEGORY_STATIC (gst_gl_color_convert_debug);
484 #define GST_CAT_DEFAULT gst_gl_color_convert_debug
485
486 #define DEBUG_INIT \
487 GST_DEBUG_CATEGORY_INIT (gst_gl_color_convert_debug, "glconvert", 0, "convert");
488
489 G_DEFINE_TYPE_WITH_CODE (GstGLColorConvert, gst_gl_color_convert,
490 GST_TYPE_OBJECT, G_ADD_PRIVATE (GstGLColorConvert) DEBUG_INIT);
491
492 static void gst_gl_color_convert_finalize (GObject * object);
493 static void gst_gl_color_convert_reset (GstGLColorConvert * convert);
494
495 #define GST_GL_COLOR_CONVERT_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), \
496 GST_TYPE_GL_COLOR_CONVERT, GstGLColorConvertPrivate))
497
498 static void
gst_gl_color_convert_class_init(GstGLColorConvertClass * klass)499 gst_gl_color_convert_class_init (GstGLColorConvertClass * klass)
500 {
501 G_OBJECT_CLASS (klass)->finalize = gst_gl_color_convert_finalize;
502 }
503
504 static void
gst_gl_color_convert_init(GstGLColorConvert * convert)505 gst_gl_color_convert_init (GstGLColorConvert * convert)
506 {
507 convert->priv = gst_gl_color_convert_get_instance_private (convert);
508
509 gst_gl_color_convert_reset (convert);
510 }
511
512 /**
513 * gst_gl_color_convert_new:
514 * @context: a #GstGLContext
515 *
516 * Returns: (transfer full): a new #GstGLColorConvert object
517 *
518 * Since: 1.4
519 */
520 GstGLColorConvert *
gst_gl_color_convert_new(GstGLContext * context)521 gst_gl_color_convert_new (GstGLContext * context)
522 {
523 GstGLColorConvert *convert;
524
525 convert = g_object_new (GST_TYPE_GL_COLOR_CONVERT, NULL);
526 gst_object_ref_sink (convert);
527
528 convert->context = gst_object_ref (context);
529
530 gst_video_info_set_format (&convert->in_info, GST_VIDEO_FORMAT_ENCODED, 0, 0);
531 gst_video_info_set_format (&convert->out_info, GST_VIDEO_FORMAT_ENCODED, 0,
532 0);
533
534 GST_DEBUG_OBJECT (convert,
535 "Created new colorconvert for context %" GST_PTR_FORMAT, context);
536
537 return convert;
538 }
539
540 static void
gst_gl_color_convert_finalize(GObject * object)541 gst_gl_color_convert_finalize (GObject * object)
542 {
543 GstGLColorConvert *convert;
544
545 convert = GST_GL_COLOR_CONVERT (object);
546
547 gst_gl_color_convert_reset (convert);
548
549 if (convert->context) {
550 gst_object_unref (convert->context);
551 convert->context = NULL;
552 }
553
554 G_OBJECT_CLASS (gst_gl_color_convert_parent_class)->finalize (object);
555 }
556
557 static void
_reset_gl(GstGLContext * context,GstGLColorConvert * convert)558 _reset_gl (GstGLContext * context, GstGLColorConvert * convert)
559 {
560 const GstGLFuncs *gl = context->gl_vtable;
561
562 if (convert->priv->vao) {
563 gl->DeleteVertexArrays (1, &convert->priv->vao);
564 convert->priv->vao = 0;
565 }
566
567 if (convert->priv->vertex_buffer) {
568 gl->DeleteBuffers (1, &convert->priv->vertex_buffer);
569 convert->priv->vertex_buffer = 0;
570 }
571
572 if (convert->priv->vbo_indices) {
573 gl->DeleteBuffers (1, &convert->priv->vbo_indices);
574 convert->priv->vbo_indices = 0;
575 }
576 }
577
578 static void
gst_gl_color_convert_reset_shader(GstGLColorConvert * convert)579 gst_gl_color_convert_reset_shader (GstGLColorConvert * convert)
580 {
581 convert->priv->convert_info.chroma_sampling[0] = 1.0f;
582 convert->priv->convert_info.chroma_sampling[1] = 1.0f;
583
584 if (convert->priv->convert_info.frag_prog) {
585 g_free (convert->priv->convert_info.frag_prog);
586 convert->priv->convert_info.frag_prog = NULL;
587 }
588 if (convert->priv->convert_info.frag_body) {
589 g_free (convert->priv->convert_info.frag_body);
590 convert->priv->convert_info.frag_body = NULL;
591 }
592 if (convert->shader) {
593 gst_object_unref (convert->shader);
594 convert->shader = NULL;
595 }
596
597 convert->initted = FALSE;
598 }
599
600 static void
gst_gl_color_convert_reset(GstGLColorConvert * convert)601 gst_gl_color_convert_reset (GstGLColorConvert * convert)
602 {
603 guint i;
604
605 if (convert->fbo) {
606 gst_object_unref (convert->fbo);
607 convert->fbo = NULL;
608 }
609
610 for (i = 0; i < convert->priv->convert_info.out_n_textures; i++) {
611 if (convert->priv->out_tex[i])
612 gst_memory_unref ((GstMemory *) convert->priv->out_tex[i]);
613 convert->priv->out_tex[i] = NULL;
614 }
615
616 if (convert->priv->pool) {
617 convert->priv->pool_started = FALSE;
618
619 gst_object_unref (convert->priv->pool);
620 convert->priv->pool = NULL;
621 }
622
623 gst_caps_replace (&convert->priv->in_caps, NULL);
624 gst_caps_replace (&convert->priv->out_caps, NULL);
625
626 if (convert->context) {
627 gst_gl_context_thread_add (convert->context,
628 (GstGLContextThreadFunc) _reset_gl, convert);
629 }
630
631 gst_gl_color_convert_reset_shader (convert);
632 }
633
634 static gboolean
_gst_gl_color_convert_can_passthrough_info(GstVideoInfo * in,GstVideoInfo * out)635 _gst_gl_color_convert_can_passthrough_info (GstVideoInfo * in,
636 GstVideoInfo * out)
637 {
638 gint i;
639
640 if (GST_VIDEO_INFO_FORMAT (in) != GST_VIDEO_INFO_FORMAT (out))
641 return FALSE;
642 if (GST_VIDEO_INFO_WIDTH (in) != GST_VIDEO_INFO_WIDTH (out))
643 return FALSE;
644 if (GST_VIDEO_INFO_HEIGHT (in) != GST_VIDEO_INFO_HEIGHT (out))
645 return FALSE;
646 if (GST_VIDEO_INFO_SIZE (in) != GST_VIDEO_INFO_SIZE (out))
647 return FALSE;
648
649 for (i = 0; i < in->finfo->n_planes; i++) {
650 if (in->stride[i] != out->stride[i])
651 return FALSE;
652 if (in->offset[i] != out->offset[i])
653 return FALSE;
654 }
655
656 if (!gst_video_colorimetry_is_equal (&in->colorimetry, &out->colorimetry))
657 return FALSE;
658 if (in->chroma_site != out->chroma_site)
659 return FALSE;
660
661 return TRUE;
662 }
663
664 static gboolean
_gst_gl_color_convert_set_caps_unlocked(GstGLColorConvert * convert,GstCaps * in_caps,GstCaps * out_caps)665 _gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert,
666 GstCaps * in_caps, GstCaps * out_caps)
667 {
668 GstVideoInfo in_info, out_info;
669 GstCapsFeatures *in_features, *out_features;
670 GstGLTextureTarget from_target, to_target;
671 gboolean passthrough;
672
673 g_return_val_if_fail (convert != NULL, FALSE);
674 g_return_val_if_fail (in_caps, FALSE);
675 g_return_val_if_fail (out_caps, FALSE);
676
677 GST_LOG_OBJECT (convert, "Setting caps in %" GST_PTR_FORMAT
678 " out %" GST_PTR_FORMAT, in_caps, out_caps);
679
680 if (!gst_video_info_from_caps (&in_info, in_caps))
681 g_assert_not_reached ();
682
683 if (!gst_video_info_from_caps (&out_info, out_caps))
684 g_assert_not_reached ();
685
686 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&in_info) !=
687 GST_VIDEO_FORMAT_UNKNOWN, FALSE);
688 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&in_info) !=
689 GST_VIDEO_FORMAT_ENCODED, FALSE);
690 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&out_info) !=
691 GST_VIDEO_FORMAT_UNKNOWN, FALSE);
692 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&out_info) !=
693 GST_VIDEO_FORMAT_ENCODED, FALSE);
694
695 in_features = gst_caps_get_features (in_caps, 0);
696 out_features = gst_caps_get_features (out_caps, 0);
697
698 if (!gst_caps_features_contains (in_features,
699 GST_CAPS_FEATURE_MEMORY_GL_MEMORY)
700 || !gst_caps_features_contains (out_features,
701 GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
702 return FALSE;
703 }
704
705 {
706 GstStructure *in_s = gst_caps_get_structure (in_caps, 0);
707 GstStructure *out_s = gst_caps_get_structure (out_caps, 0);
708
709 if (gst_structure_has_field_typed (in_s, "texture-target", G_TYPE_STRING))
710 from_target =
711 gst_gl_texture_target_from_string (gst_structure_get_string (in_s,
712 "texture-target"));
713 else
714 from_target = GST_GL_TEXTURE_TARGET_2D;
715
716 if (gst_structure_has_field_typed (out_s, "texture-target", G_TYPE_STRING))
717 to_target =
718 gst_gl_texture_target_from_string (gst_structure_get_string (out_s,
719 "texture-target"));
720 else
721 to_target = GST_GL_TEXTURE_TARGET_2D;
722
723 if (to_target == GST_GL_TEXTURE_TARGET_NONE
724 || from_target == GST_GL_TEXTURE_TARGET_NONE)
725 /* invalid caps */
726 return FALSE;
727 }
728
729 if (gst_video_info_is_equal (&convert->in_info, &in_info) &&
730 gst_video_info_is_equal (&convert->out_info, &out_info) &&
731 convert->priv->from_texture_target == from_target &&
732 convert->priv->to_texture_target == to_target)
733 return TRUE;
734
735 /* If input and output are identical, pass through directly */
736 passthrough =
737 _gst_gl_color_convert_can_passthrough_info (&in_info, &out_info) &&
738 from_target == to_target;
739
740 if (!passthrough && to_target != GST_GL_TEXTURE_TARGET_2D
741 && to_target != GST_GL_TEXTURE_TARGET_RECTANGLE)
742 return FALSE;
743
744 {
745 guint yuv_gray_flags, in_flags, out_flags;
746
747 in_flags = GST_VIDEO_FORMAT_INFO_FLAGS (in_info.finfo);
748 out_flags = GST_VIDEO_FORMAT_INFO_FLAGS (out_info.finfo);
749 yuv_gray_flags = GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY;
750
751 /* GRAY/YUV -> GRAY/YUV is not supported for non-passthrough */
752 if (!passthrough && (in_flags & yuv_gray_flags) != 0
753 && (out_flags & yuv_gray_flags) != 0)
754 return FALSE;
755 }
756
757 gst_gl_color_convert_reset (convert);
758 convert->in_info = in_info;
759 convert->out_info = out_info;
760 gst_caps_replace (&convert->priv->in_caps, in_caps);
761 gst_caps_replace (&convert->priv->out_caps, out_caps);
762 convert->priv->from_texture_target = from_target;
763 convert->priv->to_texture_target = to_target;
764 convert->initted = FALSE;
765
766 convert->passthrough = passthrough;
767 #ifndef GST_DISABLE_GST_DEBUG
768 if (G_UNLIKELY (convert->passthrough))
769 GST_DEBUG_OBJECT (convert,
770 "Configuring passthrough mode for same in/out caps");
771 else {
772 GST_DEBUG_OBJECT (convert, "Color converting %" GST_PTR_FORMAT
773 " to %" GST_PTR_FORMAT, in_caps, out_caps);
774 }
775 #endif
776
777 return TRUE;
778 }
779
780 /**
781 * gst_gl_color_convert_set_caps:
782 * @convert: a #GstGLColorConvert
783 * @in_caps: input #GstCaps
784 * @out_caps: output #GstCaps
785 *
786 * Initializes @convert with the information required for conversion.
787 *
788 * Since: 1.6
789 */
790 gboolean
gst_gl_color_convert_set_caps(GstGLColorConvert * convert,GstCaps * in_caps,GstCaps * out_caps)791 gst_gl_color_convert_set_caps (GstGLColorConvert * convert,
792 GstCaps * in_caps, GstCaps * out_caps)
793 {
794 gboolean ret;
795
796 GST_OBJECT_LOCK (convert);
797 ret = _gst_gl_color_convert_set_caps_unlocked (convert, in_caps, out_caps);
798 GST_OBJECT_UNLOCK (convert);
799
800 return ret;
801 }
802
803 /**
804 * gst_gl_color_convert_decide_allocation:
805 * @convert: a #GstGLColorConvert
806 * @query: a completed ALLOCATION #GstQuery
807 *
808 * Provides an implementation of #GstBaseTransformClass.decide_allocation()
809 *
810 * Returns: whether the allocation parameters were successfully chosen
811 *
812 * Since: 1.8
813 */
814 gboolean
gst_gl_color_convert_decide_allocation(GstGLColorConvert * convert,GstQuery * query)815 gst_gl_color_convert_decide_allocation (GstGLColorConvert * convert,
816 GstQuery * query)
817 {
818 GstBufferPool *pool = NULL;
819 GstStructure *config;
820 GstCaps *caps;
821 guint min, max, size, n, i;
822 gboolean update_pool;
823 GstGLVideoAllocationParams *params;
824 GstVideoInfo vinfo;
825
826 gst_query_parse_allocation (query, &caps, NULL);
827 if (!caps)
828 return FALSE;
829
830 gst_video_info_from_caps (&vinfo, caps);
831
832 n = gst_query_get_n_allocation_pools (query);
833 if (n > 0) {
834 update_pool = TRUE;
835 for (i = 0; i < n; i++) {
836 gst_query_parse_nth_allocation_pool (query, i, &pool, &size, &min, &max);
837
838 if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) {
839 if (pool)
840 gst_object_unref (pool);
841 pool = NULL;
842 }
843 }
844 }
845
846 if (!pool) {
847 GstVideoInfo vinfo;
848
849 gst_video_info_init (&vinfo);
850 size = vinfo.size;
851 min = max = 0;
852 update_pool = FALSE;
853 }
854
855 if (!pool)
856 pool = gst_gl_buffer_pool_new (convert->context);
857
858 config = gst_buffer_pool_get_config (pool);
859
860 gst_buffer_pool_config_set_params (config, caps, size, min, max);
861 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
862 if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL))
863 gst_buffer_pool_config_add_option (config,
864 GST_BUFFER_POOL_OPTION_GL_SYNC_META);
865
866 params = gst_gl_video_allocation_params_new (convert->context, NULL, &vinfo,
867 0, NULL, convert->priv->to_texture_target, 0);
868 gst_buffer_pool_config_set_gl_allocation_params (config,
869 (GstGLAllocationParams *) params);
870 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
871
872 if (!gst_buffer_pool_set_config (pool, config))
873 GST_WARNING_OBJECT (convert, "Failed to set buffer pool config");
874
875 if (update_pool)
876 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
877 else
878 gst_query_add_allocation_pool (query, pool, size, min, max);
879
880 if (convert->priv->pool) {
881 gst_object_unref (convert->priv->pool);
882 convert->priv->pool_started = FALSE;
883 }
884 convert->priv->pool = pool;
885
886 return TRUE;
887 }
888
889 static void
_init_value_string_list(GValue * list,...)890 _init_value_string_list (GValue * list, ...)
891 {
892 GValue item = G_VALUE_INIT;
893 gchar *str;
894 va_list args;
895
896 g_value_init (list, GST_TYPE_LIST);
897
898 va_start (args, list);
899 while ((str = va_arg (args, gchar *))) {
900 g_value_init (&item, G_TYPE_STRING);
901 g_value_set_string (&item, str);
902
903 gst_value_list_append_value (list, &item);
904 g_value_unset (&item);
905 }
906 va_end (args);
907 }
908
909 static void
_append_value_string_list(GValue * list,...)910 _append_value_string_list (GValue * list, ...)
911 {
912 GValue item = G_VALUE_INIT;
913 gchar *str;
914 va_list args;
915
916 va_start (args, list);
917 while ((str = va_arg (args, gchar *))) {
918 g_value_init (&item, G_TYPE_STRING);
919 g_value_set_string (&item, str);
920
921 gst_value_list_append_value (list, &item);
922 g_value_unset (&item);
923 }
924 va_end (args);
925 }
926
927 static void
_init_supported_formats(GstGLContext * context,gboolean output,GValue * supported_formats)928 _init_supported_formats (GstGLContext * context, gboolean output,
929 GValue * supported_formats)
930 {
931 /* Assume if context == NULL that we don't have a GL context and can
932 * do the conversion */
933
934 /* Always supported input and output formats */
935 _init_value_string_list (supported_formats, "RGBA", "RGB", "RGBx", "BGR",
936 "BGRx", "BGRA", "xRGB", "xBGR", "ARGB", "ABGR", "GRAY8", "GRAY16_LE",
937 "GRAY16_BE", "AYUV", "VUYA", "YUY2", "UYVY", NULL);
938
939 /* Always supported input formats or output with multiple draw buffers */
940 if (!output || (!context || context->gl_vtable->DrawBuffers))
941 _append_value_string_list (supported_formats, "Y444", "I420", "YV12",
942 "Y42B", "Y41B", "NV12", "NV21", NULL);
943
944 /* Requires reading from a RG/LA framebuffer... */
945 if (!context || (USING_GLES3 (context) || USING_OPENGL (context)))
946 _append_value_string_list (supported_formats, "YUY2", "UYVY", NULL);
947
948 if (!context || gst_gl_format_is_supported (context, GST_GL_RGBA16))
949 _append_value_string_list (supported_formats, "ARGB64", NULL);
950
951 if (!context || gst_gl_format_is_supported (context, GST_GL_RGB565))
952 _append_value_string_list (supported_formats, "RGB16", "BGR16", NULL);
953 }
954
955 /* copies the given caps */
956 static GstCaps *
gst_gl_color_convert_caps_transform_format_info(GstGLContext * context,gboolean output,GstCaps * caps)957 gst_gl_color_convert_caps_transform_format_info (GstGLContext * context,
958 gboolean output, GstCaps * caps)
959 {
960 GstStructure *st;
961 GstCapsFeatures *f;
962 gint i, n;
963 GstCaps *res;
964 GValue supported_formats = G_VALUE_INIT;
965 GValue rgb_formats = G_VALUE_INIT;
966 GValue supported_rgb_formats = G_VALUE_INIT;
967
968 /* There are effectively two modes here with the RGB/YUV transition:
969 * 1. There is a RGB-like format as input and we can transform to YUV or,
970 * 2. No RGB-like format as input so we can only transform to RGB-like formats
971 *
972 * We also filter down the list of formats depending on what the OpenGL
973 * context supports (when provided).
974 */
975
976 _init_value_string_list (&rgb_formats, "RGBA", "ARGB", "BGRA", "ABGR", "RGBx",
977 "xRGB", "BGRx", "xBGR", "RGB", "BGR", "ARGB64", NULL);
978 _init_supported_formats (context, output, &supported_formats);
979 gst_value_intersect (&supported_rgb_formats, &rgb_formats,
980 &supported_formats);
981
982 res = gst_caps_new_empty ();
983
984 n = gst_caps_get_size (caps);
985 for (i = 0; i < n; i++) {
986 const GValue *format;
987
988 st = gst_caps_get_structure (caps, i);
989 f = gst_caps_get_features (caps, i);
990
991 format = gst_structure_get_value (st, "format");
992 st = gst_structure_copy (st);
993 if (GST_VALUE_HOLDS_LIST (format)) {
994 gboolean have_rgb_formats = FALSE;
995 GValue passthrough_formats = G_VALUE_INIT;
996 gint j, len;
997
998 g_value_init (&passthrough_formats, GST_TYPE_LIST);
999 len = gst_value_list_get_size (format);
1000 for (j = 0; j < len; j++) {
1001 const GValue *val;
1002
1003 val = gst_value_list_get_value (format, j);
1004 if (G_VALUE_HOLDS_STRING (val)) {
1005 const gchar *format_str = g_value_get_string (val);
1006 GstVideoFormat v_format = gst_video_format_from_string (format_str);
1007 const GstVideoFormatInfo *t_info =
1008 gst_video_format_get_info (v_format);
1009 if (GST_VIDEO_FORMAT_INFO_FLAGS (t_info) & (GST_VIDEO_FORMAT_FLAG_YUV
1010 | GST_VIDEO_FORMAT_FLAG_GRAY)) {
1011 gst_value_list_append_value (&passthrough_formats, val);
1012 } else if (GST_VIDEO_FORMAT_INFO_FLAGS (t_info) &
1013 GST_VIDEO_FORMAT_FLAG_RGB) {
1014 have_rgb_formats = TRUE;
1015 break;
1016 }
1017 }
1018 }
1019 if (have_rgb_formats) {
1020 gst_structure_set_value (st, "format", &supported_formats);
1021 } else {
1022 /* add passthrough structure, then the rgb conversion structure */
1023 gst_structure_set_value (st, "format", &passthrough_formats);
1024 gst_caps_append_structure_full (res, gst_structure_copy (st),
1025 gst_caps_features_copy (f));
1026 gst_structure_set_value (st, "format", &supported_rgb_formats);
1027 }
1028 g_value_unset (&passthrough_formats);
1029 } else if (G_VALUE_HOLDS_STRING (format)) {
1030 const gchar *format_str = g_value_get_string (format);
1031 GstVideoFormat v_format = gst_video_format_from_string (format_str);
1032 const GstVideoFormatInfo *t_info = gst_video_format_get_info (v_format);
1033 if (GST_VIDEO_FORMAT_INFO_FLAGS (t_info) & (GST_VIDEO_FORMAT_FLAG_YUV |
1034 GST_VIDEO_FORMAT_FLAG_GRAY)) {
1035 /* add passthrough structure, then the rgb conversion structure */
1036 gst_structure_set_value (st, "format", format);
1037 gst_caps_append_structure_full (res, gst_structure_copy (st),
1038 gst_caps_features_copy (f));
1039 gst_structure_set_value (st, "format", &supported_rgb_formats);
1040 } else { /* RGB */
1041 gst_structure_set_value (st, "format", &supported_formats);
1042 }
1043 }
1044 gst_structure_remove_fields (st, "colorimetry", "chroma-site",
1045 "texture-target", NULL);
1046
1047 gst_caps_append_structure_full (res, st, gst_caps_features_copy (f));
1048 }
1049
1050 g_value_unset (&supported_formats);
1051 g_value_unset (&rgb_formats);
1052 g_value_unset (&supported_rgb_formats);
1053
1054 return res;
1055 }
1056
1057 /**
1058 * gst_gl_color_convert_transform_caps:
1059 * @context: a #GstGLContext to use for transforming @caps
1060 * @direction: a #GstPadDirection
1061 * @caps: (transfer none): the #GstCaps to transform
1062 * @filter: (transfer none): a set of filter #GstCaps
1063 *
1064 * Provides an implementation of #GstBaseTransformClass.transform_caps()
1065 *
1066 * Returns: (transfer full): the converted #GstCaps
1067 *
1068 * Since: 1.6
1069 */
1070 GstCaps *
gst_gl_color_convert_transform_caps(GstGLContext * context,GstPadDirection direction,GstCaps * caps,GstCaps * filter)1071 gst_gl_color_convert_transform_caps (GstGLContext * context,
1072 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
1073 {
1074 caps = gst_gl_color_convert_caps_transform_format_info (context,
1075 direction == GST_PAD_SRC, caps);
1076
1077 if (filter) {
1078 GstCaps *tmp;
1079
1080 tmp = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
1081 gst_caps_unref (caps);
1082 caps = tmp;
1083 }
1084
1085 return caps;
1086 }
1087
1088 /* fixation from videoconvert */
1089 #define SCORE_FORMAT_CHANGE 1
1090 #define SCORE_DEPTH_CHANGE 1
1091 #define SCORE_ALPHA_CHANGE 1
1092 #define SCORE_CHROMA_W_CHANGE 1
1093 #define SCORE_CHROMA_H_CHANGE 1
1094 #define SCORE_PALETTE_CHANGE 1
1095
1096 #define SCORE_COLORSPACE_LOSS 2 /* RGB <-> YUV */
1097 #define SCORE_DEPTH_LOSS 4 /* change bit depth */
1098 #define SCORE_ALPHA_LOSS 8 /* lose the alpha channel */
1099 #define SCORE_CHROMA_W_LOSS 16 /* vertical subsample */
1100 #define SCORE_CHROMA_H_LOSS 32 /* horizontal subsample */
1101 #define SCORE_PALETTE_LOSS 64 /* convert to palette format */
1102 #define SCORE_COLOR_LOSS 128 /* convert to GRAY */
1103
1104 #define COLORSPACE_MASK (GST_VIDEO_FORMAT_FLAG_YUV | \
1105 GST_VIDEO_FORMAT_FLAG_RGB | GST_VIDEO_FORMAT_FLAG_GRAY)
1106 #define ALPHA_MASK (GST_VIDEO_FORMAT_FLAG_ALPHA)
1107 #define PALETTE_MASK (GST_VIDEO_FORMAT_FLAG_PALETTE)
1108
1109 static GstGLTextureTarget
_texture_target_demask(guint target_mask)1110 _texture_target_demask (guint target_mask)
1111 {
1112 if (target_mask & (1 << GST_GL_TEXTURE_TARGET_2D)) {
1113 return GST_GL_TEXTURE_TARGET_2D;
1114 }
1115 if (target_mask & (1 << GST_GL_TEXTURE_TARGET_RECTANGLE)) {
1116 return GST_GL_TEXTURE_TARGET_RECTANGLE;
1117 }
1118 if (target_mask & (1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES)) {
1119 return GST_GL_TEXTURE_TARGET_EXTERNAL_OES;
1120 }
1121
1122 return 0;
1123 }
1124
1125 /* calculate how much loss a conversion would be */
1126 static void
score_format_target(const GstVideoFormatInfo * in_info,guint targets_mask,GstVideoFormat v_format,guint other_targets_mask,gint * min_loss,const GstVideoFormatInfo ** out_info,GstGLTextureTarget * result)1127 score_format_target (const GstVideoFormatInfo * in_info, guint targets_mask,
1128 GstVideoFormat v_format, guint other_targets_mask, gint * min_loss,
1129 const GstVideoFormatInfo ** out_info, GstGLTextureTarget * result)
1130 {
1131 const GstVideoFormatInfo *t_info;
1132 GstVideoFormatFlags in_flags, t_flags;
1133 gint loss;
1134
1135 t_info = gst_video_format_get_info (v_format);
1136 if (!t_info)
1137 return;
1138
1139 /* accept input format immediately without loss */
1140 if (in_info == t_info && (targets_mask & other_targets_mask) != 0) {
1141 *min_loss = 0;
1142 *out_info = t_info;
1143 *result = _texture_target_demask (targets_mask & other_targets_mask);
1144 return;
1145 }
1146
1147 /* can only passthrough external-oes textures */
1148 other_targets_mask &= ~(1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES);
1149 if (other_targets_mask == 0)
1150 return;
1151 /* try to keep the same target */
1152 if (targets_mask & other_targets_mask)
1153 other_targets_mask = targets_mask & other_targets_mask;
1154
1155 loss = SCORE_FORMAT_CHANGE;
1156
1157 in_flags = GST_VIDEO_FORMAT_INFO_FLAGS (in_info);
1158 in_flags &= ~GST_VIDEO_FORMAT_FLAG_LE;
1159 in_flags &= ~GST_VIDEO_FORMAT_FLAG_COMPLEX;
1160 in_flags &= ~GST_VIDEO_FORMAT_FLAG_UNPACK;
1161
1162 t_flags = GST_VIDEO_FORMAT_INFO_FLAGS (t_info);
1163 t_flags &= ~GST_VIDEO_FORMAT_FLAG_LE;
1164 t_flags &= ~GST_VIDEO_FORMAT_FLAG_COMPLEX;
1165 t_flags &= ~GST_VIDEO_FORMAT_FLAG_UNPACK;
1166
1167 /* GRAY/YUV -> GRAY/YUV is not supported */
1168 if ((in_flags & (GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY)) != 0
1169 && (t_flags & (GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY)) !=
1170 0)
1171 return;
1172
1173 if ((t_flags & PALETTE_MASK) != (in_flags & PALETTE_MASK)) {
1174 loss += SCORE_PALETTE_CHANGE;
1175 if (t_flags & PALETTE_MASK)
1176 loss += SCORE_PALETTE_LOSS;
1177 }
1178
1179 if ((t_flags & COLORSPACE_MASK) != (in_flags & COLORSPACE_MASK)) {
1180 loss += SCORE_COLORSPACE_LOSS;
1181 if (t_flags & GST_VIDEO_FORMAT_FLAG_GRAY)
1182 loss += SCORE_COLOR_LOSS;
1183 }
1184
1185 if ((t_flags & ALPHA_MASK) != (in_flags & ALPHA_MASK)) {
1186 loss += SCORE_ALPHA_CHANGE;
1187 if (in_flags & ALPHA_MASK)
1188 loss += SCORE_ALPHA_LOSS;
1189 }
1190
1191 if ((in_info->h_sub[1]) != (t_info->h_sub[1])) {
1192 loss += SCORE_CHROMA_H_CHANGE;
1193 if ((in_info->h_sub[1]) < (t_info->h_sub[1]))
1194 loss += SCORE_CHROMA_H_LOSS;
1195 }
1196 if ((in_info->w_sub[1]) != (t_info->w_sub[1])) {
1197 loss += SCORE_CHROMA_W_CHANGE;
1198 if ((in_info->w_sub[1]) < (t_info->w_sub[1]))
1199 loss += SCORE_CHROMA_W_LOSS;
1200 }
1201
1202 if ((in_info->bits) != (t_info->bits)) {
1203 loss += SCORE_DEPTH_CHANGE;
1204 if ((in_info->bits) > (t_info->bits))
1205 loss += SCORE_DEPTH_LOSS;
1206 }
1207
1208 if (loss < *min_loss) {
1209 GstGLTextureTarget target = _texture_target_demask (other_targets_mask);
1210
1211 if (target != 0) {
1212 *out_info = t_info;
1213 *min_loss = loss;
1214 *result = target;
1215 }
1216 }
1217 }
1218
1219 static void
gst_gl_color_convert_fixate_format_target(GstCaps * caps,GstCaps * result)1220 gst_gl_color_convert_fixate_format_target (GstCaps * caps, GstCaps * result)
1221 {
1222 GstStructure *ins, *outs;
1223 const gchar *in_format;
1224 const GstVideoFormatInfo *in_info, *out_info = NULL;
1225 const GValue *targets;
1226 guint targets_mask = 0;
1227 GstGLTextureTarget target;
1228 gint min_loss = G_MAXINT;
1229 guint i, capslen;
1230
1231 ins = gst_caps_get_structure (caps, 0);
1232 in_format = gst_structure_get_string (ins, "format");
1233 if (!in_format)
1234 return;
1235 targets = gst_structure_get_value (ins, "texture-target");
1236 targets_mask = gst_gl_value_get_texture_target_mask (targets);
1237 if (!targets_mask)
1238 return;
1239
1240 in_info =
1241 gst_video_format_get_info (gst_video_format_from_string (in_format));
1242 if (!in_info)
1243 return;
1244
1245 outs = gst_caps_get_structure (result, 0);
1246
1247 capslen = gst_caps_get_size (result);
1248 for (i = 0; i < capslen; i++) {
1249 GstStructure *tests;
1250 const GValue *format;
1251 const GValue *other_targets;
1252 guint other_targets_mask = 0;
1253
1254 tests = gst_caps_get_structure (result, i);
1255
1256 format = gst_structure_get_value (tests, "format");
1257 other_targets = gst_structure_get_value (tests, "texture-target");
1258 /* should not happen */
1259 if (format == NULL || other_targets == NULL)
1260 continue;
1261
1262 other_targets_mask = gst_gl_value_get_texture_target_mask (other_targets);
1263
1264 if (GST_VALUE_HOLDS_LIST (format)) {
1265 gint j, len;
1266
1267 len = gst_value_list_get_size (format);
1268 for (j = 0; j < len; j++) {
1269 const GValue *val;
1270
1271 val = gst_value_list_get_value (format, j);
1272 if (G_VALUE_HOLDS_STRING (val)) {
1273 const gchar *format_str = g_value_get_string (val);
1274 GstVideoFormat v_format = gst_video_format_from_string (format_str);
1275 score_format_target (in_info, targets_mask, v_format,
1276 other_targets_mask, &min_loss, &out_info, &target);
1277 if (min_loss == 0)
1278 break;
1279 }
1280 }
1281 } else if (G_VALUE_HOLDS_STRING (format)) {
1282 const gchar *format_str = g_value_get_string (format);
1283 GstVideoFormat v_format = gst_video_format_from_string (format_str);
1284 score_format_target (in_info, targets_mask, v_format, other_targets_mask,
1285 &min_loss, &out_info, &target);
1286 }
1287 }
1288 if (out_info)
1289 gst_structure_set (outs, "format", G_TYPE_STRING,
1290 GST_VIDEO_FORMAT_INFO_NAME (out_info), NULL);
1291 if (target)
1292 gst_structure_set (outs, "texture-target", G_TYPE_STRING,
1293 gst_gl_texture_target_to_string (target), NULL);
1294 }
1295
1296 /**
1297 * gst_gl_color_convert_fixate_caps:
1298 * @context: a #GstGLContext to use for transforming @caps
1299 * @direction: a #GstPadDirection
1300 * @caps: (transfer none): the #GstCaps of @direction
1301 * @other: (transfer full): the #GstCaps to fixate
1302 *
1303 * Provides an implementation of #GstBaseTransformClass.fixate_caps()
1304 *
1305 * Returns: (transfer full): the fixated #GstCaps
1306 *
1307 * Since: 1.8
1308 */
1309 GstCaps *
gst_gl_color_convert_fixate_caps(GstGLContext * context,GstPadDirection direction,GstCaps * caps,GstCaps * other)1310 gst_gl_color_convert_fixate_caps (GstGLContext * context,
1311 GstPadDirection direction, GstCaps * caps, GstCaps * other)
1312 {
1313 GstCaps *result;
1314
1315 result = gst_caps_intersect (other, caps);
1316 if (gst_caps_is_empty (result)) {
1317 gst_caps_unref (result);
1318 result = other;
1319 } else {
1320 gst_caps_unref (other);
1321 }
1322
1323 result = gst_caps_make_writable (result);
1324 gst_gl_color_convert_fixate_format_target (caps, result);
1325
1326 result = gst_caps_fixate (result);
1327
1328 if (direction == GST_PAD_SINK) {
1329 if (gst_caps_is_subset (caps, result)) {
1330 gst_caps_replace (&result, caps);
1331 }
1332 }
1333
1334 return result;
1335 }
1336
1337 /**
1338 * gst_gl_color_convert_perform:
1339 * @convert: a #GstGLColorConvert
1340 * @inbuf: (transfer none): the #GstGLMemory filled #GstBuffer to convert
1341 *
1342 * Converts the data contained by @inbuf using the formats specified by the
1343 * #GstCaps passed to gst_gl_color_convert_set_caps()
1344 *
1345 * Returns: (transfer full): a converted #GstBuffer or %NULL
1346 *
1347 * Since: 1.4
1348 */
1349 GstBuffer *
gst_gl_color_convert_perform(GstGLColorConvert * convert,GstBuffer * inbuf)1350 gst_gl_color_convert_perform (GstGLColorConvert * convert, GstBuffer * inbuf)
1351 {
1352 GstBuffer *ret;
1353
1354 g_return_val_if_fail (convert != NULL, FALSE);
1355
1356 GST_OBJECT_LOCK (convert);
1357 ret = _gst_gl_color_convert_perform_unlocked (convert, inbuf);
1358 GST_OBJECT_UNLOCK (convert);
1359
1360 return ret;
1361 }
1362
1363 static GstBuffer *
_gst_gl_color_convert_perform_unlocked(GstGLColorConvert * convert,GstBuffer * inbuf)1364 _gst_gl_color_convert_perform_unlocked (GstGLColorConvert * convert,
1365 GstBuffer * inbuf)
1366 {
1367 g_return_val_if_fail (convert != NULL, FALSE);
1368 g_return_val_if_fail (inbuf, FALSE);
1369
1370 if (G_UNLIKELY (convert->passthrough))
1371 return gst_buffer_ref (inbuf);
1372
1373 convert->inbuf = inbuf;
1374
1375 gst_gl_context_thread_add (convert->context,
1376 (GstGLContextThreadFunc) _do_convert, convert);
1377
1378 if (!convert->priv->result) {
1379 if (convert->outbuf)
1380 gst_buffer_unref (convert->outbuf);
1381 convert->outbuf = NULL;
1382 return NULL;
1383 }
1384
1385 return convert->outbuf;
1386 }
1387
1388 static inline gboolean
_is_RGBx(GstVideoFormat v_format)1389 _is_RGBx (GstVideoFormat v_format)
1390 {
1391 switch (v_format) {
1392 case GST_VIDEO_FORMAT_RGBx:
1393 case GST_VIDEO_FORMAT_xRGB:
1394 case GST_VIDEO_FORMAT_BGRx:
1395 case GST_VIDEO_FORMAT_xBGR:
1396 return TRUE;
1397 default:
1398 return FALSE;
1399 }
1400 }
1401
1402 static inline gchar
_index_to_shader_swizzle(int idx)1403 _index_to_shader_swizzle (int idx)
1404 {
1405 switch (idx) {
1406 case 0:
1407 return 'r';
1408 case 1:
1409 return 'g';
1410 case 2:
1411 return 'b';
1412 case 3:
1413 return 'a';
1414 default:
1415 return '#';
1416 }
1417 }
1418
1419 /* attempts to transform expected to want using swizzling */
1420 static gchar *
_RGB_pixel_order(const gchar * expected,const gchar * wanted)1421 _RGB_pixel_order (const gchar * expected, const gchar * wanted)
1422 {
1423 GString *ret = g_string_sized_new (4);
1424 gchar *expect, *want;
1425 gchar *orig_want;
1426 int len;
1427 gboolean discard_output = TRUE;
1428
1429 if (g_ascii_strcasecmp (expected, wanted) == 0) {
1430 g_string_free (ret, TRUE);
1431 return g_ascii_strdown (expected, -1);
1432 }
1433
1434 expect = g_ascii_strdown (expected, -1);
1435 orig_want = want = g_ascii_strdown (wanted, -1);
1436
1437 if (strcmp (expect, "rgb16") == 0 || strcmp (expect, "bgr16") == 0) {
1438 gchar *temp = expect;
1439 expect = g_strndup (temp, 3);
1440 g_free (temp);
1441 }
1442
1443 if (strcmp (want, "rgb16") == 0 || strcmp (want, "bgr16") == 0) {
1444 gchar *temp = want;
1445 orig_want = want = g_strndup (temp, 3);
1446 g_free (temp);
1447 }
1448
1449 /* pad want with 'a's */
1450 if ((len = strlen (want)) < 4) {
1451 gchar *new_want = g_strndup (want, 4);
1452 while (len < 4) {
1453 new_want[len] = 'a';
1454 len++;
1455 }
1456 g_free (want);
1457 orig_want = want = new_want;
1458 }
1459
1460 /* pad expect with 'a's */
1461 if ((len = strlen (expect)) < 4) {
1462 gchar *new_expect = g_strndup (expect, 4);
1463 while (len < 4) {
1464 new_expect[len] = 'a';
1465 len++;
1466 }
1467 g_free (expect);
1468 expect = new_expect;
1469 }
1470
1471 /* build the swizzle format */
1472 while (want && want[0] != '\0') {
1473 gchar *val;
1474 gint idx;
1475 gchar needle = want[0];
1476
1477 if (needle == 'x')
1478 needle = 'a';
1479
1480 if (!(val = strchr (expect, needle))
1481 && needle == 'a' && !(val = strchr (expect, 'x')))
1482 goto out;
1483
1484 idx = (gint) (val - expect);
1485
1486 ret = g_string_append_c (ret, _index_to_shader_swizzle (idx));
1487 want = &want[1];
1488 }
1489
1490 discard_output = FALSE;
1491 out:
1492 g_free (orig_want);
1493 g_free (expect);
1494
1495 return g_string_free (ret, discard_output);
1496 }
1497
1498 static guint
_get_n_textures(GstVideoFormat v_format)1499 _get_n_textures (GstVideoFormat v_format)
1500 {
1501 switch (v_format) {
1502 case GST_VIDEO_FORMAT_RGBA:
1503 case GST_VIDEO_FORMAT_RGBx:
1504 case GST_VIDEO_FORMAT_ARGB:
1505 case GST_VIDEO_FORMAT_xRGB:
1506 case GST_VIDEO_FORMAT_BGRA:
1507 case GST_VIDEO_FORMAT_BGRx:
1508 case GST_VIDEO_FORMAT_ABGR:
1509 case GST_VIDEO_FORMAT_xBGR:
1510 case GST_VIDEO_FORMAT_RGB:
1511 case GST_VIDEO_FORMAT_BGR:
1512 case GST_VIDEO_FORMAT_AYUV:
1513 case GST_VIDEO_FORMAT_VUYA:
1514 case GST_VIDEO_FORMAT_GRAY8:
1515 case GST_VIDEO_FORMAT_GRAY16_LE:
1516 case GST_VIDEO_FORMAT_GRAY16_BE:
1517 case GST_VIDEO_FORMAT_YUY2:
1518 case GST_VIDEO_FORMAT_UYVY:
1519 case GST_VIDEO_FORMAT_RGB16:
1520 case GST_VIDEO_FORMAT_BGR16:
1521 case GST_VIDEO_FORMAT_ARGB64:
1522 return 1;
1523 case GST_VIDEO_FORMAT_NV12:
1524 case GST_VIDEO_FORMAT_NV21:
1525 return 2;
1526 case GST_VIDEO_FORMAT_I420:
1527 case GST_VIDEO_FORMAT_Y444:
1528 case GST_VIDEO_FORMAT_Y42B:
1529 case GST_VIDEO_FORMAT_Y41B:
1530 case GST_VIDEO_FORMAT_YV12:
1531 return 3;
1532 default:
1533 g_assert_not_reached ();
1534 return 0;
1535 }
1536 }
1537
1538 static void
_RGB_to_RGB(GstGLColorConvert * convert)1539 _RGB_to_RGB (GstGLColorConvert * convert)
1540 {
1541 struct ConvertInfo *info = &convert->priv->convert_info;
1542 GstVideoFormat in_format = GST_VIDEO_INFO_FORMAT (&convert->in_info);
1543 const gchar *in_format_str = gst_video_format_to_string (in_format);
1544 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1545 const gchar *out_format_str = gst_video_format_to_string (out_format);
1546 gchar *pixel_order = _RGB_pixel_order (in_format_str, out_format_str);
1547 gchar *alpha = NULL;
1548
1549 if (_is_RGBx (in_format)) {
1550 int i;
1551 char input_alpha_channel = 'a';
1552 for (i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
1553 if (in_format_str[i] == 'X' || in_format_str[i] == 'x') {
1554 input_alpha_channel = _index_to_shader_swizzle (i);
1555 break;
1556 }
1557 }
1558 alpha = g_strdup_printf ("t.%c = 1.0;", input_alpha_channel);
1559 }
1560 info->templ = &templ_REORDER;
1561 info->frag_body = g_strdup_printf (templ_REORDER_BODY, alpha ? alpha : "",
1562 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1563 info->shader_tex_names[0] = "tex";
1564
1565 g_free (alpha);
1566 g_free (pixel_order);
1567 }
1568
1569 static void
_YUV_to_RGB(GstGLColorConvert * convert)1570 _YUV_to_RGB (GstGLColorConvert * convert)
1571 {
1572 struct ConvertInfo *info = &convert->priv->convert_info;
1573 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1574 const gchar *out_format_str = gst_video_format_to_string (out_format);
1575 gchar *pixel_order = _RGB_pixel_order ("rgba", out_format_str);
1576 gboolean apple_ycbcr = gst_gl_context_check_feature (convert->context,
1577 "GL_APPLE_ycbcr_422");
1578 gboolean in_tex_rectangular = FALSE;
1579
1580 #if GST_GL_HAVE_OPENGL
1581 GstMemory *memory = gst_buffer_peek_memory (convert->inbuf, 0);
1582 if (gst_is_gl_memory (memory) && (USING_OPENGL (convert->context)
1583 || USING_OPENGL3 (convert->context))) {
1584 in_tex_rectangular =
1585 convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE;
1586 }
1587 #endif
1588
1589 info->out_n_textures = 1;
1590
1591 if (in_tex_rectangular && apple_ycbcr
1592 && gst_buffer_n_memory (convert->inbuf) == 1) {
1593 /* FIXME: We should probably also check if tex_target actually is using
1594 * the Apple YCbCr422 extension. It could also be a normal UYVY texture
1595 * with RB or Lum/Alpha
1596 */
1597 /* The mangling will change this to the correct texture2DRect, sampler2DRect
1598 * for us */
1599 info->templ = &templ_REORDER;
1600 info->frag_body =
1601 g_strdup_printf (templ_REORDER_BODY, "", pixel_order[0], pixel_order[1],
1602 pixel_order[2], pixel_order[3]);
1603 info->shader_tex_names[0] = "tex";
1604 } else {
1605 switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) {
1606 case GST_VIDEO_FORMAT_AYUV:
1607 info->templ = &templ_AYUV_to_RGB;
1608 info->frag_body = g_strdup_printf (templ_AYUV_to_RGB_BODY,
1609 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1610 info->shader_tex_names[0] = "tex";
1611 break;
1612 case GST_VIDEO_FORMAT_VUYA:
1613 info->templ = &templ_VUYA_to_RGB;
1614 info->frag_body = g_strdup_printf (templ_VUYA_to_RGB_BODY,
1615 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1616 info->shader_tex_names[0] = "tex";
1617 break;
1618 case GST_VIDEO_FORMAT_I420:
1619 case GST_VIDEO_FORMAT_Y444:
1620 case GST_VIDEO_FORMAT_Y42B:
1621 case GST_VIDEO_FORMAT_Y41B:
1622 info->templ = &templ_PLANAR_YUV_to_RGB;
1623 info->frag_body =
1624 g_strdup_printf (templ_PLANAR_YUV_to_RGB_BODY, pixel_order[0],
1625 pixel_order[1], pixel_order[2], pixel_order[3]);
1626 info->shader_tex_names[0] = "Ytex";
1627 info->shader_tex_names[1] = "Utex";
1628 info->shader_tex_names[2] = "Vtex";
1629 break;
1630 case GST_VIDEO_FORMAT_YV12:
1631 info->templ = &templ_PLANAR_YUV_to_RGB;
1632 info->frag_body =
1633 g_strdup_printf (templ_PLANAR_YUV_to_RGB_BODY, pixel_order[0],
1634 pixel_order[1], pixel_order[2], pixel_order[3]);
1635 info->shader_tex_names[0] = "Ytex";
1636 info->shader_tex_names[1] = "Vtex";
1637 info->shader_tex_names[2] = "Utex";
1638 break;
1639 case GST_VIDEO_FORMAT_YUY2:
1640 {
1641 char uv_val = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1642 info->templ = &templ_YUY2_UYVY_to_RGB;
1643 info->frag_body =
1644 g_strdup_printf (templ_YUY2_UYVY_to_RGB_BODY, 'r', uv_val, uv_val,
1645 'g', 'a', pixel_order[0], pixel_order[1], pixel_order[2],
1646 pixel_order[3]);
1647 info->shader_tex_names[0] = "Ytex";
1648 break;
1649 }
1650 case GST_VIDEO_FORMAT_UYVY:
1651 {
1652 char y_val = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1653 info->templ = &templ_YUY2_UYVY_to_RGB;
1654 info->frag_body =
1655 g_strdup_printf (templ_YUY2_UYVY_to_RGB_BODY, y_val, 'g', 'g', 'r',
1656 'b', pixel_order[0], pixel_order[1], pixel_order[2],
1657 pixel_order[3]);
1658 info->shader_tex_names[0] = "Ytex";
1659 break;
1660 }
1661 case GST_VIDEO_FORMAT_NV12:
1662 {
1663 char val2 = convert->priv->in_tex_formats[1] == GST_GL_RG ? 'g' : 'a';
1664 info->templ = &templ_NV12_NV21_to_RGB;
1665 info->frag_body =
1666 g_strdup_printf (templ_NV12_NV21_to_RGB_BODY, 'r', val2,
1667 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1668 info->shader_tex_names[0] = "Ytex";
1669 info->shader_tex_names[1] = "UVtex";
1670 break;
1671 }
1672 case GST_VIDEO_FORMAT_NV21:
1673 {
1674 char val2 = convert->priv->in_tex_formats[1] == GST_GL_RG ? 'g' : 'a';
1675 info->templ = &templ_NV12_NV21_to_RGB;
1676 info->frag_body =
1677 g_strdup_printf (templ_NV12_NV21_to_RGB_BODY, val2, 'r',
1678 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1679 info->shader_tex_names[0] = "Ytex";
1680 info->shader_tex_names[1] = "UVtex";
1681 break;
1682 }
1683 default:
1684 break;
1685 }
1686 }
1687
1688 if (gst_video_colorimetry_matches (&convert->in_info.colorimetry,
1689 GST_VIDEO_COLORIMETRY_BT709)) {
1690 info->cms_offset = (gfloat *) from_yuv_bt709_offset;
1691 info->cms_coeff1 = (gfloat *) from_yuv_bt709_rcoeff;
1692 info->cms_coeff2 = (gfloat *) from_yuv_bt709_gcoeff;
1693 info->cms_coeff3 = (gfloat *) from_yuv_bt709_bcoeff;
1694 } else {
1695 /* defaults/bt601 */
1696 info->cms_offset = (gfloat *) from_yuv_bt601_offset;
1697 info->cms_coeff1 = (gfloat *) from_yuv_bt601_rcoeff;
1698 info->cms_coeff2 = (gfloat *) from_yuv_bt601_gcoeff;
1699 info->cms_coeff3 = (gfloat *) from_yuv_bt601_bcoeff;
1700 }
1701
1702 g_free (pixel_order);
1703 }
1704
1705 static void
_RGB_to_YUV(GstGLColorConvert * convert)1706 _RGB_to_YUV (GstGLColorConvert * convert)
1707 {
1708 struct ConvertInfo *info = &convert->priv->convert_info;
1709 GstVideoFormat in_format = GST_VIDEO_INFO_FORMAT (&convert->in_info);
1710 const gchar *in_format_str = gst_video_format_to_string (in_format);
1711 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1712 gchar *pixel_order = _RGB_pixel_order (in_format_str, "rgba");
1713 const gchar *alpha;
1714
1715 info->frag_prog = NULL;
1716
1717 info->shader_tex_names[0] = "tex";
1718
1719 switch (out_format) {
1720 case GST_VIDEO_FORMAT_AYUV:
1721 alpha = _is_RGBx (in_format) ? "1.0" : "texel.a";
1722 info->templ = &templ_RGB_to_AYUV;
1723 info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, pixel_order[0],
1724 pixel_order[1], pixel_order[2], pixel_order[3], alpha);
1725 info->out_n_textures = 1;
1726 break;
1727 case GST_VIDEO_FORMAT_VUYA:
1728 alpha = _is_RGBx (in_format) ? "1.0" : "texel.a";
1729 info->templ = &templ_RGB_to_VUYA;
1730 info->frag_body = g_strdup_printf (templ_RGB_to_VUYA_BODY, pixel_order[0],
1731 pixel_order[1], pixel_order[2], pixel_order[3], alpha);
1732 info->out_n_textures = 1;
1733 break;
1734 case GST_VIDEO_FORMAT_I420:
1735 case GST_VIDEO_FORMAT_YV12:
1736 case GST_VIDEO_FORMAT_Y444:
1737 case GST_VIDEO_FORMAT_Y42B:
1738 case GST_VIDEO_FORMAT_Y41B:
1739 info->templ = &templ_RGB_to_PLANAR_YUV;
1740 info->frag_body = g_strdup_printf (templ_RGB_to_PLANAR_YUV_BODY,
1741 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1742 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1743 info->out_n_textures = 3;
1744 if (out_format == GST_VIDEO_FORMAT_Y444) {
1745 info->chroma_sampling[0] = info->chroma_sampling[1] = 1.0f;
1746 } else if (out_format == GST_VIDEO_FORMAT_Y42B) {
1747 info->chroma_sampling[0] = 2.0f;
1748 info->chroma_sampling[1] = 1.0f;
1749 } else if (out_format == GST_VIDEO_FORMAT_Y41B) {
1750 info->chroma_sampling[0] = 4.0f;
1751 info->chroma_sampling[1] = 1.0f;
1752 } else {
1753 info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f;
1754 }
1755 break;
1756 case GST_VIDEO_FORMAT_YUY2:
1757 info->templ = &templ_RGB_to_YUY2_UYVY;
1758 info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY,
1759 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1760 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1761 'x', 'y', 'x', 'z');
1762 info->out_n_textures = 1;
1763 break;
1764 case GST_VIDEO_FORMAT_UYVY:
1765 info->templ = &templ_RGB_to_YUY2_UYVY,
1766 info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY,
1767 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1768 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1769 'y', 'x', 'z', 'x');
1770 info->out_n_textures = 1;
1771 break;
1772 case GST_VIDEO_FORMAT_NV12:
1773 info->templ = &templ_RGB_to_NV12_NV21,
1774 info->frag_body = g_strdup_printf (templ_RGB_to_NV12_NV21_BODY,
1775 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1776 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1777 'y', 'z');
1778 info->out_n_textures = 2;
1779 break;
1780 case GST_VIDEO_FORMAT_NV21:
1781 info->templ = &templ_RGB_to_NV12_NV21,
1782 info->frag_body = g_strdup_printf (templ_RGB_to_NV12_NV21_BODY,
1783 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1784 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1785 'z', 'y');
1786 info->out_n_textures = 2;
1787 break;
1788 default:
1789 break;
1790 }
1791
1792 if (gst_video_colorimetry_matches (&convert->in_info.colorimetry,
1793 GST_VIDEO_COLORIMETRY_BT709)) {
1794 info->cms_offset = (gfloat *) from_rgb_bt709_offset;
1795 info->cms_coeff1 = (gfloat *) from_rgb_bt709_ycoeff;
1796 info->cms_coeff2 = (gfloat *) from_rgb_bt709_ucoeff;
1797 info->cms_coeff3 = (gfloat *) from_rgb_bt709_vcoeff;
1798 } else {
1799 /* defaults/bt601 */
1800 info->cms_offset = (gfloat *) from_rgb_bt601_offset;
1801 info->cms_coeff1 = (gfloat *) from_rgb_bt601_ycoeff;
1802 info->cms_coeff2 = (gfloat *) from_rgb_bt601_ucoeff;
1803 info->cms_coeff3 = (gfloat *) from_rgb_bt601_vcoeff;
1804 }
1805
1806 g_free (pixel_order);
1807 }
1808
1809 static void
_RGB_to_GRAY(GstGLColorConvert * convert)1810 _RGB_to_GRAY (GstGLColorConvert * convert)
1811 {
1812 struct ConvertInfo *info = &convert->priv->convert_info;
1813 GstVideoFormat in_format = GST_VIDEO_INFO_FORMAT (&convert->in_info);
1814 const gchar *in_format_str = gst_video_format_to_string (in_format);
1815 gchar *pixel_order = _RGB_pixel_order (in_format_str, "rgba");
1816 gchar *alpha = NULL;
1817
1818 info->out_n_textures = 1;
1819 info->shader_tex_names[0] = "tex";
1820
1821 if (_is_RGBx (in_format))
1822 alpha = g_strdup_printf ("t.%c = 1.0;", pixel_order[3]);
1823
1824 switch (GST_VIDEO_INFO_FORMAT (&convert->out_info)) {
1825 case GST_VIDEO_FORMAT_GRAY8:
1826 info->templ = &templ_REORDER;
1827 info->frag_body = g_strdup_printf (templ_REORDER_BODY, alpha ? alpha : "",
1828 pixel_order[0], pixel_order[0], pixel_order[0], pixel_order[3]);
1829 break;
1830 default:
1831 break;
1832 }
1833
1834 g_free (alpha);
1835 g_free (pixel_order);
1836 }
1837
1838 static void
_GRAY_to_RGB(GstGLColorConvert * convert)1839 _GRAY_to_RGB (GstGLColorConvert * convert)
1840 {
1841 struct ConvertInfo *info = &convert->priv->convert_info;
1842 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1843 const gchar *out_format_str = gst_video_format_to_string (out_format);
1844 gchar *pixel_order = _RGB_pixel_order ("rgba", out_format_str);
1845
1846 info->shader_tex_names[0] = "tex";
1847
1848 switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) {
1849 case GST_VIDEO_FORMAT_GRAY8:
1850 info->templ = &templ_REORDER;
1851 info->frag_body = g_strdup_printf (templ_REORDER_BODY, "", pixel_order[0],
1852 pixel_order[0], pixel_order[0], pixel_order[3]);
1853 break;
1854 case GST_VIDEO_FORMAT_GRAY16_LE:
1855 {
1856 char val2 = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1857 info->templ = &templ_COMPOSE;
1858 info->frag_body = g_strdup_printf (templ_COMPOSE_BODY, val2, 'r',
1859 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1860 break;
1861 }
1862 case GST_VIDEO_FORMAT_GRAY16_BE:
1863 {
1864 char val2 = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1865 info->templ = &templ_COMPOSE;
1866 info->frag_body = g_strdup_printf (templ_COMPOSE_BODY, 'r', val2,
1867 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1868 break;
1869 }
1870 default:
1871 break;
1872 }
1873
1874 g_free (pixel_order);
1875 }
1876
1877 static void
_bind_buffer(GstGLColorConvert * convert)1878 _bind_buffer (GstGLColorConvert * convert)
1879 {
1880 const GstGLFuncs *gl = convert->context->gl_vtable;
1881
1882 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, convert->priv->vbo_indices);
1883 gl->BindBuffer (GL_ARRAY_BUFFER, convert->priv->vertex_buffer);
1884
1885 /* Load the vertex position */
1886 gl->VertexAttribPointer (convert->priv->attr_position, 3, GL_FLOAT, GL_FALSE,
1887 5 * sizeof (GLfloat), (void *) 0);
1888
1889 /* Load the texture coordinate */
1890 gl->VertexAttribPointer (convert->priv->attr_texture, 2, GL_FLOAT, GL_FALSE,
1891 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
1892
1893 gl->EnableVertexAttribArray (convert->priv->attr_position);
1894 gl->EnableVertexAttribArray (convert->priv->attr_texture);
1895 }
1896
1897 static void
_unbind_buffer(GstGLColorConvert * convert)1898 _unbind_buffer (GstGLColorConvert * convert)
1899 {
1900 const GstGLFuncs *gl = convert->context->gl_vtable;
1901
1902 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1903 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1904
1905 gl->DisableVertexAttribArray (convert->priv->attr_position);
1906 gl->DisableVertexAttribArray (convert->priv->attr_texture);
1907 }
1908
1909 static GstGLShader *
_create_shader(GstGLColorConvert * convert)1910 _create_shader (GstGLColorConvert * convert)
1911 {
1912 struct ConvertInfo *info = &convert->priv->convert_info;
1913 GString *str = g_string_new (NULL);
1914 GstGLShader *ret = NULL;
1915 GstGLSLStage *stage;
1916 GstGLSLVersion version;
1917 GstGLSLProfile profile;
1918 gchar *version_str, *tmp, *tmp1;
1919 const gchar *strings[3];
1920 GError *error = NULL;
1921 int i;
1922
1923 ret = gst_gl_shader_new (convert->context);
1924
1925 tmp =
1926 _gst_glsl_mangle_shader (text_vertex_shader, GL_VERTEX_SHADER,
1927 info->templ->target, convert->priv->from_texture_target, convert->context,
1928 &version, &profile);
1929
1930 tmp1 = gst_glsl_version_profile_to_string (version, profile);
1931 version_str = g_strdup_printf ("#version %s\n", tmp1);
1932 g_free (tmp1);
1933
1934 strings[0] = version_str;
1935 strings[1] = tmp;
1936 if (!(stage = gst_glsl_stage_new_with_strings (convert->context,
1937 GL_VERTEX_SHADER, version, profile, 2, strings))) {
1938 GST_ERROR_OBJECT (convert, "Failed to create vertex stage");
1939 g_free (version_str);
1940 g_free (tmp);
1941 gst_object_unref (ret);
1942 return NULL;
1943 }
1944 g_free (tmp);
1945
1946 if (!gst_gl_shader_compile_attach_stage (ret, stage, &error)) {
1947 GST_ERROR_OBJECT (convert, "Failed to compile vertex shader %s",
1948 error->message);
1949 g_clear_error (&error);
1950 g_free (version_str);
1951 gst_object_unref (stage);
1952 gst_object_unref (ret);
1953 return NULL;
1954 }
1955
1956 if (info->templ->extensions)
1957 g_string_append (str, info->templ->extensions);
1958
1959 if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES
1960 && info->templ->target != GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
1961 g_string_append (str, glsl_OES_extension_string);
1962
1963 g_string_append (str,
1964 gst_gl_shader_string_get_highest_precision (convert->context, version,
1965 profile));
1966
1967 if (info->templ->uniforms)
1968 g_string_append (str, info->templ->uniforms);
1969
1970 g_string_append_c (str, '\n');
1971
1972 /* GL 3.3+ and GL ES 3.x */
1973 if ((profile == GST_GLSL_PROFILE_CORE && version >= GST_GLSL_VERSION_330)
1974 || (profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)) {
1975 if (info->out_n_textures > 1) {
1976 gint i;
1977
1978 for (i = 0; i < info->out_n_textures; i++) {
1979 g_string_append_printf (str,
1980 "layout(location = %d) out vec4 fragColor_%d;\n", i, i);
1981 }
1982 } else {
1983 g_string_append (str, "layout (location = 0) out vec4 fragColor;\n");
1984 }
1985 } else if (profile == GST_GLSL_PROFILE_CORE
1986 && version >= GST_GLSL_VERSION_150) {
1987 /* no layout specifiers, use glBindFragDataLocation instead */
1988 if (info->out_n_textures > 1) {
1989 gint i;
1990
1991 for (i = 0; i < info->out_n_textures; i++) {
1992 gchar *var_name = g_strdup_printf ("fragColor_%d", i);
1993 g_string_append_printf (str, "out vec4 %s;\n", var_name);
1994 gst_gl_shader_bind_frag_data_location (ret, i, var_name);
1995 g_free (var_name);
1996 }
1997 } else {
1998 g_string_append (str, "out vec4 fragColor;\n");
1999 gst_gl_shader_bind_frag_data_location (ret, 0, "fragColor");
2000 }
2001 }
2002
2003 for (i = 0; i < MAX_FUNCTIONS; i++) {
2004 if (info->templ->functions[i] == NULL)
2005 break;
2006
2007 g_string_append_c (str, '\n');
2008 g_string_append (str, info->templ->functions[i]);
2009 g_string_append_c (str, '\n');
2010 }
2011
2012 {
2013 const gchar *varying = NULL;
2014
2015 if ((profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)
2016 || (profile == GST_GLSL_PROFILE_CORE
2017 && version >= GST_GLSL_VERSION_150)) {
2018 varying = "in";
2019 } else {
2020 varying = "varying";
2021 }
2022 g_string_append_printf (str, "\n%s vec2 v_texcoord;\nvoid main (void) {\n",
2023 varying);
2024 }
2025 if (info->frag_body) {
2026 g_string_append (str, "vec2 texcoord;\n");
2027 if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE
2028 && info->templ->target != GST_GL_TEXTURE_TARGET_RECTANGLE) {
2029 g_string_append (str, "texcoord = v_texcoord * vec2 (width, height);\n");
2030 } else {
2031 g_string_append (str, "texcoord = v_texcoord;\n");
2032 }
2033
2034 g_string_append (str, info->frag_body);
2035 }
2036 g_string_append (str, "\n}");
2037 tmp = g_string_free (str, FALSE);
2038 info->frag_prog = _gst_glsl_mangle_shader (tmp, GL_FRAGMENT_SHADER,
2039 info->templ->target, convert->priv->from_texture_target, convert->context,
2040 &version, &profile);
2041 g_free (tmp);
2042
2043 strings[1] = info->frag_prog;
2044 if (!(stage = gst_glsl_stage_new_with_strings (convert->context,
2045 GL_FRAGMENT_SHADER, version, profile, 2, strings))) {
2046 GST_ERROR_OBJECT (convert, "Failed to create fragment stage");
2047 g_free (info->frag_prog);
2048 info->frag_prog = NULL;
2049 g_free (version_str);
2050 gst_object_unref (ret);
2051 return NULL;
2052 }
2053 g_free (version_str);
2054 if (!gst_gl_shader_compile_attach_stage (ret, stage, &error)) {
2055 GST_ERROR_OBJECT (convert, "Failed to compile fragment shader %s",
2056 error->message);
2057 g_clear_error (&error);
2058 g_free (info->frag_prog);
2059 info->frag_prog = NULL;
2060 gst_object_unref (stage);
2061 gst_object_unref (ret);
2062 return NULL;
2063 }
2064
2065 if (!gst_gl_shader_link (ret, &error)) {
2066 GST_ERROR_OBJECT (convert, "Failed to link shader %s", error->message);
2067 g_clear_error (&error);
2068 g_free (info->frag_prog);
2069 info->frag_prog = NULL;
2070 gst_object_unref (ret);
2071 return NULL;
2072 }
2073
2074 return ret;
2075 }
2076
2077 /* Called in the gl thread */
2078 static gboolean
_init_convert(GstGLColorConvert * convert)2079 _init_convert (GstGLColorConvert * convert)
2080 {
2081 GstGLFuncs *gl;
2082 struct ConvertInfo *info = &convert->priv->convert_info;
2083 gint i;
2084
2085 gl = convert->context->gl_vtable;
2086
2087 if (convert->initted)
2088 return TRUE;
2089
2090 GST_INFO ("Initializing color conversion from %s to %s",
2091 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->in_info)),
2092 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->out_info)));
2093
2094 if (!gl->CreateProgramObject && !gl->CreateProgram) {
2095 GST_ERROR_OBJECT (convert, "Cannot perform color conversion without "
2096 "OpenGL shaders");
2097 goto error;
2098 }
2099
2100 if (GST_VIDEO_INFO_IS_RGB (&convert->in_info)) {
2101 if (GST_VIDEO_INFO_IS_RGB (&convert->out_info)) {
2102 _RGB_to_RGB (convert);
2103 }
2104 }
2105
2106 if (GST_VIDEO_INFO_IS_YUV (&convert->in_info)) {
2107 if (GST_VIDEO_INFO_IS_RGB (&convert->out_info)) {
2108 _YUV_to_RGB (convert);
2109 }
2110 }
2111
2112 if (GST_VIDEO_INFO_IS_RGB (&convert->in_info)) {
2113 if (GST_VIDEO_INFO_IS_YUV (&convert->out_info)) {
2114 _RGB_to_YUV (convert);
2115 }
2116 }
2117
2118 if (GST_VIDEO_INFO_IS_RGB (&convert->in_info)) {
2119 if (GST_VIDEO_INFO_IS_GRAY (&convert->out_info)) {
2120 _RGB_to_GRAY (convert);
2121 }
2122 }
2123
2124 if (GST_VIDEO_INFO_IS_GRAY (&convert->in_info)) {
2125 if (GST_VIDEO_INFO_IS_RGB (&convert->out_info)) {
2126 _GRAY_to_RGB (convert);
2127 }
2128 }
2129
2130 if (!info->frag_body || info->in_n_textures == 0 || info->out_n_textures == 0)
2131 goto unhandled_format;
2132
2133 /* multiple draw targets not supported on GLES2... */
2134 if (info->out_n_textures > 1 && !gl->DrawBuffers) {
2135 GST_ERROR ("Conversion requires output to multiple draw buffers");
2136 goto incompatible_api;
2137 }
2138
2139 /* Requires reading from a RG/LA framebuffer... */
2140 if (USING_GLES2 (convert->context) && !USING_GLES3 (convert->context) &&
2141 (GST_VIDEO_INFO_FORMAT (&convert->out_info) == GST_VIDEO_FORMAT_YUY2 ||
2142 GST_VIDEO_INFO_FORMAT (&convert->out_info) ==
2143 GST_VIDEO_FORMAT_UYVY)) {
2144 GST_ERROR ("Conversion requires reading with an unsupported format");
2145 goto incompatible_api;
2146 }
2147
2148 if (!(convert->shader = _create_shader (convert)))
2149 goto error;
2150
2151 convert->priv->attr_position =
2152 gst_gl_shader_get_attribute_location (convert->shader, "a_position");
2153 convert->priv->attr_texture =
2154 gst_gl_shader_get_attribute_location (convert->shader, "a_texcoord");
2155
2156 gst_gl_shader_use (convert->shader);
2157
2158 if (info->cms_offset && info->cms_coeff1
2159 && info->cms_coeff2 && info->cms_coeff3) {
2160 gst_gl_shader_set_uniform_3fv (convert->shader, "offset", 1,
2161 info->cms_offset);
2162 gst_gl_shader_set_uniform_3fv (convert->shader, "coeff1", 1,
2163 info->cms_coeff1);
2164 gst_gl_shader_set_uniform_3fv (convert->shader, "coeff2", 1,
2165 info->cms_coeff2);
2166 gst_gl_shader_set_uniform_3fv (convert->shader, "coeff3", 1,
2167 info->cms_coeff3);
2168 }
2169
2170 for (i = info->in_n_textures; i >= 0; i--) {
2171 if (info->shader_tex_names[i])
2172 gst_gl_shader_set_uniform_1i (convert->shader, info->shader_tex_names[i],
2173 i);
2174 }
2175
2176 gst_gl_shader_set_uniform_1f (convert->shader, "width",
2177 GST_VIDEO_INFO_WIDTH (&convert->in_info));
2178 gst_gl_shader_set_uniform_1f (convert->shader, "height",
2179 GST_VIDEO_INFO_HEIGHT (&convert->in_info));
2180
2181 if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE) {
2182 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_x", 1.);
2183 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_y", 1.);
2184 } else {
2185 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_x",
2186 1. / (gfloat) GST_VIDEO_INFO_WIDTH (&convert->in_info));
2187 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_y",
2188 1. / (gfloat) GST_VIDEO_INFO_HEIGHT (&convert->in_info));
2189 }
2190
2191 if (info->chroma_sampling[0] > 0.0f && info->chroma_sampling[1] > 0.0f) {
2192 gst_gl_shader_set_uniform_2fv (convert->shader, "chroma_sampling", 1,
2193 info->chroma_sampling);
2194 }
2195
2196 gst_gl_context_clear_shader (convert->context);
2197
2198 if (convert->fbo == NULL && !_init_convert_fbo (convert)) {
2199 goto error;
2200 }
2201
2202 if (!convert->priv->vertex_buffer) {
2203 if (gl->GenVertexArrays) {
2204 gl->GenVertexArrays (1, &convert->priv->vao);
2205 gl->BindVertexArray (convert->priv->vao);
2206 }
2207
2208 gl->GenBuffers (1, &convert->priv->vertex_buffer);
2209 gl->BindBuffer (GL_ARRAY_BUFFER, convert->priv->vertex_buffer);
2210 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
2211 GL_STATIC_DRAW);
2212
2213 gl->GenBuffers (1, &convert->priv->vbo_indices);
2214 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, convert->priv->vbo_indices);
2215 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
2216 GL_STATIC_DRAW);
2217
2218 if (gl->GenVertexArrays) {
2219 _bind_buffer (convert);
2220 gl->BindVertexArray (0);
2221 }
2222
2223 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
2224 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
2225 }
2226
2227 gl->BindTexture (GL_TEXTURE_2D, 0);
2228
2229 convert->initted = TRUE;
2230
2231 return TRUE;
2232
2233 unhandled_format:
2234 GST_ERROR_OBJECT (convert, "Don't know how to convert from %s to %s",
2235 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->in_info)),
2236 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->out_info)));
2237
2238 error:
2239 return FALSE;
2240
2241 incompatible_api:
2242 {
2243 GST_ERROR_OBJECT (convert, "Converting from %s to %s requires "
2244 "functionality that the current OpenGL setup does not support",
2245 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->in_info)),
2246 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
2247 (&convert->out_info)));
2248 return FALSE;
2249 }
2250 }
2251
2252 /* called by _init_convert (in the gl thread) */
2253 static gboolean
_init_convert_fbo(GstGLColorConvert * convert)2254 _init_convert_fbo (GstGLColorConvert * convert)
2255 {
2256 guint out_width, out_height;
2257
2258 out_width = GST_VIDEO_INFO_WIDTH (&convert->out_info);
2259 out_height = GST_VIDEO_INFO_HEIGHT (&convert->out_info);
2260
2261 convert->fbo =
2262 gst_gl_framebuffer_new_with_default_depth (convert->context, out_width,
2263 out_height);
2264
2265 return convert->fbo != NULL;
2266 }
2267
2268 static gboolean
_do_convert_one_view(GstGLContext * context,GstGLColorConvert * convert,guint view_num)2269 _do_convert_one_view (GstGLContext * context, GstGLColorConvert * convert,
2270 guint view_num)
2271 {
2272 guint in_width, in_height, out_width, out_height;
2273 struct ConvertInfo *c_info = &convert->priv->convert_info;
2274 GstMapInfo out_info[GST_VIDEO_MAX_PLANES], in_info[GST_VIDEO_MAX_PLANES];
2275 gboolean res = TRUE;
2276 gint i, j = 0;
2277 const gint in_plane_offset = view_num * c_info->in_n_textures;
2278 const gint out_plane_offset = view_num * c_info->out_n_textures;
2279
2280 out_width = GST_VIDEO_INFO_WIDTH (&convert->out_info);
2281 out_height = GST_VIDEO_INFO_HEIGHT (&convert->out_info);
2282 in_width = GST_VIDEO_INFO_WIDTH (&convert->in_info);
2283 in_height = GST_VIDEO_INFO_HEIGHT (&convert->in_info);
2284
2285 for (i = 0; i < c_info->in_n_textures; i++) {
2286 convert->priv->in_tex[i] =
2287 (GstGLMemory *) gst_buffer_peek_memory (convert->inbuf,
2288 i + in_plane_offset);
2289 if (!gst_is_gl_memory ((GstMemory *) convert->priv->in_tex[i])) {
2290 GST_ERROR_OBJECT (convert, "input must be GstGLMemory");
2291 res = FALSE;
2292 goto out;
2293 }
2294 if (!gst_memory_map ((GstMemory *) convert->priv->in_tex[i], &in_info[i],
2295 GST_MAP_READ | GST_MAP_GL)) {
2296 GST_ERROR_OBJECT (convert, "failed to map input memory %p",
2297 convert->priv->in_tex[i]);
2298 res = FALSE;
2299 goto out;
2300 }
2301 }
2302
2303 for (j = 0; j < c_info->out_n_textures; j++) {
2304 GstGLMemory *out_tex =
2305 (GstGLMemory *) gst_buffer_peek_memory (convert->outbuf,
2306 j + out_plane_offset);
2307 gint mem_width, mem_height;
2308
2309 if (!gst_is_gl_memory ((GstMemory *) out_tex)) {
2310 GST_ERROR_OBJECT (convert, "output must be GstGLMemory");
2311 res = FALSE;
2312 goto out;
2313 }
2314
2315 mem_width = gst_gl_memory_get_texture_width (out_tex);
2316 mem_height = gst_gl_memory_get_texture_height (out_tex);
2317
2318 if (out_tex->tex_format == GST_GL_LUMINANCE
2319 || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
2320 || out_width != mem_width || out_height != mem_height) {
2321 /* Luminance formats are not color renderable */
2322 /* renderering to a framebuffer only renders the intersection of all
2323 * the attachments i.e. the smallest attachment size */
2324 if (!convert->priv->out_tex[j]) {
2325 GstGLVideoAllocationParams *params;
2326 GstGLBaseMemoryAllocator *base_mem_allocator;
2327 GstAllocator *allocator;
2328 GstVideoInfo temp_info;
2329
2330 gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, out_width,
2331 out_height);
2332
2333 allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR_NAME);
2334 base_mem_allocator = GST_GL_BASE_MEMORY_ALLOCATOR (allocator);
2335 params = gst_gl_video_allocation_params_new (context, NULL, &temp_info,
2336 0, NULL, convert->priv->to_texture_target, GST_GL_RGBA);
2337
2338 convert->priv->out_tex[j] =
2339 (GstGLMemory *) gst_gl_base_memory_alloc (base_mem_allocator,
2340 (GstGLAllocationParams *) params);
2341
2342 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
2343 gst_object_unref (allocator);
2344 }
2345 } else {
2346 convert->priv->out_tex[j] = out_tex;
2347 }
2348
2349 if (!gst_memory_map ((GstMemory *) convert->priv->out_tex[j], &out_info[j],
2350 GST_MAP_WRITE | GST_MAP_GL)) {
2351 GST_ERROR_OBJECT (convert, "failed to map output memory %p",
2352 convert->priv->out_tex[i]);
2353 res = FALSE;
2354 goto out;
2355 }
2356 }
2357
2358 GST_LOG_OBJECT (convert, "converting to textures:%p,%p,%p,%p "
2359 "dimensions:%ux%u, from textures:%p,%p,%p,%p dimensions:%ux%u",
2360 convert->priv->out_tex[0], convert->priv->out_tex[1],
2361 convert->priv->out_tex[2], convert->priv->out_tex[3], out_width,
2362 out_height, convert->priv->in_tex[0], convert->priv->in_tex[1],
2363 convert->priv->in_tex[2], convert->priv->in_tex[3], in_width, in_height);
2364
2365 if (!_do_convert_draw (context, convert))
2366 res = FALSE;
2367
2368 out:
2369 for (j--; j >= 0; j--) {
2370 GstGLMemory *out_tex =
2371 (GstGLMemory *) gst_buffer_peek_memory (convert->outbuf,
2372 j + out_plane_offset);
2373 gint mem_width, mem_height;
2374
2375 gst_memory_unmap ((GstMemory *) convert->priv->out_tex[j], &out_info[j]);
2376
2377 mem_width = gst_gl_memory_get_texture_width (out_tex);
2378 mem_height = gst_gl_memory_get_texture_height (out_tex);
2379
2380 if (out_tex->tex_format == GST_GL_LUMINANCE
2381 || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
2382 || out_width != mem_width || out_height != mem_height) {
2383 GstMapInfo to_info, from_info;
2384
2385 if (!gst_memory_map ((GstMemory *) convert->priv->out_tex[j], &from_info,
2386 GST_MAP_READ | GST_MAP_GL)) {
2387 GST_ERROR_OBJECT (convert, "Failed to map intermediate memory");
2388 res = FALSE;
2389 continue;
2390 }
2391 if (!gst_memory_map ((GstMemory *) out_tex, &to_info,
2392 GST_MAP_WRITE | GST_MAP_GL)) {
2393 GST_ERROR_OBJECT (convert, "Failed to map intermediate memory");
2394 res = FALSE;
2395 continue;
2396 }
2397 gst_gl_memory_copy_into (convert->priv->out_tex[j],
2398 out_tex->tex_id, convert->priv->to_texture_target,
2399 out_tex->tex_format, mem_width, mem_height);
2400 gst_memory_unmap ((GstMemory *) convert->priv->out_tex[j], &from_info);
2401 gst_memory_unmap ((GstMemory *) out_tex, &to_info);
2402 } else {
2403 convert->priv->out_tex[j] = NULL;
2404 }
2405 }
2406
2407 /* YV12 the same as I420 except planes 1+2 swapped */
2408 if (GST_VIDEO_INFO_FORMAT (&convert->out_info) == GST_VIDEO_FORMAT_YV12) {
2409 GstMemory *mem1 =
2410 gst_buffer_get_memory (convert->outbuf, 1 + out_plane_offset);
2411 GstMemory *mem2 =
2412 gst_buffer_get_memory (convert->outbuf, 2 + out_plane_offset);
2413
2414 gst_buffer_replace_memory (convert->outbuf, 1 + out_plane_offset, mem2);
2415 gst_buffer_replace_memory (convert->outbuf, 2 + out_plane_offset, mem1);
2416 }
2417
2418 for (i--; i >= 0; i--) {
2419 gst_memory_unmap ((GstMemory *) convert->priv->in_tex[i], &in_info[i]);
2420 }
2421
2422 return res;
2423 }
2424
2425 /* Called by the idle function in the gl thread */
2426 void
_do_convert(GstGLContext * context,GstGLColorConvert * convert)2427 _do_convert (GstGLContext * context, GstGLColorConvert * convert)
2428 {
2429 GstVideoInfo *in_info = &convert->in_info;
2430 struct ConvertInfo *c_info = &convert->priv->convert_info;
2431 gboolean res = TRUE;
2432 gint views, v;
2433 GstGLSyncMeta *sync_meta;
2434 GstFlowReturn ret;
2435
2436 convert->outbuf = NULL;
2437
2438 if (GST_VIDEO_INFO_MULTIVIEW_MODE (in_info) ==
2439 GST_VIDEO_MULTIVIEW_MODE_SEPARATED)
2440 views = GST_VIDEO_INFO_VIEWS (in_info);
2441 else
2442 views = 1;
2443
2444 c_info->in_n_textures =
2445 _get_n_textures (GST_VIDEO_INFO_FORMAT (&convert->in_info));
2446 c_info->out_n_textures =
2447 _get_n_textures (GST_VIDEO_INFO_FORMAT (&convert->out_info));
2448
2449 {
2450 gboolean tex_format_change = FALSE;
2451 guint i, v;
2452
2453 for (v = 0; v < views; v++) {
2454 for (i = 0; i < c_info->in_n_textures; i++) {
2455 guint j = v * c_info->in_n_textures + i;
2456 GstGLMemory *in_tex =
2457 (GstGLMemory *) gst_buffer_peek_memory (convert->inbuf, j);
2458 if (!gst_is_gl_memory ((GstMemory *) in_tex)) {
2459 GST_ERROR_OBJECT (convert, "input must be GstGLMemory");
2460 convert->priv->result = FALSE;
2461 return;
2462 }
2463
2464 if (j >= GST_VIDEO_MAX_PLANES)
2465 /* our arrays aren't that big */
2466 g_assert_not_reached ();
2467
2468 if (v > 0 && in_tex->tex_format != convert->priv->in_tex_formats[i]) {
2469 GST_ERROR_OBJECT (convert, "Cannot convert textures with "
2470 "different types");
2471 convert->priv->result = FALSE;
2472 return;
2473 }
2474
2475 if (convert->priv->in_tex_formats[j] != in_tex->tex_format)
2476 tex_format_change = TRUE;
2477
2478 convert->priv->in_tex_formats[j] = in_tex->tex_format;
2479 }
2480 }
2481
2482 if (tex_format_change)
2483 gst_gl_color_convert_reset_shader (convert);
2484 }
2485
2486 if (!_init_convert (convert)) {
2487 convert->priv->result = FALSE;
2488 return;
2489 }
2490
2491 sync_meta = gst_buffer_get_gl_sync_meta (convert->inbuf);
2492 if (sync_meta)
2493 gst_gl_sync_meta_wait (sync_meta, convert->context);
2494
2495 if (!convert->priv->pool) {
2496 gboolean ret;
2497 /* No pool! */
2498 GstQuery *query = gst_query_new_allocation (convert->priv->out_caps, TRUE);
2499 ret = gst_gl_color_convert_decide_allocation (convert, query);
2500 gst_query_unref (query);
2501
2502 if (!ret) {
2503 GST_ERROR_OBJECT (convert, "Failed to choose allocation parameters");
2504 convert->priv->result = FALSE;
2505 return;
2506 }
2507
2508 if (!convert->priv->pool) {
2509 GST_ERROR_OBJECT (convert, "Failed to create a buffer pool");
2510 convert->priv->result = FALSE;
2511 return;
2512 }
2513 }
2514
2515 if (!convert->priv->pool_started) {
2516 if (!gst_buffer_pool_set_active (convert->priv->pool, TRUE)) {
2517 GST_ERROR_OBJECT (convert, "Failed to start buffer pool");
2518 convert->priv->result = FALSE;
2519 return;
2520 }
2521 convert->priv->pool_started = TRUE;
2522 }
2523
2524 ret =
2525 gst_buffer_pool_acquire_buffer (convert->priv->pool, &convert->outbuf,
2526 NULL);
2527 if (ret != GST_FLOW_OK) {
2528 GST_ERROR_OBJECT (convert, "Failed to acquire buffer from pool: %s",
2529 gst_flow_get_name (ret));
2530 convert->priv->result = FALSE;
2531 return;
2532 }
2533
2534 gst_gl_insert_debug_marker (context, "%s converting from %s to %s",
2535 GST_OBJECT_NAME (convert),
2536 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (in_info)),
2537 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->out_info)));
2538 /* Handle all views on input and output one at a time */
2539 for (v = 0; res && v < views; v++)
2540 res = _do_convert_one_view (context, convert, v);
2541
2542 if (!res) {
2543 gst_buffer_unref (convert->outbuf);
2544 convert->outbuf = NULL;
2545 }
2546
2547 if (convert->outbuf) {
2548 GstVideoOverlayCompositionMeta *composition_meta;
2549 GstGLSyncMeta *sync_meta =
2550 gst_buffer_add_gl_sync_meta (convert->context, convert->outbuf);
2551
2552 if (sync_meta)
2553 gst_gl_sync_meta_set_sync_point (sync_meta, convert->context);
2554
2555 composition_meta =
2556 gst_buffer_get_video_overlay_composition_meta (convert->inbuf);
2557 if (composition_meta) {
2558 GST_DEBUG ("found video overlay composition meta, applying on output.");
2559 gst_buffer_add_video_overlay_composition_meta
2560 (convert->outbuf, composition_meta->overlay);
2561 }
2562 }
2563
2564 convert->priv->result = res;
2565 return;
2566 }
2567
2568 static gboolean
_do_convert_draw(GstGLContext * context,GstGLColorConvert * convert)2569 _do_convert_draw (GstGLContext * context, GstGLColorConvert * convert)
2570 {
2571 GstGLFuncs *gl;
2572 struct ConvertInfo *c_info = &convert->priv->convert_info;
2573 guint out_width, out_height;
2574 gint i;
2575 gboolean ret = TRUE;
2576
2577 GLenum multipleRT[] = {
2578 GL_COLOR_ATTACHMENT0,
2579 GL_COLOR_ATTACHMENT1,
2580 GL_COLOR_ATTACHMENT2
2581 };
2582
2583 gl = context->gl_vtable;
2584
2585 gst_gl_framebuffer_bind (convert->fbo);
2586
2587 /* attach the texture to the FBO to renderer to */
2588 for (i = 0; i < c_info->out_n_textures; i++) {
2589 GstGLBaseMemory *tex = (GstGLBaseMemory *) convert->priv->out_tex[i];
2590
2591 gst_gl_framebuffer_attach (convert->fbo, GL_COLOR_ATTACHMENT0 + i, tex);
2592 }
2593
2594 if (gl->DrawBuffers)
2595 gl->DrawBuffers (c_info->out_n_textures, multipleRT);
2596 else if (gl->DrawBuffer)
2597 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
2598
2599 gst_gl_framebuffer_get_effective_dimensions (convert->fbo, &out_width,
2600 &out_height);
2601 gl->Viewport (0, 0, out_width, out_height);
2602
2603 gst_gl_shader_use (convert->shader);
2604
2605 if (gl->BindVertexArray)
2606 gl->BindVertexArray (convert->priv->vao);
2607 _bind_buffer (convert);
2608
2609 for (i = c_info->in_n_textures - 1; i >= 0; i--) {
2610 gchar *scale_name = g_strdup_printf ("tex_scale%u", i);
2611 guint gl_target =
2612 gst_gl_texture_target_to_gl (convert->priv->from_texture_target);
2613
2614 gl->ActiveTexture (GL_TEXTURE0 + i);
2615 gl->BindTexture (gl_target, convert->priv->in_tex[i]->tex_id);
2616 gl->TexParameteri (gl_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
2617 gl->TexParameteri (gl_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
2618 gl->TexParameteri (gl_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
2619 gl->TexParameteri (gl_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
2620
2621 gst_gl_shader_set_uniform_2fv (convert->shader, scale_name, 1,
2622 convert->priv->in_tex[i]->tex_scaling);
2623
2624 g_free (scale_name);
2625 }
2626
2627 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
2628
2629 if (gl->BindVertexArray)
2630 gl->BindVertexArray (0);
2631 else
2632 _unbind_buffer (convert);
2633
2634 if (gl->DrawBuffer)
2635 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
2636
2637 /* we are done with the shader */
2638 gst_gl_context_clear_shader (context);
2639
2640 if (!gst_gl_context_check_framebuffer_status (context, GL_FRAMEBUFFER))
2641 ret = FALSE;
2642
2643 gst_gl_context_clear_framebuffer (context);
2644
2645 return ret;
2646 }
2647