1 /*
2 * gloverlaycompositor element
3 * Copyrithg (C) 2018 Matthew Waters <matthew@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 /**
22 * SECTION:element-glcompositoroverlay
23 * @title: glcompositoroverlay
24 *
25 */
26 #ifdef HAVE_CONFIG_H
27 #include "config.h"
28 #endif
29
30 #include <gst/gl/gstglfuncs.h>
31 #include <gst/video/video.h>
32
33 #include "gstgloverlaycompositorelement.h"
34
35 enum
36 {
37 PROP_0,
38 PROP_LAST,
39 };
40
41 #define GST_CAT_DEFAULT gst_gl_overlay_compositor_element_debug
42 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
43
44 #define DEBUG_INIT \
45 GST_DEBUG_CATEGORY_INIT (gst_gl_overlay_compositor_element_debug, "gloverlaycompositorelement", 0, "gloverlaycompositor element");
46 #define gst_gl_overlay_compositor_element_parent_class parent_class
47 G_DEFINE_TYPE_WITH_CODE (GstGLOverlayCompositorElement,
48 gst_gl_overlay_compositor_element, GST_TYPE_GL_FILTER, DEBUG_INIT);
49
50 static GstStaticPadTemplate overlay_sink_pad_template =
51 GST_STATIC_PAD_TEMPLATE ("sink",
52 GST_PAD_SINK,
53 GST_PAD_ALWAYS,
54 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
55 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY ","
56 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
57 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
58 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
59 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
60 GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
61 "RGBA") ", texture-target=(string) { 2D, rectangle } "));
62
63 static GstStaticPadTemplate overlay_src_pad_template =
64 GST_STATIC_PAD_TEMPLATE ("src",
65 GST_PAD_SRC,
66 GST_PAD_ALWAYS,
67 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
68 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY ","
69 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
70 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
71 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
72 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
73 GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
74 "RGBA") ", texture-target=(string) { 2D, rectangle } "));
75
76 static gboolean
77 gst_gl_overlay_compositor_element_propose_allocation (GstBaseTransform * trans,
78 GstQuery * decide_query, GstQuery * query);
79 static GstFlowReturn _oce_prepare_output_buffer (GstBaseTransform * bt,
80 GstBuffer * buffer, GstBuffer ** outbuf);
81
82 static gboolean gst_gl_overlay_compositor_element_gl_start (GstGLBaseFilter *
83 base);
84 static void gst_gl_overlay_compositor_element_gl_stop (GstGLBaseFilter * base);
85
86 static GstCaps *_oce_transform_internal_caps (GstGLFilter *
87 filter, GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps);
88 static gboolean gst_gl_overlay_compositor_element_filter (GstGLFilter * filter,
89 GstBuffer * inbuf, GstBuffer * outbuf);
90 static gboolean gst_gl_overlay_compositor_element_filter_texture (GstGLFilter *
91 filter, GstGLMemory * in_tex, GstGLMemory * out_tex);
92 static gboolean gst_gl_overlay_compositor_element_callback (GstGLFilter *
93 filter, GstGLMemory * in_tex, gpointer stuff);
94
95 static void
gst_gl_overlay_compositor_element_class_init(GstGLOverlayCompositorElementClass * klass)96 gst_gl_overlay_compositor_element_class_init (GstGLOverlayCompositorElementClass
97 * klass)
98 {
99 GstElementClass *element_class;
100
101 element_class = GST_ELEMENT_CLASS (klass);
102
103 gst_element_class_set_metadata (element_class,
104 "OpenGL overlaying filter", "Filter/Effect",
105 "Flatten a stream containing GstVideoOverlayCompositionMeta",
106 "<matthew@centricular.com>");
107
108 gst_element_class_add_static_pad_template (element_class,
109 &overlay_src_pad_template);
110 gst_element_class_add_static_pad_template (element_class,
111 &overlay_sink_pad_template);
112
113 GST_BASE_TRANSFORM_CLASS (klass)->passthrough_on_same_caps = TRUE;
114 GST_BASE_TRANSFORM_CLASS (klass)->propose_allocation =
115 gst_gl_overlay_compositor_element_propose_allocation;
116 GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
117 _oce_prepare_output_buffer;
118
119 GST_GL_FILTER_CLASS (klass)->filter =
120 gst_gl_overlay_compositor_element_filter;
121 GST_GL_FILTER_CLASS (klass)->filter_texture =
122 gst_gl_overlay_compositor_element_filter_texture;
123 GST_GL_FILTER_CLASS (klass)->transform_internal_caps =
124 _oce_transform_internal_caps;
125
126 GST_GL_BASE_FILTER_CLASS (klass)->gl_start =
127 gst_gl_overlay_compositor_element_gl_start;
128 GST_GL_BASE_FILTER_CLASS (klass)->gl_stop =
129 gst_gl_overlay_compositor_element_gl_stop;
130 GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
131 GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3;
132 }
133
134 static void
gst_gl_overlay_compositor_element_init(GstGLOverlayCompositorElement * overlay_compositor_element)135 gst_gl_overlay_compositor_element_init (GstGLOverlayCompositorElement *
136 overlay_compositor_element)
137 {
138 }
139
140 static GstCaps *
_oce_transform_internal_caps(GstGLFilter * filter,GstPadDirection direction,GstCaps * caps,GstCaps * filter_caps)141 _oce_transform_internal_caps (GstGLFilter * filter,
142 GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps)
143 {
144 GstCaps *ret;
145
146 /* add/remove the composition overlay meta as necessary */
147 if (direction == GST_PAD_SRC) {
148 ret = gst_gl_overlay_compositor_add_caps (gst_caps_copy (caps));
149 } else {
150 guint i, n;
151 GstCaps *removed;
152
153 ret = gst_caps_copy (caps);
154 removed = gst_caps_copy (caps);
155 n = gst_caps_get_size (removed);
156 for (i = 0; i < n; i++) {
157 GstCapsFeatures *feat = gst_caps_get_features (removed, i);
158
159 if (feat) {
160 feat = gst_caps_features_copy (feat);
161
162 if (gst_caps_features_contains (feat,
163 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) {
164 /* prefer the passthrough case */
165 gst_caps_features_remove (feat,
166 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
167 gst_caps_set_features (removed, i, feat);
168 }
169 }
170 }
171
172 ret = gst_caps_merge (ret, removed);
173 }
174
175 GST_DEBUG_OBJECT (filter, "meta modifications returned caps %" GST_PTR_FORMAT,
176 ret);
177 return ret;
178 }
179
180 static gboolean
gst_gl_overlay_compositor_element_propose_allocation(GstBaseTransform * trans,GstQuery * decide_query,GstQuery * query)181 gst_gl_overlay_compositor_element_propose_allocation (GstBaseTransform * trans,
182 GstQuery * decide_query, GstQuery * query)
183 {
184 GstStructure *allocation_meta = NULL;
185 guint width = 0, height = 0;
186
187 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->propose_allocation (trans,
188 decide_query, query))
189 return FALSE;
190
191 if ((width == 0 || height == 0) && decide_query) {
192 GstCaps *decide_caps;
193 gst_query_parse_allocation (decide_query, &decide_caps, NULL);
194
195 if (decide_caps) {
196 GstVideoInfo vinfo;
197
198 if (gst_video_info_from_caps (&vinfo, decide_caps)) {
199 width = GST_VIDEO_INFO_WIDTH (&vinfo);
200 height = GST_VIDEO_INFO_HEIGHT (&vinfo);
201 }
202 }
203 }
204
205 if ((width == 0 || height == 0) && query) {
206 GstCaps *caps;
207 gst_query_parse_allocation (decide_query, &caps, NULL);
208
209 if (caps) {
210 GstVideoInfo vinfo;
211
212 if (gst_video_info_from_caps (&vinfo, caps)) {
213 width = GST_VIDEO_INFO_WIDTH (&vinfo);
214 height = GST_VIDEO_INFO_HEIGHT (&vinfo);
215 }
216 }
217 }
218
219 if (width != 0 && height != 0) {
220 allocation_meta =
221 gst_structure_new ("GstVideoOverlayCompositionMeta",
222 "width", G_TYPE_UINT, width, "height", G_TYPE_UINT, height, NULL);
223 }
224
225 GST_DEBUG_OBJECT (trans, "Adding overlay composition meta with size %ux%u",
226 width, height);
227 gst_query_add_allocation_meta (query,
228 GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, allocation_meta);
229
230 return TRUE;
231 }
232
233 static void
gst_gl_overlay_compositor_element_gl_stop(GstGLBaseFilter * base)234 gst_gl_overlay_compositor_element_gl_stop (GstGLBaseFilter * base)
235 {
236 GstGLOverlayCompositorElement *self =
237 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (base);
238
239 if (self->shader)
240 gst_object_unref (self->shader);
241 self->shader = NULL;
242
243 if (self->overlay_compositor) {
244 gst_gl_overlay_compositor_free_overlays (self->overlay_compositor);
245 gst_object_unref (self->overlay_compositor);
246 }
247 self->overlay_compositor = NULL;
248
249 GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base);
250 }
251
252 static gboolean
gst_gl_overlay_compositor_element_gl_start(GstGLBaseFilter * base)253 gst_gl_overlay_compositor_element_gl_start (GstGLBaseFilter * base)
254 {
255 GstGLOverlayCompositorElement *self =
256 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (base);
257 GError *error = NULL;
258
259 self->overlay_compositor = gst_gl_overlay_compositor_new (base->context);
260 g_object_set (self->overlay_compositor, "yinvert", TRUE, NULL);
261
262 if (!(self->shader = gst_gl_shader_new_default (base->context, &error))) {
263 GST_ELEMENT_ERROR (base, RESOURCE, NOT_FOUND, ("%s",
264 "Failed to compile identity shader"), ("%s", error->message));
265 return FALSE;
266 }
267
268 return GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base);
269 }
270
271 static GstFlowReturn
_oce_prepare_output_buffer(GstBaseTransform * bt,GstBuffer * buffer,GstBuffer ** outbuf)272 _oce_prepare_output_buffer (GstBaseTransform * bt,
273 GstBuffer * buffer, GstBuffer ** outbuf)
274 {
275 GstGLOverlayCompositorElement *self = GST_GL_OVERLAY_COMPOSITOR_ELEMENT (bt);
276 GstVideoOverlayCompositionMeta *comp_meta;
277
278 if (gst_base_transform_is_passthrough (bt))
279 goto passthrough;
280
281 if (!self->overlay_compositor)
282 return GST_FLOW_NOT_NEGOTIATED;
283
284 comp_meta = gst_buffer_get_video_overlay_composition_meta (buffer);
285 if (!comp_meta)
286 goto passthrough;
287
288 if (gst_video_overlay_composition_n_rectangles (comp_meta->overlay) == 0)
289 goto passthrough;
290
291 return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (bt,
292 buffer, outbuf);
293
294 passthrough:
295 GST_LOG_OBJECT (bt, "passthrough detected, forwarding input buffer");
296 *outbuf = buffer;
297 return GST_FLOW_OK;
298 }
299
300 static gboolean
gst_gl_overlay_compositor_element_filter(GstGLFilter * filter,GstBuffer * inbuf,GstBuffer * outbuf)301 gst_gl_overlay_compositor_element_filter (GstGLFilter * filter,
302 GstBuffer * inbuf, GstBuffer * outbuf)
303 {
304 GstGLOverlayCompositorElement *self =
305 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter);
306
307 if (inbuf == outbuf)
308 return TRUE;
309
310 gst_gl_overlay_compositor_upload_overlays (self->overlay_compositor, inbuf);
311
312 return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
313 }
314
315 static gboolean
gst_gl_overlay_compositor_element_filter_texture(GstGLFilter * filter,GstGLMemory * in_tex,GstGLMemory * out_tex)316 gst_gl_overlay_compositor_element_filter_texture (GstGLFilter * filter,
317 GstGLMemory * in_tex, GstGLMemory * out_tex)
318 {
319 GstGLOverlayCompositorElement *self =
320 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter);
321
322 gst_gl_filter_render_to_target_with_shader (filter, in_tex, out_tex,
323 self->shader);
324
325 gst_gl_filter_render_to_target (filter, NULL, out_tex,
326 gst_gl_overlay_compositor_element_callback, NULL);
327
328 return TRUE;
329 }
330
331 static gboolean
gst_gl_overlay_compositor_element_callback(GstGLFilter * filter,GstGLMemory * in_tex,gpointer stuff)332 gst_gl_overlay_compositor_element_callback (GstGLFilter * filter,
333 GstGLMemory * in_tex, gpointer stuff)
334 {
335 GstGLOverlayCompositorElement *self =
336 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter);
337
338 GST_LOG_OBJECT (self, "drawing overlays");
339
340 gst_gl_overlay_compositor_draw_overlays (self->overlay_compositor);
341
342 return TRUE;
343 }
344