1 /* GStreamer faceoverlay plugin
2 * Copyright (C) 2011 Laura Lucas Alday <lauralucas@gmail.com>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 *
22 * Alternatively, the contents of this file may be used under the
23 * GNU Lesser General Public License Version 2.1 (the "LGPL"), in
24 * which case the following provisions apply instead of the ones
25 * mentioned above:
26 *
27 * This library is free software; you can redistribute it and/or
28 * modify it under the terms of the GNU Library General Public
29 * License as published by the Free Software Foundation; either
30 * version 2 of the License, or (at your option) any later version.
31 *
32 * This library is distributed in the hope that it will be useful,
33 * but WITHOUT ANY WARRANTY; without even the implied warranty of
34 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
35 * Library General Public License for more details.
36 *
37 * You should have received a copy of the GNU Library General Public
38 * License along with this library; if not, write to the
39 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
40 * Boston, MA 02110-1301, USA.
41 */
42
43 /**
44 * SECTION:element-faceoverlay
45 *
46 * Overlays a SVG image over a detected face in a video stream.
47 * x, y, w, and h properties are optional, and change the image position and
48 * size relative to the detected face position and size.
49 *
50 * <refsect2>
51 * <title>Example launch line</title>
52 * |[
53 * gst-launch-1.0 autovideosrc ! videoconvert ! faceoverlay location=/path/to/gnome-video-effects/pixmaps/bow.svg x=0.5 y=0.5 w=0.7 h=0.7 ! videoconvert ! autovideosink
54 * ]|
55 * </refsect2>
56 */
57
58 #ifdef HAVE_CONFIG_H
59 # include <config.h>
60 #endif
61
62 #include <gst/gst.h>
63 #include <gst/video/video.h>
64 #include <string.h>
65
66 #include "gstfaceoverlay.h"
67
68 GST_DEBUG_CATEGORY_STATIC (gst_face_overlay_debug);
69 #define GST_CAT_DEFAULT gst_face_overlay_debug
70
71 enum
72 {
73 PROP_0,
74 PROP_LOCATION,
75 PROP_X,
76 PROP_Y,
77 PROP_W,
78 PROP_H
79 };
80
81 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
82 GST_PAD_SINK,
83 GST_PAD_ALWAYS,
84 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{RGB}")));
85
86 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
87 GST_PAD_SRC,
88 GST_PAD_ALWAYS,
89 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{BGRA}")));
90
91 #define gst_face_overlay_parent_class parent_class
92 G_DEFINE_TYPE (GstFaceOverlay, gst_face_overlay, GST_TYPE_BIN);
93
94 static void gst_face_overlay_set_property (GObject * object, guint prop_id,
95 const GValue * value, GParamSpec * pspec);
96 static void gst_face_overlay_get_property (GObject * object, guint prop_id,
97 GValue * value, GParamSpec * pspec);
98 static void gst_face_overlay_message_handler (GstBin * bin,
99 GstMessage * message);
100 static GstStateChangeReturn gst_face_overlay_change_state (GstElement * element,
101 GstStateChange transition);
102 static gboolean gst_face_overlay_create_children (GstFaceOverlay * filter);
103
104 static gboolean
gst_face_overlay_create_children(GstFaceOverlay * filter)105 gst_face_overlay_create_children (GstFaceOverlay * filter)
106 {
107 GstElement *csp, *face_detect, *overlay;
108 GstPad *pad;
109
110 csp = gst_element_factory_make ("videoconvert", NULL);
111 face_detect = gst_element_factory_make ("facedetect", NULL);
112 overlay = gst_element_factory_make ("rsvgoverlay", NULL);
113
114 /* FIXME: post missing-plugin messages on NULL->READY if needed */
115 if (csp == NULL || face_detect == NULL || overlay == NULL)
116 goto missing_element;
117
118 g_object_set (face_detect, "display", FALSE, NULL);
119
120 gst_bin_add_many (GST_BIN (filter), face_detect, csp, overlay, NULL);
121 filter->svg_overlay = overlay;
122
123 if (!gst_element_link_many (face_detect, csp, overlay, NULL))
124 GST_ERROR_OBJECT (filter, "couldn't link elements");
125
126 pad = gst_element_get_static_pad (face_detect, "sink");
127 if (!gst_ghost_pad_set_target (GST_GHOST_PAD (filter->sinkpad), pad))
128 GST_ERROR_OBJECT (filter->sinkpad, "couldn't set sinkpad target");
129 gst_object_unref (pad);
130
131 pad = gst_element_get_static_pad (overlay, "src");
132 if (!gst_ghost_pad_set_target (GST_GHOST_PAD (filter->srcpad), pad))
133 GST_ERROR_OBJECT (filter->srcpad, "couldn't set srcpad target");
134 gst_object_unref (pad);
135
136 return TRUE;
137
138 /* ERRORS */
139 missing_element:
140 {
141 /* clean up */
142 if (csp == NULL)
143 GST_ERROR_OBJECT (filter, "videoconvert element not found");
144 else
145 gst_object_unref (csp);
146
147 if (face_detect == NULL)
148 GST_ERROR_OBJECT (filter, "facedetect element not found (opencv plugin)");
149 else
150 gst_object_unref (face_detect);
151
152 if (overlay == NULL)
153 GST_ERROR_OBJECT (filter, "rsvgoverlay element not found (rsvg plugin)");
154 else
155 gst_object_unref (overlay);
156
157 return FALSE;
158 }
159 }
160
161 static GstStateChangeReturn
gst_face_overlay_change_state(GstElement * element,GstStateChange transition)162 gst_face_overlay_change_state (GstElement * element, GstStateChange transition)
163 {
164 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
165 GstFaceOverlay *filter = GST_FACEOVERLAY (element);
166
167 switch (transition) {
168 case GST_STATE_CHANGE_NULL_TO_READY:
169 if (filter->svg_overlay == NULL) {
170 GST_ELEMENT_ERROR (filter, CORE, MISSING_PLUGIN, (NULL),
171 ("Some required plugins are missing, probably either the opencv "
172 "facedetect element or rsvgoverlay"));
173 return GST_STATE_CHANGE_FAILURE;
174 }
175 filter->update_svg = TRUE;
176 break;
177 default:
178 break;
179 }
180
181 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
182
183 switch (transition) {
184 default:
185 break;
186 }
187
188 return ret;
189 }
190
191 static void
gst_face_overlay_handle_faces(GstFaceOverlay * filter,const GstStructure * s)192 gst_face_overlay_handle_faces (GstFaceOverlay * filter, const GstStructure * s)
193 {
194 guint x, y, width, height;
195 gint svg_x, svg_y, svg_width, svg_height;
196 const GstStructure *face;
197 const GValue *faces_list, *face_val;
198 gchar *new_location = NULL;
199 gint face_count;
200
201 #if 0
202 /* optionally draw the image once every two messages for better performance */
203 filter->process_message = !filter->process_message;
204 if (!filter->process_message)
205 return;
206 #endif
207
208 faces_list = gst_structure_get_value (s, "faces");
209 face_count = gst_value_list_get_size (faces_list);
210 GST_LOG_OBJECT (filter, "face count: %d", face_count);
211
212 if (face_count == 0) {
213 GST_DEBUG_OBJECT (filter, "no face, clearing overlay");
214 g_object_set (filter->svg_overlay, "location", NULL, NULL);
215 GST_OBJECT_LOCK (filter);
216 filter->update_svg = TRUE;
217 GST_OBJECT_UNLOCK (filter);
218 return;
219 }
220
221 /* The last face in the list seems to be the right one, objects mistakenly
222 * detected as faces for a couple of frames seem to be in the list
223 * beginning. TODO: needs confirmation. */
224 face_val = gst_value_list_get_value (faces_list, face_count - 1);
225 face = gst_value_get_structure (face_val);
226 gst_structure_get_uint (face, "x", &x);
227 gst_structure_get_uint (face, "y", &y);
228 gst_structure_get_uint (face, "width", &width);
229 gst_structure_get_uint (face, "height", &height);
230
231 /* Apply x and y offsets relative to face position and size.
232 * Set image width and height as a fraction of face width and height.
233 * Cast to int since face position and size will never be bigger than
234 * G_MAX_INT and we may have negative values as svg_x or svg_y */
235
236 GST_OBJECT_LOCK (filter);
237
238 svg_x = (gint) x + (gint) (filter->x * width);
239 svg_y = (gint) y + (gint) (filter->y * height);
240
241 svg_width = (gint) (filter->w * width);
242 svg_height = (gint) (filter->h * height);
243
244 if (filter->update_svg) {
245 new_location = g_strdup (filter->location);
246 filter->update_svg = FALSE;
247 }
248 GST_OBJECT_UNLOCK (filter);
249
250 if (new_location != NULL) {
251 GST_DEBUG_OBJECT (filter, "set rsvgoverlay location=%s", new_location);
252 g_object_set (filter->svg_overlay, "location", new_location, NULL);
253 g_free (new_location);
254 }
255
256 GST_LOG_OBJECT (filter, "overlay dimensions: %d x %d @ %d,%d",
257 svg_width, svg_height, svg_x, svg_y);
258
259 g_object_set (filter->svg_overlay,
260 "x", svg_x, "y", svg_y, "width", svg_width, "height", svg_height, NULL);
261 }
262
263 static void
gst_face_overlay_message_handler(GstBin * bin,GstMessage * message)264 gst_face_overlay_message_handler (GstBin * bin, GstMessage * message)
265 {
266 if (GST_MESSAGE_TYPE (message) == GST_MESSAGE_ELEMENT) {
267 const GstStructure *s = gst_message_get_structure (message);
268
269 if (gst_structure_has_name (s, "facedetect")) {
270 gst_face_overlay_handle_faces (GST_FACEOVERLAY (bin), s);
271 }
272 }
273
274 GST_BIN_CLASS (parent_class)->handle_message (bin, message);
275 }
276
277 static void
gst_face_overlay_class_init(GstFaceOverlayClass * klass)278 gst_face_overlay_class_init (GstFaceOverlayClass * klass)
279 {
280 GObjectClass *gobject_class;
281 GstBinClass *gstbin_class;
282 GstElementClass *gstelement_class;
283
284 gobject_class = G_OBJECT_CLASS (klass);
285 gstbin_class = GST_BIN_CLASS (klass);
286 gstelement_class = GST_ELEMENT_CLASS (klass);
287
288 gobject_class->set_property = gst_face_overlay_set_property;
289 gobject_class->get_property = gst_face_overlay_get_property;
290
291 g_object_class_install_property (gobject_class, PROP_LOCATION,
292 g_param_spec_string ("location", "Location",
293 "Location of SVG file to use for face overlay",
294 "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
295 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_X,
296 g_param_spec_float ("x", "face x offset",
297 "Specify image x relative to detected face x.", -G_MAXFLOAT,
298 G_MAXFLOAT, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
299 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_Y,
300 g_param_spec_float ("y", "face y offset",
301 "Specify image y relative to detected face y.", -G_MAXFLOAT,
302 G_MAXFLOAT, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
303 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_W,
304 g_param_spec_float ("w", "face width percent",
305 "Specify image width relative to face width.", 0, G_MAXFLOAT, 1,
306 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
307 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_H,
308 g_param_spec_float ("h", "face height percent",
309 "Specify image height relative to face height.", 0, G_MAXFLOAT, 1,
310 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
311
312 gst_element_class_set_static_metadata (gstelement_class,
313 "faceoverlay",
314 "Filter/Editor/Video",
315 "Overlays SVG graphics over a detected face in a video stream",
316 "Laura Lucas Alday <lauralucas@gmail.com>");
317
318 gst_element_class_add_pad_template (gstelement_class,
319 gst_static_pad_template_get (&src_factory));
320 gst_element_class_add_pad_template (gstelement_class,
321 gst_static_pad_template_get (&sink_factory));
322
323 gstbin_class->handle_message =
324 GST_DEBUG_FUNCPTR (gst_face_overlay_message_handler);
325 gstelement_class->change_state =
326 GST_DEBUG_FUNCPTR (gst_face_overlay_change_state);
327 }
328
329 static void
gst_face_overlay_init(GstFaceOverlay * filter)330 gst_face_overlay_init (GstFaceOverlay * filter)
331 {
332 GstPadTemplate *tmpl;
333
334 filter->x = 0;
335 filter->y = 0;
336 filter->w = 1;
337 filter->h = 1;
338 filter->svg_overlay = NULL;
339 filter->location = NULL;
340 filter->process_message = TRUE;
341
342 tmpl = gst_static_pad_template_get (&sink_factory);
343 filter->sinkpad = gst_ghost_pad_new_no_target_from_template ("sink", tmpl);
344 gst_object_unref (tmpl);
345 gst_element_add_pad (GST_ELEMENT (filter), filter->sinkpad);
346
347 tmpl = gst_static_pad_template_get (&src_factory);
348 filter->srcpad = gst_ghost_pad_new_no_target_from_template ("src", tmpl);
349 gst_object_unref (tmpl);
350 gst_element_add_pad (GST_ELEMENT (filter), filter->srcpad);
351
352 gst_face_overlay_create_children (filter);
353 }
354
355 static void
gst_face_overlay_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)356 gst_face_overlay_set_property (GObject * object, guint prop_id,
357 const GValue * value, GParamSpec * pspec)
358 {
359 GstFaceOverlay *filter = GST_FACEOVERLAY (object);
360
361 switch (prop_id) {
362 case PROP_LOCATION:
363 GST_OBJECT_LOCK (filter);
364 g_free (filter->location);
365 filter->location = g_value_dup_string (value);
366 filter->update_svg = TRUE;
367 GST_OBJECT_UNLOCK (filter);
368 break;
369 case PROP_X:
370 GST_OBJECT_LOCK (filter);
371 filter->x = g_value_get_float (value);
372 GST_OBJECT_UNLOCK (filter);
373 break;
374 case PROP_Y:
375 GST_OBJECT_LOCK (filter);
376 filter->y = g_value_get_float (value);
377 GST_OBJECT_UNLOCK (filter);
378 break;
379 case PROP_W:
380 GST_OBJECT_LOCK (filter);
381 filter->w = g_value_get_float (value);
382 GST_OBJECT_UNLOCK (filter);
383 break;
384 case PROP_H:
385 GST_OBJECT_LOCK (filter);
386 filter->h = g_value_get_float (value);
387 GST_OBJECT_UNLOCK (filter);
388 break;
389 default:
390 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
391 break;
392 }
393 }
394
395 static void
gst_face_overlay_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)396 gst_face_overlay_get_property (GObject * object, guint prop_id,
397 GValue * value, GParamSpec * pspec)
398 {
399 GstFaceOverlay *filter = GST_FACEOVERLAY (object);
400
401 switch (prop_id) {
402 case PROP_LOCATION:
403 GST_OBJECT_LOCK (filter);
404 g_value_set_string (value, filter->location);
405 GST_OBJECT_UNLOCK (filter);
406 break;
407 case PROP_X:
408 GST_OBJECT_LOCK (filter);
409 g_value_set_float (value, filter->x);
410 GST_OBJECT_UNLOCK (filter);
411 break;
412 case PROP_Y:
413 GST_OBJECT_LOCK (filter);
414 g_value_set_float (value, filter->y);
415 GST_OBJECT_UNLOCK (filter);
416 break;
417 case PROP_W:
418 GST_OBJECT_LOCK (filter);
419 g_value_set_float (value, filter->w);
420 GST_OBJECT_UNLOCK (filter);
421 break;
422 case PROP_H:
423 GST_OBJECT_LOCK (filter);
424 g_value_set_float (value, filter->h);
425 GST_OBJECT_UNLOCK (filter);
426 break;
427 default:
428 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
429 break;
430 }
431 }
432
433 static gboolean
faceoverlay_init(GstPlugin * faceoverlay)434 faceoverlay_init (GstPlugin * faceoverlay)
435 {
436 GST_DEBUG_CATEGORY_INIT (gst_face_overlay_debug, "faceoverlay",
437 0, "SVG Face Overlay");
438
439 return gst_element_register (faceoverlay, "faceoverlay", GST_RANK_NONE,
440 GST_TYPE_FACEOVERLAY);
441 }
442
443 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
444 GST_VERSION_MINOR,
445 faceoverlay,
446 "SVG Face Overlay",
447 faceoverlay_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)
448