1 /* GStreamer
2 * Copyright (C) <2011> Jon Nordby <jononor@gmail.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 /**
21 * SECTION:element-cairooverlay
22 *
23 * cairooverlay renders an overlay using a application provided render function.
24 *
25 * The full example can be found in tests/examples/cairo/cairo_overlay.c
26 * <refsect2>
27 * <title>Example code</title>
28 * |[
29 *
30 * #include <gst/gst.h>
31 * #include <gst/video/video.h>
32 *
33 * ...
34 *
35 * typedef struct {
36 * gboolean valid;
37 * int width;
38 * int height;
39 * } CairoOverlayState;
40 *
41 * ...
42 *
43 * static void
44 * prepare_overlay (GstElement * overlay, GstCaps * caps, gpointer user_data)
45 * {
46 * CairoOverlayState *state = (CairoOverlayState *)user_data;
47 *
48 * gst_video_format_parse_caps (caps, NULL, &state->width, &state->height);
49 * state->valid = TRUE;
50 * }
51 *
52 * static void
53 * draw_overlay (GstElement * overlay, cairo_t * cr, guint64 timestamp,
54 * guint64 duration, gpointer user_data)
55 * {
56 * CairoOverlayState *s = (CairoOverlayState *)user_data;
57 * double scale;
58 *
59 * if (!s->valid)
60 * return;
61 *
62 * scale = 2*(((timestamp/(int)1e7) % 70)+30)/100.0;
63 * cairo_translate(cr, s->width/2, (s->height/2)-30);
64 * cairo_scale (cr, scale, scale);
65 *
66 * cairo_move_to (cr, 0, 0);
67 * cairo_curve_to (cr, 0,-30, -50,-30, -50,0);
68 * cairo_curve_to (cr, -50,30, 0,35, 0,60 );
69 * cairo_curve_to (cr, 0,35, 50,30, 50,0 ); *
70 * cairo_curve_to (cr, 50,-30, 0,-30, 0,0 );
71 * cairo_set_source_rgba (cr, 0.9, 0.0, 0.1, 0.7);
72 * cairo_fill (cr);
73 * }
74 *
75 * ...
76 *
77 * cairo_overlay = gst_element_factory_make ("cairooverlay", "overlay");
78 *
79 * g_signal_connect (cairo_overlay, "draw", G_CALLBACK (draw_overlay),
80 * overlay_state);
81 * g_signal_connect (cairo_overlay, "caps-changed",
82 * G_CALLBACK (prepare_overlay), overlay_state);
83 * ...
84 *
85 * ]|
86 * </refsect2>
87 */
88
89 #ifdef HAVE_CONFIG_H
90 #include "config.h"
91 #endif
92
93 #include "gstcairooverlay.h"
94
95 #include <gst/video/video.h>
96
97 #include <cairo.h>
98
99 /* RGB16 is native-endianness in GStreamer */
100 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
101 #define TEMPLATE_CAPS GST_VIDEO_CAPS_MAKE("{ BGRx, BGRA, RGB16 }")
102 #else
103 #define TEMPLATE_CAPS GST_VIDEO_CAPS_MAKE("{ xRGB, ARGB, RGB16 }")
104 #endif
105
106 static GstStaticPadTemplate gst_cairo_overlay_src_template =
107 GST_STATIC_PAD_TEMPLATE ("src",
108 GST_PAD_SRC,
109 GST_PAD_ALWAYS,
110 GST_STATIC_CAPS (TEMPLATE_CAPS)
111 );
112
113 static GstStaticPadTemplate gst_cairo_overlay_sink_template =
114 GST_STATIC_PAD_TEMPLATE ("sink",
115 GST_PAD_SINK,
116 GST_PAD_ALWAYS,
117 GST_STATIC_CAPS (TEMPLATE_CAPS)
118 );
119
120 G_DEFINE_TYPE (GstCairoOverlay, gst_cairo_overlay, GST_TYPE_BASE_TRANSFORM);
121
122 enum
123 {
124 PROP_0,
125 PROP_DRAW_ON_TRANSPARENT_SURFACE,
126 };
127
128 #define DEFAULT_DRAW_ON_TRANSPARENT_SURFACE (FALSE)
129
130 enum
131 {
132 SIGNAL_DRAW,
133 SIGNAL_CAPS_CHANGED,
134 N_SIGNALS
135 };
136
137 static guint gst_cairo_overlay_signals[N_SIGNALS];
138
139 static void
gst_cairo_overlay_set_property(GObject * object,guint property_id,const GValue * value,GParamSpec * pspec)140 gst_cairo_overlay_set_property (GObject * object, guint property_id,
141 const GValue * value, GParamSpec * pspec)
142 {
143 GstCairoOverlay *overlay = GST_CAIRO_OVERLAY (object);
144
145 GST_OBJECT_LOCK (overlay);
146
147 switch (property_id) {
148 case PROP_DRAW_ON_TRANSPARENT_SURFACE:
149 overlay->draw_on_transparent_surface = g_value_get_boolean (value);
150 break;
151 default:
152 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
153 break;
154 }
155
156 GST_OBJECT_UNLOCK (overlay);
157 }
158
159 static void
gst_cairo_overlay_get_property(GObject * object,guint property_id,GValue * value,GParamSpec * pspec)160 gst_cairo_overlay_get_property (GObject * object, guint property_id,
161 GValue * value, GParamSpec * pspec)
162 {
163 GstCairoOverlay *overlay = GST_CAIRO_OVERLAY (object);
164
165 GST_OBJECT_LOCK (overlay);
166
167 switch (property_id) {
168 case PROP_DRAW_ON_TRANSPARENT_SURFACE:
169 g_value_set_boolean (value, overlay->draw_on_transparent_surface);
170 break;
171 default:
172 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
173 break;
174 }
175
176 GST_OBJECT_UNLOCK (overlay);
177 }
178
179 static gboolean
gst_cairo_overlay_query(GstBaseTransform * trans,GstPadDirection direction,GstQuery * query)180 gst_cairo_overlay_query (GstBaseTransform * trans, GstPadDirection direction,
181 GstQuery * query)
182 {
183 GstCairoOverlay *overlay = GST_CAIRO_OVERLAY (trans);
184
185 switch (GST_QUERY_TYPE (query)) {
186 case GST_QUERY_ALLOCATION:
187 {
188 /* We're always running in passthrough mode, which means that
189 * basetransform just passes through ALLOCATION queries and
190 * never ever calls BaseTransform::decide_allocation().
191 *
192 * We hook into the query handling for that reason
193 */
194 overlay->attach_compo_to_buffer = FALSE;
195
196 if (!GST_BASE_TRANSFORM_CLASS (gst_cairo_overlay_parent_class)->query
197 (trans, direction, query)) {
198 return FALSE;
199 }
200
201 overlay->attach_compo_to_buffer = gst_query_find_allocation_meta (query,
202 GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
203
204 return TRUE;
205 }
206 default:
207 return
208 GST_BASE_TRANSFORM_CLASS (gst_cairo_overlay_parent_class)->query
209 (trans, direction, query);
210 }
211 }
212
213 static gboolean
gst_cairo_overlay_set_caps(GstBaseTransform * trans,GstCaps * in_caps,GstCaps * out_caps)214 gst_cairo_overlay_set_caps (GstBaseTransform * trans, GstCaps * in_caps,
215 GstCaps * out_caps)
216 {
217 GstCairoOverlay *overlay = GST_CAIRO_OVERLAY (trans);
218
219 if (!gst_video_info_from_caps (&overlay->info, in_caps))
220 return FALSE;
221
222 g_signal_emit (overlay, gst_cairo_overlay_signals[SIGNAL_CAPS_CHANGED], 0,
223 in_caps, NULL);
224
225 return TRUE;
226 }
227
228 /* Copy from video-overlay-composition.c */
229 static void
gst_video_overlay_rectangle_premultiply_0(GstVideoFrame * frame)230 gst_video_overlay_rectangle_premultiply_0 (GstVideoFrame * frame)
231 {
232 int i, j;
233 int width = GST_VIDEO_FRAME_WIDTH (frame);
234 int height = GST_VIDEO_FRAME_HEIGHT (frame);
235 int stride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
236 guint8 *data = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
237
238 for (j = 0; j < height; ++j) {
239 guint8 *line;
240
241 line = data;
242 line += stride * j;
243 for (i = 0; i < width; ++i) {
244 int a = line[0];
245 line[1] = line[1] * a / 255;
246 line[2] = line[2] * a / 255;
247 line[3] = line[3] * a / 255;
248 line += 4;
249 }
250 }
251 }
252
253 /* Copy from video-overlay-composition.c */
254 static void
gst_video_overlay_rectangle_premultiply_3(GstVideoFrame * frame)255 gst_video_overlay_rectangle_premultiply_3 (GstVideoFrame * frame)
256 {
257 int i, j;
258 int width = GST_VIDEO_FRAME_WIDTH (frame);
259 int height = GST_VIDEO_FRAME_HEIGHT (frame);
260 int stride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
261 guint8 *data = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
262
263 for (j = 0; j < height; ++j) {
264 guint8 *line;
265
266 line = data;
267 line += stride * j;
268 for (i = 0; i < width; ++i) {
269 int a = line[3];
270 line[0] = line[0] * a / 255;
271 line[1] = line[1] * a / 255;
272 line[2] = line[2] * a / 255;
273 line += 4;
274 }
275 }
276 }
277
278 /* Copy from video-overlay-composition.c */
279 static void
gst_video_overlay_rectangle_premultiply(GstVideoFrame * frame)280 gst_video_overlay_rectangle_premultiply (GstVideoFrame * frame)
281 {
282 gint alpha_offset;
283
284 alpha_offset = GST_VIDEO_FRAME_COMP_POFFSET (frame, 3);
285 switch (alpha_offset) {
286 case 0:
287 gst_video_overlay_rectangle_premultiply_0 (frame);
288 break;
289 case 3:
290 gst_video_overlay_rectangle_premultiply_3 (frame);
291 break;
292 default:
293 g_assert_not_reached ();
294 break;
295 }
296 }
297
298 /* Copy from video-overlay-composition.c */
299 static void
gst_video_overlay_rectangle_unpremultiply_0(GstVideoFrame * frame)300 gst_video_overlay_rectangle_unpremultiply_0 (GstVideoFrame * frame)
301 {
302 int i, j;
303 int width = GST_VIDEO_FRAME_WIDTH (frame);
304 int height = GST_VIDEO_FRAME_HEIGHT (frame);
305 int stride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
306 guint8 *data = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
307
308 for (j = 0; j < height; ++j) {
309 guint8 *line;
310
311 line = data;
312 line += stride * j;
313 for (i = 0; i < width; ++i) {
314 int a = line[0];
315 if (a) {
316 line[1] = MIN ((line[1] * 255 + a / 2) / a, 255);
317 line[2] = MIN ((line[2] * 255 + a / 2) / a, 255);
318 line[3] = MIN ((line[3] * 255 + a / 2) / a, 255);
319 }
320 line += 4;
321 }
322 }
323 }
324
325 /* Copy from video-overlay-composition.c */
326 static void
gst_video_overlay_rectangle_unpremultiply_3(GstVideoFrame * frame)327 gst_video_overlay_rectangle_unpremultiply_3 (GstVideoFrame * frame)
328 {
329 int i, j;
330 int width = GST_VIDEO_FRAME_WIDTH (frame);
331 int height = GST_VIDEO_FRAME_HEIGHT (frame);
332 int stride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
333 guint8 *data = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
334
335 for (j = 0; j < height; ++j) {
336 guint8 *line;
337
338 line = data;
339 line += stride * j;
340 for (i = 0; i < width; ++i) {
341 int a = line[3];
342 if (a) {
343 line[0] = MIN ((line[0] * 255 + a / 2) / a, 255);
344 line[1] = MIN ((line[1] * 255 + a / 2) / a, 255);
345 line[2] = MIN ((line[2] * 255 + a / 2) / a, 255);
346 }
347 line += 4;
348 }
349 }
350 }
351
352 /* Copy from video-overlay-composition.c */
353 static void
gst_video_overlay_rectangle_unpremultiply(GstVideoFrame * frame)354 gst_video_overlay_rectangle_unpremultiply (GstVideoFrame * frame)
355 {
356 gint alpha_offset;
357
358 alpha_offset = GST_VIDEO_FRAME_COMP_POFFSET (frame, 3);
359 switch (alpha_offset) {
360 case 0:
361 gst_video_overlay_rectangle_unpremultiply_0 (frame);
362 break;
363 case 3:
364 gst_video_overlay_rectangle_unpremultiply_3 (frame);
365 break;
366 default:
367 g_assert_not_reached ();
368 break;
369 }
370 }
371
372 static GstFlowReturn
gst_cairo_overlay_transform_ip(GstBaseTransform * trans,GstBuffer * buf)373 gst_cairo_overlay_transform_ip (GstBaseTransform * trans, GstBuffer * buf)
374 {
375 GstCairoOverlay *overlay = GST_CAIRO_OVERLAY (trans);
376 GstVideoFrame frame;
377 cairo_surface_t *surface;
378 cairo_t *cr;
379 cairo_format_t format;
380 gboolean draw_on_transparent_surface = overlay->draw_on_transparent_surface;
381
382 switch (GST_VIDEO_INFO_FORMAT (&overlay->info)) {
383 case GST_VIDEO_FORMAT_ARGB:
384 case GST_VIDEO_FORMAT_BGRA:
385 format = CAIRO_FORMAT_ARGB32;
386 break;
387 case GST_VIDEO_FORMAT_xRGB:
388 case GST_VIDEO_FORMAT_BGRx:
389 format = CAIRO_FORMAT_RGB24;
390 break;
391 case GST_VIDEO_FORMAT_RGB16:
392 format = CAIRO_FORMAT_RGB16_565;
393 break;
394 default:
395 {
396 GST_WARNING ("No matching cairo format for %s",
397 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&overlay->info)));
398 return GST_FLOW_ERROR;
399 }
400 }
401
402 /* If we need to map the buffer writable, do so */
403 if (!draw_on_transparent_surface || !overlay->attach_compo_to_buffer) {
404 if (!gst_video_frame_map (&frame, &overlay->info, buf, GST_MAP_READWRITE)) {
405 return GST_FLOW_ERROR;
406 }
407 } else {
408 frame.buffer = NULL;
409 }
410
411 if (draw_on_transparent_surface) {
412 surface =
413 cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
414 GST_VIDEO_INFO_WIDTH (&overlay->info),
415 GST_VIDEO_INFO_HEIGHT (&overlay->info));
416 } else {
417 if (format == CAIRO_FORMAT_ARGB32)
418 gst_video_overlay_rectangle_premultiply (&frame);
419
420 surface =
421 cairo_image_surface_create_for_data (GST_VIDEO_FRAME_PLANE_DATA (&frame,
422 0), format, GST_VIDEO_FRAME_WIDTH (&frame),
423 GST_VIDEO_FRAME_HEIGHT (&frame), GST_VIDEO_FRAME_PLANE_STRIDE (&frame,
424 0));
425 }
426
427 if (G_UNLIKELY (!surface))
428 return GST_FLOW_ERROR;
429
430 cr = cairo_create (surface);
431 if (G_UNLIKELY (!cr)) {
432 cairo_surface_destroy (surface);
433 return GST_FLOW_ERROR;
434 }
435
436 g_signal_emit (overlay, gst_cairo_overlay_signals[SIGNAL_DRAW], 0,
437 cr, GST_BUFFER_PTS (buf), GST_BUFFER_DURATION (buf), NULL);
438
439 cairo_destroy (cr);
440
441 if (draw_on_transparent_surface) {
442 guint size;
443 GstBuffer *surface_buffer;
444 GstVideoOverlayRectangle *rect;
445 GstVideoOverlayComposition *composition;
446 gsize offset[GST_VIDEO_MAX_PLANES] = { 0, };
447 gint stride[GST_VIDEO_MAX_PLANES] = { 0, };
448
449 size =
450 cairo_image_surface_get_height (surface) *
451 cairo_image_surface_get_stride (surface);
452 stride[0] = cairo_image_surface_get_stride (surface);
453
454 /* Create a GstVideoOverlayComposition for blending, this handles
455 * pre-multiplied alpha correctly */
456 surface_buffer =
457 gst_buffer_new_wrapped_full (0, cairo_image_surface_get_data (surface),
458 size, 0, size, surface, (GDestroyNotify) cairo_surface_destroy);
459 gst_buffer_add_video_meta_full (surface_buffer, GST_VIDEO_FRAME_FLAG_NONE,
460 (G_BYTE_ORDER ==
461 G_LITTLE_ENDIAN ? GST_VIDEO_FORMAT_BGRA : GST_VIDEO_FORMAT_ARGB),
462 GST_VIDEO_INFO_WIDTH (&overlay->info),
463 GST_VIDEO_INFO_HEIGHT (&overlay->info), 1, offset, stride);
464 rect =
465 gst_video_overlay_rectangle_new_raw (surface_buffer, 0, 0,
466 GST_VIDEO_INFO_WIDTH (&overlay->info),
467 GST_VIDEO_INFO_HEIGHT (&overlay->info),
468 GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);
469 gst_buffer_unref (surface_buffer);
470
471 if (overlay->attach_compo_to_buffer) {
472 GstVideoOverlayCompositionMeta *composition_meta;
473
474 composition_meta = gst_buffer_get_video_overlay_composition_meta (buf);
475 if (composition_meta) {
476 GstVideoOverlayComposition *merged_composition =
477 gst_video_overlay_composition_copy (composition_meta->overlay);
478 gst_video_overlay_composition_add_rectangle (merged_composition, rect);
479 gst_video_overlay_composition_unref (composition_meta->overlay);
480 composition_meta->overlay = merged_composition;
481 gst_video_overlay_rectangle_unref (rect);
482 } else {
483 composition = gst_video_overlay_composition_new (rect);
484 gst_video_overlay_rectangle_unref (rect);
485 gst_buffer_add_video_overlay_composition_meta (buf, composition);
486 gst_video_overlay_composition_unref (composition);
487 }
488 } else {
489 composition = gst_video_overlay_composition_new (rect);
490 gst_video_overlay_rectangle_unref (rect);
491 gst_video_overlay_composition_blend (composition, &frame);
492 gst_video_overlay_composition_unref (composition);
493 }
494 } else {
495 cairo_surface_destroy (surface);
496 if (format == CAIRO_FORMAT_ARGB32)
497 gst_video_overlay_rectangle_unpremultiply (&frame);
498 }
499
500 if (frame.buffer) {
501 gst_video_frame_unmap (&frame);
502 }
503
504 return GST_FLOW_OK;
505 }
506
507 static void
gst_cairo_overlay_class_init(GstCairoOverlayClass * klass)508 gst_cairo_overlay_class_init (GstCairoOverlayClass * klass)
509 {
510 GstBaseTransformClass *btrans_class;
511 GstElementClass *element_class;
512 GObjectClass *gobject_class;
513
514 btrans_class = (GstBaseTransformClass *) klass;
515 element_class = (GstElementClass *) klass;
516 gobject_class = (GObjectClass *) klass;
517
518 btrans_class->set_caps = gst_cairo_overlay_set_caps;
519 btrans_class->transform_ip = gst_cairo_overlay_transform_ip;
520 btrans_class->query = gst_cairo_overlay_query;
521
522 gobject_class->set_property = gst_cairo_overlay_set_property;
523 gobject_class->get_property = gst_cairo_overlay_get_property;
524
525 g_object_class_install_property (gobject_class,
526 PROP_DRAW_ON_TRANSPARENT_SURFACE,
527 g_param_spec_boolean ("draw-on-transparent-surface",
528 "Draw on transparent surface",
529 "Let the draw signal work on a transparent surface "
530 "and blend the results with the video at a later time",
531 DEFAULT_DRAW_ON_TRANSPARENT_SURFACE,
532 GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
533 | G_PARAM_STATIC_STRINGS));
534
535 /**
536 * GstCairoOverlay::draw:
537 * @overlay: Overlay element emitting the signal.
538 * @cr: Cairo context to draw to.
539 * @timestamp: Timestamp (see #GstClockTime) of the current buffer.
540 * @duration: Duration (see #GstClockTime) of the current buffer.
541 *
542 * This signal is emitted when the overlay should be drawn.
543 */
544 gst_cairo_overlay_signals[SIGNAL_DRAW] =
545 g_signal_new ("draw",
546 G_TYPE_FROM_CLASS (klass),
547 0,
548 0,
549 NULL,
550 NULL,
551 g_cclosure_marshal_generic,
552 G_TYPE_NONE, 3, CAIRO_GOBJECT_TYPE_CONTEXT, G_TYPE_UINT64, G_TYPE_UINT64);
553
554 /**
555 * GstCairoOverlay::caps-changed:
556 * @overlay: Overlay element emitting the signal.
557 * @caps: The #GstCaps of the element.
558 *
559 * This signal is emitted when the caps of the element has changed.
560 */
561 gst_cairo_overlay_signals[SIGNAL_CAPS_CHANGED] =
562 g_signal_new ("caps-changed",
563 G_TYPE_FROM_CLASS (klass),
564 0,
565 0, NULL, NULL, g_cclosure_marshal_generic, G_TYPE_NONE, 1, GST_TYPE_CAPS);
566
567 gst_element_class_set_static_metadata (element_class, "Cairo overlay",
568 "Filter/Editor/Video",
569 "Render overlay on a video stream using Cairo",
570 "Jon Nordby <jononor@gmail.com>");
571
572 gst_element_class_add_static_pad_template (element_class,
573 &gst_cairo_overlay_sink_template);
574 gst_element_class_add_static_pad_template (element_class,
575 &gst_cairo_overlay_src_template);
576 }
577
578 static void
gst_cairo_overlay_init(GstCairoOverlay * overlay)579 gst_cairo_overlay_init (GstCairoOverlay * overlay)
580 {
581 /* nothing to do */
582 }
583