1 /* vi:set et ai sw=2 sts=2 ts=2: */
2 /*
3 * Copyright (C) 2003,2004 Bastien Nocera <hadess@hadess.net>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General
16 * Public License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 *
20 * Most of the code is taken from the totem-video-thumbnailer and
21 * made suitable for Tumbler by Nick Schermer.
22 */
23
24 #ifdef HAVE_CONFIG_H
25 #include <config.h>
26 #endif
27
28 #include <string.h>
29 #include <math.h>
30
31 #include <glib.h>
32 #include <glib-object.h>
33 #include <gdk-pixbuf/gdk-pixbuf.h>
34 #include <tumbler/tumbler.h>
35
36 #include <gst/gst.h>
37 #include <gst/tag/tag.h>
38
39 #include "gst-thumbnailer.h"
40
41
42
43 #define BORING_IMAGE_VARIANCE 256.0 /* tweak this if necessary */
44 #define TUMBLER_GST_PLAY_FLAG_VIDEO (1 << 0) /* from GstPlayFlags */
45 #define TUMBLER_GST_PLAY_FLAG_AUDIO (1 << 1) /* from GstPlayFlags */
46
47
48
49 static void gst_thumbnailer_create (TumblerAbstractThumbnailer *thumbnailer,
50 GCancellable *cancellable,
51 TumblerFileInfo *info);
52
53
54
55 struct _GstThumbnailerClass
56 {
57 TumblerAbstractThumbnailerClass __parent__;
58 };
59
60 struct _GstThumbnailer
61 {
62 TumblerAbstractThumbnailer __parent__;
63 };
64
65
66
67 G_DEFINE_DYNAMIC_TYPE (GstThumbnailer,
68 gst_thumbnailer,
69 TUMBLER_TYPE_ABSTRACT_THUMBNAILER);
70
71
72
73 void
gst_thumbnailer_register(TumblerProviderPlugin * plugin)74 gst_thumbnailer_register (TumblerProviderPlugin *plugin)
75 {
76 gst_thumbnailer_register_type (G_TYPE_MODULE (plugin));
77 }
78
79
80
81 static void
gst_thumbnailer_class_init(GstThumbnailerClass * klass)82 gst_thumbnailer_class_init (GstThumbnailerClass *klass)
83 {
84 TumblerAbstractThumbnailerClass *abstractthumbnailer_class;
85
86 abstractthumbnailer_class = TUMBLER_ABSTRACT_THUMBNAILER_CLASS (klass);
87 abstractthumbnailer_class->create = gst_thumbnailer_create;
88 }
89
90
91
92 static void
gst_thumbnailer_class_finalize(GstThumbnailerClass * klass)93 gst_thumbnailer_class_finalize (GstThumbnailerClass *klass)
94 {
95 }
96
97
98
99 static void
gst_thumbnailer_init(GstThumbnailer * thumbnailer)100 gst_thumbnailer_init (GstThumbnailer *thumbnailer)
101 {
102 }
103
104
105
106 static GdkPixbuf *
gst_thumbnailer_buffer_to_pixbuf(GstBuffer * buffer)107 gst_thumbnailer_buffer_to_pixbuf (GstBuffer *buffer)
108 {
109 GstMapInfo info;
110 GdkPixbuf *pixbuf = NULL;
111 GdkPixbufLoader *loader;
112
113 if (!gst_buffer_map (buffer, &info, GST_MAP_READ))
114 return NULL;
115
116 loader = gdk_pixbuf_loader_new ();
117
118 if (gdk_pixbuf_loader_write (loader, info.data, info.size, NULL)
119 && gdk_pixbuf_loader_close (loader, NULL))
120 {
121 pixbuf = gdk_pixbuf_loader_get_pixbuf (loader);
122 if (pixbuf != NULL)
123 g_object_ref (pixbuf);
124 }
125
126 g_object_unref (loader);
127
128 gst_buffer_unmap (buffer, &info);
129
130 return pixbuf;
131 }
132
133
134
135 static GdkPixbuf *
gst_thumbnailer_cover_from_tags(GstTagList * tags,GCancellable * cancellable)136 gst_thumbnailer_cover_from_tags (GstTagList *tags,
137 GCancellable *cancellable)
138 {
139 GstSample *cover = NULL;
140 guint i;
141 GstCaps *caps;
142 const GstStructure *caps_struct;
143 gint type = GST_TAG_IMAGE_TYPE_UNDEFINED;
144 GstBuffer *buffer;
145 GdkPixbuf *pixbuf = NULL;
146
147 for (i = 0; ; i++)
148 {
149 GstSample *sample;
150
151 if (g_cancellable_is_cancelled (cancellable))
152 break;
153
154 /* look for image in the tags */
155 if (!gst_tag_list_get_sample_index (tags, GST_TAG_IMAGE, i, &sample))
156 break;
157
158 caps = gst_sample_get_caps (sample);
159 caps_struct = gst_caps_get_structure (caps, 0);
160 gst_structure_get_enum (caps_struct,
161 "image-type",
162 GST_TYPE_TAG_IMAGE_TYPE,
163 &type);
164
165 if (cover != NULL)
166 gst_sample_unref (cover);
167 cover = sample;
168
169 /* prefer the from cover image if specified */
170 if (type == GST_TAG_IMAGE_TYPE_FRONT_COVER)
171 break;
172 }
173
174 if (cover == NULL
175 && !g_cancellable_is_cancelled (cancellable))
176 {
177 /* look for preview image */
178 gst_tag_list_get_sample_index (tags, GST_TAG_PREVIEW_IMAGE, 0, &cover);
179 }
180
181 if (cover != NULL)
182 {
183 /* create image */
184 buffer = gst_sample_get_buffer (cover);
185 pixbuf = gst_thumbnailer_buffer_to_pixbuf (buffer);
186 gst_sample_unref (cover);
187 }
188
189 return pixbuf;
190 }
191
192
193
194 static GdkPixbuf *
gst_thumbnailer_cover_by_name(GstElement * play,const gchar * signal_name,GCancellable * cancellable)195 gst_thumbnailer_cover_by_name (GstElement *play,
196 const gchar *signal_name,
197 GCancellable *cancellable)
198 {
199 GstTagList *tags = NULL;
200 GdkPixbuf *cover;
201
202 g_signal_emit_by_name (G_OBJECT (play), signal_name, 0, &tags);
203
204 if (tags == NULL)
205 return FALSE;
206
207 /* check the tags for a cover */
208 cover = gst_thumbnailer_cover_from_tags (tags, cancellable);
209 gst_tag_list_free (tags);
210
211 return cover;
212 }
213
214
215
216 static GdkPixbuf *
gst_thumbnailer_cover(GstElement * play,GCancellable * cancellable)217 gst_thumbnailer_cover (GstElement *play,
218 GCancellable *cancellable)
219 {
220 GdkPixbuf *cover;
221
222 cover = gst_thumbnailer_cover_by_name (play, "get-audio-tags", cancellable);
223 if (cover == NULL)
224 cover = gst_thumbnailer_cover_by_name (play, "get-video-tags", cancellable);
225
226 return cover;
227 }
228
229
230
231 static gboolean
gst_thumbnailer_has_video(GstElement * play)232 gst_thumbnailer_has_video (GstElement *play)
233 {
234 guint n_video;
235 g_object_get (play, "n-video", &n_video, NULL);
236 return n_video > 0;
237 }
238
239
240
241 static void
gst_thumbnailer_destroy_pixbuf(guchar * pixbuf,gpointer data)242 gst_thumbnailer_destroy_pixbuf (guchar *pixbuf,
243 gpointer data)
244 {
245 gst_sample_unref (GST_SAMPLE (data));
246 }
247
248
249
250 static gboolean
gst_thumbnailer_pixbuf_interesting(GdkPixbuf * pixbuf)251 gst_thumbnailer_pixbuf_interesting (GdkPixbuf *pixbuf)
252 {
253 gint rowstride;
254 gint height;
255 guchar *buffer;
256 gint length;
257 gint i;
258 gfloat x_bar = 0.0f;
259 gfloat variance = 0.0f;
260 gfloat temp;
261
262 rowstride = gdk_pixbuf_get_rowstride (pixbuf);
263 height = gdk_pixbuf_get_height (pixbuf);
264 length = (rowstride * height);
265
266 buffer = gdk_pixbuf_get_pixels (pixbuf);
267
268 /* calculate the x-bar */
269 for (i = 0; i < length; i++)
270 x_bar += (gfloat) buffer[i];
271 x_bar /= (gfloat) length;
272
273 /* calculate the variance */
274 for (i = 0; i < length; i++)
275 {
276 temp = ((gfloat) buffer[i] - x_bar);
277 variance += temp * temp;
278 }
279
280 return (variance > BORING_IMAGE_VARIANCE);
281 }
282
283
284
285 static GdkPixbuf *
gst_thumbnailer_capture_frame(GstElement * play,gint width)286 gst_thumbnailer_capture_frame (GstElement *play,
287 gint width)
288 {
289 GstCaps *to_caps;
290 GstSample *sample = NULL;
291 GdkPixbuf *pixbuf = NULL;
292 GstStructure *s;
293 GstCaps *sample_caps;
294 gint outwidth = 0, outheight = 0;
295 GstMemory *memory;
296 GstMapInfo info;
297
298 /* desired output format (RGB24) */
299 to_caps = gst_caps_new_simple ("video/x-raw",
300 "format", G_TYPE_STRING, "RGB",
301 "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
302 "width", G_TYPE_INT, width,
303 NULL);
304
305 /* get the frame */
306 g_signal_emit_by_name (play, "convert-sample", to_caps, &sample);
307 gst_caps_unref (to_caps);
308
309 if (sample == NULL)
310 return NULL;
311
312 sample_caps = gst_sample_get_caps (sample);
313 if (sample_caps == NULL)
314 {
315 /* no caps on output buffer */
316 gst_sample_unref (sample);
317 return NULL;
318 }
319
320 /* size of the frame */
321 s = gst_caps_get_structure (sample_caps, 0);
322 gst_structure_get_int (s, "width", &outwidth);
323 gst_structure_get_int (s, "height", &outheight);
324 if (outwidth <= 0 || outheight <= 0)
325 {
326 /* invalid size */
327 gst_sample_unref (sample);
328 return NULL;
329 }
330
331 /* get the memory block of the buffer */
332 memory = gst_buffer_get_memory (gst_sample_get_buffer (sample), 0);
333 if (gst_memory_map (memory, &info, GST_MAP_READ))
334 {
335 /* create pixmap for the data */
336 pixbuf = gdk_pixbuf_new_from_data (info.data,
337 GDK_COLORSPACE_RGB, FALSE, 8,
338 outwidth, outheight,
339 GST_ROUND_UP_4 (width * 3),
340 gst_thumbnailer_destroy_pixbuf,
341 sample);
342
343 /* release memory */
344 gst_memory_unmap (memory, &info);
345 }
346
347 gst_memory_unref (memory);
348
349 /* release sample if pixbuf failed */
350 if (pixbuf == NULL)
351 gst_sample_unref (sample);
352
353 return pixbuf;
354 }
355
356
357
358 static GdkPixbuf *
gst_thumbnailer_capture_interesting_frame(GstElement * play,gint64 duration,gint width,GCancellable * cancellable)359 gst_thumbnailer_capture_interesting_frame (GstElement *play,
360 gint64 duration,
361 gint width,
362 GCancellable *cancellable)
363 {
364 GdkPixbuf *pixbuf = NULL;
365 guint n;
366 const gdouble offsets[] = { 1.0 / 3.0, 2.0 / 3.0, 0.1, 0.9, 0.5 };
367 gint64 seek_time;
368
369 /* video has no duration, capture 1st frame */
370 if (duration == -1)
371 {
372 if (!g_cancellable_is_cancelled (cancellable))
373 return gst_thumbnailer_capture_frame (play, width);
374 else
375 return NULL;
376 }
377
378 for (n = 0; n < G_N_ELEMENTS (offsets); n++)
379 {
380 /* check if we should abort */
381 if (g_cancellable_is_cancelled (cancellable))
382 break;
383
384 /* seek to offset */
385 seek_time = offsets[n] * duration;
386 gst_element_seek (play, 1.0,
387 GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT,
388 GST_SEEK_TYPE_SET, seek_time * GST_MSECOND,
389 GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE);
390
391 /* wait for the seek to complete */
392 gst_element_get_state (play, NULL, NULL, GST_CLOCK_TIME_NONE);
393
394 /* check if we should abort */
395 if (g_cancellable_is_cancelled (cancellable))
396 break;
397
398 /* get the frame */
399 pixbuf = gst_thumbnailer_capture_frame (play, width);
400 if (pixbuf == NULL)
401 continue;
402
403 /* check if image is interesting or end of loop */
404 if (n + 1 == G_N_ELEMENTS (offsets)
405 || gst_thumbnailer_pixbuf_interesting (pixbuf))
406 break;
407
408 /* continue looking for something better */
409 g_object_unref (pixbuf);
410 pixbuf = NULL;
411 }
412
413 return pixbuf;
414 }
415
416
417
418 static GstBusSyncReply
gst_thumbnailer_error_handler(GstBus * bus,GstMessage * message,gpointer user_data)419 gst_thumbnailer_error_handler (GstBus *bus,
420 GstMessage *message,
421 gpointer user_data)
422 {
423 GCancellable *cancellable = user_data;
424
425 switch (GST_MESSAGE_TYPE (message))
426 {
427 case GST_MESSAGE_ERROR:
428 case GST_MESSAGE_EOS:
429 /* stop */
430 g_cancellable_cancel (cancellable);
431 return GST_BUS_DROP;
432
433 default:
434 return GST_BUS_PASS;
435 }
436 }
437
438
439
440 static gboolean
gst_thumbnailer_play_start(GstElement * play,GCancellable * cancellable)441 gst_thumbnailer_play_start (GstElement *play,
442 GCancellable *cancellable)
443 {
444 GstBus *bus;
445 gboolean terminate = FALSE;
446 GstMessage *message;
447 gboolean async_received = FALSE;
448
449 /* pause to prepare for seeking */
450 gst_element_set_state (play, GST_STATE_PAUSED);
451
452 bus = gst_element_get_bus (play);
453
454 while (!terminate
455 && !g_cancellable_is_cancelled (cancellable))
456 {
457 message = gst_bus_timed_pop_filtered (bus,
458 GST_CLOCK_TIME_NONE,
459 GST_MESSAGE_ASYNC_DONE | GST_MESSAGE_ERROR);
460
461 switch (GST_MESSAGE_TYPE (message))
462 {
463 case GST_MESSAGE_ASYNC_DONE:
464 if (GST_MESSAGE_SRC (message) == GST_OBJECT (play))
465 {
466 async_received = TRUE;
467 terminate = TRUE;
468 }
469 break;
470
471 case GST_MESSAGE_ERROR:
472 terminate = TRUE;
473 break;
474
475 default:
476 break;
477 }
478
479 gst_message_unref (message);
480 }
481
482 /* setup the error handler */
483 if (async_received)
484 gst_bus_set_sync_handler (bus, gst_thumbnailer_error_handler, cancellable, NULL);
485
486 gst_object_unref (bus);
487
488 return async_received;
489 }
490
491
492
493 static GstElement *
gst_thumbnailer_play_init(TumblerFileInfo * info)494 gst_thumbnailer_play_init (TumblerFileInfo *info)
495 {
496 GstElement *play;
497 GstElement *audio_sink;
498 GstElement *video_sink;
499
500 /* prepare play factory */
501 play = gst_element_factory_make ("playbin", "play");
502 audio_sink = gst_element_factory_make ("fakesink", "audio-fake-sink");
503 video_sink = gst_element_factory_make ("fakesink", "video-fake-sink");
504 g_object_set (video_sink, "sync", TRUE, NULL);
505
506 g_object_set (play,
507 "uri", tumbler_file_info_get_uri (info),
508 "audio-sink", audio_sink,
509 "video-sink", video_sink,
510 "flags", TUMBLER_GST_PLAY_FLAG_VIDEO | TUMBLER_GST_PLAY_FLAG_AUDIO,
511 NULL);
512
513 return play;
514 }
515
516
517
518 static GdkPixbuf *
gst_thumbnailer_scale_pixbuf(GdkPixbuf * source,gint dest_width,gint dest_height)519 gst_thumbnailer_scale_pixbuf (GdkPixbuf *source,
520 gint dest_width,
521 gint dest_height)
522 {
523 gdouble wratio;
524 gdouble hratio;
525 gint source_width;
526 gint source_height;
527
528 /* determine source pixbuf dimensions */
529 source_width = gdk_pixbuf_get_width (source);
530 source_height = gdk_pixbuf_get_height (source);
531
532 /* don't do anything if there is no need to resize */
533 if (source_width <= dest_width && source_height <= dest_height)
534 return g_object_ref (source);
535
536 /* determine which axis needs to be scaled down more */
537 wratio = (gdouble) source_width / (gdouble) dest_width;
538 hratio = (gdouble) source_height / (gdouble) dest_height;
539
540 /* adjust the other axis */
541 if (hratio > wratio)
542 dest_width = rint (source_width / hratio);
543 else
544 dest_height = rint (source_height / wratio);
545
546 /* scale the pixbuf down to the desired size */
547 return gdk_pixbuf_scale_simple (source, MAX (dest_width, 1),
548 MAX (dest_height, 1),
549 GDK_INTERP_BILINEAR);
550 }
551
552
553
554 static void
gst_thumbnailer_create(TumblerAbstractThumbnailer * thumbnailer,GCancellable * cancellable,TumblerFileInfo * info)555 gst_thumbnailer_create (TumblerAbstractThumbnailer *thumbnailer,
556 GCancellable *cancellable,
557 TumblerFileInfo *info)
558 {
559 GstElement *play;
560 GdkPixbuf *pixbuf = NULL;
561 gint64 duration;
562 TumblerImageData data;
563 GError *error = NULL;
564 TumblerThumbnail *thumbnail;
565 gint width, height;
566 TumblerThumbnailFlavor *flavor;
567 GdkPixbuf *scaled;
568
569 /* check for early cancellation */
570 if (g_cancellable_is_cancelled (cancellable))
571 return;
572
573 /* Check if is a sparse video file */
574 if (tumbler_util_guess_is_sparse (info))
575 {
576 g_debug ("Video file '%s' is probably sparse, skipping\n",
577 tumbler_file_info_get_uri (info));
578 return;
579 }
580
581 /* get size of dest thumb */
582 thumbnail = tumbler_file_info_get_thumbnail (info);
583 flavor = tumbler_thumbnail_get_flavor (thumbnail);
584 tumbler_thumbnail_flavor_get_size (flavor, &width, &height);
585
586 /* prepare factory */
587 play = gst_thumbnailer_play_init (info);
588
589 if (gst_thumbnailer_play_start (play, cancellable))
590 {
591 /* check for covers in the file */
592 pixbuf = gst_thumbnailer_cover (play, cancellable);
593
594 /* extract cover from video stream */
595 if (pixbuf == NULL
596 && gst_thumbnailer_has_video (play))
597 {
598 /* get the length of the video track */
599 if (gst_element_query_duration (play, GST_FORMAT_TIME, &duration)
600 && duration != -1)
601 duration /= GST_MSECOND;
602 else
603 duration = -1;
604
605 pixbuf = gst_thumbnailer_capture_interesting_frame (play, duration, width, cancellable);
606 }
607 }
608
609 /* stop factory */
610 gst_element_set_state (play, GST_STATE_NULL);
611 g_object_unref (play);
612
613 if (G_LIKELY (pixbuf != NULL))
614 {
615 /* scale to correct size if required */
616 scaled = gst_thumbnailer_scale_pixbuf (pixbuf, width, height);
617 g_object_unref (pixbuf);
618 pixbuf = scaled;
619
620 data.data = gdk_pixbuf_get_pixels (pixbuf);
621 data.has_alpha = gdk_pixbuf_get_has_alpha (pixbuf);
622 data.bits_per_sample = gdk_pixbuf_get_bits_per_sample (pixbuf);
623 data.width = gdk_pixbuf_get_width (pixbuf);
624 data.height = gdk_pixbuf_get_height (pixbuf);
625 data.rowstride = gdk_pixbuf_get_rowstride (pixbuf);
626 data.colorspace = (TumblerColorspace) gdk_pixbuf_get_colorspace (pixbuf);
627
628 tumbler_thumbnail_save_image_data (thumbnail, &data,
629 tumbler_file_info_get_mtime (info),
630 NULL, &error);
631
632 g_object_unref (pixbuf);
633
634 if (error != NULL)
635 {
636 g_signal_emit_by_name (thumbnailer, "error",
637 tumbler_file_info_get_uri (info),
638 error->code, error->message);
639 g_error_free (error);
640 }
641 else
642 {
643 g_signal_emit_by_name (thumbnailer, "ready",
644 tumbler_file_info_get_uri (info));
645 }
646 }
647
648 g_object_unref (thumbnail);
649 }
650