1 /* GStreamer
2  * Copyright (C) 2011 Entropy Wave Inc <ds@entropywave.com>
3  *
4  * This library is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Library General Public
6  * License as published by the Free Software Foundation; either
7  * version 2 of the License, or (at your option) any later version.
8  *
9  * This library is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Library General Public License for more details.
13  *
14  * You should have received a copy of the GNU Library General Public
15  * License along with this library; if not, write to the
16  * Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
17  * Boston, MA 02110-1335, USA.
18  */
19 /**
20  * SECTION:element-gstscenechange
21  * @title: gstscenechange
22  *
23  * The scenechange element detects scene changes (also known as shot
24  * changes) in a video stream, and sends a signal when this occurs.
25  * Applications can listen to this signal and make changes to the
26  * pipeline such as cutting the stream.  In addition, whenever a
27  * scene change is detected, a custom downstream "GstForceKeyUnit"
28  * event is sent to downstream elements.  Most video encoder elements
29  * will insert synchronization points into the stream when this event
30  * is received.  When used with a tee element, the scenechange element
31  * can be used to align the synchronization points among multiple
32  * video encoders, which is useful for segmented streaming.
33  *
34  * The scenechange element does not work with compressed video.
35  *
36  * ## Example launch line
37  * |[
38  * gst-launch-1.0 -v filesrc location=some_file.ogv ! decodebin !
39  *   scenechange ! theoraenc ! fakesink
40  * ]|
41  *
42  */
43 /*
44  * The algorithm used for scene change detection is a modification
45  * of Jim Easterbrook's shot change detector.  I'm not aware of a
46  * research paper, but the code I got the idea from is here:
47  *  http://sourceforge.net/projects/shot-change/
48  *
49  * The method is relatively simple.  Calculate the sum of absolute
50  * differences of a picture and the previous picture, and compare this
51  * picture difference value with neighboring pictures.  In the original
52  * algorithm, the value is compared to a configurable number of past
53  * and future pictures.  However, comparing to future frames requires
54  * introducing latency into the stream, which I did not want.  So this
55  * implementation only compared to previous frames.
56  *
57  * This code is more directly derived from the scene change detection
58  * implementation in Schroedinger.  Schro's implementation is closer
59  * to the Easterbrook algorithm, comparing to future pictures.  In
60  * terms of accuracy, schro's implementation has about 2-3 false positives
61  * or false negatives per 100 scene changes.  This implementation has
62  * about 5 per 100.  The threshold is tuned for minimum total false
63  * positives or negatives, on the assumption that the badness of a
64  * false negative is the same as a false positive.
65  *
66  * This algorithm is pretty much at its limit for error rate.  I
67  * recommend any future work in this area to increase the complexity
68  * of detection, and then write an automatic tuning system as opposed
69  * to the manual tuning I did here.
70  *
71  * Inside the TESTING define are some hard-coded (mostly hand-written)
72  * scene change frame numbers for some easily available sequences.
73  *
74  */
75 
76 #ifdef HAVE_CONFIG_H
77 #include "config.h"
78 #endif
79 
80 #include <gst/gst.h>
81 #include <gst/video/video.h>
82 #include <gst/video/gstvideofilter.h>
83 #include <string.h>
84 #include "gstscenechange.h"
85 
86 GST_DEBUG_CATEGORY_STATIC (gst_scene_change_debug_category);
87 #define GST_CAT_DEFAULT gst_scene_change_debug_category
88 
89 /* prototypes */
90 
91 
92 static GstFlowReturn gst_scene_change_transform_frame_ip (GstVideoFilter *
93     filter, GstVideoFrame * frame);
94 
95 #undef TESTING
96 #ifdef TESTING
97 static gboolean is_shot_change (int frame_number);
98 #endif
99 
100 enum
101 {
102   PROP_0
103 };
104 
105 #define VIDEO_CAPS \
106     GST_VIDEO_CAPS_MAKE("{ I420, Y42B, Y41B, Y444 }")
107 
108 /* class initialization */
109 
110 G_DEFINE_TYPE_WITH_CODE (GstSceneChange, gst_scene_change,
111     GST_TYPE_VIDEO_FILTER,
112     GST_DEBUG_CATEGORY_INIT (gst_scene_change_debug_category, "scenechange", 0,
113         "debug category for scenechange element"));
114 
115 static void
gst_scene_change_class_init(GstSceneChangeClass * klass)116 gst_scene_change_class_init (GstSceneChangeClass * klass)
117 {
118   GstVideoFilterClass *video_filter_class = GST_VIDEO_FILTER_CLASS (klass);
119 
120   gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass),
121       gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
122           gst_caps_from_string (VIDEO_CAPS)));
123   gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass),
124       gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
125           gst_caps_from_string (VIDEO_CAPS)));
126 
127   gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass),
128       "Scene change detector",
129       "Video/Filter", "Detects scene changes in video",
130       "David Schleef <ds@entropywave.com>");
131 
132   video_filter_class->transform_frame_ip =
133       GST_DEBUG_FUNCPTR (gst_scene_change_transform_frame_ip);
134 
135 }
136 
137 static void
gst_scene_change_init(GstSceneChange * scenechange)138 gst_scene_change_init (GstSceneChange * scenechange)
139 {
140 }
141 
142 
143 static double
get_frame_score(GstVideoFrame * f1,GstVideoFrame * f2)144 get_frame_score (GstVideoFrame * f1, GstVideoFrame * f2)
145 {
146   int i;
147   int j;
148   int score = 0;
149   int width, height;
150   guint8 *s1;
151   guint8 *s2;
152 
153   width = f1->info.width;
154   height = f1->info.height;
155 
156   for (j = 0; j < height; j++) {
157     s1 = (guint8 *) f1->data[0] + f1->info.stride[0] * j;
158     s2 = (guint8 *) f2->data[0] + f2->info.stride[0] * j;
159     for (i = 0; i < width; i++) {
160       score += ABS (s1[i] - s2[i]);
161     }
162   }
163 
164   return ((double) score) / (width * height);
165 }
166 
167 static GstFlowReturn
gst_scene_change_transform_frame_ip(GstVideoFilter * filter,GstVideoFrame * frame)168 gst_scene_change_transform_frame_ip (GstVideoFilter * filter,
169     GstVideoFrame * frame)
170 {
171   GstSceneChange *scenechange = GST_SCENE_CHANGE (filter);
172   GstVideoFrame oldframe;
173   double score_min;
174   double score_max;
175   double threshold;
176   double score;
177   gboolean change;
178   gboolean ret;
179   int i;
180 
181   GST_DEBUG_OBJECT (scenechange, "transform_frame_ip");
182 
183   if (!scenechange->oldbuf) {
184     scenechange->n_diffs = 0;
185     memset (scenechange->diffs, 0, sizeof (double) * SC_N_DIFFS);
186     scenechange->oldbuf = gst_buffer_ref (frame->buffer);
187     memcpy (&scenechange->oldinfo, &frame->info, sizeof (GstVideoInfo));
188     return GST_FLOW_OK;
189   }
190 
191   ret =
192       gst_video_frame_map (&oldframe, &scenechange->oldinfo,
193       scenechange->oldbuf, GST_MAP_READ);
194   if (!ret) {
195     GST_ERROR_OBJECT (scenechange, "failed to map old video frame");
196     return GST_FLOW_ERROR;
197   }
198 
199   score = get_frame_score (&oldframe, frame);
200 
201   gst_video_frame_unmap (&oldframe);
202 
203   gst_buffer_unref (scenechange->oldbuf);
204   scenechange->oldbuf = gst_buffer_ref (frame->buffer);
205   memcpy (&scenechange->oldinfo, &frame->info, sizeof (GstVideoInfo));
206 
207   memmove (scenechange->diffs, scenechange->diffs + 1,
208       sizeof (double) * (SC_N_DIFFS - 1));
209   scenechange->diffs[SC_N_DIFFS - 1] = score;
210   scenechange->n_diffs++;
211 
212   score_min = scenechange->diffs[0];
213   score_max = scenechange->diffs[0];
214   for (i = 1; i < SC_N_DIFFS - 1; i++) {
215     score_min = MIN (score_min, scenechange->diffs[i]);
216     score_max = MAX (score_max, scenechange->diffs[i]);
217   }
218 
219   threshold = 1.8 * score_max - 0.8 * score_min;
220 
221   if (scenechange->n_diffs > (SC_N_DIFFS - 1)) {
222     if (score < 5) {
223       change = FALSE;
224     } else if (score / threshold < 1.0) {
225       change = FALSE;
226     } else if ((score > 30)
227         && (score / scenechange->diffs[SC_N_DIFFS - 2] > 1.4)) {
228       change = TRUE;
229     } else if (score / threshold > 2.3) {
230       change = TRUE;
231     } else if (score > 50) {
232       change = TRUE;
233     } else {
234       change = FALSE;
235     }
236   } else {
237     change = FALSE;
238   }
239 
240   if (change == TRUE) {
241     memset (scenechange->diffs, 0, sizeof (double) * SC_N_DIFFS);
242     scenechange->n_diffs = 0;
243   }
244 #ifdef TESTING
245   if (change != is_shot_change (scenechange->n_diffs)) {
246     g_print ("%d %g %g %g %d\n", scenechange->n_diffs, score / threshold,
247         score, threshold, change);
248   }
249 #endif
250 
251   if (change) {
252     GstEvent *event;
253 
254     GST_INFO_OBJECT (scenechange, "%d %g %g %g %d",
255         scenechange->n_diffs, score / threshold, score, threshold, change);
256 
257     event =
258         gst_video_event_new_downstream_force_key_unit (GST_BUFFER_PTS
259         (frame->buffer), GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, FALSE,
260         scenechange->count++);
261 
262     gst_pad_push_event (GST_BASE_TRANSFORM_SRC_PAD (scenechange), event);
263   }
264 
265   return GST_FLOW_OK;
266 }
267 
268 
269 
270 
271 
272 
273 #ifdef TESTING
274 /* This is from ds's personal collection.  No, you can't have it. */
275 int showreel_changes[] = {
276   242, 483, 510, 550, 579, 603, 609, 1056, 1067, 1074, 1079, 1096,
277   1106, 1113, 1127, 1145, 1156, 1170, 1212, 1228, 1243, 1269, 1274,
278   1322, 1349, 1370, 1378, 1423, 1456, 1458, 1508, 1519, 1542, 1679,
279   1767, 1837, 1895, 1962, 2006, 2035, 2102, 2139, 2196, 2561, 2664,
280   2837, 2895, 2985, 3035, 3077, 3128, 3176, 3218, 3306, 3351, 3388,
281   3421, 3470, 3711, 3832, 4029, 4184, 4444, 4686, 4719, 4825, 4941,
282   5009, 5091, 5194, 5254, 5286, 5287, 5343, 5431, 5501, 5634, 5695, 5788,
283   5839, 5861, 5930, 6030, 6168, 6193, 6237, 6336, 6376, 6421, 6495,
284   6550, 6611, 6669, 6733, 6819, 6852, 6944, 7087, 7148, 7189, 7431,
285   7540, 7599, 7632, 7661, 7693, 7930, 7963, 8003, 8076, 8109, 8147,
286   8177, 8192, 8219, 8278, 8322, 8370, 8409, 8566, 8603, 8747, 8775,
287   8873, 8907, 8955, 8969, 8983, 8997, 9026, 9079, 9140, 9165, 9206,
288   9276, 9378, 9449, 9523, 9647, 9703, 9749, 9790, 9929, 10056, 10216,
289   10307, 10411, 10487, 10557, 10695, 10770, 10854, 11095, 11265, 11517, 11589,
290   11686, 11825, 11940, 12004, 12047, 12113, 12179, 12233, 12532, 12586, 12708,
291   12793, 12877, 12954, 13030, 13105, 13177, 13279, 13396, 13486, 13538, 13561,
292   13591, 13627, 13656, 13709, 13763, 13815, 13842, 13876, 13906, 13929, 13955,
293   14003, 14070, 14097, 14127, 14153, 14198, 14269, 14348, 14367, 14440, 14488,
294   14548, 14573, 14599, 14630, 14665, 14907, 14962, 15013, 15089, 15148, 15227,
295   15314, 15355, 15369, 15451, 15470, 15542, 15570, 15640, 15684, 15781, 15869,
296   15938, 16172, 16266, 16429, 16479, 16521, 16563, 16612, 16671, 16692, 16704,
297   16720, 16756, 16789, 16802, 16815, 16867, 16908, 16939, 16953, 16977, 17006,
298   17014, 17026, 17040, 17062, 17121, 17176, 17226, 17322, 17444, 17496, 17641,
299   17698, 17744, 17826, 17913, 17993, 18073, 18219, 18279, 18359, 18475, 18544,
300   18587, 18649, 18698, 18756, 18826, 18853, 18866, 19108, 19336, 19481, 19544,
301   19720, 19816, 19908, 19982, 20069, 20310, 20355, 20374, 20409, 20469, 20599,
302   20607, 20652, 20805, 20822, 20882, 20982, 21029, 21433, 21468, 21561, 21602,
303   21661, 21720, 21909, 22045, 22166, 22225, 22323, 22362, 22433, 22477, 22529,
304   22571, 22617, 22642, 22676, 22918, 22978, 23084, 23161, 23288, 23409, 23490,
305   23613, 23721, 23815, 24131, 24372, 24468, 24507, 24555, 24568, 24616, 24634,
306   24829, 24843, 24919, 24992, 25040, 25160, 25288, 25607, 25684, 25717, 25764,
307   25821, 25866, 25901, 25925, 25941, 25978, 25998, 26011, 26030, 26055, 26118,
308   26133, 26145, 26159, 26175, 26182, 26195, 26205, 26238, 26258, 26316, 26340,
309   26581, 26725, 26834, 26874, 26995, 27065, 27178, 27238, 27365, 27607, 27669,
310   27694,
311   27774, 27800, 27841, 27930, 27985, 28057, 28091, 28132, 28189, 28270, 28545,
312   28653, 28711, 28770, 28886, 28966, 29139, 29241, 29356, 29415, 29490, 29576,
313   29659, 29776, 29842, 29910, 30029, 30056, 30100, 30129, 30175, 30316, 30376,
314   30441, 30551, 30666, 30784, 30843, 30948, 31045, 31286, 31315, 31534, 31607,
315   31742,
316   31817, 31853, 31984, 32009, 32112, 32162, 32210, 32264
317 };
318 
319 /* Sintel */
320 int sintel_changes[] = {
321   752, 1018, 1036, 1056, 1078, 1100, 1169, 1319, 1339, 1370,
322   1425, 1455, 1494, 1552, 1572, 1637, 1663, 1777, 1955, 2060,
323   2125, 2429, 2624, 2780, 2835, 2881, 2955, 3032, 3144, 3217,
324   3315, 3384, 3740, 3890, 4234, 4261, 4322, 4368, 4425, 4481,
325   4555, 4605, 4671, 4714, 4743, 4875, 4920, 5082, 5158, 5267,
326   5379, 5956, 6021, 6071, 6112, 6139, 6221, 6318, 6374, 6519,
327   6558, 6615, 6691, 6803, 6900, 6944, 7134, 7266, 7351, 7414,
328   7467, 7503, 7559, 7573, 7656, 7733, 7876, 7929, 7971, 7985,
329   8047, 8099, 8144, 8215, 8394, 8435, 8480, 9133, 9190, 9525,
330   9962,
331 };
332 
333 /* Breathe Out video, http://media.xiph.org/video/misc/ */
334 int breatheout_changes[] = {
335   143, 263, 334, 426, 462, 563, 583, 618, 655, 707,
336   818, 823, 858, 913, 956, 977, 999, 1073, 1124, 1144,
337   1166, 1187, 1206, 1227, 1240, 1264, 1289, 1312, 1477, 1535,
338   1646, 1692, 1739, 1757, 1798, 1855, 1974, 2048, 2129, 2212,
339   2369, 2412, 2463, 2578, 2649, 2699, 2778, 2857, 2923, 3014,
340   3107, 3246, 3321, 3350, 3459, 3498, 3541, 3567, 3613, 3636,
341   3673, 3709, 3747, 3834, 3862, 3902, 3922, 4022, 4117, 4262,
342   4303, 4357, 4556, 4578, 4617, 4716, 4792, 4873, 4895, 4917,
343   4932, 4972, 5015, 5034, 5058, 5090, 5162, 5180, 5202, 5222,
344   5239, 5258, 5281, 5298, 5397, 5430,
345   485, 507, 534, 665, 685, 755, 1023, 1379, 1441, 1503,
346   1584, 1621, 1903, 2081, 2281, 2511, 2958, 3071, 3185, 3214,
347   3271, 3424, 3479, 3588, 3879, 3979, 4043, 4062, 4143, 4207,
348   4237, 4336, 4461, 4476, 4533, 4647, 4815, 4853, 4949, 5075,
349   5142, 5316, 5376,
350   3514, 3952, 4384, 5337
351 };
352 
353 #define changes showreel_changes
354 
355 static gboolean
is_shot_change(int frame_number)356 is_shot_change (int frame_number)
357 {
358   int i;
359   for (i = 0; i < sizeof (changes) / sizeof (changes[0]); i++) {
360     if (changes[i] == frame_number)
361       return TRUE;
362   }
363   return FALSE;
364 }
365 #endif
366