1 /* GStreamer
2 * Copyright (C) 2011 Entropy Wave Inc <ds@entropywave.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
17 * Boston, MA 02110-1335, USA.
18 */
19 /**
20 * SECTION:element-gstscenechange
21 * @title: gstscenechange
22 *
23 * The scenechange element detects scene changes (also known as shot
24 * changes) in a video stream, and sends a signal when this occurs.
25 * Applications can listen to this signal and make changes to the
26 * pipeline such as cutting the stream. In addition, whenever a
27 * scene change is detected, a custom downstream "GstForceKeyUnit"
28 * event is sent to downstream elements. Most video encoder elements
29 * will insert synchronization points into the stream when this event
30 * is received. When used with a tee element, the scenechange element
31 * can be used to align the synchronization points among multiple
32 * video encoders, which is useful for segmented streaming.
33 *
34 * The scenechange element does not work with compressed video.
35 *
36 * ## Example launch line
37 * |[
38 * gst-launch-1.0 -v filesrc location=some_file.ogv ! decodebin !
39 * scenechange ! theoraenc ! fakesink
40 * ]|
41 *
42 */
43 /*
44 * The algorithm used for scene change detection is a modification
45 * of Jim Easterbrook's shot change detector. I'm not aware of a
46 * research paper, but the code I got the idea from is here:
47 * http://sourceforge.net/projects/shot-change/
48 *
49 * The method is relatively simple. Calculate the sum of absolute
50 * differences of a picture and the previous picture, and compare this
51 * picture difference value with neighboring pictures. In the original
52 * algorithm, the value is compared to a configurable number of past
53 * and future pictures. However, comparing to future frames requires
54 * introducing latency into the stream, which I did not want. So this
55 * implementation only compared to previous frames.
56 *
57 * This code is more directly derived from the scene change detection
58 * implementation in Schroedinger. Schro's implementation is closer
59 * to the Easterbrook algorithm, comparing to future pictures. In
60 * terms of accuracy, schro's implementation has about 2-3 false positives
61 * or false negatives per 100 scene changes. This implementation has
62 * about 5 per 100. The threshold is tuned for minimum total false
63 * positives or negatives, on the assumption that the badness of a
64 * false negative is the same as a false positive.
65 *
66 * This algorithm is pretty much at its limit for error rate. I
67 * recommend any future work in this area to increase the complexity
68 * of detection, and then write an automatic tuning system as opposed
69 * to the manual tuning I did here.
70 *
71 * Inside the TESTING define are some hard-coded (mostly hand-written)
72 * scene change frame numbers for some easily available sequences.
73 *
74 */
75
76 #ifdef HAVE_CONFIG_H
77 #include "config.h"
78 #endif
79
80 #include <gst/gst.h>
81 #include <gst/video/video.h>
82 #include <gst/video/gstvideofilter.h>
83 #include <string.h>
84 #include "gstscenechange.h"
85
86 GST_DEBUG_CATEGORY_STATIC (gst_scene_change_debug_category);
87 #define GST_CAT_DEFAULT gst_scene_change_debug_category
88
89 /* prototypes */
90
91
92 static GstFlowReturn gst_scene_change_transform_frame_ip (GstVideoFilter *
93 filter, GstVideoFrame * frame);
94
95 #undef TESTING
96 #ifdef TESTING
97 static gboolean is_shot_change (int frame_number);
98 #endif
99
100 enum
101 {
102 PROP_0
103 };
104
105 #define VIDEO_CAPS \
106 GST_VIDEO_CAPS_MAKE("{ I420, Y42B, Y41B, Y444 }")
107
108 /* class initialization */
109
110 G_DEFINE_TYPE_WITH_CODE (GstSceneChange, gst_scene_change,
111 GST_TYPE_VIDEO_FILTER,
112 GST_DEBUG_CATEGORY_INIT (gst_scene_change_debug_category, "scenechange", 0,
113 "debug category for scenechange element"));
114
115 static void
gst_scene_change_class_init(GstSceneChangeClass * klass)116 gst_scene_change_class_init (GstSceneChangeClass * klass)
117 {
118 GstVideoFilterClass *video_filter_class = GST_VIDEO_FILTER_CLASS (klass);
119
120 gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass),
121 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
122 gst_caps_from_string (VIDEO_CAPS)));
123 gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass),
124 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
125 gst_caps_from_string (VIDEO_CAPS)));
126
127 gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass),
128 "Scene change detector",
129 "Video/Filter", "Detects scene changes in video",
130 "David Schleef <ds@entropywave.com>");
131
132 video_filter_class->transform_frame_ip =
133 GST_DEBUG_FUNCPTR (gst_scene_change_transform_frame_ip);
134
135 }
136
137 static void
gst_scene_change_init(GstSceneChange * scenechange)138 gst_scene_change_init (GstSceneChange * scenechange)
139 {
140 }
141
142
143 static double
get_frame_score(GstVideoFrame * f1,GstVideoFrame * f2)144 get_frame_score (GstVideoFrame * f1, GstVideoFrame * f2)
145 {
146 int i;
147 int j;
148 int score = 0;
149 int width, height;
150 guint8 *s1;
151 guint8 *s2;
152
153 width = f1->info.width;
154 height = f1->info.height;
155
156 for (j = 0; j < height; j++) {
157 s1 = (guint8 *) f1->data[0] + f1->info.stride[0] * j;
158 s2 = (guint8 *) f2->data[0] + f2->info.stride[0] * j;
159 for (i = 0; i < width; i++) {
160 score += ABS (s1[i] - s2[i]);
161 }
162 }
163
164 return ((double) score) / (width * height);
165 }
166
167 static GstFlowReturn
gst_scene_change_transform_frame_ip(GstVideoFilter * filter,GstVideoFrame * frame)168 gst_scene_change_transform_frame_ip (GstVideoFilter * filter,
169 GstVideoFrame * frame)
170 {
171 GstSceneChange *scenechange = GST_SCENE_CHANGE (filter);
172 GstVideoFrame oldframe;
173 double score_min;
174 double score_max;
175 double threshold;
176 double score;
177 gboolean change;
178 gboolean ret;
179 int i;
180
181 GST_DEBUG_OBJECT (scenechange, "transform_frame_ip");
182
183 if (!scenechange->oldbuf) {
184 scenechange->n_diffs = 0;
185 memset (scenechange->diffs, 0, sizeof (double) * SC_N_DIFFS);
186 scenechange->oldbuf = gst_buffer_ref (frame->buffer);
187 memcpy (&scenechange->oldinfo, &frame->info, sizeof (GstVideoInfo));
188 return GST_FLOW_OK;
189 }
190
191 ret =
192 gst_video_frame_map (&oldframe, &scenechange->oldinfo,
193 scenechange->oldbuf, GST_MAP_READ);
194 if (!ret) {
195 GST_ERROR_OBJECT (scenechange, "failed to map old video frame");
196 return GST_FLOW_ERROR;
197 }
198
199 score = get_frame_score (&oldframe, frame);
200
201 gst_video_frame_unmap (&oldframe);
202
203 gst_buffer_unref (scenechange->oldbuf);
204 scenechange->oldbuf = gst_buffer_ref (frame->buffer);
205 memcpy (&scenechange->oldinfo, &frame->info, sizeof (GstVideoInfo));
206
207 memmove (scenechange->diffs, scenechange->diffs + 1,
208 sizeof (double) * (SC_N_DIFFS - 1));
209 scenechange->diffs[SC_N_DIFFS - 1] = score;
210 scenechange->n_diffs++;
211
212 score_min = scenechange->diffs[0];
213 score_max = scenechange->diffs[0];
214 for (i = 1; i < SC_N_DIFFS - 1; i++) {
215 score_min = MIN (score_min, scenechange->diffs[i]);
216 score_max = MAX (score_max, scenechange->diffs[i]);
217 }
218
219 threshold = 1.8 * score_max - 0.8 * score_min;
220
221 if (scenechange->n_diffs > (SC_N_DIFFS - 1)) {
222 if (score < 5) {
223 change = FALSE;
224 } else if (score / threshold < 1.0) {
225 change = FALSE;
226 } else if ((score > 30)
227 && (score / scenechange->diffs[SC_N_DIFFS - 2] > 1.4)) {
228 change = TRUE;
229 } else if (score / threshold > 2.3) {
230 change = TRUE;
231 } else if (score > 50) {
232 change = TRUE;
233 } else {
234 change = FALSE;
235 }
236 } else {
237 change = FALSE;
238 }
239
240 if (change == TRUE) {
241 memset (scenechange->diffs, 0, sizeof (double) * SC_N_DIFFS);
242 scenechange->n_diffs = 0;
243 }
244 #ifdef TESTING
245 if (change != is_shot_change (scenechange->n_diffs)) {
246 g_print ("%d %g %g %g %d\n", scenechange->n_diffs, score / threshold,
247 score, threshold, change);
248 }
249 #endif
250
251 if (change) {
252 GstEvent *event;
253
254 GST_INFO_OBJECT (scenechange, "%d %g %g %g %d",
255 scenechange->n_diffs, score / threshold, score, threshold, change);
256
257 event =
258 gst_video_event_new_downstream_force_key_unit (GST_BUFFER_PTS
259 (frame->buffer), GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, FALSE,
260 scenechange->count++);
261
262 gst_pad_push_event (GST_BASE_TRANSFORM_SRC_PAD (scenechange), event);
263 }
264
265 return GST_FLOW_OK;
266 }
267
268
269
270
271
272
273 #ifdef TESTING
274 /* This is from ds's personal collection. No, you can't have it. */
275 int showreel_changes[] = {
276 242, 483, 510, 550, 579, 603, 609, 1056, 1067, 1074, 1079, 1096,
277 1106, 1113, 1127, 1145, 1156, 1170, 1212, 1228, 1243, 1269, 1274,
278 1322, 1349, 1370, 1378, 1423, 1456, 1458, 1508, 1519, 1542, 1679,
279 1767, 1837, 1895, 1962, 2006, 2035, 2102, 2139, 2196, 2561, 2664,
280 2837, 2895, 2985, 3035, 3077, 3128, 3176, 3218, 3306, 3351, 3388,
281 3421, 3470, 3711, 3832, 4029, 4184, 4444, 4686, 4719, 4825, 4941,
282 5009, 5091, 5194, 5254, 5286, 5287, 5343, 5431, 5501, 5634, 5695, 5788,
283 5839, 5861, 5930, 6030, 6168, 6193, 6237, 6336, 6376, 6421, 6495,
284 6550, 6611, 6669, 6733, 6819, 6852, 6944, 7087, 7148, 7189, 7431,
285 7540, 7599, 7632, 7661, 7693, 7930, 7963, 8003, 8076, 8109, 8147,
286 8177, 8192, 8219, 8278, 8322, 8370, 8409, 8566, 8603, 8747, 8775,
287 8873, 8907, 8955, 8969, 8983, 8997, 9026, 9079, 9140, 9165, 9206,
288 9276, 9378, 9449, 9523, 9647, 9703, 9749, 9790, 9929, 10056, 10216,
289 10307, 10411, 10487, 10557, 10695, 10770, 10854, 11095, 11265, 11517, 11589,
290 11686, 11825, 11940, 12004, 12047, 12113, 12179, 12233, 12532, 12586, 12708,
291 12793, 12877, 12954, 13030, 13105, 13177, 13279, 13396, 13486, 13538, 13561,
292 13591, 13627, 13656, 13709, 13763, 13815, 13842, 13876, 13906, 13929, 13955,
293 14003, 14070, 14097, 14127, 14153, 14198, 14269, 14348, 14367, 14440, 14488,
294 14548, 14573, 14599, 14630, 14665, 14907, 14962, 15013, 15089, 15148, 15227,
295 15314, 15355, 15369, 15451, 15470, 15542, 15570, 15640, 15684, 15781, 15869,
296 15938, 16172, 16266, 16429, 16479, 16521, 16563, 16612, 16671, 16692, 16704,
297 16720, 16756, 16789, 16802, 16815, 16867, 16908, 16939, 16953, 16977, 17006,
298 17014, 17026, 17040, 17062, 17121, 17176, 17226, 17322, 17444, 17496, 17641,
299 17698, 17744, 17826, 17913, 17993, 18073, 18219, 18279, 18359, 18475, 18544,
300 18587, 18649, 18698, 18756, 18826, 18853, 18866, 19108, 19336, 19481, 19544,
301 19720, 19816, 19908, 19982, 20069, 20310, 20355, 20374, 20409, 20469, 20599,
302 20607, 20652, 20805, 20822, 20882, 20982, 21029, 21433, 21468, 21561, 21602,
303 21661, 21720, 21909, 22045, 22166, 22225, 22323, 22362, 22433, 22477, 22529,
304 22571, 22617, 22642, 22676, 22918, 22978, 23084, 23161, 23288, 23409, 23490,
305 23613, 23721, 23815, 24131, 24372, 24468, 24507, 24555, 24568, 24616, 24634,
306 24829, 24843, 24919, 24992, 25040, 25160, 25288, 25607, 25684, 25717, 25764,
307 25821, 25866, 25901, 25925, 25941, 25978, 25998, 26011, 26030, 26055, 26118,
308 26133, 26145, 26159, 26175, 26182, 26195, 26205, 26238, 26258, 26316, 26340,
309 26581, 26725, 26834, 26874, 26995, 27065, 27178, 27238, 27365, 27607, 27669,
310 27694,
311 27774, 27800, 27841, 27930, 27985, 28057, 28091, 28132, 28189, 28270, 28545,
312 28653, 28711, 28770, 28886, 28966, 29139, 29241, 29356, 29415, 29490, 29576,
313 29659, 29776, 29842, 29910, 30029, 30056, 30100, 30129, 30175, 30316, 30376,
314 30441, 30551, 30666, 30784, 30843, 30948, 31045, 31286, 31315, 31534, 31607,
315 31742,
316 31817, 31853, 31984, 32009, 32112, 32162, 32210, 32264
317 };
318
319 /* Sintel */
320 int sintel_changes[] = {
321 752, 1018, 1036, 1056, 1078, 1100, 1169, 1319, 1339, 1370,
322 1425, 1455, 1494, 1552, 1572, 1637, 1663, 1777, 1955, 2060,
323 2125, 2429, 2624, 2780, 2835, 2881, 2955, 3032, 3144, 3217,
324 3315, 3384, 3740, 3890, 4234, 4261, 4322, 4368, 4425, 4481,
325 4555, 4605, 4671, 4714, 4743, 4875, 4920, 5082, 5158, 5267,
326 5379, 5956, 6021, 6071, 6112, 6139, 6221, 6318, 6374, 6519,
327 6558, 6615, 6691, 6803, 6900, 6944, 7134, 7266, 7351, 7414,
328 7467, 7503, 7559, 7573, 7656, 7733, 7876, 7929, 7971, 7985,
329 8047, 8099, 8144, 8215, 8394, 8435, 8480, 9133, 9190, 9525,
330 9962,
331 };
332
333 /* Breathe Out video, http://media.xiph.org/video/misc/ */
334 int breatheout_changes[] = {
335 143, 263, 334, 426, 462, 563, 583, 618, 655, 707,
336 818, 823, 858, 913, 956, 977, 999, 1073, 1124, 1144,
337 1166, 1187, 1206, 1227, 1240, 1264, 1289, 1312, 1477, 1535,
338 1646, 1692, 1739, 1757, 1798, 1855, 1974, 2048, 2129, 2212,
339 2369, 2412, 2463, 2578, 2649, 2699, 2778, 2857, 2923, 3014,
340 3107, 3246, 3321, 3350, 3459, 3498, 3541, 3567, 3613, 3636,
341 3673, 3709, 3747, 3834, 3862, 3902, 3922, 4022, 4117, 4262,
342 4303, 4357, 4556, 4578, 4617, 4716, 4792, 4873, 4895, 4917,
343 4932, 4972, 5015, 5034, 5058, 5090, 5162, 5180, 5202, 5222,
344 5239, 5258, 5281, 5298, 5397, 5430,
345 485, 507, 534, 665, 685, 755, 1023, 1379, 1441, 1503,
346 1584, 1621, 1903, 2081, 2281, 2511, 2958, 3071, 3185, 3214,
347 3271, 3424, 3479, 3588, 3879, 3979, 4043, 4062, 4143, 4207,
348 4237, 4336, 4461, 4476, 4533, 4647, 4815, 4853, 4949, 5075,
349 5142, 5316, 5376,
350 3514, 3952, 4384, 5337
351 };
352
353 #define changes showreel_changes
354
355 static gboolean
is_shot_change(int frame_number)356 is_shot_change (int frame_number)
357 {
358 int i;
359 for (i = 0; i < sizeof (changes) / sizeof (changes[0]); i++) {
360 if (changes[i] == frame_number)
361 return TRUE;
362 }
363 return FALSE;
364 }
365 #endif
366