1 /*
2 * GStreamer
3 * Copyright (C) 2013 Miguel Casas-Sanchez <miguelecasassanchez@gmail.com>
4 * Except: Parts of code inside the preprocessor define CODE_FROM_OREILLY_BOOK,
5 * which are downloaded from O'Reilly website
6 * [http://examples.oreilly.com/9780596516130/]
7 * and adapted. Its license reads:
8 * "Oct. 3, 2008
9 * Right to use this code in any way you want without warrenty, support or
10 * any guarentee of it working. "
11 *
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a
14 * copy of this software and associated documentation files (the "Software"),
15 * to deal in the Software without restriction, including without limitation
16 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17 * and/or sell copies of the Software, and to permit persons to whom the
18 * Software is furnished to do so, subject to the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
29 * DEALINGS IN THE SOFTWARE.
30 *
31 * Alternatively, the contents of this file may be used under the
32 * GNU Lesser General Public License Version 2.1 (the "LGPL"), in
33 * which case the following provisions apply instead of the ones
34 * mentioned above:
35 *
36 * This library is free software; you can redistribute it and/or
37 * modify it under the terms of the GNU Library General Public
38 * License as published by the Free Software Foundation; either
39 * version 2 of the License, or (at your option) any later version.
40 *
41 * This library is distributed in the hope that it will be useful,
42 * but WITHOUT ANY WARRANTY; without even the implied warranty of
43 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
44 * Library General Public License for more details.
45 *
46 * You should have received a copy of the GNU Library General Public
47 * License along with this library; if not, write to the
48 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
49 * Boston, MA 02110-1301, USA.
50 */
51 #define CODE_FROM_OREILLY_BOOK
52
53 /**
54 * SECTION:element-segmentation
55 *
56 * This element creates and updates a fg/bg model using one of several approaches.
57 * The one called "codebook" refers to the codebook approach following the opencv
58 * O'Reilly book [1] implementation of the algorithm described in K. Kim,
59 * T. H. Chalidabhongse, D. Harwood and L. Davis [2]. BackgroundSubtractorMOG [3],
60 * or MOG for shorts, refers to a Gaussian Mixture-based Background/Foreground
61 * Segmentation Algorithm. OpenCV MOG implements the algorithm described in [4].
62 * BackgroundSubtractorMOG2 [5], refers to another Gaussian Mixture-based
63 * Background/Foreground segmentation algorithm. OpenCV MOG2 implements the
64 * algorithm described in [6] and [7].
65 *
66 * [1] Learning OpenCV: Computer Vision with the OpenCV Library by Gary Bradski
67 * and Adrian Kaehler, Published by O'Reilly Media, October 3, 2008
68 * [2] "Real-time Foreground-Background Segmentation using Codebook Model",
69 * Real-time Imaging, Volume 11, Issue 3, Pages 167-256, June 2005.
70 * [3] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog
71 * [4] P. KadewTraKuPong and R. Bowden, "An improved adaptive background
72 * mixture model for real-time tracking with shadow detection", Proc. 2nd
73 * European Workshop on Advanced Video-Based Surveillance Systems, 2001
74 * [5] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog2
75 * [6] Z.Zivkovic, "Improved adaptive Gausian mixture model for background
76 * subtraction", International Conference Pattern Recognition, UK, August, 2004.
77 * [7] Z.Zivkovic, F. van der Heijden, "Efficient Adaptive Density Estimation
78 * per Image Pixel for the Task of Background Subtraction", Pattern Recognition
79 * Letters, vol. 27, no. 7, pages 773-780, 2006.
80 *
81 * <refsect2>
82 * <title>Example launch line</title>
83 * |[
84 * gst-launch-1.0 v4l2src device=/dev/video0 ! videoconvert ! segmentation test-mode=true method=2 ! videoconvert ! ximagesink
85 * ]|
86 * </refsect2>
87 */
88
89 #ifdef HAVE_CONFIG_H
90 #include <config.h>
91 #endif
92
93 #include "gstsegmentation.h"
94 #include <opencv2/imgproc.hpp>
95
96 GST_DEBUG_CATEGORY_STATIC (gst_segmentation_debug);
97 #define GST_CAT_DEFAULT gst_segmentation_debug
98
99 using namespace cv;
100
101 /* Filter signals and args */
102 enum
103 {
104 /* FILL ME */
105 LAST_SIGNAL
106 };
107
108 enum
109 {
110 PROP_0,
111 PROP_TEST_MODE,
112 PROP_METHOD,
113 PROP_LEARNING_RATE
114 };
115 typedef enum
116 {
117 METHOD_BOOK,
118 METHOD_MOG,
119 METHOD_MOG2
120 } GstSegmentationMethod;
121
122 #define DEFAULT_TEST_MODE FALSE
123 #define DEFAULT_METHOD METHOD_MOG2
124 #define DEFAULT_LEARNING_RATE 0.01
125
126 #define GST_TYPE_SEGMENTATION_METHOD (gst_segmentation_method_get_type ())
127 static GType
gst_segmentation_method_get_type(void)128 gst_segmentation_method_get_type (void)
129 {
130 static GType etype = 0;
131 if (etype == 0) {
132 static const GEnumValue values[] = {
133 {METHOD_BOOK, "Codebook-based segmentation (Bradski2008)", "codebook"},
134 {METHOD_MOG, "Mixture-of-Gaussians segmentation (Bowden2001)", "mog"},
135 {METHOD_MOG2, "Mixture-of-Gaussians segmentation (Zivkovic2004)", "mog2"},
136 {0, NULL, NULL},
137 };
138 etype = g_enum_register_static ("GstSegmentationMethod", values);
139 }
140 return etype;
141 }
142
143 G_DEFINE_TYPE (GstSegmentation, gst_segmentation, GST_TYPE_OPENCV_VIDEO_FILTER);
144
145 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
146 GST_PAD_SINK,
147 GST_PAD_ALWAYS,
148 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA")));
149
150 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
151 GST_PAD_SRC,
152 GST_PAD_ALWAYS,
153 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA")));
154
155
156 static void
157 gst_segmentation_set_property (GObject * object, guint prop_id,
158 const GValue * value, GParamSpec * pspec);
159 static void
160 gst_segmentation_get_property (GObject * object, guint prop_id,
161 GValue * value, GParamSpec * pspec);
162
163 static GstFlowReturn gst_segmentation_transform_ip (GstOpencvVideoFilter *
164 filter, GstBuffer * buffer, Mat img);
165
166 static void gst_segmentation_finalize (GObject * object);
167 static gboolean gst_segmentation_set_caps (GstOpencvVideoFilter * filter,
168 gint in_width, gint in_height, int in_cv_type, gint out_width,
169 gint out_height, int out_cv_type);
170
171 /* Codebook algorithm + connected components functions*/
172 static int update_codebook (unsigned char *p, codeBook * c,
173 unsigned *cbBounds, int numChannels);
174 static int clear_stale_entries (codeBook * c);
175 static unsigned char background_diff (unsigned char *p, codeBook * c,
176 int numChannels, int *minMod, int *maxMod);
177 static void find_connected_components (Mat mask, int poly1_hull0,
178 float perimScale);
179
180 /* MOG (Mixture-of-Gaussians functions */
181 static int run_mog_iteration (GstSegmentation * filter);
182 static int run_mog2_iteration (GstSegmentation * filter);
183
184 /* initialize the segmentation's class */
185 static void
gst_segmentation_class_init(GstSegmentationClass * klass)186 gst_segmentation_class_init (GstSegmentationClass * klass)
187 {
188 GObjectClass *gobject_class;
189 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
190 GstOpencvVideoFilterClass *cvfilter_class =
191 (GstOpencvVideoFilterClass *) klass;
192
193 gobject_class = (GObjectClass *) klass;
194
195 gobject_class->finalize = gst_segmentation_finalize;
196 gobject_class->set_property = gst_segmentation_set_property;
197 gobject_class->get_property = gst_segmentation_get_property;
198
199
200 cvfilter_class->cv_trans_ip_func = gst_segmentation_transform_ip;
201 cvfilter_class->cv_set_caps = gst_segmentation_set_caps;
202
203 g_object_class_install_property (gobject_class, PROP_METHOD,
204 g_param_spec_enum ("method",
205 "Segmentation method to use",
206 "Segmentation method to use",
207 GST_TYPE_SEGMENTATION_METHOD, DEFAULT_METHOD,
208 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
209
210 g_object_class_install_property (gobject_class, PROP_TEST_MODE,
211 g_param_spec_boolean ("test-mode", "test-mode",
212 "If true, the output RGB is overwritten with the calculated foreground (white color)",
213 DEFAULT_TEST_MODE, (GParamFlags)
214 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
215
216 g_object_class_install_property (gobject_class, PROP_LEARNING_RATE,
217 g_param_spec_float ("learning-rate", "learning-rate",
218 "Speed with which a motionless foreground pixel would become background (inverse of number of frames)",
219 0, 1, DEFAULT_LEARNING_RATE, (GParamFlags) (G_PARAM_READWRITE)));
220
221 gst_element_class_set_static_metadata (element_class,
222 "Foreground/background video sequence segmentation",
223 "Filter/Effect/Video",
224 "Create a Foregound/Background mask applying a particular algorithm",
225 "Miguel Casas-Sanchez <miguelecasassanchez@gmail.com>");
226
227 gst_element_class_add_static_pad_template (element_class, &src_factory);
228 gst_element_class_add_static_pad_template (element_class, &sink_factory);
229
230 }
231
232 /* initialize the new element
233 * instantiate pads and add them to element
234 * set pad calback functions
235 * initialize instance structure
236 */
237 static void
gst_segmentation_init(GstSegmentation * filter)238 gst_segmentation_init (GstSegmentation * filter)
239 {
240 filter->method = DEFAULT_METHOD;
241 filter->test_mode = DEFAULT_TEST_MODE;
242 filter->framecount = 0;
243 filter->learning_rate = DEFAULT_LEARNING_RATE;
244 gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER (filter), TRUE);
245 }
246
247 static void
gst_segmentation_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)248 gst_segmentation_set_property (GObject * object, guint prop_id,
249 const GValue * value, GParamSpec * pspec)
250 {
251 GstSegmentation *filter = GST_SEGMENTATION (object);
252
253 switch (prop_id) {
254 case PROP_METHOD:
255 filter->method = g_value_get_enum (value);
256 break;
257 case PROP_TEST_MODE:
258 filter->test_mode = g_value_get_boolean (value);
259 break;
260 case PROP_LEARNING_RATE:
261 filter->learning_rate = g_value_get_float (value);
262 break;
263 default:
264 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
265 break;
266 }
267 }
268
269 static void
gst_segmentation_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)270 gst_segmentation_get_property (GObject * object, guint prop_id,
271 GValue * value, GParamSpec * pspec)
272 {
273 GstSegmentation *filter = GST_SEGMENTATION (object);
274
275 switch (prop_id) {
276 case PROP_METHOD:
277 g_value_set_enum (value, filter->method);
278 break;
279 case PROP_TEST_MODE:
280 g_value_set_boolean (value, filter->test_mode);
281 break;
282 case PROP_LEARNING_RATE:
283 g_value_set_float (value, filter->learning_rate);
284 break;
285 default:
286 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
287 break;
288 }
289 }
290
291 static gboolean
gst_segmentation_set_caps(GstOpencvVideoFilter * filter,gint in_width,gint in_height,int in_cv_type,gint out_width,gint out_height,int out_cv_type)292 gst_segmentation_set_caps (GstOpencvVideoFilter * filter, gint in_width,
293 gint in_height, int in_cv_type,
294 gint out_width, gint out_height, int out_cv_type)
295 {
296 GstSegmentation *segmentation = GST_SEGMENTATION (filter);
297 Size size;
298
299 size = Size (in_width, in_height);
300 segmentation->width = in_width;
301 segmentation->height = in_height;
302
303 segmentation->cvRGB.create (size, CV_8UC3);
304 segmentation->cvYUV.create (size, CV_8UC3);
305
306 segmentation->cvFG = Mat::zeros (size, CV_8UC1);
307
308 segmentation->ch1.create (size, CV_8UC1);
309 segmentation->ch2.create (size, CV_8UC1);
310 segmentation->ch3.create (size, CV_8UC1);
311
312 /* Codebook method */
313 segmentation->TcodeBook = (codeBook *)
314 g_malloc (sizeof (codeBook) *
315 (segmentation->width * segmentation->height + 1));
316 for (int j = 0; j < segmentation->width * segmentation->height; j++) {
317 segmentation->TcodeBook[j].numEntries = 0;
318 segmentation->TcodeBook[j].t = 0;
319 }
320 segmentation->learning_interval = (int) (1.0 / segmentation->learning_rate);
321
322 /* Mixture-of-Gaussians (mog) methods */
323 segmentation->mog = bgsegm::createBackgroundSubtractorMOG ();
324 segmentation->mog2 = createBackgroundSubtractorMOG2 ();
325
326 return TRUE;
327 }
328
329 /* Clean up */
330 static void
gst_segmentation_finalize(GObject * object)331 gst_segmentation_finalize (GObject * object)
332 {
333 GstSegmentation *filter = GST_SEGMENTATION (object);
334
335 filter->cvRGB.release ();
336 filter->cvYUV.release ();
337 filter->cvFG.release ();
338 filter->ch1.release ();
339 filter->ch2.release ();
340 filter->ch3.release ();
341 filter->mog.release ();
342 filter->mog2.release ();
343 g_free (filter->TcodeBook);
344
345 G_OBJECT_CLASS (gst_segmentation_parent_class)->finalize (object);
346 }
347
348 static GstFlowReturn
gst_segmentation_transform_ip(GstOpencvVideoFilter * cvfilter,GstBuffer * buffer,Mat img)349 gst_segmentation_transform_ip (GstOpencvVideoFilter * cvfilter,
350 GstBuffer * buffer, Mat img)
351 {
352 GstSegmentation *filter = GST_SEGMENTATION (cvfilter);
353 int j;
354
355 filter->framecount++;
356
357 /* Image preprocessing: color space conversion etc */
358 cvtColor (img, filter->cvRGB, COLOR_RGBA2RGB);
359 cvtColor (filter->cvRGB, filter->cvYUV, COLOR_RGB2YCrCb);
360
361 /* Create and update a fg/bg model using a codebook approach following the
362 * opencv O'Reilly book [1] implementation of the algo described in [2].
363 *
364 * [1] Learning OpenCV: Computer Vision with the OpenCV Library by Gary
365 * Bradski and Adrian Kaehler, Published by O'Reilly Media, October 3, 2008
366 * [2] "Real-time Foreground-Background Segmentation using Codebook Model",
367 * Real-time Imaging, Volume 11, Issue 3, Pages 167-256, June 2005. */
368 if (METHOD_BOOK == filter->method) {
369 unsigned cbBounds[3] = { 10, 5, 5 };
370 int minMod[3] = { 20, 20, 20 }, maxMod[3] = {
371 20, 20, 20
372 };
373
374 if (filter->framecount < 30) {
375 /* Learning background phase: update_codebook on every frame */
376 for (j = 0; j < filter->width * filter->height; j++) {
377 update_codebook (filter->cvYUV.data + j * 3,
378 (codeBook *) & (filter->TcodeBook[j]), cbBounds, 3);
379 }
380 } else {
381 /* this updating is responsible for FG becoming BG again */
382 if (filter->framecount % filter->learning_interval == 0) {
383 for (j = 0; j < filter->width * filter->height; j++) {
384 update_codebook (filter->cvYUV.data + j * 3,
385 (codeBook *) & (filter->TcodeBook[j]), cbBounds, 3);
386 }
387 }
388 if (filter->framecount % 60 == 0) {
389 for (j = 0; j < filter->width * filter->height; j++)
390 clear_stale_entries ((codeBook *) & (filter->TcodeBook[j]));
391 }
392
393 for (j = 0; j < filter->width * filter->height; j++) {
394 if (background_diff
395 (filter->cvYUV.data + j * 3,
396 (codeBook *) & (filter->TcodeBook[j]), 3, minMod, maxMod)) {
397 filter->cvFG.data[j] = (char) 255;
398 } else {
399 filter->cvFG.data[j] = 0;
400 }
401 }
402 }
403
404 /* 3rd param is the smallest area to show: (w+h)/param , in pixels */
405 find_connected_components (filter->cvFG, 1, 10000);
406
407 }
408 /* Create the foreground and background masks using BackgroundSubtractorMOG [1],
409 * Gaussian Mixture-based Background/Foreground segmentation algorithm. OpenCV
410 * MOG implements the algorithm described in [2].
411 *
412 * [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog
413 * [2] P. KadewTraKuPong and R. Bowden, "An improved adaptive background
414 * mixture model for real-time tracking with shadow detection", Proc. 2nd
415 * European Workshop on Advanced Video-Based Surveillance Systems, 2001
416 */
417 else if (METHOD_MOG == filter->method) {
418 run_mog_iteration (filter);
419 }
420 /* Create the foreground and background masks using BackgroundSubtractorMOG2
421 * [1], Gaussian Mixture-based Background/Foreground segmentation algorithm.
422 * OpenCV MOG2 implements the algorithm described in [2] and [3].
423 *
424 * [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog2
425 * [2] Z.Zivkovic, "Improved adaptive Gausian mixture model for background
426 * subtraction", International Conference Pattern Recognition, UK, Aug 2004.
427 * [3] Z.Zivkovic, F. van der Heijden, "Efficient Adaptive Density Estimation
428 * per Image Pixel for the Task of Background Subtraction", Pattern
429 * Recognition Letters, vol. 27, no. 7, pages 773-780, 2006. */
430 else if (METHOD_MOG2 == filter->method) {
431 run_mog2_iteration (filter);
432 }
433
434 /* if we want to test_mode, just overwrite the output */
435 std::vector < cv::Mat > channels (3);
436
437 if (filter->test_mode) {
438 cvtColor (filter->cvFG, filter->cvRGB, COLOR_GRAY2RGB);
439
440 split (filter->cvRGB, channels);
441 } else
442 split (img, channels);
443
444 channels.push_back (filter->cvFG);
445
446 /* copy anyhow the fg/bg to the alpha channel in the output image */
447 merge (channels, img);
448
449
450 return GST_FLOW_OK;
451 }
452
453 /* entry point to initialize the plug-in
454 * initialize the plug-in itself
455 * register the element factories and other features
456 */
457 gboolean
gst_segmentation_plugin_init(GstPlugin * plugin)458 gst_segmentation_plugin_init (GstPlugin * plugin)
459 {
460 GST_DEBUG_CATEGORY_INIT (gst_segmentation_debug, "segmentation",
461 0, "Performs Foreground/Background segmentation in video sequences");
462
463 return gst_element_register (plugin, "segmentation", GST_RANK_NONE,
464 GST_TYPE_SEGMENTATION);
465 }
466
467
468
469 #ifdef CODE_FROM_OREILLY_BOOK /* See license at the beginning of the page */
470 /*
471 int update_codebook(uchar *p, codeBook &c, unsigned cbBounds)
472 Updates the codebook entry with a new data point
473
474 p Pointer to a YUV or HSI pixel
475 c Codebook for this pixel
476 cbBounds Learning bounds for codebook (Rule of thumb: 10)
477 numChannels Number of color channels we¡¯re learning
478
479 NOTES:
480 cvBounds must be of length equal to numChannels
481
482 RETURN
483 codebook index
484 */
485 int
update_codebook(unsigned char * p,codeBook * c,unsigned * cbBounds,int numChannels)486 update_codebook (unsigned char *p, codeBook * c, unsigned *cbBounds,
487 int numChannels)
488 {
489 /* c->t+=1; */
490 unsigned int high[3], low[3];
491 int n, i;
492 int matchChannel;
493
494 for (n = 0; n < numChannels; n++) {
495 high[n] = p[n] + cbBounds[n];
496 if (high[n] > 255)
497 high[n] = 255;
498
499 if (p[n] > cbBounds[n])
500 low[n] = p[n] - cbBounds[n];
501 else
502 low[n] = 0;
503 }
504
505 /* SEE IF THIS FITS AN EXISTING CODEWORD */
506 for (i = 0; i < c->numEntries; i++) {
507 matchChannel = 0;
508 for (n = 0; n < numChannels; n++) {
509 if ((c->cb[i]->learnLow[n] <= *(p + n)) &&
510 /* Found an entry for this channel */
511 (*(p + n) <= c->cb[i]->learnHigh[n])) {
512 matchChannel++;
513 }
514 }
515 if (matchChannel == numChannels) { /* If an entry was found */
516 c->cb[i]->t_last_update = c->t;
517 /* adjust this codeword for the first channel */
518 for (n = 0; n < numChannels; n++) {
519 if (c->cb[i]->max[n] < *(p + n)) {
520 c->cb[i]->max[n] = *(p + n);
521 } else if (c->cb[i]->min[n] > *(p + n)) {
522 c->cb[i]->min[n] = *(p + n);
523 }
524 }
525 break;
526 }
527 }
528 /* OVERHEAD TO TRACK POTENTIAL STALE ENTRIES */
529 for (int s = 0; s < c->numEntries; s++) {
530 /* Track which codebook entries are going stale: */
531 int negRun = c->t - c->cb[s]->t_last_update;
532 if (c->cb[s]->stale < negRun)
533 c->cb[s]->stale = negRun;
534 }
535 /* ENTER A NEW CODEWORD IF NEEDED */
536 if (i == c->numEntries) { /* if no existing codeword found, make one */
537 code_element **foo =
538 (code_element **) g_malloc (sizeof (code_element *) *
539 (c->numEntries + 1));
540 for (int ii = 0; ii < c->numEntries; ii++) {
541 foo[ii] = c->cb[ii]; /* copy all pointers */
542 }
543 foo[c->numEntries] = (code_element *) g_malloc (sizeof (code_element));
544 if (c->numEntries)
545 g_free (c->cb);
546 c->cb = foo;
547 for (n = 0; n < numChannels; n++) {
548 c->cb[c->numEntries]->learnHigh[n] = high[n];
549 c->cb[c->numEntries]->learnLow[n] = low[n];
550 c->cb[c->numEntries]->max[n] = *(p + n);
551 c->cb[c->numEntries]->min[n] = *(p + n);
552 }
553 c->cb[c->numEntries]->t_last_update = c->t;
554 c->cb[c->numEntries]->stale = 0;
555 c->numEntries += 1;
556 }
557 /* SLOWLY ADJUST LEARNING BOUNDS */
558 for (n = 0; n < numChannels; n++) {
559 if (c->cb[i]->learnHigh[n] < high[n])
560 c->cb[i]->learnHigh[n] += 1;
561 if (c->cb[i]->learnLow[n] > low[n])
562 c->cb[i]->learnLow[n] -= 1;
563 }
564 return (i);
565 }
566
567
568
569
570
571 /*
572 int clear_stale_entries(codeBook &c)
573 During learning, after you've learned for some period of time,
574 periodically call this to clear out stale codebook entries
575
576 c Codebook to clean up
577
578 Return
579 number of entries cleared
580 */
581 int
clear_stale_entries(codeBook * c)582 clear_stale_entries (codeBook * c)
583 {
584 int staleThresh = c->t >> 1;
585 int *keep = (int *) g_malloc (sizeof (int) * (c->numEntries));
586 int keepCnt = 0;
587 code_element **foo;
588 int k;
589 int numCleared;
590 /* SEE WHICH CODEBOOK ENTRIES ARE TOO STALE */
591 for (int i = 0; i < c->numEntries; i++) {
592 if (c->cb[i]->stale > staleThresh)
593 keep[i] = 0; /* Mark for destruction */
594 else {
595 keep[i] = 1; /* Mark to keep */
596 keepCnt += 1;
597 }
598 }
599 /* KEEP ONLY THE GOOD */
600 c->t = 0; /* Full reset on stale tracking */
601 foo = (code_element **) g_malloc (sizeof (code_element *) * keepCnt);
602 k = 0;
603 for (int ii = 0; ii < c->numEntries; ii++) {
604 if (keep[ii]) {
605 foo[k] = c->cb[ii];
606 /* We have to refresh these entries for next clearStale */
607 foo[k]->t_last_update = 0;
608 k++;
609 }
610 }
611 /* CLEAN UP */
612 g_free (keep);
613 g_free (c->cb);
614 c->cb = foo;
615 numCleared = c->numEntries - keepCnt;
616 c->numEntries = keepCnt;
617 return (numCleared);
618 }
619
620
621
622 /*
623 uchar background_diff( uchar *p, codeBook &c,
624 int minMod, int maxMod)
625 Given a pixel and a codebook, determine if the pixel is
626 covered by the codebook
627
628 p Pixel pointer (YUV interleaved)
629 c Codebook reference
630 numChannels Number of channels we are testing
631 maxMod Add this (possibly negative) number onto
632
633 max level when determining if new pixel is foreground
634 minMod Subract this (possibly negative) number from
635 min level when determining if new pixel is foreground
636
637 NOTES:
638 minMod and maxMod must have length numChannels,
639 e.g. 3 channels => minMod[3], maxMod[3]. There is one min and
640 one max threshold per channel.
641
642 Return
643 0 => background, 255 => foreground
644 */
645 unsigned char
background_diff(unsigned char * p,codeBook * c,int numChannels,int * minMod,int * maxMod)646 background_diff (unsigned char *p, codeBook * c, int numChannels,
647 int *minMod, int *maxMod)
648 {
649 int matchChannel;
650 /* SEE IF THIS FITS AN EXISTING CODEWORD */
651 int i;
652 for (i = 0; i < c->numEntries; i++) {
653 matchChannel = 0;
654 for (int n = 0; n < numChannels; n++) {
655 if ((c->cb[i]->min[n] - minMod[n] <= *(p + n)) &&
656 (*(p + n) <= c->cb[i]->max[n] + maxMod[n])) {
657 matchChannel++; /* Found an entry for this channel */
658 } else {
659 break;
660 }
661 }
662 if (matchChannel == numChannels) {
663 break; /* Found an entry that matched all channels */
664 }
665 }
666 if (i >= c->numEntries)
667 return (255);
668 return (0);
669 }
670
671
672
673
674 /*
675 void find_connected_components(IplImage *mask, int poly1_hull0,
676 float perimScale, int *num,
677 CvRect *bbs, CvPoint *centers)
678 This cleans up the foreground segmentation mask derived from calls
679 to backgroundDiff
680
681 mask Is a grayscale (8-bit depth) “raw†mask image that
682 will be cleaned up
683
684 OPTIONAL PARAMETERS:
685 poly1_hull0 If set, approximate connected component by
686 (DEFAULT) polygon, or else convex hull (0)
687 perimScale Len = image (width+height)/perimScale. If contour
688 len < this, delete that contour (DEFAULT: 4)
689 num Maximum number of rectangles and/or centers to
690 return; on return, will contain number filled
691 (DEFAULT: NULL)
692 bbs Pointer to bounding box rectangle vector of
693 length num. (DEFAULT SETTING: NULL)
694 centers Pointer to contour centers vector of length
695 num (DEFAULT: NULL)
696 */
697
698 /* Approx.threshold - the bigger it is, the simpler is the boundary */
699 #define CVCONTOUR_APPROX_LEVEL 1
700 /* How many iterations of erosion and/or dilation there should be */
701 #define CVCLOSE_ITR 1
702 static void
find_connected_components(Mat mask,int poly1_hull0,float perimScale)703 find_connected_components (Mat mask, int poly1_hull0, float perimScale)
704 {
705 /* Just some convenience variables */
706 const Scalar CVX_WHITE = CV_RGB (0xff, 0xff, 0xff);
707 //const Scalar CVX_BLACK = CV_RGB (0x00, 0x00, 0x00);
708 int idx = 0;
709
710 /* CLEAN UP RAW MASK */
711 morphologyEx (mask, mask, MORPH_OPEN, Mat (), Point (-1, -1), CVCLOSE_ITR);
712 morphologyEx (mask, mask, MORPH_CLOSE, Mat (), Point (-1, -1), CVCLOSE_ITR);
713 /* FIND CONTOURS AROUND ONLY BIGGER REGIONS */
714
715 std::vector < std::vector < Point > >contours;
716 std::vector < std::vector < Point > >to_draw;
717 std::vector < Vec4i > hierarchy;
718 findContours (mask, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE,
719 Point (0, 0));
720 if (contours.size () == 0)
721 return;
722
723 for (; idx >= 0; idx = hierarchy[idx][0]) {
724 const std::vector < Point > &c = contours[idx];
725 double len = fabs (contourArea (Mat (c)));
726 double q = (mask.size ().height + mask.size ().width) / perimScale;
727 if (len >= q) {
728 std::vector < Point > c_new;
729 if (poly1_hull0) {
730 approxPolyDP (c, c_new, CVCONTOUR_APPROX_LEVEL, (hierarchy[idx][2] < 0
731 && hierarchy[idx][3] < 0));
732 } else {
733 convexHull (c, c_new, true, true);
734 }
735 to_draw.push_back (c_new);
736 }
737 }
738
739 mask.setTo (Scalar::all (0));
740 if (to_draw.size () > 0) {
741 drawContours (mask, to_draw, -1, CVX_WHITE, FILLED);
742 }
743
744 }
745 #endif /*ifdef CODE_FROM_OREILLY_BOOK */
746
747 int
run_mog_iteration(GstSegmentation * filter)748 run_mog_iteration (GstSegmentation * filter)
749 {
750 /*
751 BackgroundSubtractorMOG [1], Gaussian Mixture-based Background/Foreground
752 Segmentation Algorithm. OpenCV MOG implements the algorithm described in [2].
753
754 [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog
755 [2] P. KadewTraKuPong and R. Bowden, "An improved adaptive background
756 mixture model for real-time tracking with shadow detection", Proc. 2nd
757 European Workshop on Advanced Video-Based Surveillance Systems, 2001
758 */
759
760 filter->mog->apply (filter->cvYUV, filter->cvFG, filter->learning_rate);
761
762 return (0);
763 }
764
765 int
run_mog2_iteration(GstSegmentation * filter)766 run_mog2_iteration (GstSegmentation * filter)
767 {
768 /*
769 BackgroundSubtractorMOG2 [1], Gaussian Mixture-based Background/Foreground
770 segmentation algorithm. OpenCV MOG2 implements the algorithm described in
771 [2] and [3].
772
773 [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog2
774 [2] Z.Zivkovic, "Improved adaptive Gausian mixture model for background
775 subtraction", International Conference Pattern Recognition, UK, August, 2004.
776 [3] Z.Zivkovic, F. van der Heijden, "Efficient Adaptive Density Estimation per
777 Image Pixel for the Task of Background Subtraction", Pattern Recognition
778 Letters, vol. 27, no. 7, pages 773-780, 2006.
779 */
780
781 filter->mog2->apply (filter->cvYUV, filter->cvFG, filter->learning_rate);
782
783 return (0);
784 }
785