1 /*
2 * filter_tracker.cpp -- Motion tracker
3 * Copyright (C) 2016 Jean-Baptiste Mardelle
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <framework/mlt.h>
21 #include <opencv2/tracking.hpp>
22 #include <opencv2/core/version.hpp>
23
24 #define CV_VERSION_INT (CV_VERSION_MAJOR << 16 | CV_VERSION_MINOR << 8 | CV_VERSION_REVISION)
25
26 typedef struct
27 {
28 cv::Ptr<cv::Tracker> tracker;
29 #if CV_VERSION_INT < 0x040500
30 cv::Rect2d boundingBox;
31 #else
32 cv::Rect boundingBox;
33 #endif
34 char * algo;
35 mlt_rect startRect;
36 bool initialized;
37 bool playback;
38 bool analyze;
39 int last_position;
40 int analyse_width;
41 int analyse_height;
42 mlt_position producer_in;
43 mlt_position producer_length;
44 } private_data;
45
46
property_changed(mlt_service owner,mlt_filter filter,mlt_event_data event_data)47 static void property_changed( mlt_service owner, mlt_filter filter, mlt_event_data event_data )
48 {
49 private_data* pdata = (private_data*)filter->child;
50 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
51 const char *name = mlt_event_data_to_string(event_data);
52
53 if ( name && !strcmp( name, "results" ) )
54 {
55 mlt_properties_anim_get_int( filter_properties, "results", 0, -1 );
56 mlt_animation anim = mlt_properties_get_animation( filter_properties, "results" );
57 if ( anim && mlt_animation_key_count( anim ) > 0 )
58 {
59 // We received valid analysis data
60 pdata->initialized = true;
61 pdata->playback = true;
62 return;
63 }
64 else
65 {
66 // Analysis data was discarded
67 pdata->initialized = false;
68 pdata->producer_length = 0;
69 pdata->playback = false;
70 mlt_properties_set( filter_properties, "_results", NULL );
71 }
72 }
73 if ( !pdata->initialized )
74 {
75 return;
76 }
77 if ( !strcmp( name, "rect" ) )
78 {
79 // The initial rect was changed, we need to reset the tracker with this new rect
80 mlt_rect rect = mlt_properties_get_rect( filter_properties, "rect" );
81 if ( rect.x != pdata->startRect.x || rect.y != pdata->startRect.y || rect.w != pdata->startRect.w || rect.h != pdata->startRect.h )
82 {
83 pdata->playback = false;
84 pdata->initialized = false;
85 }
86 }
87 else if ( !strcmp( name, "algo" ) )
88 {
89 char *algo = mlt_properties_get( filter_properties, "algo" );
90 if ( pdata->algo && *pdata->algo != '\0' && strcmp( algo, pdata->algo ) )
91 {
92 pdata->playback = false;
93 pdata->initialized = false;
94 }
95 }
96 else if ( !strcmp( name, "_reset" ) )
97 {
98 mlt_properties_set( filter_properties, "results", NULL );
99 }
100 }
101
apply(mlt_filter filter,private_data * data,int width,int height,int position,int length)102 static void apply( mlt_filter filter, private_data* data, int width, int height, int position, int length )
103 {
104 mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
105 mlt_rect rect = mlt_properties_anim_get_rect( properties, "results", position, length );
106 mlt_profile profile = mlt_service_profile( MLT_FILTER_SERVICE( filter ) );
107 // Calculate the region now
108 double scale_width = mlt_profile_scale_width( profile, width );
109 double scale_height = mlt_profile_scale_height( profile, height );
110 rect.x *= scale_width;
111 rect.w *= scale_width;
112 rect.y *= scale_height;
113 rect.h *= scale_height;
114 data->boundingBox.x = rect.x;
115 data->boundingBox.y= rect.y;
116 data->boundingBox.width = rect.w;
117 data->boundingBox.height = rect.h;
118 }
119
120
analyze(mlt_filter filter,cv::Mat cvFrame,private_data * data,int width,int height,int position,int length)121 static void analyze( mlt_filter filter, cv::Mat cvFrame, private_data* data, int width, int height, int position, int length )
122 {
123 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
124
125 if ( data->analyse_width == -1 )
126 {
127 // Store analyze width/height
128 data->analyse_width = width;
129 data->analyse_height = height;
130 }
131 else if ( data->analyse_width != width || data->analyse_height != height )
132 {
133 // Frame size changed, reset all stored data
134 data->initialized = false;
135 data->analyse_width = width;
136 data->analyse_height = height;
137 }
138 // Create tracker and initialize it
139 if (!data->initialized)
140 {
141 // Build tracker
142 data->algo = mlt_properties_get( filter_properties, "algo" );
143 #if CV_VERSION_MAJOR > 3 || (CV_VERSION_MAJOR == 3 && CV_VERSION_MINOR >= 3)
144 if ( !data->algo || *data->algo == '\0' || !strcmp(data->algo, "KCF" ) )
145 {
146 data->tracker = cv::TrackerKCF::create();
147 }
148 #if CV_VERSION_INT >= 0x030402 && CV_VERSION_INT < 0x040500
149 else if ( !strcmp(data->algo, "CSRT" ) )
150 {
151 data->tracker = cv::TrackerCSRT::create();
152 }
153 else if ( !strcmp(data->algo, "MOSSE" ) )
154 {
155 data->tracker = cv::TrackerMOSSE::create();
156 }
157 #endif
158 else if ( !strcmp(data->algo, "MIL" ) )
159 {
160 data->tracker = cv::TrackerMIL::create();
161 }
162 #if CV_VERSION_INT >= 0x030402 && CV_VERSION_INT < 0x040500
163 else if ( !strcmp(data->algo, "TLD" ) )
164 {
165 data->tracker = cv::TrackerTLD::create();
166 }
167 else
168 {
169 data->tracker = cv::TrackerBoosting::create();
170 }
171 #endif // CV_VERSION_INT >= 0x030402 && CV_VERSION_INT < 0x040500
172 #else
173 if ( data->algo == NULL || !strcmp(data->algo, "" ) )
174 {
175 data->tracker = cv::Tracker::create( "KCF" );
176 }
177 else
178 {
179 data->tracker = cv::Tracker::create( data->algo );
180 }
181 #endif
182
183 // Discard previous results
184 mlt_properties_set( filter_properties, "_results", "" );
185 if( data->tracker == NULL )
186 {
187 fprintf( stderr, "Tracker initialized FAILED\n" );
188 }
189 else
190 {
191 data->startRect = mlt_properties_get_rect( filter_properties, "rect" );
192 mlt_profile profile = mlt_service_profile( MLT_FILTER_SERVICE( filter ) );
193 double scale_width = mlt_profile_scale_width( profile, width );
194 double scale_height = mlt_profile_scale_height( profile, height );
195 data->startRect.x *= scale_width;
196 data->startRect.w *= scale_width;
197 data->startRect.y *= scale_height;
198 data->startRect.h *= scale_height;
199
200 // Ensure startRect is within frame boundaries
201 if ( data->startRect.x > width )
202 {
203 data->startRect.x = width - 10;
204 }
205 else
206 {
207 data->startRect.x = MAX( 0., data->startRect.x );
208 }
209 if ( data->startRect.y > height )
210 {
211 data->startRect.y = height - 10;
212 }
213 else
214 {
215 data->startRect.y = MAX( 0., data->startRect.y );
216 }
217 if ( data->startRect.x + data->startRect.w > width )
218 {
219 data->startRect.w = width - data->startRect.x;
220 }
221 if ( data->startRect.y + data->startRect.h > height )
222 {
223 data->startRect.h = height - data->startRect.y;
224 }
225
226 data->boundingBox.x = data->startRect.x;
227 data->boundingBox.y= data->startRect.y;
228 data->boundingBox.width = data->startRect.w;
229 data->boundingBox.height = data->startRect.h;
230 if ( data->boundingBox.width <1 ) {
231 data->boundingBox.width = 50;
232 }
233 if ( data->boundingBox.height <1 ) {
234 data->boundingBox.height = 50;
235 }
236 #if CV_VERSION_INT >= 0x030402 && CV_VERSION_INT < 0x040500
237 if ( data->tracker->init( cvFrame, data->boundingBox ) ) {
238 #else
239 {
240 data->tracker->init( cvFrame, data->boundingBox );
241 #endif
242 data->initialized = true;
243 data->analyze = true;
244 data->last_position = position - 1;
245 }
246 // init anim property
247 mlt_properties_anim_get_int( filter_properties, "_results", 0, length );
248 mlt_animation anim = mlt_properties_get_animation( filter_properties, "_results" );
249 if ( anim == NULL ) {
250 fprintf( stderr, "animation initialized FAILED\n" );
251 }
252 }
253 }
254 else
255 {
256 data->tracker->update( cvFrame, data->boundingBox );
257 }
258 if( data->analyze && position != data->last_position + 1 )
259 {
260 // We are in real time, do not try to store data
261 data->analyze = false;
262 }
263 if ( !data->analyze )
264 {
265 return;
266 }
267 // Store results in temp variable
268 mlt_rect rect;
269 rect.x = data->boundingBox.x;
270 rect.y = data->boundingBox.y;
271 rect.w = data->boundingBox.width;
272 rect.h = data->boundingBox.height;
273 rect.o = 0;
274
275 int steps = mlt_properties_get_int(filter_properties, "steps");
276 if ( steps > 1 && position > 0 && position < length - 1 )
277 {
278 if ( position % steps == 0 )
279 mlt_properties_anim_set_rect( filter_properties, "_results", rect, position, length, mlt_keyframe_smooth );
280 }
281 else
282 {
283 mlt_properties_anim_set_rect( filter_properties, "_results", rect, position, length, mlt_keyframe_smooth );
284 }
285 if ( position + 1 == length )
286 {
287 //Analysis finished, store results
288 mlt_animation anim = mlt_properties_get_animation( filter_properties, "_results");
289 mlt_properties_set( filter_properties, "results", strdup( mlt_animation_serialize( anim ) ) );
290 // Discard temporary data
291 mlt_properties_set( filter_properties, "_results", (char*) NULL );
292
293 data->playback = true;
294 }
295 data->last_position = position;
296 }
297
298
299 /** Get the image.
300 */
301 static int filter_get_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable )
302 {
303 int error = 0;
304 mlt_filter filter = (mlt_filter) mlt_frame_pop_service( frame );
305 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
306 mlt_position position = mlt_filter_get_position( filter, frame );
307 mlt_service_lock( MLT_FILTER_SERVICE( filter ) );
308 int shape_width = mlt_properties_get_int( filter_properties, "shape_width" );
309 int blur = mlt_properties_get_int( filter_properties, "blur" );
310 cv::Mat cvFrame;
311
312 private_data* data = (private_data*) filter->child;
313 if ( shape_width == 0 && blur == 0 && !data->playback ) {
314 error = mlt_frame_get_image( frame, image, format, width, height, 1 );
315 }
316 else
317 {
318 *format = mlt_image_rgb;
319 error = mlt_frame_get_image( frame, image, format, width, height, 1 );
320 cvFrame = cv::Mat( *height, *width, CV_8UC3, *image );
321 }
322 if ( data->producer_length == 0 )
323 {
324 mlt_producer producer = mlt_frame_get_original_producer( frame );
325 if ( producer )
326 {
327 data->producer_in = mlt_producer_get_in( producer );
328 data->producer_length = mlt_producer_get_playtime( producer );
329 }
330 }
331 if ( !data->initialized )
332 {
333 mlt_properties_anim_get_int( filter_properties, "results", 0, data->producer_length );
334 mlt_animation anim = mlt_properties_get_animation( filter_properties, "results" );
335 if ( anim && mlt_animation_key_count(anim) > 0 )
336 {
337 data->initialized = true;
338 data->playback = true;
339 }
340 }
341 if( data->playback )
342 {
343 // Clip already analysed, don't re-process
344 apply( filter, data, *width, *height, position, data->producer_in + data->producer_length );
345 }
346 else
347 {
348 analyze( filter, cvFrame, data, *width, *height, position, data->producer_in + data->producer_length );
349 }
350 // ensure bounding box is within the frame boundaries or OpenCV will crash
351 if ( data->boundingBox.x > *width )
352 {
353 data->boundingBox.x = *width - 10;
354 }
355 else
356 {
357 data->boundingBox.x = MAX(0., data->boundingBox.x);
358 }
359 if ( data->boundingBox.y > *height )
360 {
361 data->boundingBox.y = *height - 10;
362 }
363 else
364 {
365 data->boundingBox.y = MAX(0., data->boundingBox.y);
366 }
367 if ( data->boundingBox.x + data->boundingBox.width > *width )
368 {
369 data->boundingBox.width = *width - data->boundingBox.x;
370 }
371 if ( data->boundingBox.y + data->boundingBox.height > *height )
372 {
373 data->boundingBox.height = *height - data->boundingBox.y;
374 }
375
376 if ( blur > 0 )
377 {
378 switch ( mlt_properties_get_int( filter_properties, "blur_type" ) )
379 {
380 case 1:
381 // Gaussian Blur
382 cv::GaussianBlur( cvFrame( data->boundingBox ), cvFrame( data->boundingBox ), cv::Size( 0, 0 ), blur );
383 break;
384 case 0:
385 default:
386 // Median Blur
387 ++blur;
388 if ( blur % 2 == 0 )
389 {
390 // median blur param must be odd and, minimum 3
391 ++blur;
392 }
393 cv::medianBlur( cvFrame( data->boundingBox ), cvFrame( data->boundingBox ), blur );
394 break;
395 }
396 }
397
398 // Paint overlay shape
399 if ( shape_width != 0 )
400 {
401 // Get the OpenCV image
402 mlt_color shape_color = mlt_properties_get_color( filter_properties, "shape_color" );
403 switch ( mlt_properties_get_int( filter_properties, "shape" ) )
404 {
405 case 2:
406 // Arrow
407 cv::arrowedLine( cvFrame, cv::Point( data->boundingBox.x + data->boundingBox.width/2, data->boundingBox.y - data->boundingBox.height/2 ), cv::Point( data->boundingBox.x + data->boundingBox.width/2, data->boundingBox.y ), cv::Scalar( shape_color.r, shape_color.g, shape_color.b ), MAX( shape_width, 1 ), 4, 0, .2 );
408 break;
409 case 1:
410 // Ellipse
411 {
412 cv::RotatedRect bounding = cv::RotatedRect( cv::Point2f( data->boundingBox.x + data->boundingBox.width/2, data->boundingBox.y + data->boundingBox.height/2 ), cv::Size2f( data->boundingBox.width, data->boundingBox.height ), 0);
413 cv::ellipse( cvFrame, bounding, cv::Scalar( shape_color.r, shape_color.g, shape_color.b ), shape_width, 1 );
414 }
415 break;
416 case 0:
417 default:
418 // Rectangle
419 cv::rectangle( cvFrame, data->boundingBox, cv::Scalar( shape_color.r, shape_color.g, shape_color.b ), shape_width, 1 );
420 break;
421 }
422 }
423
424 mlt_service_unlock( MLT_FILTER_SERVICE( filter ) );
425 return error;
426 }
427
428
429 /** Filter processing.
430 */
431 static mlt_frame filter_process( mlt_filter filter, mlt_frame frame )
432 {
433 mlt_frame_push_service( frame, filter );
434 mlt_frame_push_get_image( frame, filter_get_image );
435 return frame;
436 }
437
438 static void filter_close( mlt_filter filter )
439 {
440 private_data* data = (private_data*) filter->child;
441 free ( data );
442 filter->child = NULL;
443 filter->close = NULL;
444 filter->parent.close = NULL;
445 mlt_service_close( &filter->parent );
446 }
447
448 /** Constructor for the filter.
449 */
450
451 extern "C" {
452
453 mlt_filter filter_tracker_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
454 {
455 mlt_filter filter = mlt_filter_new();
456 private_data* data = (private_data*) calloc( 1, sizeof(private_data) );
457
458 if ( filter && data)
459 {
460 mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
461 mlt_properties_set_int( properties, "shape_width", 1 );
462 mlt_properties_set_int( properties, "steps", 5 );
463 mlt_properties_set( properties, "algo", "KCF" );
464 data->initialized = false;
465 data->playback = false;
466 data->boundingBox.x = 0;
467 data->boundingBox.y= 0;
468 data->boundingBox.width = 0;
469 data->boundingBox.height = 0;
470 data->analyze = false;
471 data->last_position = -1;
472 data->analyse_width = -1;
473 data->analyse_height = -1;
474 data->producer_in = 0;
475 data->producer_length = 0;
476 filter->child = data;
477
478 // Create a unique ID for storing data on the frame
479 filter->close = filter_close;
480 filter->process = filter_process;
481
482 mlt_events_listen( properties, filter, "property-changed", (mlt_listener)property_changed );
483 }
484 else
485 {
486 mlt_log_error( MLT_FILTER_SERVICE( filter ), "Filter tracker failed\n" );
487
488 if( filter )
489 {
490 mlt_filter_close( filter );
491 }
492
493 if( data )
494 {
495 free( data );
496 }
497
498 filter = NULL;
499 }
500 return filter;
501 }
502
503 }
504
505