1 #include "image_opencv.h"
2 #include <iostream>
3
4 #ifdef OPENCV
5 #include "utils.h"
6
7 #include <cstdio>
8 #include <cstdlib>
9 #include <cmath>
10 #include <string>
11 #include <vector>
12 #include <fstream>
13 #include <algorithm>
14 #include <atomic>
15
16 #include <opencv2/core/version.hpp>
17 #include <opencv2/imgproc/imgproc.hpp>
18 #include <opencv2/opencv.hpp>
19 #include <opencv2/opencv_modules.hpp>
20
21 #include <opencv2/highgui/highgui.hpp>
22 #include <opencv2/video/video.hpp>
23
24 // includes for OpenCV >= 3.x
25 #ifndef CV_VERSION_EPOCH
26 #include <opencv2/core/types.hpp>
27 #include <opencv2/videoio/videoio.hpp>
28 #include <opencv2/imgcodecs/imgcodecs.hpp>
29 #endif
30
31 // OpenCV includes for OpenCV 2.x
32 #ifdef CV_VERSION_EPOCH
33 #include <opencv2/highgui/highgui_c.h>
34 #include <opencv2/imgproc/imgproc_c.h>
35 #include <opencv2/core/types_c.h>
36 #include <opencv2/core/version.hpp>
37 #endif
38
39 //using namespace cv;
40
41 using std::cerr;
42 using std::endl;
43
44 #ifdef DEBUG
45 #define OCV_D "d"
46 #else
47 #define OCV_D
48 #endif//DEBUG
49
50
51 // OpenCV libraries
52 #ifndef CV_VERSION_EPOCH
53 #define OPENCV_VERSION CVAUX_STR(CV_VERSION_MAJOR)"" CVAUX_STR(CV_VERSION_MINOR)"" CVAUX_STR(CV_VERSION_REVISION) OCV_D
54 #ifndef USE_CMAKE_LIBS
55 #pragma comment(lib, "opencv_world" OPENCV_VERSION ".lib")
56 #endif // USE_CMAKE_LIBS
57 #else // CV_VERSION_EPOCH
58 #define OPENCV_VERSION CVAUX_STR(CV_VERSION_EPOCH)"" CVAUX_STR(CV_VERSION_MAJOR)"" CVAUX_STR(CV_VERSION_MINOR) OCV_D
59 #ifndef USE_CMAKE_LIBS
60 #pragma comment(lib, "opencv_core" OPENCV_VERSION ".lib")
61 #pragma comment(lib, "opencv_imgproc" OPENCV_VERSION ".lib")
62 #pragma comment(lib, "opencv_highgui" OPENCV_VERSION ".lib")
63 #endif // USE_CMAKE_LIBS
64 #endif // CV_VERSION_EPOCH
65
66 #include "http_stream.h"
67
68 #ifndef CV_RGB
69 #define CV_RGB(r, g, b) cvScalar( (b), (g), (r), 0 )
70 #endif
71
72 #ifndef CV_FILLED
73 #define CV_FILLED cv::FILLED
74 #endif
75
76 #ifndef CV_AA
77 #define CV_AA cv::LINE_AA
78 #endif
79
80 extern "C" {
81
82 //struct mat_cv : cv::Mat { };
83 //struct cap_cv : cv::VideoCapture { };
84 //struct write_cv : cv::VideoWriter { };
85
86 //struct mat_cv : cv::Mat { int a[0]; };
87 //struct cap_cv : cv::VideoCapture { int a[0]; };
88 //struct write_cv : cv::VideoWriter { int a[0]; };
89
90 // ====================================================================
91 // cv::Mat
92 // ====================================================================
93 image mat_to_image(cv::Mat mat);
94 cv::Mat image_to_mat(image img);
95 // image ipl_to_image(mat_cv* src);
96 // mat_cv *image_to_ipl(image img);
97 // cv::Mat ipl_to_mat(IplImage *ipl);
98 // IplImage *mat_to_ipl(cv::Mat mat);
99
100
load_image_mat_cv(const char * filename,int flag)101 extern "C" mat_cv *load_image_mat_cv(const char *filename, int flag)
102 {
103 cv::Mat *mat_ptr = NULL;
104 try {
105 cv::Mat mat = cv::imread(filename, flag);
106 if (mat.empty())
107 {
108 std::string shrinked_filename = filename;
109 if (shrinked_filename.length() > 1024) {
110 shrinked_filename.resize(1024);
111 shrinked_filename = std::string("name is too long: ") + shrinked_filename;
112 }
113 cerr << "Cannot load image " << shrinked_filename << std::endl;
114 std::ofstream bad_list("bad.list", std::ios::out | std::ios::app);
115 bad_list << shrinked_filename << std::endl;
116 //if (check_mistakes) getchar();
117 return NULL;
118 }
119 cv::Mat dst;
120 if (mat.channels() == 3) cv::cvtColor(mat, dst, cv::COLOR_RGB2BGR);
121 else if (mat.channels() == 4) cv::cvtColor(mat, dst, cv::COLOR_RGBA2BGRA);
122 else dst = mat;
123
124 mat_ptr = new cv::Mat(dst);
125
126 return (mat_cv *)mat_ptr;
127 }
128 catch (...) {
129 cerr << "OpenCV exception: load_image_mat_cv \n";
130 }
131 if (mat_ptr) delete mat_ptr;
132 return NULL;
133 }
134 // ----------------------------------------
135
load_image_mat(char * filename,int channels)136 cv::Mat load_image_mat(char *filename, int channels)
137 {
138 int flag = cv::IMREAD_UNCHANGED;
139 if (channels == 0) flag = cv::IMREAD_COLOR;
140 else if (channels == 1) flag = cv::IMREAD_GRAYSCALE;
141 else if (channels == 3) flag = cv::IMREAD_COLOR;
142 else {
143 fprintf(stderr, "OpenCV can't force load with %d channels\n", channels);
144 }
145 //flag |= IMREAD_IGNORE_ORIENTATION; // un-comment it if you want
146
147 cv::Mat *mat_ptr = (cv::Mat *)load_image_mat_cv(filename, flag);
148
149 if (mat_ptr == NULL) {
150 return cv::Mat();
151 }
152 cv::Mat mat = *mat_ptr;
153 delete mat_ptr;
154
155 return mat;
156 }
157 // ----------------------------------------
158
load_image_cv(char * filename,int channels)159 extern "C" image load_image_cv(char *filename, int channels)
160 {
161 cv::Mat mat = load_image_mat(filename, channels);
162
163 if (mat.empty()) {
164 return make_image(10, 10, channels);
165 }
166 return mat_to_image(mat);
167 }
168 // ----------------------------------------
169
load_image_resize(char * filename,int w,int h,int c,image * im)170 extern "C" image load_image_resize(char *filename, int w, int h, int c, image *im)
171 {
172 image out;
173 try {
174 cv::Mat loaded_image = load_image_mat(filename, c);
175
176 *im = mat_to_image(loaded_image);
177
178 cv::Mat resized(h, w, CV_8UC3);
179 cv::resize(loaded_image, resized, cv::Size(w, h), 0, 0, cv::INTER_LINEAR);
180 out = mat_to_image(resized);
181 }
182 catch (...) {
183 cerr << " OpenCV exception: load_image_resize() can't load image %s " << filename << " \n";
184 out = make_image(w, h, c);
185 *im = make_image(w, h, c);
186 }
187 return out;
188 }
189 // ----------------------------------------
190
get_width_mat(mat_cv * mat)191 extern "C" int get_width_mat(mat_cv *mat)
192 {
193 if (mat == NULL) {
194 cerr << " Pointer is NULL in get_width_mat() \n";
195 return 0;
196 }
197 return ((cv::Mat *)mat)->cols;
198 }
199 // ----------------------------------------
200
get_height_mat(mat_cv * mat)201 extern "C" int get_height_mat(mat_cv *mat)
202 {
203 if (mat == NULL) {
204 cerr << " Pointer is NULL in get_height_mat() \n";
205 return 0;
206 }
207 return ((cv::Mat *)mat)->rows;
208 }
209 // ----------------------------------------
210
release_mat(mat_cv ** mat)211 extern "C" void release_mat(mat_cv **mat)
212 {
213 try {
214 cv::Mat **mat_ptr = (cv::Mat **)mat;
215 if (*mat_ptr) delete *mat_ptr;
216 *mat_ptr = NULL;
217 }
218 catch (...) {
219 cerr << "OpenCV exception: release_mat \n";
220 }
221 }
222
223 // ====================================================================
224 // IplImage
225 // ====================================================================
226 /*
227 extern "C" int get_width_cv(mat_cv *ipl_src)
228 {
229 IplImage *ipl = (IplImage *)ipl_src;
230 return ipl->width;
231 }
232 // ----------------------------------------
233
234 extern "C" int get_height_cv(mat_cv *ipl_src)
235 {
236 IplImage *ipl = (IplImage *)ipl_src;
237 return ipl->height;
238 }
239 // ----------------------------------------
240
241 extern "C" void release_ipl(mat_cv **ipl)
242 {
243 IplImage **ipl_img = (IplImage **)ipl;
244 if (*ipl_img) cvReleaseImage(ipl_img);
245 *ipl_img = NULL;
246 }
247 // ----------------------------------------
248
249 // ====================================================================
250 // image-to-ipl, ipl-to-image, image_to_mat, mat_to_image
251 // ====================================================================
252
253 extern "C" mat_cv *image_to_ipl(image im)
254 {
255 int x, y, c;
256 IplImage *disp = cvCreateImage(cvSize(im.w, im.h), IPL_DEPTH_8U, im.c);
257 int step = disp->widthStep;
258 for (y = 0; y < im.h; ++y) {
259 for (x = 0; x < im.w; ++x) {
260 for (c = 0; c < im.c; ++c) {
261 float val = im.data[c*im.h*im.w + y*im.w + x];
262 disp->imageData[y*step + x*im.c + c] = (unsigned char)(val * 255);
263 }
264 }
265 }
266 return (mat_cv *)disp;
267 }
268 // ----------------------------------------
269
270 extern "C" image ipl_to_image(mat_cv* src_ptr)
271 {
272 IplImage* src = (IplImage*)src_ptr;
273 int h = src->height;
274 int w = src->width;
275 int c = src->nChannels;
276 image im = make_image(w, h, c);
277 unsigned char *data = (unsigned char *)src->imageData;
278 int step = src->widthStep;
279 int i, j, k;
280
281 for (i = 0; i < h; ++i) {
282 for (k = 0; k < c; ++k) {
283 for (j = 0; j < w; ++j) {
284 im.data[k*w*h + i*w + j] = data[i*step + j*c + k] / 255.;
285 }
286 }
287 }
288 return im;
289 }
290 // ----------------------------------------
291
292 cv::Mat ipl_to_mat(IplImage *ipl)
293 {
294 Mat m = cvarrToMat(ipl, true);
295 return m;
296 }
297 // ----------------------------------------
298
299 IplImage *mat_to_ipl(cv::Mat mat)
300 {
301 IplImage *ipl = new IplImage;
302 *ipl = mat;
303 return ipl;
304 }
305 // ----------------------------------------
306 */
307
image_to_mat(image img)308 extern "C" cv::Mat image_to_mat(image img)
309 {
310 int channels = img.c;
311 int width = img.w;
312 int height = img.h;
313 cv::Mat mat = cv::Mat(height, width, CV_8UC(channels));
314 int step = mat.step;
315
316 for (int y = 0; y < img.h; ++y) {
317 for (int x = 0; x < img.w; ++x) {
318 for (int c = 0; c < img.c; ++c) {
319 float val = img.data[c*img.h*img.w + y*img.w + x];
320 mat.data[y*step + x*img.c + c] = (unsigned char)(val * 255);
321 }
322 }
323 }
324 return mat;
325 }
326 // ----------------------------------------
327
mat_to_image(cv::Mat mat)328 extern "C" image mat_to_image(cv::Mat mat)
329 {
330 int w = mat.cols;
331 int h = mat.rows;
332 int c = mat.channels();
333 image im = make_image(w, h, c);
334 unsigned char *data = (unsigned char *)mat.data;
335 int step = mat.step;
336 for (int y = 0; y < h; ++y) {
337 for (int k = 0; k < c; ++k) {
338 for (int x = 0; x < w; ++x) {
339 //uint8_t val = mat.ptr<uint8_t>(y)[c * x + k];
340 //uint8_t val = mat.at<Vec3b>(y, x).val[k];
341 //im.data[k*w*h + y*w + x] = val / 255.0f;
342
343 im.data[k*w*h + y*w + x] = data[y*step + x*c + k] / 255.0f;
344 }
345 }
346 }
347 return im;
348 }
349
mat_to_image_cv(mat_cv * mat)350 image mat_to_image_cv(mat_cv *mat)
351 {
352 return mat_to_image(*(cv::Mat*)mat);
353 }
354
355 // ====================================================================
356 // Window
357 // ====================================================================
create_window_cv(char const * window_name,int full_screen,int width,int height)358 extern "C" void create_window_cv(char const* window_name, int full_screen, int width, int height)
359 {
360 try {
361 int window_type = cv::WINDOW_NORMAL;
362 #ifdef CV_VERSION_EPOCH // OpenCV 2.x
363 if (full_screen) window_type = CV_WINDOW_FULLSCREEN;
364 #else
365 if (full_screen) window_type = cv::WINDOW_FULLSCREEN;
366 #endif
367 cv::namedWindow(window_name, window_type);
368 cv::moveWindow(window_name, 0, 0);
369 cv::resizeWindow(window_name, width, height);
370 }
371 catch (...) {
372 cerr << "OpenCV exception: create_window_cv \n";
373 }
374 }
375 // ----------------------------------------
376
destroy_all_windows_cv()377 extern "C" void destroy_all_windows_cv()
378 {
379 try {
380 cv::destroyAllWindows();
381 }
382 catch (...) {
383 cerr << "OpenCV exception: destroy_all_windows_cv \n";
384 }
385 }
386 // ----------------------------------------
387
wait_key_cv(int delay)388 extern "C" int wait_key_cv(int delay)
389 {
390 try {
391 return cv::waitKey(delay);
392 }
393 catch (...) {
394 cerr << "OpenCV exception: wait_key_cv \n";
395 }
396 return -1;
397 }
398 // ----------------------------------------
399
wait_until_press_key_cv()400 extern "C" int wait_until_press_key_cv()
401 {
402 return wait_key_cv(0);
403 }
404 // ----------------------------------------
405
make_window(char * name,int w,int h,int fullscreen)406 extern "C" void make_window(char *name, int w, int h, int fullscreen)
407 {
408 try {
409 cv::namedWindow(name, cv::WINDOW_NORMAL);
410 if (fullscreen) {
411 #ifdef CV_VERSION_EPOCH // OpenCV 2.x
412 cv::setWindowProperty(name, cv::WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
413 #else
414 cv::setWindowProperty(name, cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN);
415 #endif
416 }
417 else {
418 cv::resizeWindow(name, w, h);
419 if (strcmp(name, "Demo") == 0) cv::moveWindow(name, 0, 0);
420 }
421 }
422 catch (...) {
423 cerr << "OpenCV exception: make_window \n";
424 }
425 }
426 // ----------------------------------------
427
get_pixel(image m,int x,int y,int c)428 static float get_pixel(image m, int x, int y, int c)
429 {
430 assert(x < m.w && y < m.h && c < m.c);
431 return m.data[c*m.h*m.w + y*m.w + x];
432 }
433 // ----------------------------------------
434
show_image_cv(image p,const char * name)435 extern "C" void show_image_cv(image p, const char *name)
436 {
437 try {
438 image copy = copy_image(p);
439 constrain_image(copy);
440
441 cv::Mat mat = image_to_mat(copy);
442 if (mat.channels() == 3) cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR);
443 else if (mat.channels() == 4) cv::cvtColor(mat, mat, cv::COLOR_RGBA2BGR);
444 cv::namedWindow(name, cv::WINDOW_NORMAL);
445 cv::imshow(name, mat);
446 free_image(copy);
447 }
448 catch (...) {
449 cerr << "OpenCV exception: show_image_cv \n";
450 }
451 }
452 // ----------------------------------------
453
454 /*
455 extern "C" void show_image_cv_ipl(mat_cv *disp, const char *name)
456 {
457 if (disp == NULL) return;
458 char buff[256];
459 sprintf(buff, "%s", name);
460 cv::namedWindow(buff, WINDOW_NORMAL);
461 cvShowImage(buff, disp);
462 }
463 // ----------------------------------------
464 */
465
show_image_mat(mat_cv * mat_ptr,const char * name)466 extern "C" void show_image_mat(mat_cv *mat_ptr, const char *name)
467 {
468 try {
469 if (mat_ptr == NULL) return;
470 cv::Mat &mat = *(cv::Mat *)mat_ptr;
471 cv::namedWindow(name, cv::WINDOW_NORMAL);
472 cv::imshow(name, mat);
473 }
474 catch (...) {
475 cerr << "OpenCV exception: show_image_mat \n";
476 }
477 }
478
479 // ====================================================================
480 // Video Writer
481 // ====================================================================
create_video_writer(char * out_filename,char c1,char c2,char c3,char c4,int fps,int width,int height,int is_color)482 extern "C" write_cv *create_video_writer(char *out_filename, char c1, char c2, char c3, char c4, int fps, int width, int height, int is_color)
483 {
484 try {
485 cv::VideoWriter * output_video_writer =
486 #ifdef CV_VERSION_EPOCH
487 new cv::VideoWriter(out_filename, CV_FOURCC(c1, c2, c3, c4), fps, cv::Size(width, height), is_color);
488 #else
489 new cv::VideoWriter(out_filename, cv::VideoWriter::fourcc(c1, c2, c3, c4), fps, cv::Size(width, height), is_color);
490 #endif
491
492 return (write_cv *)output_video_writer;
493 }
494 catch (...) {
495 cerr << "OpenCV exception: create_video_writer \n";
496 }
497 return NULL;
498 }
499
write_frame_cv(write_cv * output_video_writer,mat_cv * mat)500 extern "C" void write_frame_cv(write_cv *output_video_writer, mat_cv *mat)
501 {
502 try {
503 cv::VideoWriter *out = (cv::VideoWriter *)output_video_writer;
504 out->write(*(cv::Mat*)mat);
505 }
506 catch (...) {
507 cerr << "OpenCV exception: write_frame_cv \n";
508 }
509 }
510
release_video_writer(write_cv ** output_video_writer)511 extern "C" void release_video_writer(write_cv **output_video_writer)
512 {
513 try {
514 if (output_video_writer) {
515 std::cout << " closing...";
516 cv::VideoWriter *out = *(cv::VideoWriter **)output_video_writer;
517 out->release();
518 delete out;
519 output_video_writer = NULL;
520 std::cout << " closed!";
521 }
522 else {
523 cerr << "OpenCV exception: output_video_writer isn't created \n";
524 }
525 }
526 catch (...) {
527 cerr << "OpenCV exception: release_video_writer \n";
528 }
529 }
530
531 /*
532 extern "C" void *open_video_stream(const char *f, int c, int w, int h, int fps)
533 {
534 VideoCapture *cap;
535 if(f) cap = new VideoCapture(f);
536 else cap = new VideoCapture(c);
537 if(!cap->isOpened()) return 0;
538 if(w) cap->set(CV_CAP_PROP_FRAME_WIDTH, w);
539 if(h) cap->set(CV_CAP_PROP_FRAME_HEIGHT, w);
540 if(fps) cap->set(CV_CAP_PROP_FPS, w);
541 return (void *) cap;
542 }
543
544
545 extern "C" image get_image_from_stream(void *p)
546 {
547 VideoCapture *cap = (VideoCapture *)p;
548 Mat m;
549 *cap >> m;
550 if(m.empty()) return make_empty_image(0,0,0);
551 return mat_to_image(m);
552 }
553
554 extern "C" int show_image_cv(image im, const char* name, int ms)
555 {
556 Mat m = image_to_mat(im);
557 imshow(name, m);
558 int c = waitKey(ms);
559 if (c != -1) c = c%256;
560 return c;
561 }
562 */
563
564
565 // ====================================================================
566 // Video Capture
567 // ====================================================================
568
get_capture_video_stream(const char * path)569 extern "C" cap_cv* get_capture_video_stream(const char *path) {
570 cv::VideoCapture* cap = NULL;
571 try {
572 cap = new cv::VideoCapture(path);
573 }
574 catch (...) {
575 cerr << " OpenCV exception: video-stream " << path << " can't be opened! \n";
576 }
577 return (cap_cv*)cap;
578 }
579 // ----------------------------------------
580
get_capture_webcam(int index)581 extern "C" cap_cv* get_capture_webcam(int index)
582 {
583 cv::VideoCapture* cap = NULL;
584 try {
585 cap = new cv::VideoCapture(index);
586 //cap->set(CV_CAP_PROP_FRAME_WIDTH, 1280);
587 //cap->set(CV_CAP_PROP_FRAME_HEIGHT, 960);
588 }
589 catch (...) {
590 cerr << " OpenCV exception: Web-camera " << index << " can't be opened! \n";
591 }
592 return (cap_cv*)cap;
593 }
594 // ----------------------------------------
595
release_capture(cap_cv * cap)596 extern "C" void release_capture(cap_cv* cap)
597 {
598 try {
599 cv::VideoCapture *cpp_cap = (cv::VideoCapture *)cap;
600 delete cpp_cap;
601 }
602 catch (...) {
603 cerr << " OpenCV exception: cv::VideoCapture " << cap << " can't be released! \n";
604 }
605 }
606 // ----------------------------------------
607
get_capture_frame_cv(cap_cv * cap)608 extern "C" mat_cv* get_capture_frame_cv(cap_cv *cap) {
609 cv::Mat *mat = NULL;
610 try {
611 mat = new cv::Mat();
612 if (cap) {
613 cv::VideoCapture &cpp_cap = *(cv::VideoCapture *)cap;
614 if (cpp_cap.isOpened())
615 {
616 cpp_cap >> *mat;
617 }
618 else std::cout << " Video-stream stopped! \n";
619 }
620 else cerr << " cv::VideoCapture isn't created \n";
621 }
622 catch (...) {
623 std::cout << " OpenCV exception: Video-stream stoped! \n";
624 }
625 return (mat_cv *)mat;
626 }
627 // ----------------------------------------
628
get_stream_fps_cpp_cv(cap_cv * cap)629 extern "C" int get_stream_fps_cpp_cv(cap_cv *cap)
630 {
631 int fps = 25;
632 try {
633 cv::VideoCapture &cpp_cap = *(cv::VideoCapture *)cap;
634 #ifndef CV_VERSION_EPOCH // OpenCV 3.x
635 fps = cpp_cap.get(cv::CAP_PROP_FPS);
636 #else // OpenCV 2.x
637 fps = cpp_cap.get(CV_CAP_PROP_FPS);
638 #endif
639 }
640 catch (...) {
641 cerr << " Can't get FPS of source videofile. For output video FPS = 25 by default. \n";
642 }
643 return fps;
644 }
645 // ----------------------------------------
646
get_capture_property_cv(cap_cv * cap,int property_id)647 extern "C" double get_capture_property_cv(cap_cv *cap, int property_id)
648 {
649 try {
650 cv::VideoCapture &cpp_cap = *(cv::VideoCapture *)cap;
651 return cpp_cap.get(property_id);
652 }
653 catch (...) {
654 cerr << " OpenCV exception: Can't get property of source video-stream. \n";
655 }
656 return 0;
657 }
658 // ----------------------------------------
659
get_capture_frame_count_cv(cap_cv * cap)660 extern "C" double get_capture_frame_count_cv(cap_cv *cap)
661 {
662 try {
663 cv::VideoCapture &cpp_cap = *(cv::VideoCapture *)cap;
664 #ifndef CV_VERSION_EPOCH // OpenCV 3.x
665 return cpp_cap.get(cv::CAP_PROP_FRAME_COUNT);
666 #else // OpenCV 2.x
667 return cpp_cap.get(CV_CAP_PROP_FRAME_COUNT);
668 #endif
669 }
670 catch (...) {
671 cerr << " OpenCV exception: Can't get CAP_PROP_FRAME_COUNT of source videofile. \n";
672 }
673 return 0;
674 }
675 // ----------------------------------------
676
set_capture_property_cv(cap_cv * cap,int property_id,double value)677 extern "C" int set_capture_property_cv(cap_cv *cap, int property_id, double value)
678 {
679 try {
680 cv::VideoCapture &cpp_cap = *(cv::VideoCapture *)cap;
681 return cpp_cap.set(property_id, value);
682 }
683 catch (...) {
684 cerr << " Can't set property of source video-stream. \n";
685 }
686 return false;
687 }
688 // ----------------------------------------
689
set_capture_position_frame_cv(cap_cv * cap,int index)690 extern "C" int set_capture_position_frame_cv(cap_cv *cap, int index)
691 {
692 try {
693 cv::VideoCapture &cpp_cap = *(cv::VideoCapture *)cap;
694 #ifndef CV_VERSION_EPOCH // OpenCV 3.x
695 return cpp_cap.set(cv::CAP_PROP_POS_FRAMES, index);
696 #else // OpenCV 2.x
697 return cpp_cap.set(CV_CAP_PROP_POS_FRAMES, index);
698 #endif
699 }
700 catch (...) {
701 cerr << " Can't set CAP_PROP_POS_FRAMES of source videofile. \n";
702 }
703 return false;
704 }
705 // ----------------------------------------
706
707
708
709 // ====================================================================
710 // ... Video Capture
711 // ====================================================================
712
get_image_from_stream_cpp(cap_cv * cap)713 extern "C" image get_image_from_stream_cpp(cap_cv *cap)
714 {
715 cv::Mat *src = NULL;
716 static int once = 1;
717 if (once) {
718 once = 0;
719 do {
720 if (src) delete src;
721 src = (cv::Mat*)get_capture_frame_cv(cap);
722 if (!src) return make_empty_image(0, 0, 0);
723 } while (src->cols < 1 || src->rows < 1 || src->channels() < 1);
724 printf("Video stream: %d x %d \n", src->cols, src->rows);
725 }
726 else
727 src = (cv::Mat*)get_capture_frame_cv(cap);
728
729 if (!src) return make_empty_image(0, 0, 0);
730 image im = mat_to_image(*src);
731 rgbgr_image(im);
732 if (src) delete src;
733 return im;
734 }
735 // ----------------------------------------
736
wait_for_stream(cap_cv * cap,cv::Mat * src,int dont_close)737 extern "C" int wait_for_stream(cap_cv *cap, cv::Mat* src, int dont_close)
738 {
739 if (!src) {
740 if (dont_close) src = new cv::Mat(416, 416, CV_8UC(3)); // cvCreateImage(cvSize(416, 416), IPL_DEPTH_8U, 3);
741 else return 0;
742 }
743 if (src->cols < 1 || src->rows < 1 || src->channels() < 1) {
744 if (dont_close) {
745 delete src;// cvReleaseImage(&src);
746 int z = 0;
747 for (z = 0; z < 20; ++z) {
748 src = (cv::Mat*)get_capture_frame_cv(cap);
749 delete src;// cvReleaseImage(&src);
750 }
751 src = new cv::Mat(416, 416, CV_8UC(3)); // cvCreateImage(cvSize(416, 416), IPL_DEPTH_8U, 3);
752 }
753 else return 0;
754 }
755 return 1;
756 }
757 // ----------------------------------------
758
get_image_from_stream_resize(cap_cv * cap,int w,int h,int c,mat_cv ** in_img,int dont_close)759 extern "C" image get_image_from_stream_resize(cap_cv *cap, int w, int h, int c, mat_cv** in_img, int dont_close)
760 {
761 c = c ? c : 3;
762 cv::Mat *src = NULL;
763
764 static int once = 1;
765 if (once) {
766 once = 0;
767 do {
768 if (src) delete src;
769 src = (cv::Mat*)get_capture_frame_cv(cap);
770 if (!src) return make_empty_image(0, 0, 0);
771 } while (src->cols < 1 || src->rows < 1 || src->channels() < 1);
772 printf("Video stream: %d x %d \n", src->cols, src->rows);
773 }
774 else
775 src = (cv::Mat*)get_capture_frame_cv(cap);
776
777 if (!wait_for_stream(cap, src, dont_close)) return make_empty_image(0, 0, 0);
778
779 *(cv::Mat **)in_img = src;
780
781 cv::Mat new_img = cv::Mat(h, w, CV_8UC(c));
782 cv::resize(*src, new_img, new_img.size(), 0, 0, cv::INTER_LINEAR);
783 if (c>1) cv::cvtColor(new_img, new_img, cv::COLOR_RGB2BGR);
784 image im = mat_to_image(new_img);
785
786 //show_image_cv(im, "im");
787 //show_image_mat(*in_img, "in_img");
788 return im;
789 }
790 // ----------------------------------------
791
get_image_from_stream_letterbox(cap_cv * cap,int w,int h,int c,mat_cv ** in_img,int dont_close)792 extern "C" image get_image_from_stream_letterbox(cap_cv *cap, int w, int h, int c, mat_cv** in_img, int dont_close)
793 {
794 c = c ? c : 3;
795 cv::Mat *src = NULL;
796 static int once = 1;
797 if (once) {
798 once = 0;
799 do {
800 if (src) delete src;
801 src = (cv::Mat*)get_capture_frame_cv(cap);
802 if (!src) return make_empty_image(0, 0, 0);
803 } while (src->cols < 1 || src->rows < 1 || src->channels() < 1);
804 printf("Video stream: %d x %d \n", src->cols, src->rows);
805 }
806 else
807 src = (cv::Mat*)get_capture_frame_cv(cap);
808
809 if (!wait_for_stream(cap, src, dont_close)) return make_empty_image(0, 0, 0); // passes (cv::Mat *)src while should be (cv::Mat **)src
810
811 *in_img = (mat_cv *)new cv::Mat(src->rows, src->cols, CV_8UC(c));
812 cv::resize(*src, **(cv::Mat**)in_img, (*(cv::Mat**)in_img)->size(), 0, 0, cv::INTER_LINEAR);
813
814 if (c>1) cv::cvtColor(*src, *src, cv::COLOR_RGB2BGR);
815 image tmp = mat_to_image(*src);
816 image im = letterbox_image(tmp, w, h);
817 free_image(tmp);
818 release_mat((mat_cv **)&src);
819
820 //show_image_cv(im, "im");
821 //show_image_mat(*in_img, "in_img");
822 return im;
823 }
824 // ----------------------------------------
825
826 // ====================================================================
827 // Image Saving
828 // ====================================================================
829 extern int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes);
830 extern int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality);
831
save_mat_png(cv::Mat img_src,const char * name)832 extern "C" void save_mat_png(cv::Mat img_src, const char *name)
833 {
834 cv::Mat img_rgb;
835 if (img_src.channels() >= 3) cv::cvtColor(img_src, img_rgb, cv::COLOR_RGB2BGR);
836 stbi_write_png(name, img_rgb.cols, img_rgb.rows, 3, (char *)img_rgb.data, 0);
837 }
838 // ----------------------------------------
839
save_mat_jpg(cv::Mat img_src,const char * name)840 extern "C" void save_mat_jpg(cv::Mat img_src, const char *name)
841 {
842 cv::Mat img_rgb;
843 if (img_src.channels() >= 3) cv::cvtColor(img_src, img_rgb, cv::COLOR_RGB2BGR);
844 stbi_write_jpg(name, img_rgb.cols, img_rgb.rows, 3, (char *)img_rgb.data, 80);
845 }
846 // ----------------------------------------
847
848
save_cv_png(mat_cv * img_src,const char * name)849 extern "C" void save_cv_png(mat_cv *img_src, const char *name)
850 {
851 cv::Mat* img = (cv::Mat* )img_src;
852 save_mat_png(*img, name);
853 }
854 // ----------------------------------------
855
save_cv_jpg(mat_cv * img_src,const char * name)856 extern "C" void save_cv_jpg(mat_cv *img_src, const char *name)
857 {
858 cv::Mat* img = (cv::Mat*)img_src;
859 save_mat_jpg(*img, name);
860 }
861 // ----------------------------------------
862
863
864 // ====================================================================
865 // Draw Detection
866 // ====================================================================
draw_detections_cv_v3(mat_cv * mat,detection * dets,int num,float thresh,char ** names,image ** alphabet,int classes,int ext_output)867 extern "C" void draw_detections_cv_v3(mat_cv* mat, detection *dets, int num, float thresh, char **names, image **alphabet, int classes, int ext_output)
868 {
869 try {
870 cv::Mat *show_img = (cv::Mat*)mat;
871 int i, j;
872 if (!show_img) return;
873 static int frame_id = 0;
874 frame_id++;
875
876 for (i = 0; i < num; ++i) {
877 char labelstr[4096] = { 0 };
878 int class_id = -1;
879 for (j = 0; j < classes; ++j) {
880 int show = strncmp(names[j], "dont_show", 9);
881 if (dets[i].prob[j] > thresh && show) {
882 if (class_id < 0) {
883 strcat(labelstr, names[j]);
884 class_id = j;
885 char buff[10];
886 sprintf(buff, " (%2.0f%%)", dets[i].prob[j] * 100);
887 strcat(labelstr, buff);
888 printf("%s: %.0f%% ", names[j], dets[i].prob[j] * 100);
889 }
890 else {
891 strcat(labelstr, ", ");
892 strcat(labelstr, names[j]);
893 printf(", %s: %.0f%% ", names[j], dets[i].prob[j] * 100);
894 }
895 }
896 }
897 if (class_id >= 0) {
898 int width = std::max(1.0f, show_img->rows * .002f);
899
900 //if(0){
901 //width = pow(prob, 1./2.)*10+1;
902 //alphabet = 0;
903 //}
904
905 //printf("%d %s: %.0f%%\n", i, names[class_id], prob*100);
906 int offset = class_id * 123457 % classes;
907 float red = get_color(2, offset, classes);
908 float green = get_color(1, offset, classes);
909 float blue = get_color(0, offset, classes);
910 float rgb[3];
911
912 //width = prob*20+2;
913
914 rgb[0] = red;
915 rgb[1] = green;
916 rgb[2] = blue;
917 box b = dets[i].bbox;
918 if (std::isnan(b.w) || std::isinf(b.w)) b.w = 0.5;
919 if (std::isnan(b.h) || std::isinf(b.h)) b.h = 0.5;
920 if (std::isnan(b.x) || std::isinf(b.x)) b.x = 0.5;
921 if (std::isnan(b.y) || std::isinf(b.y)) b.y = 0.5;
922 b.w = (b.w < 1) ? b.w : 1;
923 b.h = (b.h < 1) ? b.h : 1;
924 b.x = (b.x < 1) ? b.x : 1;
925 b.y = (b.y < 1) ? b.y : 1;
926 //printf("%f %f %f %f\n", b.x, b.y, b.w, b.h);
927
928 int left = (b.x - b.w / 2.)*show_img->cols;
929 int right = (b.x + b.w / 2.)*show_img->cols;
930 int top = (b.y - b.h / 2.)*show_img->rows;
931 int bot = (b.y + b.h / 2.)*show_img->rows;
932
933 if (left < 0) left = 0;
934 if (right > show_img->cols - 1) right = show_img->cols - 1;
935 if (top < 0) top = 0;
936 if (bot > show_img->rows - 1) bot = show_img->rows - 1;
937
938 //int b_x_center = (left + right) / 2;
939 //int b_y_center = (top + bot) / 2;
940 //int b_width = right - left;
941 //int b_height = bot - top;
942 //sprintf(labelstr, "%d x %d - w: %d, h: %d", b_x_center, b_y_center, b_width, b_height);
943
944 float const font_size = show_img->rows / 1000.F;
945 cv::Size const text_size = cv::getTextSize(labelstr, cv::FONT_HERSHEY_COMPLEX_SMALL, font_size, 1, 0);
946 cv::Point pt1, pt2, pt_text, pt_text_bg1, pt_text_bg2;
947 pt1.x = left;
948 pt1.y = top;
949 pt2.x = right;
950 pt2.y = bot;
951 pt_text.x = left;
952 pt_text.y = top - 4;// 12;
953 pt_text_bg1.x = left;
954 pt_text_bg1.y = top - (3 + 18 * font_size);
955 pt_text_bg2.x = right;
956 if ((right - left) < text_size.width) pt_text_bg2.x = left + text_size.width;
957 pt_text_bg2.y = top;
958 cv::Scalar color;
959 color.val[0] = red * 256;
960 color.val[1] = green * 256;
961 color.val[2] = blue * 256;
962
963 // you should create directory: result_img
964 //static int copied_frame_id = -1;
965 //static IplImage* copy_img = NULL;
966 //if (copied_frame_id != frame_id) {
967 // copied_frame_id = frame_id;
968 // if(copy_img == NULL) copy_img = cvCreateImage(cvSize(show_img->width, show_img->height), show_img->depth, show_img->nChannels);
969 // cvCopy(show_img, copy_img, 0);
970 //}
971 //static int img_id = 0;
972 //img_id++;
973 //char image_name[1024];
974 //sprintf(image_name, "result_img/img_%d_%d_%d_%s.jpg", frame_id, img_id, class_id, names[class_id]);
975 //CvRect rect = cvRect(pt1.x, pt1.y, pt2.x - pt1.x, pt2.y - pt1.y);
976 //cvSetImageROI(copy_img, rect);
977 //cvSaveImage(image_name, copy_img, 0);
978 //cvResetImageROI(copy_img);
979
980 cv::rectangle(*show_img, pt1, pt2, color, width, 8, 0);
981 if (ext_output)
982 printf("\t(left_x: %4.0f top_y: %4.0f width: %4.0f height: %4.0f)\n",
983 (float)left, (float)top, b.w*show_img->cols, b.h*show_img->rows);
984 else
985 printf("\n");
986
987 cv::rectangle(*show_img, pt_text_bg1, pt_text_bg2, color, width, 8, 0);
988 cv::rectangle(*show_img, pt_text_bg1, pt_text_bg2, color, CV_FILLED, 8, 0); // filled
989 cv::Scalar black_color = CV_RGB(0, 0, 0);
990 cv::putText(*show_img, labelstr, pt_text, cv::FONT_HERSHEY_COMPLEX_SMALL, font_size, black_color, 2 * font_size, CV_AA);
991 // cv::FONT_HERSHEY_COMPLEX_SMALL, cv::FONT_HERSHEY_SIMPLEX
992 }
993 }
994 if (ext_output) {
995 fflush(stdout);
996 }
997 }
998 catch (...) {
999 cerr << "OpenCV exception: draw_detections_cv_v3() \n";
1000 }
1001 }
1002 // ----------------------------------------
1003
1004 // ====================================================================
1005 // Draw Loss & Accuracy chart
1006 // ====================================================================
draw_train_chart(char * windows_name,float max_img_loss,int max_batches,int number_of_lines,int img_size,int dont_show,char * chart_path)1007 extern "C" mat_cv* draw_train_chart(char *windows_name, float max_img_loss, int max_batches, int number_of_lines, int img_size, int dont_show, char* chart_path)
1008 {
1009 int img_offset = 60;
1010 int draw_size = img_size - img_offset;
1011 cv::Mat *img_ptr = new cv::Mat(img_size, img_size, CV_8UC3, CV_RGB(255, 255, 255));
1012 cv::Mat &img = *img_ptr;
1013 cv::Point pt1, pt2, pt_text;
1014
1015 try {
1016 // load chart from file
1017 if (chart_path != NULL && chart_path[0] != '\0') {
1018 *img_ptr = cv::imread(chart_path);
1019 }
1020 else {
1021 // draw new chart
1022 char char_buff[100];
1023 int i;
1024 // vertical lines
1025 pt1.x = img_offset; pt2.x = img_size, pt_text.x = 30;
1026 for (i = 1; i <= number_of_lines; ++i) {
1027 pt1.y = pt2.y = (float)i * draw_size / number_of_lines;
1028 cv::line(img, pt1, pt2, CV_RGB(224, 224, 224), 1, 8, 0);
1029 if (i % 10 == 0) {
1030 sprintf(char_buff, "%2.1f", max_img_loss*(number_of_lines - i) / number_of_lines);
1031 pt_text.y = pt1.y + 3;
1032
1033 cv::putText(img, char_buff, pt_text, cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 0), 1, CV_AA);
1034 cv::line(img, pt1, pt2, CV_RGB(128, 128, 128), 1, 8, 0);
1035 }
1036 }
1037 // horizontal lines
1038 pt1.y = draw_size; pt2.y = 0, pt_text.y = draw_size + 15;
1039 for (i = 0; i <= number_of_lines; ++i) {
1040 pt1.x = pt2.x = img_offset + (float)i * draw_size / number_of_lines;
1041 cv::line(img, pt1, pt2, CV_RGB(224, 224, 224), 1, 8, 0);
1042 if (i % 10 == 0) {
1043 sprintf(char_buff, "%d", max_batches * i / number_of_lines);
1044 pt_text.x = pt1.x - 20;
1045 cv::putText(img, char_buff, pt_text, cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 0), 1, CV_AA);
1046 cv::line(img, pt1, pt2, CV_RGB(128, 128, 128), 1, 8, 0);
1047 }
1048 }
1049
1050 cv::putText(img, "Loss", cv::Point(10, 55), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 255), 1, CV_AA);
1051 cv::putText(img, "Iteration number", cv::Point(draw_size / 2, img_size - 10), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 0), 1, CV_AA);
1052 char max_batches_buff[100];
1053 sprintf(max_batches_buff, "in cfg max_batches=%d", max_batches);
1054 cv::putText(img, max_batches_buff, cv::Point(draw_size - 195, img_size - 10), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 0), 1, CV_AA);
1055 cv::putText(img, "Press 's' to save : chart.png", cv::Point(5, img_size - 10), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 0), 1, CV_AA);
1056 }
1057
1058 if (!dont_show) {
1059 printf(" If error occurs - run training with flag: -dont_show \n");
1060 cv::namedWindow(windows_name, cv::WINDOW_NORMAL);
1061 cv::moveWindow(windows_name, 0, 0);
1062 cv::resizeWindow(windows_name, img_size, img_size);
1063 cv::imshow(windows_name, img);
1064 cv::waitKey(20);
1065 }
1066 }
1067 catch (...) {
1068 cerr << "OpenCV exception: draw_train_chart() \n";
1069 }
1070 return (mat_cv*)img_ptr;
1071 }
1072 // ----------------------------------------
1073
draw_train_loss(char * windows_name,mat_cv * img_src,int img_size,float avg_loss,float max_img_loss,int current_batch,int max_batches,float precision,int draw_precision,char * accuracy_name,int dont_show,int mjpeg_port,double time_remaining)1074 extern "C" void draw_train_loss(char *windows_name, mat_cv* img_src, int img_size, float avg_loss, float max_img_loss, int current_batch, int max_batches,
1075 float precision, int draw_precision, char *accuracy_name, int dont_show, int mjpeg_port, double time_remaining)
1076 {
1077 try {
1078 cv::Mat &img = *(cv::Mat*)img_src;
1079 int img_offset = 60;
1080 int draw_size = img_size - img_offset;
1081 char char_buff[100];
1082 cv::Point pt1, pt2;
1083 pt1.x = img_offset + draw_size * (float)current_batch / max_batches;
1084 pt1.y = draw_size * (1 - avg_loss / max_img_loss);
1085 if (pt1.y < 0) pt1.y = 1;
1086 cv::circle(img, pt1, 1, CV_RGB(0, 0, 255), CV_FILLED, 8, 0);
1087
1088 // precision
1089 if (draw_precision) {
1090 static float old_precision = 0;
1091 static float max_precision = 0;
1092 static int iteration_old = 0;
1093 static int text_iteration_old = 0;
1094 if (iteration_old == 0)
1095 cv::putText(img, accuracy_name, cv::Point(10, 12), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(255, 0, 0), 1, CV_AA);
1096
1097 if (iteration_old != 0){
1098 cv::line(img,
1099 cv::Point(img_offset + draw_size * (float)iteration_old / max_batches, draw_size * (1 - old_precision)),
1100 cv::Point(img_offset + draw_size * (float)current_batch / max_batches, draw_size * (1 - precision)),
1101 CV_RGB(255, 0, 0), 1, 8, 0);
1102 }
1103
1104 sprintf(char_buff, "%2.1f%% ", precision * 100);
1105 cv::putText(img, char_buff, cv::Point(10, 28), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(255, 255, 255), 5, CV_AA);
1106 cv::putText(img, char_buff, cv::Point(10, 28), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(200, 0, 0), 1, CV_AA);
1107
1108 if ((std::fabs(old_precision - precision) > 0.1) || (max_precision < precision) || (current_batch - text_iteration_old) >= max_batches / 10) {
1109 text_iteration_old = current_batch;
1110 max_precision = std::max(max_precision, precision);
1111 sprintf(char_buff, "%2.0f%% ", precision * 100);
1112 cv::putText(img, char_buff, cv::Point(pt1.x - 30, draw_size * (1 - precision) + 15), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(255, 255, 255), 5, CV_AA);
1113 cv::putText(img, char_buff, cv::Point(pt1.x - 30, draw_size * (1 - precision) + 15), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(200, 0, 0), 1, CV_AA);
1114 }
1115 old_precision = precision;
1116 iteration_old = current_batch;
1117 }
1118 sprintf(char_buff, "current avg loss = %2.4f iteration = %d approx. time left = %2.2f hours", avg_loss, current_batch, time_remaining);
1119 pt1.x = 15, pt1.y = draw_size + 18;
1120 pt2.x = pt1.x + 800, pt2.y = pt1.y + 20;
1121 cv::rectangle(img, pt1, pt2, CV_RGB(255, 255, 255), CV_FILLED, 8, 0);
1122 pt1.y += 15;
1123 cv::putText(img, char_buff, pt1, cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 100), 1, CV_AA);
1124
1125 int k = 0;
1126 if (!dont_show) {
1127 cv::imshow(windows_name, img);
1128 k = cv::waitKey(20);
1129 }
1130 static int old_batch = 0;
1131 if (k == 's' || current_batch == (max_batches - 1) || (current_batch / 100 > old_batch / 100)) {
1132 old_batch = current_batch;
1133 save_mat_png(img, "chart.png");
1134 save_mat_png(img, windows_name);
1135 cv::putText(img, "- Saved", cv::Point(260, img_size - 10), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(255, 0, 0), 1, CV_AA);
1136 }
1137 else
1138 cv::putText(img, "- Saved", cv::Point(260, img_size - 10), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(255, 255, 255), 1, CV_AA);
1139
1140 if (mjpeg_port > 0) send_mjpeg((mat_cv *)&img, mjpeg_port, 500000, 70);
1141 }
1142 catch (...) {
1143 cerr << "OpenCV exception: draw_train_loss() \n";
1144 }
1145 }
1146 // ----------------------------------------
1147
1148
1149 // ====================================================================
1150 // Data augmentation
1151 // ====================================================================
1152
image_data_augmentation(mat_cv * mat,int w,int h,int pleft,int ptop,int swidth,int sheight,int flip,float dhue,float dsat,float dexp,int gaussian_noise,int blur,int num_boxes,float * truth)1153 extern "C" image image_data_augmentation(mat_cv* mat, int w, int h,
1154 int pleft, int ptop, int swidth, int sheight, int flip,
1155 float dhue, float dsat, float dexp,
1156 int gaussian_noise, int blur, int num_boxes, float *truth)
1157 {
1158 image out;
1159 try {
1160 cv::Mat img = *(cv::Mat *)mat;
1161
1162 // crop
1163 cv::Rect src_rect(pleft, ptop, swidth, sheight);
1164 cv::Rect img_rect(cv::Point2i(0, 0), img.size());
1165 cv::Rect new_src_rect = src_rect & img_rect;
1166
1167 cv::Rect dst_rect(cv::Point2i(std::max<int>(0, -pleft), std::max<int>(0, -ptop)), new_src_rect.size());
1168 cv::Mat sized;
1169
1170 if (src_rect.x == 0 && src_rect.y == 0 && src_rect.size() == img.size()) {
1171 cv::resize(img, sized, cv::Size(w, h), 0, 0, cv::INTER_LINEAR);
1172 }
1173 else {
1174 cv::Mat cropped(src_rect.size(), img.type());
1175 //cropped.setTo(cv::Scalar::all(0));
1176 cropped.setTo(cv::mean(img));
1177
1178 img(new_src_rect).copyTo(cropped(dst_rect));
1179
1180 // resize
1181 cv::resize(cropped, sized, cv::Size(w, h), 0, 0, cv::INTER_LINEAR);
1182 }
1183
1184 // flip
1185 if (flip) {
1186 cv::Mat cropped;
1187 cv::flip(sized, cropped, 1); // 0 - x-axis, 1 - y-axis, -1 - both axes (x & y)
1188 sized = cropped.clone();
1189 }
1190
1191 // HSV augmentation
1192 // cv::COLOR_BGR2HSV, cv::COLOR_RGB2HSV, cv::COLOR_HSV2BGR, cv::COLOR_HSV2RGB
1193 if (dsat != 1 || dexp != 1 || dhue != 0) {
1194 if (img.channels() >= 3)
1195 {
1196 cv::Mat hsv_src;
1197 cvtColor(sized, hsv_src, cv::COLOR_RGB2HSV); // RGB to HSV
1198
1199 std::vector<cv::Mat> hsv;
1200 cv::split(hsv_src, hsv);
1201
1202 hsv[1] *= dsat;
1203 hsv[2] *= dexp;
1204 hsv[0] += 179 * dhue;
1205
1206 cv::merge(hsv, hsv_src);
1207
1208 cvtColor(hsv_src, sized, cv::COLOR_HSV2RGB); // HSV to RGB (the same as previous)
1209 }
1210 else
1211 {
1212 sized *= dexp;
1213 }
1214 }
1215
1216 //std::stringstream window_name;
1217 //window_name << "augmentation - " << ipl;
1218 //cv::imshow(window_name.str(), sized);
1219 //cv::waitKey(0);
1220
1221 if (blur) {
1222 cv::Mat dst(sized.size(), sized.type());
1223 if (blur == 1) {
1224 cv::GaussianBlur(sized, dst, cv::Size(17, 17), 0);
1225 //cv::bilateralFilter(sized, dst, 17, 75, 75);
1226 }
1227 else {
1228 int ksize = (blur / 2) * 2 + 1;
1229 cv::Size kernel_size = cv::Size(ksize, ksize);
1230 cv::GaussianBlur(sized, dst, kernel_size, 0);
1231 //cv::medianBlur(sized, dst, ksize);
1232 //cv::bilateralFilter(sized, dst, ksize, 75, 75);
1233
1234 // sharpen
1235 //cv::Mat img_tmp;
1236 //cv::GaussianBlur(dst, img_tmp, cv::Size(), 3);
1237 //cv::addWeighted(dst, 1.5, img_tmp, -0.5, 0, img_tmp);
1238 //dst = img_tmp;
1239 }
1240 //std::cout << " blur num_boxes = " << num_boxes << std::endl;
1241
1242 if (blur == 1) {
1243 cv::Rect img_rect(0, 0, sized.cols, sized.rows);
1244 int t;
1245 for (t = 0; t < num_boxes; ++t) {
1246 box b = float_to_box_stride(truth + t*(4 + 1), 1);
1247 if (!b.x) break;
1248 int left = (b.x - b.w / 2.)*sized.cols;
1249 int width = b.w*sized.cols;
1250 int top = (b.y - b.h / 2.)*sized.rows;
1251 int height = b.h*sized.rows;
1252 cv::Rect roi(left, top, width, height);
1253 roi = roi & img_rect;
1254
1255 sized(roi).copyTo(dst(roi));
1256 }
1257 }
1258 dst.copyTo(sized);
1259 }
1260
1261 if (gaussian_noise) {
1262 cv::Mat noise = cv::Mat(sized.size(), sized.type());
1263 gaussian_noise = std::min(gaussian_noise, 127);
1264 gaussian_noise = std::max(gaussian_noise, 0);
1265 cv::randn(noise, 0, gaussian_noise); //mean and variance
1266 cv::Mat sized_norm = sized + noise;
1267 //cv::normalize(sized_norm, sized_norm, 0.0, 255.0, cv::NORM_MINMAX, sized.type());
1268 //cv::imshow("source", sized);
1269 //cv::imshow("gaussian noise", sized_norm);
1270 //cv::waitKey(0);
1271 sized = sized_norm;
1272 }
1273
1274 //char txt[100];
1275 //sprintf(txt, "blur = %d", blur);
1276 //cv::putText(sized, txt, cv::Point(100, 100), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.7, CV_RGB(255, 0, 0), 1, CV_AA);
1277
1278 // Mat -> image
1279 out = mat_to_image(sized);
1280 }
1281 catch (...) {
1282 cerr << "OpenCV can't augment image: " << w << " x " << h << " \n";
1283 out = mat_to_image(*(cv::Mat*)mat);
1284 }
1285 return out;
1286 }
1287
1288 // blend two images with (alpha and beta)
blend_images_cv(image new_img,float alpha,image old_img,float beta)1289 extern "C" void blend_images_cv(image new_img, float alpha, image old_img, float beta)
1290 {
1291 cv::Mat new_mat(cv::Size(new_img.w, new_img.h), CV_32FC(new_img.c), new_img.data);// , size_t step = AUTO_STEP)
1292 cv::Mat old_mat(cv::Size(old_img.w, old_img.h), CV_32FC(old_img.c), old_img.data);
1293 cv::addWeighted(new_mat, alpha, old_mat, beta, 0.0, new_mat);
1294 }
1295
1296 // bilateralFilter bluring
blur_image(image src_img,int ksize)1297 extern "C" image blur_image(image src_img, int ksize)
1298 {
1299 cv::Mat src = image_to_mat(src_img);
1300 cv::Mat dst;
1301 cv::Size kernel_size = cv::Size(ksize, ksize);
1302 cv::GaussianBlur(src, dst, kernel_size, 0);
1303 //cv::bilateralFilter(src, dst, ksize, 75, 75);
1304 image dst_img = mat_to_image(dst);
1305 return dst_img;
1306 }
1307
1308 // ====================================================================
1309 // Draw object - adversarial attack dnn
1310 // ====================================================================
1311
1312 std::atomic<int> x_start, y_start;
1313 std::atomic<int> x_end, y_end;
1314 std::atomic<int> x_size, y_size;
1315 std::atomic<bool> draw_select, selected;
1316
callback_mouse_click(int event,int x,int y,int flags,void * user_data)1317 void callback_mouse_click(int event, int x, int y, int flags, void* user_data)
1318 {
1319 if (event == cv::EVENT_LBUTTONDOWN)
1320 {
1321 draw_select = true;
1322 selected = false;
1323 x_start = x;
1324 y_start = y;
1325
1326 //if (prev_img_rect.contains(Point2i(x, y))) add_id_img = -1;
1327 //else if (next_img_rect.contains(Point2i(x, y))) add_id_img = 1;
1328 //else add_id_img = 0;
1329 //std::cout << "cv::EVENT_LBUTTONDOWN \n";
1330 }
1331 else if (event == cv::EVENT_LBUTTONUP)
1332 {
1333 x_size = abs(x - x_start);
1334 y_size = abs(y - y_start);
1335 x_end = std::max(x, 0);
1336 y_end = std::max(y, 0);
1337 draw_select = false;
1338 selected = true;
1339 //std::cout << "cv::EVENT_LBUTTONUP \n";
1340 }
1341 else if (event == cv::EVENT_MOUSEMOVE)
1342 {
1343 x_size = abs(x - x_start);
1344 y_size = abs(y - y_start);
1345 x_end = std::max(x, 0);
1346 y_end = std::max(y, 0);
1347 }
1348 }
1349
cv_draw_object(image sized,float * truth_cpu,int max_boxes,int num_truth,int * it_num_set,float * lr_set,int * boxonly,int classes,char ** names)1350 extern "C" void cv_draw_object(image sized, float *truth_cpu, int max_boxes, int num_truth, int *it_num_set, float *lr_set, int *boxonly, int classes, char **names)
1351 {
1352 cv::Mat frame = image_to_mat(sized);
1353 if(frame.channels() == 3) cv::cvtColor(frame, frame, cv::COLOR_RGB2BGR);
1354 cv::Mat frame_clone = frame.clone();
1355
1356
1357 std::string const window_name = "Marking image";
1358 cv::namedWindow(window_name, cv::WINDOW_NORMAL);
1359 cv::resizeWindow(window_name, 1280, 720);
1360 cv::imshow(window_name, frame);
1361 cv::moveWindow(window_name, 0, 0);
1362 cv::setMouseCallback(window_name, callback_mouse_click);
1363
1364
1365 int it_trackbar_value = 200;
1366 std::string const it_trackbar_name = "iterations";
1367 int it_tb_res = cv::createTrackbar(it_trackbar_name, window_name, &it_trackbar_value, 1000);
1368
1369 int lr_trackbar_value = 10;
1370 std::string const lr_trackbar_name = "learning_rate exp";
1371 int lr_tb_res = cv::createTrackbar(lr_trackbar_name, window_name, &lr_trackbar_value, 20);
1372
1373 int cl_trackbar_value = 0;
1374 std::string const cl_trackbar_name = "class_id";
1375 int cl_tb_res = cv::createTrackbar(cl_trackbar_name, window_name, &cl_trackbar_value, classes-1);
1376
1377 std::string const bo_trackbar_name = "box-only";
1378 int bo_tb_res = cv::createTrackbar(bo_trackbar_name, window_name, boxonly, 1);
1379
1380 int i = 0;
1381
1382 while (!selected) {
1383 #ifndef CV_VERSION_EPOCH
1384 int pressed_key = cv::waitKeyEx(20); // OpenCV 3.x
1385 #else
1386 int pressed_key = cv::waitKey(20); // OpenCV 2.x
1387 #endif
1388 if (pressed_key == 27 || pressed_key == 1048603) break;// break; // ESC - save & exit
1389
1390 frame_clone = frame.clone();
1391 char buff[100];
1392 std::string lr_value = "learning_rate = " + std::to_string(1.0 / pow(2, lr_trackbar_value));
1393 cv::putText(frame_clone, lr_value, cv::Point2i(10, 20), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(10, 50, 10), 3);
1394 cv::putText(frame_clone, lr_value, cv::Point2i(10, 20), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(20, 120, 60), 2);
1395 cv::putText(frame_clone, lr_value, cv::Point2i(10, 20), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(50, 200, 100), 1);
1396
1397 if (names) {
1398 std::string obj_name = names[cl_trackbar_value];
1399 cv::putText(frame_clone, obj_name, cv::Point2i(10, 40), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(10, 50, 10), 3);
1400 cv::putText(frame_clone, obj_name, cv::Point2i(10, 40), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(20, 120, 60), 2);
1401 cv::putText(frame_clone, obj_name, cv::Point2i(10, 40), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(50, 200, 100), 1);
1402 }
1403
1404 if (draw_select) {
1405 cv::Rect selected_rect(
1406 cv::Point2i((int)min(x_start, x_end), (int)min(y_start, y_end)),
1407 cv::Size(x_size, y_size));
1408
1409 rectangle(frame_clone, selected_rect, cv::Scalar(150, 200, 150));
1410 }
1411
1412
1413 cv::imshow(window_name, frame_clone);
1414 }
1415
1416 if (selected) {
1417 cv::Rect selected_rect(
1418 cv::Point2i((int)min(x_start, x_end), (int)min(y_start, y_end)),
1419 cv::Size(x_size, y_size));
1420
1421 printf(" x_start = %d, y_start = %d, x_size = %d, y_size = %d \n",
1422 x_start.load(), y_start.load(), x_size.load(), y_size.load());
1423
1424 rectangle(frame, selected_rect, cv::Scalar(150, 200, 150));
1425 cv::imshow(window_name, frame);
1426 cv::waitKey(100);
1427
1428 float width = x_end - x_start;
1429 float height = y_end - y_start;
1430
1431 float const relative_center_x = (float)(x_start + width / 2) / frame.cols;
1432 float const relative_center_y = (float)(y_start + height / 2) / frame.rows;
1433 float const relative_width = (float)width / frame.cols;
1434 float const relative_height = (float)height / frame.rows;
1435
1436 truth_cpu[i * 5 + 0] = relative_center_x;
1437 truth_cpu[i * 5 + 1] = relative_center_y;
1438 truth_cpu[i * 5 + 2] = relative_width;
1439 truth_cpu[i * 5 + 3] = relative_height;
1440 truth_cpu[i * 5 + 4] = cl_trackbar_value;
1441 }
1442
1443 *it_num_set = it_trackbar_value;
1444 *lr_set = 1.0 / pow(2, lr_trackbar_value);
1445 }
1446
1447 // ====================================================================
1448 // Show Anchors
1449 // ====================================================================
show_acnhors(int number_of_boxes,int num_of_clusters,float * rel_width_height_array,model anchors_data,int width,int height)1450 extern "C" void show_acnhors(int number_of_boxes, int num_of_clusters, float *rel_width_height_array, model anchors_data, int width, int height)
1451 {
1452 cv::Mat labels = cv::Mat(number_of_boxes, 1, CV_32SC1);
1453 cv::Mat points = cv::Mat(number_of_boxes, 2, CV_32FC1);
1454 cv::Mat centers = cv::Mat(num_of_clusters, 2, CV_32FC1);
1455
1456 for (int i = 0; i < number_of_boxes; ++i) {
1457 points.at<float>(i, 0) = rel_width_height_array[i * 2];
1458 points.at<float>(i, 1) = rel_width_height_array[i * 2 + 1];
1459 }
1460
1461 for (int i = 0; i < num_of_clusters; ++i) {
1462 centers.at<float>(i, 0) = anchors_data.centers.vals[i][0];
1463 centers.at<float>(i, 1) = anchors_data.centers.vals[i][1];
1464 }
1465
1466 for (int i = 0; i < number_of_boxes; ++i) {
1467 labels.at<int>(i, 0) = anchors_data.assignments[i];
1468 }
1469
1470 size_t img_size = 700;
1471 cv::Mat img = cv::Mat(img_size, img_size, CV_8UC3);
1472
1473 for (int i = 0; i < number_of_boxes; ++i) {
1474 cv::Point pt;
1475 pt.x = points.at<float>(i, 0) * img_size / width;
1476 pt.y = points.at<float>(i, 1) * img_size / height;
1477 int cluster_idx = labels.at<int>(i, 0);
1478 int red_id = (cluster_idx * (uint64_t)123 + 55) % 255;
1479 int green_id = (cluster_idx * (uint64_t)321 + 33) % 255;
1480 int blue_id = (cluster_idx * (uint64_t)11 + 99) % 255;
1481 cv::circle(img, pt, 1, CV_RGB(red_id, green_id, blue_id), CV_FILLED, 8, 0);
1482 //if(pt.x > img_size || pt.y > img_size) printf("\n pt.x = %d, pt.y = %d \n", pt.x, pt.y);
1483 }
1484
1485 for (int j = 0; j < num_of_clusters; ++j) {
1486 cv::Point pt1, pt2;
1487 pt1.x = pt1.y = 0;
1488 pt2.x = centers.at<float>(j, 0) * img_size / width;
1489 pt2.y = centers.at<float>(j, 1) * img_size / height;
1490 cv::rectangle(img, pt1, pt2, CV_RGB(255, 255, 255), 1, 8, 0);
1491 }
1492 save_mat_png(img, "cloud.png");
1493 cv::imshow("clusters", img);
1494 cv::waitKey(0);
1495 cv::destroyAllWindows();
1496 }
1497
show_opencv_info()1498 void show_opencv_info()
1499 {
1500 std::cerr << " OpenCV version: " << CV_VERSION_MAJOR << "." << CV_VERSION_MINOR << "." << CVAUX_STR(CV_VERSION_REVISION) OCV_D
1501 << std::endl;
1502 }
1503
1504
1505
1506 } // extern "C"
1507 #else // OPENCV
show_opencv_info()1508 extern "C" void show_opencv_info()
1509 {
1510 std::cerr << " OpenCV isn't used - data augmentation will be slow \n";
1511 }
wait_key_cv(int delay)1512 extern "C" int wait_key_cv(int delay) { return 0; }
wait_until_press_key_cv()1513 extern "C" int wait_until_press_key_cv() { return 0; }
destroy_all_windows_cv()1514 extern "C" void destroy_all_windows_cv() {}
1515 #endif // OPENCV
1516