1 #include <iostream>
2 #include <iomanip>
3 #include <string>
4 #include <vector>
5 #include <queue>
6 #include <fstream>
7 #include <thread>
8 #include <future>
9 #include <atomic>
10 #include <mutex> // std::mutex, std::unique_lock
11 #include <cmath>
12
13
14 // It makes sense only for video-Camera (not for video-File)
15 // To use - uncomment the following line. Optical-flow is supported only by OpenCV 3.x - 4.x
16 //#define TRACK_OPTFLOW
17 //#define GPU
18
19 // To use 3D-stereo camera ZED - uncomment the following line. ZED_SDK should be installed.
20 //#define ZED_STEREO
21
22
23 #include "yolo_v2_class.hpp" // imported functions from DLL
24
25 #ifdef OPENCV
26 #ifdef ZED_STEREO
27 #include <sl/Camera.hpp>
28 #if ZED_SDK_MAJOR_VERSION == 2
29 #define ZED_STEREO_2_COMPAT_MODE
30 #endif
31
32 #undef GPU // avoid conflict with sl::MEM::GPU
33
34 #ifdef ZED_STEREO_2_COMPAT_MODE
35 #pragma comment(lib, "sl_core64.lib")
36 #pragma comment(lib, "sl_input64.lib")
37 #endif
38 #pragma comment(lib, "sl_zed64.lib")
39
getMedian(std::vector<float> & v)40 float getMedian(std::vector<float> &v) {
41 size_t n = v.size() / 2;
42 std::nth_element(v.begin(), v.begin() + n, v.end());
43 return v[n];
44 }
45
get_3d_coordinates(std::vector<bbox_t> bbox_vect,cv::Mat xyzrgba)46 std::vector<bbox_t> get_3d_coordinates(std::vector<bbox_t> bbox_vect, cv::Mat xyzrgba)
47 {
48 bool valid_measure;
49 int i, j;
50 const unsigned int R_max_global = 10;
51
52 std::vector<bbox_t> bbox3d_vect;
53
54 for (auto &cur_box : bbox_vect) {
55
56 const unsigned int obj_size = std::min(cur_box.w, cur_box.h);
57 const unsigned int R_max = std::min(R_max_global, obj_size / 2);
58 int center_i = cur_box.x + cur_box.w * 0.5f, center_j = cur_box.y + cur_box.h * 0.5f;
59
60 std::vector<float> x_vect, y_vect, z_vect;
61 for (int R = 0; R < R_max; R++) {
62 for (int y = -R; y <= R; y++) {
63 for (int x = -R; x <= R; x++) {
64 i = center_i + x;
65 j = center_j + y;
66 sl::float4 out(NAN, NAN, NAN, NAN);
67 if (i >= 0 && i < xyzrgba.cols && j >= 0 && j < xyzrgba.rows) {
68 cv::Vec4f &elem = xyzrgba.at<cv::Vec4f>(j, i); // x,y,z,w
69 out.x = elem[0];
70 out.y = elem[1];
71 out.z = elem[2];
72 out.w = elem[3];
73 }
74 valid_measure = std::isfinite(out.z);
75 if (valid_measure)
76 {
77 x_vect.push_back(out.x);
78 y_vect.push_back(out.y);
79 z_vect.push_back(out.z);
80 }
81 }
82 }
83 }
84
85 if (x_vect.size() * y_vect.size() * z_vect.size() > 0)
86 {
87 cur_box.x_3d = getMedian(x_vect);
88 cur_box.y_3d = getMedian(y_vect);
89 cur_box.z_3d = getMedian(z_vect);
90 }
91 else {
92 cur_box.x_3d = NAN;
93 cur_box.y_3d = NAN;
94 cur_box.z_3d = NAN;
95 }
96
97 bbox3d_vect.emplace_back(cur_box);
98 }
99
100 return bbox3d_vect;
101 }
102
slMat2cvMat(sl::Mat & input)103 cv::Mat slMat2cvMat(sl::Mat &input) {
104 int cv_type = -1; // Mapping between MAT_TYPE and CV_TYPE
105 if(input.getDataType() ==
106 #ifdef ZED_STEREO_2_COMPAT_MODE
107 sl::MAT_TYPE_32F_C4
108 #else
109 sl::MAT_TYPE::F32_C4
110 #endif
111 ) {
112 cv_type = CV_32FC4;
113 } else cv_type = CV_8UC4; // sl::Mat used are either RGBA images or XYZ (4C) point clouds
114 return cv::Mat(input.getHeight(), input.getWidth(), cv_type, input.getPtr<sl::uchar1>(
115 #ifdef ZED_STEREO_2_COMPAT_MODE
116 sl::MEM::MEM_CPU
117 #else
118 sl::MEM::CPU
119 #endif
120 ));
121 }
122
zed_capture_rgb(sl::Camera & zed)123 cv::Mat zed_capture_rgb(sl::Camera &zed) {
124 sl::Mat left;
125 zed.retrieveImage(left);
126 cv::Mat left_rgb;
127 cv::cvtColor(slMat2cvMat(left), left_rgb, CV_RGBA2RGB);
128 return left_rgb;
129 }
130
zed_capture_3d(sl::Camera & zed)131 cv::Mat zed_capture_3d(sl::Camera &zed) {
132 sl::Mat cur_cloud;
133 zed.retrieveMeasure(cur_cloud,
134 #ifdef ZED_STEREO_2_COMPAT_MODE
135 sl::MEASURE_XYZ
136 #else
137 sl::MEASURE::XYZ
138 #endif
139 );
140 return slMat2cvMat(cur_cloud).clone();
141 }
142
143 static sl::Camera zed; // ZED-camera
144
145 #else // ZED_STEREO
get_3d_coordinates(std::vector<bbox_t> bbox_vect,cv::Mat xyzrgba)146 std::vector<bbox_t> get_3d_coordinates(std::vector<bbox_t> bbox_vect, cv::Mat xyzrgba) {
147 return bbox_vect;
148 }
149 #endif // ZED_STEREO
150
151
152 #include <opencv2/opencv.hpp> // C++
153 #include <opencv2/core/version.hpp>
154 #ifndef CV_VERSION_EPOCH // OpenCV 3.x and 4.x
155 #include <opencv2/videoio/videoio.hpp>
156 #define OPENCV_VERSION CVAUX_STR(CV_VERSION_MAJOR)"" CVAUX_STR(CV_VERSION_MINOR)"" CVAUX_STR(CV_VERSION_REVISION)
157 #ifndef USE_CMAKE_LIBS
158 #pragma comment(lib, "opencv_world" OPENCV_VERSION ".lib")
159 #ifdef TRACK_OPTFLOW
160 /*
161 #pragma comment(lib, "opencv_cudaoptflow" OPENCV_VERSION ".lib")
162 #pragma comment(lib, "opencv_cudaimgproc" OPENCV_VERSION ".lib")
163 #pragma comment(lib, "opencv_core" OPENCV_VERSION ".lib")
164 #pragma comment(lib, "opencv_imgproc" OPENCV_VERSION ".lib")
165 #pragma comment(lib, "opencv_highgui" OPENCV_VERSION ".lib")
166 */
167 #endif // TRACK_OPTFLOW
168 #endif // USE_CMAKE_LIBS
169 #else // OpenCV 2.x
170 #define OPENCV_VERSION CVAUX_STR(CV_VERSION_EPOCH)"" CVAUX_STR(CV_VERSION_MAJOR)"" CVAUX_STR(CV_VERSION_MINOR)
171 #ifndef USE_CMAKE_LIBS
172 #pragma comment(lib, "opencv_core" OPENCV_VERSION ".lib")
173 #pragma comment(lib, "opencv_imgproc" OPENCV_VERSION ".lib")
174 #pragma comment(lib, "opencv_highgui" OPENCV_VERSION ".lib")
175 #pragma comment(lib, "opencv_video" OPENCV_VERSION ".lib")
176 #endif // USE_CMAKE_LIBS
177 #endif // CV_VERSION_EPOCH
178
179
draw_boxes(cv::Mat mat_img,std::vector<bbox_t> result_vec,std::vector<std::string> obj_names,int current_det_fps=-1,int current_cap_fps=-1)180 void draw_boxes(cv::Mat mat_img, std::vector<bbox_t> result_vec, std::vector<std::string> obj_names,
181 int current_det_fps = -1, int current_cap_fps = -1)
182 {
183 int const colors[6][3] = { { 1,0,1 },{ 0,0,1 },{ 0,1,1 },{ 0,1,0 },{ 1,1,0 },{ 1,0,0 } };
184
185 for (auto &i : result_vec) {
186 cv::Scalar color = obj_id_to_color(i.obj_id);
187 cv::rectangle(mat_img, cv::Rect(i.x, i.y, i.w, i.h), color, 2);
188 if (obj_names.size() > i.obj_id) {
189 std::string obj_name = obj_names[i.obj_id];
190 if (i.track_id > 0) obj_name += " - " + std::to_string(i.track_id);
191 cv::Size const text_size = getTextSize(obj_name, cv::FONT_HERSHEY_COMPLEX_SMALL, 1.2, 2, 0);
192 int max_width = (text_size.width > i.w + 2) ? text_size.width : (i.w + 2);
193 max_width = std::max(max_width, (int)i.w + 2);
194 //max_width = std::max(max_width, 283);
195 std::string coords_3d;
196 if (!std::isnan(i.z_3d)) {
197 std::stringstream ss;
198 ss << std::fixed << std::setprecision(2) << "x:" << i.x_3d << "m y:" << i.y_3d << "m z:" << i.z_3d << "m ";
199 coords_3d = ss.str();
200 cv::Size const text_size_3d = getTextSize(ss.str(), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.8, 1, 0);
201 int const max_width_3d = (text_size_3d.width > i.w + 2) ? text_size_3d.width : (i.w + 2);
202 if (max_width_3d > max_width) max_width = max_width_3d;
203 }
204
205 cv::rectangle(mat_img, cv::Point2f(std::max((int)i.x - 1, 0), std::max((int)i.y - 35, 0)),
206 cv::Point2f(std::min((int)i.x + max_width, mat_img.cols - 1), std::min((int)i.y, mat_img.rows - 1)),
207 color, CV_FILLED, 8, 0);
208 putText(mat_img, obj_name, cv::Point2f(i.x, i.y - 16), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.2, cv::Scalar(0, 0, 0), 2);
209 if(!coords_3d.empty()) putText(mat_img, coords_3d, cv::Point2f(i.x, i.y-1), cv::FONT_HERSHEY_COMPLEX_SMALL, 0.8, cv::Scalar(0, 0, 0), 1);
210 }
211 }
212 if (current_det_fps >= 0 && current_cap_fps >= 0) {
213 std::string fps_str = "FPS detection: " + std::to_string(current_det_fps) + " FPS capture: " + std::to_string(current_cap_fps);
214 putText(mat_img, fps_str, cv::Point2f(10, 20), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.2, cv::Scalar(50, 255, 0), 2);
215 }
216 }
217 #endif // OPENCV
218
219
show_console_result(std::vector<bbox_t> const result_vec,std::vector<std::string> const obj_names,int frame_id=-1)220 void show_console_result(std::vector<bbox_t> const result_vec, std::vector<std::string> const obj_names, int frame_id = -1) {
221 if (frame_id >= 0) std::cout << " Frame: " << frame_id << std::endl;
222 for (auto &i : result_vec) {
223 if (obj_names.size() > i.obj_id) std::cout << obj_names[i.obj_id] << " - ";
224 std::cout << "obj_id = " << i.obj_id << ", x = " << i.x << ", y = " << i.y
225 << ", w = " << i.w << ", h = " << i.h
226 << std::setprecision(3) << ", prob = " << i.prob << std::endl;
227 }
228 }
229
objects_names_from_file(std::string const filename)230 std::vector<std::string> objects_names_from_file(std::string const filename) {
231 std::ifstream file(filename);
232 std::vector<std::string> file_lines;
233 if (!file.is_open()) return file_lines;
234 for(std::string line; getline(file, line);) file_lines.push_back(line);
235 std::cout << "object names loaded \n";
236 return file_lines;
237 }
238
239 template<typename T>
240 class send_one_replaceable_object_t {
241 const bool sync;
242 std::atomic<T *> a_ptr;
243 public:
244
send(T const & _obj)245 void send(T const& _obj) {
246 T *new_ptr = new T;
247 *new_ptr = _obj;
248 if (sync) {
249 while (a_ptr.load()) std::this_thread::sleep_for(std::chrono::milliseconds(3));
250 }
251 std::unique_ptr<T> old_ptr(a_ptr.exchange(new_ptr));
252 }
253
receive()254 T receive() {
255 std::unique_ptr<T> ptr;
256 do {
257 while(!a_ptr.load()) std::this_thread::sleep_for(std::chrono::milliseconds(3));
258 ptr.reset(a_ptr.exchange(NULL));
259 } while (!ptr);
260 T obj = *ptr;
261 return obj;
262 }
263
is_object_present()264 bool is_object_present() {
265 return (a_ptr.load() != NULL);
266 }
267
send_one_replaceable_object_t(bool _sync)268 send_one_replaceable_object_t(bool _sync) : sync(_sync), a_ptr(NULL)
269 {}
270 };
271
main(int argc,char * argv[])272 int main(int argc, char *argv[])
273 {
274 std::string names_file = "data/coco.names";
275 std::string cfg_file = "cfg/yolov3.cfg";
276 std::string weights_file = "yolov3.weights";
277 std::string filename;
278
279 if (argc > 4) { //voc.names yolo-voc.cfg yolo-voc.weights test.mp4
280 names_file = argv[1];
281 cfg_file = argv[2];
282 weights_file = argv[3];
283 filename = argv[4];
284 }
285 else if (argc > 1) filename = argv[1];
286
287 float const thresh = (argc > 5) ? std::stof(argv[5]) : 0.2;
288
289 Detector detector(cfg_file, weights_file);
290
291 auto obj_names = objects_names_from_file(names_file);
292 std::string out_videofile = "result.avi";
293 bool const save_output_videofile = false; // true - for history
294 bool const send_network = false; // true - for remote detection
295 bool const use_kalman_filter = false; // true - for stationary camera
296
297 bool detection_sync = true; // true - for video-file
298 #ifdef TRACK_OPTFLOW // for slow GPU
299 detection_sync = false;
300 Tracker_optflow tracker_flow;
301 //detector.wait_stream = true;
302 #endif // TRACK_OPTFLOW
303
304
305 while (true)
306 {
307 std::cout << "input image or video filename: ";
308 if(filename.size() == 0) std::cin >> filename;
309 if (filename.size() == 0) break;
310
311 try {
312 #ifdef OPENCV
313 preview_boxes_t large_preview(100, 150, false), small_preview(50, 50, true);
314 bool show_small_boxes = false;
315
316 std::string const file_ext = filename.substr(filename.find_last_of(".") + 1);
317 std::string const protocol = filename.substr(0, 7);
318 if (file_ext == "avi" || file_ext == "mp4" || file_ext == "mjpg" || file_ext == "mov" || // video file
319 protocol == "rtmp://" || protocol == "rtsp://" || protocol == "http://" || protocol == "https:/" || // video network stream
320 filename == "zed_camera" || file_ext == "svo" || filename == "web_camera") // ZED stereo camera
321
322 {
323 if (protocol == "rtsp://" || protocol == "http://" || protocol == "https:/" || filename == "zed_camera" || filename == "web_camera")
324 detection_sync = false;
325
326 cv::Mat cur_frame;
327 std::atomic<int> fps_cap_counter(0), fps_det_counter(0);
328 std::atomic<int> current_fps_cap(0), current_fps_det(0);
329 std::atomic<bool> exit_flag(false);
330 std::chrono::steady_clock::time_point steady_start, steady_end;
331 int video_fps = 25;
332 bool use_zed_camera = false;
333
334 track_kalman_t track_kalman;
335
336 #ifdef ZED_STEREO
337 sl::InitParameters init_params;
338 init_params.depth_minimum_distance = 0.5;
339 #ifdef ZED_STEREO_2_COMPAT_MODE
340 init_params.depth_mode = sl::DEPTH_MODE_ULTRA;
341 init_params.camera_resolution = sl::RESOLUTION_HD720;// sl::RESOLUTION_HD1080, sl::RESOLUTION_HD720
342 init_params.coordinate_units = sl::UNIT_METER;
343 init_params.camera_buffer_count_linux = 2;
344 if (file_ext == "svo") init_params.svo_input_filename.set(filename.c_str());
345 #else
346 init_params.depth_mode = sl::DEPTH_MODE::ULTRA;
347 init_params.camera_resolution = sl::RESOLUTION::HD720;// sl::RESOLUTION::HD1080, sl::RESOLUTION::HD720
348 init_params.coordinate_units = sl::UNIT::METER;
349 if (file_ext == "svo") init_params.input.setFromSVOFile(filename.c_str());
350 #endif
351 //init_params.sdk_cuda_ctx = (CUcontext)detector.get_cuda_context();
352 init_params.sdk_gpu_id = detector.cur_gpu_id;
353
354 if (filename == "zed_camera" || file_ext == "svo") {
355 std::cout << "ZED 3D Camera " << zed.open(init_params) << std::endl;
356 if (!zed.isOpened()) {
357 std::cout << " Error: ZED Camera should be connected to USB 3.0. And ZED_SDK should be installed. \n";
358 getchar();
359 return 0;
360 }
361 cur_frame = zed_capture_rgb(zed);
362 use_zed_camera = true;
363 }
364 #endif // ZED_STEREO
365
366 cv::VideoCapture cap;
367 if (filename == "web_camera") {
368 cap.open(0);
369 cap >> cur_frame;
370 } else if (!use_zed_camera) {
371 cap.open(filename);
372 cap >> cur_frame;
373 }
374 #ifdef CV_VERSION_EPOCH // OpenCV 2.x
375 video_fps = cap.get(CV_CAP_PROP_FPS);
376 #else
377 video_fps = cap.get(cv::CAP_PROP_FPS);
378 #endif
379 cv::Size const frame_size = cur_frame.size();
380 //cv::Size const frame_size(cap.get(CV_CAP_PROP_FRAME_WIDTH), cap.get(CV_CAP_PROP_FRAME_HEIGHT));
381 std::cout << "\n Video size: " << frame_size << std::endl;
382
383 cv::VideoWriter output_video;
384 if (save_output_videofile)
385 #ifdef CV_VERSION_EPOCH // OpenCV 2.x
386 output_video.open(out_videofile, CV_FOURCC('D', 'I', 'V', 'X'), std::max(35, video_fps), frame_size, true);
387 #else
388 output_video.open(out_videofile, cv::VideoWriter::fourcc('D', 'I', 'V', 'X'), std::max(35, video_fps), frame_size, true);
389 #endif
390
391 struct detection_data_t {
392 cv::Mat cap_frame;
393 std::shared_ptr<image_t> det_image;
394 std::vector<bbox_t> result_vec;
395 cv::Mat draw_frame;
396 bool new_detection;
397 uint64_t frame_id;
398 bool exit_flag;
399 cv::Mat zed_cloud;
400 std::queue<cv::Mat> track_optflow_queue;
401 detection_data_t() : exit_flag(false), new_detection(false) {}
402 };
403
404 const bool sync = detection_sync; // sync data exchange
405 send_one_replaceable_object_t<detection_data_t> cap2prepare(sync), cap2draw(sync),
406 prepare2detect(sync), detect2draw(sync), draw2show(sync), draw2write(sync), draw2net(sync);
407
408 std::thread t_cap, t_prepare, t_detect, t_post, t_draw, t_write, t_network;
409
410 // capture new video-frame
411 if (t_cap.joinable()) t_cap.join();
412 t_cap = std::thread([&]()
413 {
414 uint64_t frame_id = 0;
415 detection_data_t detection_data;
416 do {
417 detection_data = detection_data_t();
418 #ifdef ZED_STEREO
419 if (use_zed_camera) {
420 while (zed.grab() !=
421 #ifdef ZED_STEREO_2_COMPAT_MODE
422 sl::SUCCESS
423 #else
424 sl::ERROR_CODE::SUCCESS
425 #endif
426 ) std::this_thread::sleep_for(std::chrono::milliseconds(2));
427 detection_data.cap_frame = zed_capture_rgb(zed);
428 detection_data.zed_cloud = zed_capture_3d(zed);
429 }
430 else
431 #endif // ZED_STEREO
432 {
433 cap >> detection_data.cap_frame;
434 }
435 fps_cap_counter++;
436 detection_data.frame_id = frame_id++;
437 if (detection_data.cap_frame.empty() || exit_flag) {
438 std::cout << " exit_flag: detection_data.cap_frame.size = " << detection_data.cap_frame.size() << std::endl;
439 detection_data.exit_flag = true;
440 detection_data.cap_frame = cv::Mat(frame_size, CV_8UC3);
441 }
442
443 if (!detection_sync) {
444 cap2draw.send(detection_data); // skip detection
445 }
446 cap2prepare.send(detection_data);
447 } while (!detection_data.exit_flag);
448 std::cout << " t_cap exit \n";
449 });
450
451
452 // pre-processing video frame (resize, convertion)
453 t_prepare = std::thread([&]()
454 {
455 std::shared_ptr<image_t> det_image;
456 detection_data_t detection_data;
457 do {
458 detection_data = cap2prepare.receive();
459
460 det_image = detector.mat_to_image_resize(detection_data.cap_frame);
461 detection_data.det_image = det_image;
462 prepare2detect.send(detection_data); // detection
463
464 } while (!detection_data.exit_flag);
465 std::cout << " t_prepare exit \n";
466 });
467
468
469 // detection by Yolo
470 if (t_detect.joinable()) t_detect.join();
471 t_detect = std::thread([&]()
472 {
473 std::shared_ptr<image_t> det_image;
474 detection_data_t detection_data;
475 do {
476 detection_data = prepare2detect.receive();
477 det_image = detection_data.det_image;
478 std::vector<bbox_t> result_vec;
479
480 if(det_image)
481 result_vec = detector.detect_resized(*det_image, frame_size.width, frame_size.height, thresh, true); // true
482 fps_det_counter++;
483 //std::this_thread::sleep_for(std::chrono::milliseconds(150));
484
485 detection_data.new_detection = true;
486 detection_data.result_vec = result_vec;
487 detect2draw.send(detection_data);
488 } while (!detection_data.exit_flag);
489 std::cout << " t_detect exit \n";
490 });
491
492 // draw rectangles (and track objects)
493 t_draw = std::thread([&]()
494 {
495 std::queue<cv::Mat> track_optflow_queue;
496 detection_data_t detection_data;
497 do {
498
499 // for Video-file
500 if (detection_sync) {
501 detection_data = detect2draw.receive();
502 }
503 // for Video-camera
504 else
505 {
506 // get new Detection result if present
507 if (detect2draw.is_object_present()) {
508 cv::Mat old_cap_frame = detection_data.cap_frame; // use old captured frame
509 detection_data = detect2draw.receive();
510 if (!old_cap_frame.empty()) detection_data.cap_frame = old_cap_frame;
511 }
512 // get new Captured frame
513 else {
514 std::vector<bbox_t> old_result_vec = detection_data.result_vec; // use old detections
515 detection_data = cap2draw.receive();
516 detection_data.result_vec = old_result_vec;
517 }
518 }
519
520 cv::Mat cap_frame = detection_data.cap_frame;
521 cv::Mat draw_frame = detection_data.cap_frame.clone();
522 std::vector<bbox_t> result_vec = detection_data.result_vec;
523
524 #ifdef TRACK_OPTFLOW
525 if (detection_data.new_detection) {
526 tracker_flow.update_tracking_flow(detection_data.cap_frame, detection_data.result_vec);
527 while (track_optflow_queue.size() > 0) {
528 draw_frame = track_optflow_queue.back();
529 result_vec = tracker_flow.tracking_flow(track_optflow_queue.front(), false);
530 track_optflow_queue.pop();
531 }
532 }
533 else {
534 track_optflow_queue.push(cap_frame);
535 result_vec = tracker_flow.tracking_flow(cap_frame, false);
536 }
537 detection_data.new_detection = true; // to correct kalman filter
538 #endif //TRACK_OPTFLOW
539
540 // track ID by using kalman filter
541 if (use_kalman_filter) {
542 if (detection_data.new_detection) {
543 result_vec = track_kalman.correct(result_vec);
544 }
545 else {
546 result_vec = track_kalman.predict();
547 }
548 }
549 // track ID by using custom function
550 else {
551 int frame_story = std::max(5, current_fps_cap.load());
552 result_vec = detector.tracking_id(result_vec, true, frame_story, 40);
553 }
554
555 if (use_zed_camera && !detection_data.zed_cloud.empty()) {
556 result_vec = get_3d_coordinates(result_vec, detection_data.zed_cloud);
557 }
558
559 //small_preview.set(draw_frame, result_vec);
560 //large_preview.set(draw_frame, result_vec);
561 draw_boxes(draw_frame, result_vec, obj_names, current_fps_det, current_fps_cap);
562 //show_console_result(result_vec, obj_names, detection_data.frame_id);
563 //large_preview.draw(draw_frame);
564 //small_preview.draw(draw_frame, true);
565
566 detection_data.result_vec = result_vec;
567 detection_data.draw_frame = draw_frame;
568 draw2show.send(detection_data);
569 if (send_network) draw2net.send(detection_data);
570 if (output_video.isOpened()) draw2write.send(detection_data);
571 } while (!detection_data.exit_flag);
572 std::cout << " t_draw exit \n";
573 });
574
575
576 // write frame to videofile
577 t_write = std::thread([&]()
578 {
579 if (output_video.isOpened()) {
580 detection_data_t detection_data;
581 cv::Mat output_frame;
582 do {
583 detection_data = draw2write.receive();
584 if(detection_data.draw_frame.channels() == 4) cv::cvtColor(detection_data.draw_frame, output_frame, CV_RGBA2RGB);
585 else output_frame = detection_data.draw_frame;
586 output_video << output_frame;
587 } while (!detection_data.exit_flag);
588 output_video.release();
589 }
590 std::cout << " t_write exit \n";
591 });
592
593 // send detection to the network
594 t_network = std::thread([&]()
595 {
596 if (send_network) {
597 detection_data_t detection_data;
598 do {
599 detection_data = draw2net.receive();
600
601 detector.send_json_http(detection_data.result_vec, obj_names, detection_data.frame_id, filename);
602
603 } while (!detection_data.exit_flag);
604 }
605 std::cout << " t_network exit \n";
606 });
607
608
609 // show detection
610 detection_data_t detection_data;
611 do {
612
613 steady_end = std::chrono::steady_clock::now();
614 float time_sec = std::chrono::duration<double>(steady_end - steady_start).count();
615 if (time_sec >= 1) {
616 current_fps_det = fps_det_counter.load() / time_sec;
617 current_fps_cap = fps_cap_counter.load() / time_sec;
618 steady_start = steady_end;
619 fps_det_counter = 0;
620 fps_cap_counter = 0;
621 }
622
623 detection_data = draw2show.receive();
624 cv::Mat draw_frame = detection_data.draw_frame;
625
626 //if (extrapolate_flag) {
627 // cv::putText(draw_frame, "extrapolate", cv::Point2f(10, 40), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.0, cv::Scalar(50, 50, 0), 2);
628 //}
629
630 cv::imshow("window name", draw_frame);
631 int key = cv::waitKey(3); // 3 or 16ms
632 if (key == 'f') show_small_boxes = !show_small_boxes;
633 if (key == 'p') while (true) if (cv::waitKey(100) == 'p') break;
634 //if (key == 'e') extrapolate_flag = !extrapolate_flag;
635 if (key == 27) { exit_flag = true;}
636
637 //std::cout << " current_fps_det = " << current_fps_det << ", current_fps_cap = " << current_fps_cap << std::endl;
638 } while (!detection_data.exit_flag);
639 std::cout << " show detection exit \n";
640
641 cv::destroyWindow("window name");
642 // wait for all threads
643 if (t_cap.joinable()) t_cap.join();
644 if (t_prepare.joinable()) t_prepare.join();
645 if (t_detect.joinable()) t_detect.join();
646 if (t_post.joinable()) t_post.join();
647 if (t_draw.joinable()) t_draw.join();
648 if (t_write.joinable()) t_write.join();
649 if (t_network.joinable()) t_network.join();
650
651 break;
652
653 }
654 else if (file_ext == "txt") { // list of image files
655 std::ifstream file(filename);
656 if (!file.is_open()) std::cout << "File not found! \n";
657 else
658 for (std::string line; file >> line;) {
659 std::cout << line << std::endl;
660 cv::Mat mat_img = cv::imread(line);
661 std::vector<bbox_t> result_vec = detector.detect(mat_img);
662 show_console_result(result_vec, obj_names);
663 //draw_boxes(mat_img, result_vec, obj_names);
664 //cv::imwrite("res_" + line, mat_img);
665 }
666
667 }
668 else { // image file
669 // to achive high performance for multiple images do these 2 lines in another thread
670 cv::Mat mat_img = cv::imread(filename);
671 auto det_image = detector.mat_to_image_resize(mat_img);
672
673 auto start = std::chrono::steady_clock::now();
674 std::vector<bbox_t> result_vec = detector.detect_resized(*det_image, mat_img.size().width, mat_img.size().height);
675 auto end = std::chrono::steady_clock::now();
676 std::chrono::duration<double> spent = end - start;
677 std::cout << " Time: " << spent.count() << " sec \n";
678
679 //result_vec = detector.tracking_id(result_vec); // comment it - if track_id is not required
680 draw_boxes(mat_img, result_vec, obj_names);
681 cv::imshow("window name", mat_img);
682 show_console_result(result_vec, obj_names);
683 cv::waitKey(0);
684 }
685 #else // OPENCV
686 //std::vector<bbox_t> result_vec = detector.detect(filename);
687
688 auto img = detector.load_image(filename);
689 std::vector<bbox_t> result_vec = detector.detect(img);
690 detector.free_image(img);
691 show_console_result(result_vec, obj_names);
692 #endif // OPENCV
693 }
694 catch (std::exception &e) { std::cerr << "exception: " << e.what() << "\n"; getchar(); }
695 catch (...) { std::cerr << "unknown exception \n"; getchar(); }
696 filename.clear();
697 }
698
699 return 0;
700 }
701