1 /** 2 * @file 3 * @brief Header file for CVObjectDetection class 4 * @author Jonathan Thomas <jonathan@openshot.org> 5 * @author Brenno Caldato <brenno.caldato@outlook.com> 6 * 7 * @ref License 8 */ 9 10 /* LICENSE 11 * 12 * Copyright (c) 2008-2019 OpenShot Studios, LLC 13 * <http://www.openshotstudios.com/>. This file is part of 14 * OpenShot Library (libopenshot), an open-source project dedicated to 15 * delivering high quality video editing and animation solutions to the 16 * world. For more information visit <http://www.openshot.org/>. 17 * 18 * OpenShot Library (libopenshot) is free software: you can redistribute it 19 * and/or modify it under the terms of the GNU Lesser General Public License 20 * as published by the Free Software Foundation, either version 3 of the 21 * License, or (at your option) any later version. 22 * 23 * OpenShot Library (libopenshot) is distributed in the hope that it will be 24 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of 25 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26 * GNU Lesser General Public License for more details. 27 * 28 * You should have received a copy of the GNU Lesser General Public License 29 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>. 30 */ 31 32 #pragma once 33 34 #define int64 opencv_broken_int 35 #define uint64 opencv_broken_uint 36 #include <opencv2/dnn.hpp> 37 #include <opencv2/opencv.hpp> 38 #include <opencv2/core.hpp> 39 #undef uint64 40 #undef int64 41 #include "Json.h" 42 #include "ProcessingController.h" 43 #include "Clip.h" 44 #include "protobuf_messages/objdetectdata.pb.h" 45 46 #include "sort_filter/sort.hpp" 47 48 namespace openshot 49 { 50 // Stores the detected object bounding boxes and its properties. 51 struct CVDetectionData{ CVDetectionDataCVDetectionData52 CVDetectionData(){} CVDetectionDataCVDetectionData53 CVDetectionData( 54 std::vector<int> _classIds, 55 std::vector<float> _confidences, 56 std::vector<cv::Rect_<float>> _boxes, 57 size_t _frameId, 58 std::vector<int> _objectIds) 59 { 60 classIds = _classIds; 61 confidences = _confidences; 62 boxes = _boxes; 63 frameId = _frameId; 64 objectIds = _objectIds; 65 } 66 size_t frameId; 67 std::vector<int> classIds; 68 std::vector<float> confidences; 69 std::vector<cv::Rect_<float>> boxes; 70 std::vector<int> objectIds; 71 }; 72 73 /** 74 * @brief This class runs trought a clip to detect objects and returns the bounding boxes and its properties. 75 * 76 * Object detection is performed using YoloV3 model with OpenCV DNN module 77 */ 78 class CVObjectDetection{ 79 80 private: 81 82 cv::dnn::Net net; 83 std::vector<std::string> classNames; 84 float confThreshold, nmsThreshold; 85 86 std::string classesFile; 87 std::string modelConfiguration; 88 std::string modelWeights; 89 std::string processingDevice; 90 std::string protobuf_data_path; 91 92 SortTracker sort; 93 94 uint progress; 95 96 size_t start; 97 size_t end; 98 99 bool error = false; 100 101 /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes 102 ProcessingController *processingController; 103 104 void setProcessingDevice(); 105 106 // Detect onbects on a single frame 107 void DetectObjects(const cv::Mat &frame, size_t frame_number); 108 109 bool iou(cv::Rect pred_box, cv::Rect sort_box); 110 111 // Remove the bounding boxes with low confidence using non-maxima suppression 112 void postprocess(const cv::Size &frameDims, const std::vector<cv::Mat>& out, size_t frame_number); 113 114 // Get the names of the output layers 115 std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net); 116 117 public: 118 119 std::map<size_t, CVDetectionData> detectionsData; 120 121 CVObjectDetection(std::string processInfoJson, ProcessingController &processingController); 122 123 // Iterate over a clip object and run inference for each video frame 124 void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false); 125 126 CVDetectionData GetDetectionData(size_t frameId); 127 128 /// Protobuf Save and Load methods 129 // Save protobuf file 130 bool SaveObjDetectedData(); 131 // Add frame object detection data into protobuf message. 132 void AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, CVDetectionData& dData); 133 134 // Get and Set JSON methods 135 void SetJson(const std::string value); ///< Load JSON string into this object 136 void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object 137 138 // Load protobuf file (ONLY FOR MAKE TEST) 139 bool _LoadObjDetectdData(); 140 }; 141 142 } 143