1 /**
2  * @file
3  * @brief Header file for CVObjectDetection class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  * @author Brenno Caldato <brenno.caldato@outlook.com>
6  *
7  * @ref License
8  */
9 
10 /* LICENSE
11  *
12  * Copyright (c) 2008-2019 OpenShot Studios, LLC
13  * <http://www.openshotstudios.com/>. This file is part of
14  * OpenShot Library (libopenshot), an open-source project dedicated to
15  * delivering high quality video editing and animation solutions to the
16  * world. For more information visit <http://www.openshot.org/>.
17  *
18  * OpenShot Library (libopenshot) is free software: you can redistribute it
19  * and/or modify it under the terms of the GNU Lesser General Public License
20  * as published by the Free Software Foundation, either version 3 of the
21  * License, or (at your option) any later version.
22  *
23  * OpenShot Library (libopenshot) is distributed in the hope that it will be
24  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30  */
31 
32 #pragma once
33 
34 #define int64 opencv_broken_int
35 #define uint64 opencv_broken_uint
36 #include <opencv2/dnn.hpp>
37 #include <opencv2/opencv.hpp>
38 #include <opencv2/core.hpp>
39 #undef uint64
40 #undef int64
41 #include "Json.h"
42 #include "ProcessingController.h"
43 #include "Clip.h"
44 #include "protobuf_messages/objdetectdata.pb.h"
45 
46 #include "sort_filter/sort.hpp"
47 
48 namespace openshot
49 {
50     // Stores the detected object bounding boxes and its properties.
51     struct CVDetectionData{
CVDetectionDataCVDetectionData52         CVDetectionData(){}
CVDetectionDataCVDetectionData53         CVDetectionData(
54             std::vector<int> _classIds,
55             std::vector<float> _confidences,
56             std::vector<cv::Rect_<float>> _boxes,
57             size_t _frameId,
58             std::vector<int> _objectIds)
59         {
60             classIds = _classIds;
61             confidences = _confidences;
62             boxes = _boxes;
63             frameId = _frameId;
64             objectIds = _objectIds;
65         }
66         size_t frameId;
67         std::vector<int> classIds;
68         std::vector<float> confidences;
69         std::vector<cv::Rect_<float>> boxes;
70         std::vector<int> objectIds;
71     };
72 
73     /**
74      * @brief This class runs trought a clip to detect objects and returns the bounding boxes and its properties.
75      *
76      * Object detection is performed using YoloV3 model with OpenCV DNN module
77      */
78     class CVObjectDetection{
79 
80         private:
81 
82         cv::dnn::Net net;
83         std::vector<std::string> classNames;
84         float confThreshold, nmsThreshold;
85 
86         std::string classesFile;
87         std::string modelConfiguration;
88         std::string modelWeights;
89         std::string processingDevice;
90         std::string protobuf_data_path;
91 
92         SortTracker sort;
93 
94         uint progress;
95 
96         size_t start;
97         size_t end;
98 
99         bool error = false;
100 
101         /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes
102         ProcessingController *processingController;
103 
104         void setProcessingDevice();
105 
106         // Detect onbects on a single frame
107         void DetectObjects(const cv::Mat &frame, size_t frame_number);
108 
109         bool iou(cv::Rect pred_box, cv::Rect sort_box);
110 
111         // Remove the bounding boxes with low confidence using non-maxima suppression
112         void postprocess(const cv::Size &frameDims, const std::vector<cv::Mat>& out, size_t frame_number);
113 
114         // Get the names of the output layers
115         std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net);
116 
117         public:
118 
119         std::map<size_t, CVDetectionData> detectionsData;
120 
121         CVObjectDetection(std::string processInfoJson, ProcessingController &processingController);
122 
123         // Iterate over a clip object and run inference for each video frame
124         void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false);
125 
126         CVDetectionData GetDetectionData(size_t frameId);
127 
128         /// Protobuf Save and Load methods
129         // Save protobuf file
130         bool SaveObjDetectedData();
131         // Add frame object detection data into protobuf message.
132         void AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, CVDetectionData& dData);
133 
134         // Get and Set JSON methods
135         void SetJson(const std::string value); ///< Load JSON string into this object
136         void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
137 
138         // Load protobuf file (ONLY FOR MAKE TEST)
139         bool _LoadObjDetectdData();
140     };
141 
142 }
143