1 /**
2 * @file
3 * @brief Track an object selected by the user
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 * @author Brenno Caldato <brenno.caldato@outlook.com>
6 *
7 * @ref License
8 */
9
10 /* LICENSE
11 *
12 * Copyright (c) 2008-2019 OpenShot Studios, LLC
13 * <http://www.openshotstudios.com/>. This file is part of
14 * OpenShot Library (libopenshot), an open-source project dedicated to
15 * delivering high quality video editing and animation solutions to the
16 * world. For more information visit <http://www.openshot.org/>.
17 *
18 * OpenShot Library (libopenshot) is free software: you can redistribute it
19 * and/or modify it under the terms of the GNU Lesser General Public License
20 * as published by the Free Software Foundation, either version 3 of the
21 * License, or (at your option) any later version.
22 *
23 * OpenShot Library (libopenshot) is distributed in the hope that it will be
24 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 * GNU Lesser General Public License for more details.
27 *
28 * You should have received a copy of the GNU Lesser General Public License
29 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30 */
31
32 #include <fstream>
33 #include <iomanip>
34 #include <iostream>
35
36 #include <google/protobuf/util/time_util.h>
37
38 #include "OpenCVUtilities.h"
39 #include "CVTracker.h"
40
41 using namespace openshot;
42 using google::protobuf::util::TimeUtil;
43
44 // Constructor
CVTracker(std::string processInfoJson,ProcessingController & processingController)45 CVTracker::CVTracker(std::string processInfoJson, ProcessingController &processingController)
46 : processingController(&processingController), json_interval(false){
47 SetJson(processInfoJson);
48 start = 1;
49 end = 1;
50 }
51
52 // Set desirable tracker method
selectTracker(std::string trackerType)53 cv::Ptr<OPENCV_TRACKER_TYPE> CVTracker::selectTracker(std::string trackerType){
54
55 if (trackerType == "BOOSTING")
56 return OPENCV_TRACKER_NS::TrackerBoosting::create();
57 if (trackerType == "MIL")
58 return OPENCV_TRACKER_NS::TrackerMIL::create();
59 if (trackerType == "KCF")
60 return OPENCV_TRACKER_NS::TrackerKCF::create();
61 if (trackerType == "TLD")
62 return OPENCV_TRACKER_NS::TrackerTLD::create();
63 if (trackerType == "MEDIANFLOW")
64 return OPENCV_TRACKER_NS::TrackerMedianFlow::create();
65 if (trackerType == "MOSSE")
66 return OPENCV_TRACKER_NS::TrackerMOSSE::create();
67 if (trackerType == "CSRT")
68 return OPENCV_TRACKER_NS::TrackerCSRT::create();
69
70 return nullptr;
71 }
72
73 // Track object in the hole clip or in a given interval
trackClip(openshot::Clip & video,size_t _start,size_t _end,bool process_interval)74 void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
75
76 video.Open();
77 if(!json_interval){
78 start = _start; end = _end;
79
80 if(!process_interval || end <= 1 || end-start == 0){
81 // Get total number of frames in video
82 start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
83 end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
84 }
85 }
86 else{
87 start = (int)(start + video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
88 end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
89 }
90
91 if(error){
92 return;
93 }
94
95 processingController->SetError(false, "");
96 bool trackerInit = false;
97
98 size_t frame;
99 // Loop through video
100 for (frame = start; frame <= end; frame++)
101 {
102
103 // Stop the feature tracker process
104 if(processingController->ShouldStop()){
105 return;
106 }
107
108 size_t frame_number = frame;
109 // Get current frame
110 std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
111
112 // Grab OpenCV Mat image
113 cv::Mat cvimage = f->GetImageCV();
114
115 if(frame == start){
116 // Take the normalized inital bounding box and multiply to the current video shape
117 bbox = cv::Rect2d(bbox.x*cvimage.cols,bbox.y*cvimage.rows,bbox.width*cvimage.cols,
118 bbox.height*cvimage.rows);
119 }
120
121 // Pass the first frame to initialize the tracker
122 if(!trackerInit){
123
124 // Initialize the tracker
125 initTracker(cvimage, frame_number);
126
127 trackerInit = true;
128 }
129 else{
130 // Update the object tracker according to frame
131 trackerInit = trackFrame(cvimage, frame_number);
132
133 // Draw box on image
134 FrameData fd = GetTrackedData(frame_number);
135
136 }
137 // Update progress
138 processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
139 }
140 }
141
142 // Initialize the tracker
initTracker(cv::Mat & frame,size_t frameId)143 bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){
144
145 // Create new tracker object
146 tracker = selectTracker(trackerType);
147
148 // Correct if bounding box contains negative proportions (width and/or height < 0)
149 if(bbox.width < 0){
150 bbox.x = bbox.x - abs(bbox.width);
151 bbox.width = abs(bbox.width);
152 }
153 if(bbox.height < 0){
154 bbox.y = bbox.y - abs(bbox.height);
155 bbox.height = abs(bbox.height);
156 }
157
158 // Initialize tracker
159 tracker->init(frame, bbox);
160
161 float fw = frame.size().width;
162 float fh = frame.size().height;
163
164 // Add new frame data
165 trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw,
166 (bbox.y)/fh,
167 (bbox.x+bbox.width)/fw,
168 (bbox.y+bbox.height)/fh);
169
170 return true;
171 }
172
173 // Update the object tracker according to frame
trackFrame(cv::Mat & frame,size_t frameId)174 bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
175 // Update the tracking result
176 bool ok = tracker->update(frame, bbox);
177
178 // Add frame number and box coords if tracker finds the object
179 // Otherwise add only frame number
180 if (ok)
181 {
182 float fw = frame.size().width;
183 float fh = frame.size().height;
184
185 cv::Rect2d filtered_box = filter_box_jitter(frameId);
186 // Add new frame data
187 trackedDataById[frameId] = FrameData(frameId, 0, (filtered_box.x)/fw,
188 (filtered_box.y)/fh,
189 (filtered_box.x+filtered_box.width)/fw,
190 (filtered_box.y+filtered_box.height)/fh);
191 }
192 else
193 {
194 // Copy the last frame data if the tracker get lost
195 trackedDataById[frameId] = trackedDataById[frameId-1];
196 }
197
198 return ok;
199 }
200
filter_box_jitter(size_t frameId)201 cv::Rect2d CVTracker::filter_box_jitter(size_t frameId){
202 // get tracked data for the previous frame
203 float last_box_width = trackedDataById[frameId-1].x2 - trackedDataById[frameId-1].x1;
204 float last_box_height = trackedDataById[frameId-1].y2 - trackedDataById[frameId-1].y1;
205
206 float curr_box_width = bbox.width;
207 float curr_box_height = bbox.height;
208 // keep the last width and height if the difference is less than 1%
209 float threshold = 0.01;
210
211 cv::Rect2d filtered_box = bbox;
212 if(std::abs(1-(curr_box_width/last_box_width)) <= threshold){
213 filtered_box.width = last_box_width;
214 }
215 if(std::abs(1-(curr_box_height/last_box_height)) <= threshold){
216 filtered_box.height = last_box_height;
217 }
218 return filtered_box;
219 }
220
SaveTrackedData()221 bool CVTracker::SaveTrackedData(){
222 using std::ios;
223
224 // Create tracker message
225 pb_tracker::Tracker trackerMessage;
226
227 // Iterate over all frames data and save in protobuf message
228 for(std::map<size_t,FrameData>::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){
229 FrameData fData = it->second;
230 pb_tracker::Frame* pbFrameData;
231 AddFrameDataToProto(trackerMessage.add_frame(), fData);
232 }
233
234 // Add timestamp
235 *trackerMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
236
237 {
238 // Write the new message to disk.
239 std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
240 if (!trackerMessage.SerializeToOstream(&output)) {
241 std::cerr << "Failed to write protobuf message." << std::endl;
242 return false;
243 }
244 }
245
246 // Delete all global objects allocated by libprotobuf.
247 google::protobuf::ShutdownProtobufLibrary();
248
249 return true;
250
251 }
252
253 // Add frame tracked data into protobuf message.
AddFrameDataToProto(pb_tracker::Frame * pbFrameData,FrameData & fData)254 void CVTracker::AddFrameDataToProto(pb_tracker::Frame* pbFrameData, FrameData& fData) {
255
256 // Save frame number and rotation
257 pbFrameData->set_id(fData.frame_id);
258 pbFrameData->set_rotation(0);
259
260 pb_tracker::Frame::Box* box = pbFrameData->mutable_bounding_box();
261 // Save bounding box data
262 box->set_x1(fData.x1);
263 box->set_y1(fData.y1);
264 box->set_x2(fData.x2);
265 box->set_y2(fData.y2);
266 }
267
268 // Get tracker info for the desired frame
GetTrackedData(size_t frameId)269 FrameData CVTracker::GetTrackedData(size_t frameId){
270
271 // Check if the tracker info for the requested frame exists
272 if ( trackedDataById.find(frameId) == trackedDataById.end() ) {
273
274 return FrameData();
275 } else {
276
277 return trackedDataById[frameId];
278 }
279
280 }
281
282 // Load JSON string into this object
SetJson(const std::string value)283 void CVTracker::SetJson(const std::string value) {
284 // Parse JSON string into JSON objects
285 try
286 {
287 const Json::Value root = openshot::stringToJson(value);
288 // Set all values that match
289
290 SetJsonValue(root);
291 }
292 catch (const std::exception& e)
293 {
294 // Error parsing JSON (or missing keys)
295 throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
296 }
297 }
298
299 // Load Json::Value into this object
SetJsonValue(const Json::Value root)300 void CVTracker::SetJsonValue(const Json::Value root) {
301
302 // Set data from Json (if key is found)
303 if (!root["protobuf_data_path"].isNull()){
304 protobuf_data_path = (root["protobuf_data_path"].asString());
305 }
306 if (!root["tracker-type"].isNull()){
307 trackerType = (root["tracker-type"].asString());
308 }
309
310 if (!root["region"].isNull()){
311 double x = root["region"]["normalized_x"].asDouble();
312 double y = root["region"]["normalized_y"].asDouble();
313 double w = root["region"]["normalized_width"].asDouble();
314 double h = root["region"]["normalized_height"].asDouble();
315 cv::Rect2d prev_bbox(x,y,w,h);
316 bbox = prev_bbox;
317 }
318 else{
319 processingController->SetError(true, "No initial bounding box selected");
320 error = true;
321 }
322
323 if (!root["region"]["first-frame"].isNull()){
324 start = root["region"]["first-frame"].asInt64();
325 json_interval = true;
326 }
327 else{
328 processingController->SetError(true, "No first-frame");
329 error = true;
330 }
331 }
332
333 /*
334 ||||||||||||||||||||||||||||||||||||||||||||||||||
335 ONLY FOR MAKE TEST
336 ||||||||||||||||||||||||||||||||||||||||||||||||||
337 */
338
339 // Load protobuf data file
_LoadTrackedData()340 bool CVTracker::_LoadTrackedData(){
341 using std::ios;
342
343 // Create tracker message
344 pb_tracker::Tracker trackerMessage;
345
346 {
347 // Read the existing tracker message.
348 std::fstream input(protobuf_data_path, ios::in | ios::binary);
349 if (!trackerMessage.ParseFromIstream(&input)) {
350 std::cerr << "Failed to parse protobuf message." << std::endl;
351 return false;
352 }
353 }
354
355 // Make sure the trackedData is empty
356 trackedDataById.clear();
357
358 // Iterate over all frames of the saved message
359 for (size_t i = 0; i < trackerMessage.frame_size(); i++) {
360 const pb_tracker::Frame& pbFrameData = trackerMessage.frame(i);
361
362 // Load frame and rotation data
363 size_t id = pbFrameData.id();
364 float rotation = pbFrameData.rotation();
365
366 // Load bounding box data
367 const pb_tracker::Frame::Box& box = pbFrameData.bounding_box();
368 float x1 = box.x1();
369 float y1 = box.y1();
370 float x2 = box.x2();
371 float y2 = box.y2();
372
373 // Assign data to tracker map
374 trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2);
375 }
376
377 // Delete all global objects allocated by libprotobuf.
378 google::protobuf::ShutdownProtobufLibrary();
379
380 return true;
381 }
382