OpenShot Library | libopenshot  0.2.7
CVTracker.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Track an object selected by the user
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  * @author Brenno Caldato <brenno.caldato@outlook.com>
6  *
7  * @ref License
8  */
9 
10 /* LICENSE
11  *
12  * Copyright (c) 2008-2019 OpenShot Studios, LLC
13  * <http://www.openshotstudios.com/>. This file is part of
14  * OpenShot Library (libopenshot), an open-source project dedicated to
15  * delivering high quality video editing and animation solutions to the
16  * world. For more information visit <http://www.openshot.org/>.
17  *
18  * OpenShot Library (libopenshot) is free software: you can redistribute it
19  * and/or modify it under the terms of the GNU Lesser General Public License
20  * as published by the Free Software Foundation, either version 3 of the
21  * License, or (at your option) any later version.
22  *
23  * OpenShot Library (libopenshot) is distributed in the hope that it will be
24  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30  */
31 
32 #include <fstream>
33 #include <iomanip>
34 #include <iostream>
35 
36 #include <google/protobuf/util/time_util.h>
37 
38 #include "OpenCVUtilities.h"
39 #include "CVTracker.h"
40 
41 using namespace openshot;
42 using google::protobuf::util::TimeUtil;
43 
44 // Constructor
45 CVTracker::CVTracker(std::string processInfoJson, ProcessingController &processingController)
46 : processingController(&processingController), json_interval(false){
47  SetJson(processInfoJson);
48  start = 1;
49  end = 1;
50 }
51 
52 // Set desirable tracker method
53 cv::Ptr<OPENCV_TRACKER_TYPE> CVTracker::selectTracker(std::string trackerType){
54 
55  if (trackerType == "BOOSTING")
56  return OPENCV_TRACKER_NS::TrackerBoosting::create();
57  if (trackerType == "MIL")
58  return OPENCV_TRACKER_NS::TrackerMIL::create();
59  if (trackerType == "KCF")
60  return OPENCV_TRACKER_NS::TrackerKCF::create();
61  if (trackerType == "TLD")
62  return OPENCV_TRACKER_NS::TrackerTLD::create();
63  if (trackerType == "MEDIANFLOW")
64  return OPENCV_TRACKER_NS::TrackerMedianFlow::create();
65  if (trackerType == "MOSSE")
66  return OPENCV_TRACKER_NS::TrackerMOSSE::create();
67  if (trackerType == "CSRT")
68  return OPENCV_TRACKER_NS::TrackerCSRT::create();
69 
70  return nullptr;
71 }
72 
73 // Track object in the hole clip or in a given interval
74 void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
75 
76  video.Open();
77  if(!json_interval){
78  start = _start; end = _end;
79 
80  if(!process_interval || end <= 1 || end-start == 0){
81  // Get total number of frames in video
82  start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
83  end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
84  }
85  }
86  else{
87  start = (int)(start + video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
88  end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
89  }
90 
91  if(error){
92  return;
93  }
94 
95  processingController->SetError(false, "");
96  bool trackerInit = false;
97 
98  size_t frame;
99  // Loop through video
100  for (frame = start; frame <= end; frame++)
101  {
102 
103  // Stop the feature tracker process
104  if(processingController->ShouldStop()){
105  return;
106  }
107 
108  size_t frame_number = frame;
109  // Get current frame
110  std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
111 
112  // Grab OpenCV Mat image
113  cv::Mat cvimage = f->GetImageCV();
114 
115  if(frame == start){
116  // Take the normalized inital bounding box and multiply to the current video shape
117  bbox = cv::Rect2d(bbox.x*cvimage.cols,bbox.y*cvimage.rows,bbox.width*cvimage.cols,
118  bbox.height*cvimage.rows);
119  }
120 
121  // Pass the first frame to initialize the tracker
122  if(!trackerInit){
123 
124  // Initialize the tracker
125  initTracker(cvimage, frame_number);
126 
127  trackerInit = true;
128  }
129  else{
130  // Update the object tracker according to frame
131  trackerInit = trackFrame(cvimage, frame_number);
132 
133  // Draw box on image
134  FrameData fd = GetTrackedData(frame_number);
135 
136  }
137  // Update progress
138  processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
139  }
140 }
141 
142 // Initialize the tracker
143 bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){
144 
145  // Create new tracker object
146  tracker = selectTracker(trackerType);
147 
148  // Correct if bounding box contains negative proportions (width and/or height < 0)
149  if(bbox.width < 0){
150  bbox.x = bbox.x - abs(bbox.width);
151  bbox.width = abs(bbox.width);
152  }
153  if(bbox.height < 0){
154  bbox.y = bbox.y - abs(bbox.height);
155  bbox.height = abs(bbox.height);
156  }
157 
158  // Initialize tracker
159  tracker->init(frame, bbox);
160 
161  float fw = frame.size().width;
162  float fh = frame.size().height;
163 
164  // Add new frame data
165  trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw,
166  (bbox.y)/fh,
167  (bbox.x+bbox.width)/fw,
168  (bbox.y+bbox.height)/fh);
169 
170  return true;
171 }
172 
173 // Update the object tracker according to frame
174 bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
175  // Update the tracking result
176  bool ok = tracker->update(frame, bbox);
177 
178  // Add frame number and box coords if tracker finds the object
179  // Otherwise add only frame number
180  if (ok)
181  {
182  float fw = frame.size().width;
183  float fh = frame.size().height;
184 
185  cv::Rect2d filtered_box = filter_box_jitter(frameId);
186  // Add new frame data
187  trackedDataById[frameId] = FrameData(frameId, 0, (filtered_box.x)/fw,
188  (filtered_box.y)/fh,
189  (filtered_box.x+filtered_box.width)/fw,
190  (filtered_box.y+filtered_box.height)/fh);
191  }
192  else
193  {
194  // Copy the last frame data if the tracker get lost
195  trackedDataById[frameId] = trackedDataById[frameId-1];
196  }
197 
198  return ok;
199 }
200 
201 cv::Rect2d CVTracker::filter_box_jitter(size_t frameId){
202  // get tracked data for the previous frame
203  float last_box_width = trackedDataById[frameId-1].x2 - trackedDataById[frameId-1].x1;
204  float last_box_height = trackedDataById[frameId-1].y2 - trackedDataById[frameId-1].y1;
205 
206  float curr_box_width = bbox.width;
207  float curr_box_height = bbox.height;
208  // keep the last width and height if the difference is less than 1%
209  float threshold = 0.01;
210 
211  cv::Rect2d filtered_box = bbox;
212  if(std::abs(1-(curr_box_width/last_box_width)) <= threshold){
213  filtered_box.width = last_box_width;
214  }
215  if(std::abs(1-(curr_box_height/last_box_height)) <= threshold){
216  filtered_box.height = last_box_height;
217  }
218  return filtered_box;
219 }
220 
222  using std::ios;
223 
224  // Create tracker message
225  pb_tracker::Tracker trackerMessage;
226 
227  // Iterate over all frames data and save in protobuf message
228  for(std::map<size_t,FrameData>::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){
229  FrameData fData = it->second;
230  pb_tracker::Frame* pbFrameData;
231  AddFrameDataToProto(trackerMessage.add_frame(), fData);
232  }
233 
234  // Add timestamp
235  *trackerMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
236 
237  {
238  // Write the new message to disk.
239  std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
240  if (!trackerMessage.SerializeToOstream(&output)) {
241  std::cerr << "Failed to write protobuf message." << std::endl;
242  return false;
243  }
244  }
245 
246  // Delete all global objects allocated by libprotobuf.
247  google::protobuf::ShutdownProtobufLibrary();
248 
249  return true;
250 
251 }
252 
253 // Add frame tracked data into protobuf message.
254 void CVTracker::AddFrameDataToProto(pb_tracker::Frame* pbFrameData, FrameData& fData) {
255 
256  // Save frame number and rotation
257  pbFrameData->set_id(fData.frame_id);
258  pbFrameData->set_rotation(0);
259 
260  pb_tracker::Frame::Box* box = pbFrameData->mutable_bounding_box();
261  // Save bounding box data
262  box->set_x1(fData.x1);
263  box->set_y1(fData.y1);
264  box->set_x2(fData.x2);
265  box->set_y2(fData.y2);
266 }
267 
268 // Get tracker info for the desired frame
270 
271  // Check if the tracker info for the requested frame exists
272  if ( trackedDataById.find(frameId) == trackedDataById.end() ) {
273 
274  return FrameData();
275  } else {
276 
277  return trackedDataById[frameId];
278  }
279 
280 }
281 
282 // Load JSON string into this object
283 void CVTracker::SetJson(const std::string value) {
284  // Parse JSON string into JSON objects
285  try
286  {
287  const Json::Value root = openshot::stringToJson(value);
288  // Set all values that match
289 
290  SetJsonValue(root);
291  }
292  catch (const std::exception& e)
293  {
294  // Error parsing JSON (or missing keys)
295  throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
296  }
297 }
298 
299 // Load Json::Value into this object
300 void CVTracker::SetJsonValue(const Json::Value root) {
301 
302  // Set data from Json (if key is found)
303  if (!root["protobuf_data_path"].isNull()){
304  protobuf_data_path = (root["protobuf_data_path"].asString());
305  }
306  if (!root["tracker-type"].isNull()){
307  trackerType = (root["tracker-type"].asString());
308  }
309 
310  if (!root["region"].isNull()){
311  double x = root["region"]["normalized_x"].asDouble();
312  double y = root["region"]["normalized_y"].asDouble();
313  double w = root["region"]["normalized_width"].asDouble();
314  double h = root["region"]["normalized_height"].asDouble();
315  cv::Rect2d prev_bbox(x,y,w,h);
316  bbox = prev_bbox;
317  }
318  else{
319  processingController->SetError(true, "No initial bounding box selected");
320  error = true;
321  }
322 
323  if (!root["region"]["first-frame"].isNull()){
324  start = root["region"]["first-frame"].asInt64();
325  json_interval = true;
326  }
327  else{
328  processingController->SetError(true, "No first-frame");
329  error = true;
330  }
331 }
332 
333 /*
334 ||||||||||||||||||||||||||||||||||||||||||||||||||
335  ONLY FOR MAKE TEST
336 ||||||||||||||||||||||||||||||||||||||||||||||||||
337 */
338 
339 // Load protobuf data file
341  using std::ios;
342 
343  // Create tracker message
344  pb_tracker::Tracker trackerMessage;
345 
346  {
347  // Read the existing tracker message.
348  std::fstream input(protobuf_data_path, ios::in | ios::binary);
349  if (!trackerMessage.ParseFromIstream(&input)) {
350  std::cerr << "Failed to parse protobuf message." << std::endl;
351  return false;
352  }
353  }
354 
355  // Make sure the trackedData is empty
356  trackedDataById.clear();
357 
358  // Iterate over all frames of the saved message
359  for (size_t i = 0; i < trackerMessage.frame_size(); i++) {
360  const pb_tracker::Frame& pbFrameData = trackerMessage.frame(i);
361 
362  // Load frame and rotation data
363  size_t id = pbFrameData.id();
364  float rotation = pbFrameData.rotation();
365 
366  // Load bounding box data
367  const pb_tracker::Frame::Box& box = pbFrameData.bounding_box();
368  float x1 = box.x1();
369  float y1 = box.y1();
370  float x2 = box.x2();
371  float y2 = box.y2();
372 
373  // Assign data to tracker map
374  trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2);
375  }
376 
377  // Delete all global objects allocated by libprotobuf.
378  google::protobuf::ShutdownProtobufLibrary();
379 
380  return true;
381 }
cv::Ptr< OPENCV_TRACKER_TYPE > selectTracker(std::string trackerType)
Definition: CVTracker.cpp:53
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:110
Track an object selected by the user.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CVTracker.cpp:300
cv::Rect2d filter_box_jitter(size_t frameId)
Filter current bounding box jitter.
Definition: CVTracker.cpp:201
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
FrameData GetTrackedData(size_t frameId)
Get tracked data for a given frame.
Definition: CVTracker.cpp:269
void Open() override
Open the internal reader.
Definition: Clip.cpp:302
CVTracker(std::string processInfoJson, ProcessingController &processingController)
Definition: CVTracker.cpp:45
void trackClip(openshot::Clip &video, size_t _start=0, size_t _end=0, bool process_interval=false)
Definition: CVTracker.cpp:74
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
Header file for OpenCVUtilities (set some common macros)
float End() const
Get end position (in seconds) of clip (trim end of video), which can be affected by the time curve...
Definition: Clip.cpp:338
void SetError(bool err, std::string message)
bool SaveTrackedData()
Save protobuf file.
Definition: CVTracker.cpp:221
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CVTracker.cpp:283
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
void AddFrameDataToProto(pb_tracker::Frame *pbFrameData, FrameData &fData)
Add frame tracked data into protobuf message.
Definition: CVTracker.cpp:254
Exception for invalid JSON.
Definition: Exceptions.h:205
void Reader(openshot::ReaderBase *new_reader)
Set the current reader.
Definition: Clip.cpp:279
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:360