OpenShot Library | libopenshot  0.2.7
CVStabilization.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for CVStabilization class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  * @author Brenno Caldato <brenno.caldato@outlook.com>
6  *
7  * @ref License
8  */
9 
10 /* LICENSE
11  *
12  * Copyright (c) 2008-2019 OpenShot Studios, LLC
13  * <http://www.openshotstudios.com/>. This file is part of
14  * OpenShot Library (libopenshot), an open-source project dedicated to
15  * delivering high quality video editing and animation solutions to the
16  * world. For more information visit <http://www.openshot.org/>.
17  *
18  * OpenShot Library (libopenshot) is free software: you can redistribute it
19  * and/or modify it under the terms of the GNU Lesser General Public License
20  * as published by the Free Software Foundation, either version 3 of the
21  * License, or (at your option) any later version.
22  *
23  * OpenShot Library (libopenshot) is distributed in the hope that it will be
24  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30  */
31 
32 #include <fstream>
33 #include <iomanip>
34 #include <iostream>
35 
36 #include "CVStabilization.h"
37 #include <google/protobuf/util/time_util.h>
38 
39 using namespace std;
40 using namespace openshot;
41 using google::protobuf::util::TimeUtil;
42 
43 // Set default smoothing window value to compute stabilization
44 CVStabilization::CVStabilization(std::string processInfoJson, ProcessingController &processingController)
45 : processingController(&processingController){
46  SetJson(processInfoJson);
47  start = 1;
48  end = 1;
49 }
50 
51 // Process clip and store necessary stabilization data
52 void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
53 
54  if(error){
55  return;
56  }
57  processingController->SetError(false, "");
58 
59  start = _start; end = _end;
60  // Compute max and average transformation parameters
61  avr_dx=0; avr_dy=0; avr_da=0; max_dx=0; max_dy=0; max_da=0;
62 
63  video.Open();
64  // Save original video width and height
65  cv::Size readerDims(video.Reader()->info.width, video.Reader()->info.height);
66 
67  size_t frame_number;
68  if(!process_interval || end <= 1 || end-start == 0){
69  // Get total number of frames in video
70  start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
71  end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
72  }
73 
74  // Extract and track opticalflow features for each frame
75  for (frame_number = start; frame_number <= end; frame_number++)
76  {
77  // Stop the feature tracker process
78  if(processingController->ShouldStop()){
79  return;
80  }
81 
82  std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
83 
84  // Grab OpenCV Mat image
85  cv::Mat cvimage = f->GetImageCV();
86  // Resize frame to original video width and height if they differ
87  if(cvimage.size().width != readerDims.width || cvimage.size().height != readerDims.height)
88  cv::resize(cvimage, cvimage, cv::Size(readerDims.width, readerDims.height));
89  cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
90 
91  if(!TrackFrameFeatures(cvimage, frame_number)){
92  prev_to_cur_transform.push_back(TransformParam(0, 0, 0));
93  }
94 
95  // Update progress
96  processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
97  }
98 
99  // Calculate trajectory data
100  std::vector <CamTrajectory> trajectory = ComputeFramesTrajectory();
101 
102  // Calculate and save smoothed trajectory data
103  trajectoryData = SmoothTrajectory(trajectory);
104 
105  // Calculate and save transformation data
106  transformationData = GenNewCamPosition(trajectoryData);
107 
108  // Normalize smoothed trajectory data
109  for(auto &dataToNormalize : trajectoryData){
110  dataToNormalize.second.x/=readerDims.width;
111  dataToNormalize.second.y/=readerDims.height;
112  }
113  // Normalize transformation data
114  for(auto &dataToNormalize : transformationData){
115  dataToNormalize.second.dx/=readerDims.width;
116  dataToNormalize.second.dy/=readerDims.height;
117  }
118 }
119 
120 // Track current frame features and find the relative transformation
121 bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
122  // Check if there are black frames
123  if(cv::countNonZero(frame) < 1){
124  return false;
125  }
126 
127  // Initialize prev_grey if not
128  if(prev_grey.empty()){
129  prev_grey = frame;
130  return true;
131  }
132 
133  // OpticalFlow features vector
134  std::vector <cv::Point2f> prev_corner, cur_corner;
135  std::vector <cv::Point2f> prev_corner2, cur_corner2;
136  std::vector <uchar> status;
137  std::vector <float> err;
138  // Extract new image features
139  cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30);
140  // Track features
141  cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err);
142  // Remove untracked features
143  for(size_t i=0; i < status.size(); i++) {
144  if(status[i]) {
145  prev_corner2.push_back(prev_corner[i]);
146  cur_corner2.push_back(cur_corner[i]);
147  }
148  }
149  // In case no feature was detected
150  if(prev_corner2.empty() || cur_corner2.empty()){
151  last_T = cv::Mat();
152  // prev_grey = cv::Mat();
153  return false;
154  }
155 
156  // Translation + rotation only
157  cv::Mat T = cv::estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing
158 
159  double da, dx, dy;
160  // If T has nothing inside return (probably a segment where there is nothing to stabilize)
161  if(T.size().width == 0 || T.size().height == 0){
162  return false;
163  }
164  else{
165  // If no transformation is found, just use the last known good transform
166  if(T.data == NULL){
167  if(!last_T.empty())
168  last_T.copyTo(T);
169  else
170  return false;
171  }
172  // Decompose T
173  dx = T.at<double>(0,2);
174  dy = T.at<double>(1,2);
175  da = atan2(T.at<double>(1,0), T.at<double>(0,0));
176  }
177 
178  // Filter transformations parameters, if they are higher than these: return
179  if(dx > 200 || dy > 200 || da > 0.1){
180  return false;
181  }
182 
183  // Keep computing average and max transformation parameters
184  avr_dx+=fabs(dx);
185  avr_dy+=fabs(dy);
186  avr_da+=fabs(da);
187  if(fabs(dx) > max_dx)
188  max_dx = dx;
189  if(fabs(dy) > max_dy)
190  max_dy = dy;
191  if(fabs(da) > max_da)
192  max_da = da;
193 
194  T.copyTo(last_T);
195 
196  prev_to_cur_transform.push_back(TransformParam(dx, dy, da));
197  frame.copyTo(prev_grey);
198 
199  return true;
200 }
201 
202 std::vector<CamTrajectory> CVStabilization::ComputeFramesTrajectory(){
203 
204  // Accumulated frame to frame transform
205  double a = 0;
206  double x = 0;
207  double y = 0;
208 
209  vector <CamTrajectory> trajectory; // trajectory at all frames
210 
211  // Compute global camera trajectory. First frame is the origin
212  for(size_t i=0; i < prev_to_cur_transform.size(); i++) {
213  x += prev_to_cur_transform[i].dx;
214  y += prev_to_cur_transform[i].dy;
215  a += prev_to_cur_transform[i].da;
216 
217  // Save trajectory data to vector
218  trajectory.push_back(CamTrajectory(x,y,a));
219  }
220  return trajectory;
221 }
222 
223 std::map<size_t,CamTrajectory> CVStabilization::SmoothTrajectory(std::vector <CamTrajectory> &trajectory){
224 
225  std::map <size_t,CamTrajectory> smoothed_trajectory; // trajectory at all frames
226 
227  for(size_t i=0; i < trajectory.size(); i++) {
228  double sum_x = 0;
229  double sum_y = 0;
230  double sum_a = 0;
231  int count = 0;
232 
233  for(int j=-smoothingWindow; j <= smoothingWindow; j++) {
234  if(i+j < trajectory.size()) {
235  sum_x += trajectory[i+j].x;
236  sum_y += trajectory[i+j].y;
237  sum_a += trajectory[i+j].a;
238 
239  count++;
240  }
241  }
242 
243  double avg_a = sum_a / count;
244  double avg_x = sum_x / count;
245  double avg_y = sum_y / count;
246 
247  // Add smoothed trajectory data to map
248  smoothed_trajectory[i + start] = CamTrajectory(avg_x, avg_y, avg_a);
249  }
250  return smoothed_trajectory;
251 }
252 
253 // Generate new transformations parameters for each frame to follow the smoothed trajectory
254 std::map<size_t,TransformParam> CVStabilization::GenNewCamPosition(std::map <size_t,CamTrajectory> &smoothed_trajectory){
255  std::map <size_t,TransformParam> new_prev_to_cur_transform;
256 
257  // Accumulated frame to frame transform
258  double a = 0;
259  double x = 0;
260  double y = 0;
261 
262  for(size_t i=0; i < prev_to_cur_transform.size(); i++) {
263  x += prev_to_cur_transform[i].dx;
264  y += prev_to_cur_transform[i].dy;
265  a += prev_to_cur_transform[i].da;
266 
267  // target - current
268  double diff_x = smoothed_trajectory[i + start].x - x;
269  double diff_y = smoothed_trajectory[i + start].y - y;
270  double diff_a = smoothed_trajectory[i + start].a - a;
271 
272  double dx = prev_to_cur_transform[i].dx + diff_x;
273  double dy = prev_to_cur_transform[i].dy + diff_y;
274  double da = prev_to_cur_transform[i].da + diff_a;
275 
276  // Add transformation data to map
277  new_prev_to_cur_transform[i + start] = TransformParam(dx, dy, da);
278  }
279  return new_prev_to_cur_transform;
280 }
281 
282 // Save stabilization data to protobuf file
284  using std::ios;
285 
286  // Create stabilization message
287  pb_stabilize::Stabilization stabilizationMessage;
288 
289  std::map<size_t,CamTrajectory>::iterator trajData = trajectoryData.begin();
290  std::map<size_t,TransformParam>::iterator transData = transformationData.begin();
291 
292  // Iterate over all frames data and save in protobuf message
293  for(; trajData != trajectoryData.end(); ++trajData, ++transData){
294  AddFrameDataToProto(stabilizationMessage.add_frame(), trajData->second, transData->second, trajData->first);
295  }
296  // Add timestamp
297  *stabilizationMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
298 
299  // Write the new message to disk.
300  std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
301  if (!stabilizationMessage.SerializeToOstream(&output)) {
302  std::cerr << "Failed to write protobuf message." << std::endl;
303  return false;
304  }
305 
306  // Delete all global objects allocated by libprotobuf.
307  google::protobuf::ShutdownProtobufLibrary();
308 
309  return true;
310 }
311 
312 // Add frame stabilization data into protobuf message
313 void CVStabilization::AddFrameDataToProto(pb_stabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number){
314 
315  // Save frame number
316  pbFrameData->set_id(frame_number);
317 
318  // Save camera trajectory data
319  pbFrameData->set_a(trajData.a);
320  pbFrameData->set_x(trajData.x);
321  pbFrameData->set_y(trajData.y);
322 
323  // Save transformation data
324  pbFrameData->set_da(transData.da);
325  pbFrameData->set_dx(transData.dx);
326  pbFrameData->set_dy(transData.dy);
327 }
328 
330 
331  // Check if the stabilizer info for the requested frame exists
332  if ( transformationData.find(frameId) == transformationData.end() ) {
333 
334  return TransformParam();
335  } else {
336 
337  return transformationData[frameId];
338  }
339 }
340 
342 
343  // Check if the stabilizer info for the requested frame exists
344  if ( trajectoryData.find(frameId) == trajectoryData.end() ) {
345 
346  return CamTrajectory();
347  } else {
348 
349  return trajectoryData[frameId];
350  }
351 }
352 
353 // Load JSON string into this object
354 void CVStabilization::SetJson(const std::string value) {
355  // Parse JSON string into JSON objects
356  try
357  {
358  const Json::Value root = openshot::stringToJson(value);
359  // Set all values that match
360 
361  SetJsonValue(root);
362  }
363  catch (const std::exception& e)
364  {
365  // Error parsing JSON (or missing keys)
366  throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
367  }
368 }
369 
370 // Load Json::Value into this object
371 void CVStabilization::SetJsonValue(const Json::Value root) {
372 
373  // Set data from Json (if key is found)
374  if (!root["protobuf_data_path"].isNull()){
375  protobuf_data_path = (root["protobuf_data_path"].asString());
376  }
377  if (!root["smoothing-window"].isNull()){
378  smoothingWindow = (root["smoothing-window"].asInt());
379  }
380 }
381 
382 /*
383 ||||||||||||||||||||||||||||||||||||||||||||||||||
384  ONLY FOR MAKE TEST
385 ||||||||||||||||||||||||||||||||||||||||||||||||||
386 */
387 
388 // Load protobuf data file
390  using std::ios;
391  // Create stabilization message
392  pb_stabilize::Stabilization stabilizationMessage;
393  // Read the existing tracker message.
394  std::fstream input(protobuf_data_path, ios::in | ios::binary);
395  if (!stabilizationMessage.ParseFromIstream(&input)) {
396  std::cerr << "Failed to parse protobuf message." << std::endl;
397  return false;
398  }
399 
400  // Make sure the data maps are empty
401  transformationData.clear();
402  trajectoryData.clear();
403 
404  // Iterate over all frames of the saved message and assign to the data maps
405  for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) {
406  const pb_stabilize::Frame& pbFrameData = stabilizationMessage.frame(i);
407 
408  // Load frame number
409  size_t id = pbFrameData.id();
410 
411  // Load camera trajectory data
412  float x = pbFrameData.x();
413  float y = pbFrameData.y();
414  float a = pbFrameData.a();
415 
416  // Assign data to trajectory map
417  trajectoryData[id] = CamTrajectory(x,y,a);
418 
419  // Load transformation data
420  float dx = pbFrameData.dx();
421  float dy = pbFrameData.dy();
422  float da = pbFrameData.da();
423 
424  // Assing data to transformation map
425  transformationData[id] = TransformParam(dx,dy,da);
426  }
427 
428  // Delete all global objects allocated by libprotobuf.
429  google::protobuf::ShutdownProtobufLibrary();
430 
431  return true;
432 }
CamTrajectory GetCamTrajectoryTrackedData(size_t frameId)
void stabilizeClip(openshot::Clip &video, size_t _start=0, size_t _end=0, bool process_interval=false)
Process clip and store necessary stabilization data.
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:110
void AddFrameDataToProto(pb_stabilize::Frame *pbFrameData, CamTrajectory &trajData, TransformParam &transData, size_t frame_number)
Add frame stabilization data into protobuf message.
CVStabilization(std::string processInfoJson, ProcessingController &processingController)
Set default smoothing window value to compute stabilization.
STL namespace.
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
void Open() override
Open the internal reader.
Definition: Clip.cpp:302
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
float End() const
Get end position (in seconds) of clip (trim end of video), which can be affected by the time curve...
Definition: Clip.cpp:338
void SetError(bool err, std::string message)
std::map< size_t, CamTrajectory > trajectoryData
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
TransformParam GetTransformParamData(size_t frameId)
void SetJson(const std::string value)
Load JSON string into this object.
std::map< size_t, TransformParam > transformationData
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
Exception for invalid JSON.
Definition: Exceptions.h:205
void Reader(openshot::ReaderBase *new_reader)
Set the current reader.
Definition: Clip.cpp:279
Header file for CVStabilization class.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:360