OpenShot Library | libopenshot  0.2.7
CacheDisk.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for CacheDisk class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "CacheDisk.h"
32 #include "Exceptions.h"
33 #include "QtUtilities.h"
34 #include <Qt>
35 #include <QString>
36 #include <QTextStream>
37 
38 using namespace std;
39 using namespace openshot;
40 
41 // Default constructor, no max bytes
42 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale) : CacheBase(0) {
43  // Set cache type name
44  cache_type = "CacheDisk";
45  range_version = 0;
46  needs_range_processing = false;
47  frame_size_bytes = 0;
48  image_format = format;
49  image_quality = quality;
50  image_scale = scale;
51  max_bytes = 0;
52 
53  // Init path directory
54  InitPath(cache_path);
55 }
56 
57 // Constructor that sets the max bytes to cache
58 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale, int64_t max_bytes) : CacheBase(max_bytes) {
59  // Set cache type name
60  cache_type = "CacheDisk";
61  range_version = 0;
62  needs_range_processing = false;
63  frame_size_bytes = 0;
64  image_format = format;
65  image_quality = quality;
66  image_scale = scale;
67 
68  // Init path directory
69  InitPath(cache_path);
70 }
71 
72 // Initialize cache directory
73 void CacheDisk::InitPath(std::string cache_path) {
74  QString qpath;
75 
76  if (!cache_path.empty()) {
77  // Init QDir with cache directory
78  qpath = QString(cache_path.c_str());
79 
80  } else {
81  // Init QDir with user's temp directory
82  qpath = QDir::tempPath() + QString("/preview-cache/");
83  }
84 
85  // Init QDir with cache directory
86  path = QDir(qpath);
87 
88  // Check if cache directory exists
89  if (!path.exists())
90  // Create
91  path.mkpath(qpath);
92 }
93 
94 // Calculate ranges of frames
95 void CacheDisk::CalculateRanges() {
96  // Only calculate when something has changed
97  if (needs_range_processing) {
98 
99  // Create a scoped lock, to protect the cache from multiple threads
100  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
101 
102  // Sort ordered frame #s, and calculate JSON ranges
103  std::sort(ordered_frame_numbers.begin(), ordered_frame_numbers.end());
104 
105  // Clear existing JSON variable
106  Json::Value ranges = Json::Value(Json::arrayValue);
107 
108  // Increment range version
109  range_version++;
110 
111  int64_t starting_frame = *ordered_frame_numbers.begin();
112  int64_t ending_frame = starting_frame;
113 
114  // Loop through all known frames (in sequential order)
115  for (const auto frame_number : ordered_frame_numbers) {
116  if (frame_number - ending_frame > 1) {
117  // End of range detected
118  Json::Value range;
119 
120  // Add JSON object with start/end attributes
121  // Use strings, since int64_ts are supported in JSON
122  range["start"] = std::to_string(starting_frame);
123  range["end"] = std::to_string(ending_frame);
124  ranges.append(range);
125 
126  // Set new starting range
127  starting_frame = frame_number;
128  }
129 
130  // Set current frame as end of range, and keep looping
131  ending_frame = frame_number;
132  }
133 
134  // APPEND FINAL VALUE
135  Json::Value range;
136 
137  // Add JSON object with start/end attributes
138  // Use strings, since int64_ts are supported in JSON
139  range["start"] = std::to_string(starting_frame);
140  range["end"] = std::to_string(ending_frame);
141  ranges.append(range);
142 
143  // Cache range JSON as string
144  json_ranges = ranges.toStyledString();
145 
146  // Reset needs_range_processing
147  needs_range_processing = false;
148  }
149 }
150 
151 // Default destructor
153 {
154  frames.clear();
155  frame_numbers.clear();
156  ordered_frame_numbers.clear();
157 
158  // remove critical section
159  delete cacheCriticalSection;
160  cacheCriticalSection = NULL;
161 }
162 
163 // Add a Frame to the cache
164 void CacheDisk::Add(std::shared_ptr<Frame> frame)
165 {
166  // Create a scoped lock, to protect the cache from multiple threads
167  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
168  int64_t frame_number = frame->number;
169 
170  // Freshen frame if it already exists
171  if (frames.count(frame_number))
172  // Move frame to front of queue
173  MoveToFront(frame_number);
174 
175  else
176  {
177  // Add frame to queue and map
178  frames[frame_number] = frame_number;
179  frame_numbers.push_front(frame_number);
180  ordered_frame_numbers.push_back(frame_number);
181  needs_range_processing = true;
182 
183  // Save image to disk (if needed)
184  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
185  frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
186  if (frame_size_bytes == 0) {
187  // Get compressed size of frame image (to correctly apply max size against)
188  QFile image_file(frame_path);
189  frame_size_bytes = image_file.size();
190  }
191 
192  // Save audio data (if needed)
193  if (frame->has_audio_data) {
194  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
195  QFile audio_file(audio_path);
196 
197  if (audio_file.open(QIODevice::WriteOnly)) {
198  QTextStream audio_stream(&audio_file);
199  audio_stream << frame->SampleRate() << Qt::endl;
200  audio_stream << frame->GetAudioChannelsCount() << Qt::endl;
201  audio_stream << frame->GetAudioSamplesCount() << Qt::endl;
202  audio_stream << frame->ChannelsLayout() << Qt::endl;
203 
204  // Loop through all samples
205  for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
206  {
207  // Get audio for this channel
208  float *samples = frame->GetAudioSamples(channel);
209  for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
210  audio_stream << samples[sample] << Qt::endl;
211  }
212 
213  }
214 
215  }
216 
217  // Clean up old frames
218  CleanUp();
219  }
220 }
221 
222 // Get a frame from the cache (or NULL shared_ptr if no frame is found)
223 std::shared_ptr<Frame> CacheDisk::GetFrame(int64_t frame_number)
224 {
225  // Create a scoped lock, to protect the cache from multiple threads
226  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
227 
228  // Does frame exists in cache?
229  if (frames.count(frame_number)) {
230  // Does frame exist on disk
231  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
232  if (path.exists(frame_path)) {
233 
234  // Load image file
235  auto image = std::make_shared<QImage>();
236  image->load(frame_path);
237 
238  // Set pixel formatimage->
239  image = std::make_shared<QImage>(image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
240 
241  // Create frame object
242  auto frame = std::make_shared<Frame>();
243  frame->number = frame_number;
244  frame->AddImage(image);
245 
246  // Get audio data (if found)
247  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
248  QFile audio_file(audio_path);
249  if (audio_file.exists()) {
250  // Open audio file
251  QTextStream in(&audio_file);
252  if (audio_file.open(QIODevice::ReadOnly)) {
253  int sample_rate = in.readLine().toInt();
254  int channels = in.readLine().toInt();
255  int sample_count = in.readLine().toInt();
256  int channel_layout = in.readLine().toInt();
257 
258  // Set basic audio properties
259  frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
260 
261  // Loop through audio samples and add to frame
262  int current_channel = 0;
263  int current_sample = 0;
264  float *channel_samples = new float[sample_count];
265  while (!in.atEnd()) {
266  // Add sample to channel array
267  channel_samples[current_sample] = in.readLine().toFloat();
268  current_sample++;
269 
270  if (current_sample == sample_count) {
271  // Add audio to frame
272  frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
273 
274  // Increment channel, and reset sample position
275  current_channel++;
276  current_sample = 0;
277  }
278 
279  }
280  }
281  }
282 
283  // return the Frame object
284  return frame;
285  }
286  }
287 
288  // no Frame found
289  return std::shared_ptr<Frame>();
290 }
291 
292 // Get the smallest frame number (or NULL shared_ptr if no frame is found)
293 std::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
294 {
295  // Create a scoped lock, to protect the cache from multiple threads
296  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
297  std::shared_ptr<openshot::Frame> f;
298 
299  // Loop through frame numbers
300  std::deque<int64_t>::iterator itr;
301  int64_t smallest_frame = -1;
302  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
303  {
304  if (*itr < smallest_frame || smallest_frame == -1)
305  smallest_frame = *itr;
306  }
307 
308  // Return frame
309  f = GetFrame(smallest_frame);
310 
311  return f;
312 }
313 
314 // Gets the maximum bytes value
316 {
317  // Create a scoped lock, to protect the cache from multiple threads
318  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
319 
320  int64_t total_bytes = 0;
321 
322  // Loop through frames, and calculate total bytes
323  std::deque<int64_t>::reverse_iterator itr;
324  for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
325  total_bytes += frame_size_bytes;
326 
327  return total_bytes;
328 }
329 
330 // Remove a specific frame
331 void CacheDisk::Remove(int64_t frame_number)
332 {
333  Remove(frame_number, frame_number);
334 }
335 
336 // Remove range of frames
337 void CacheDisk::Remove(int64_t start_frame_number, int64_t end_frame_number)
338 {
339  // Create a scoped lock, to protect the cache from multiple threads
340  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
341 
342  // Loop through frame numbers
343  std::deque<int64_t>::iterator itr;
344  for(itr = frame_numbers.begin(); itr != frame_numbers.end();)
345  {
346  //deque<int64_t>::iterator current = itr++;
347  if (*itr >= start_frame_number && *itr <= end_frame_number)
348  {
349  // erase frame number
350  itr = frame_numbers.erase(itr);
351  } else
352  itr++;
353  }
354 
355  // Loop through ordered frame numbers
356  std::vector<int64_t>::iterator itr_ordered;
357  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end();)
358  {
359  if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
360  {
361  // erase frame number
362  frames.erase(*itr_ordered);
363 
364  // Remove the image file (if it exists)
365  QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
366  QFile image_file(frame_path);
367  if (image_file.exists())
368  image_file.remove();
369 
370  // Remove audio file (if it exists)
371  QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
372  QFile audio_file(audio_path);
373  if (audio_file.exists())
374  audio_file.remove();
375 
376  itr_ordered = ordered_frame_numbers.erase(itr_ordered);
377  } else
378  itr_ordered++;
379  }
380 
381  // Needs range processing (since cache has changed)
382  needs_range_processing = true;
383 }
384 
385 // Move frame to front of queue (so it lasts longer)
386 void CacheDisk::MoveToFront(int64_t frame_number)
387 {
388  // Does frame exists in cache?
389  if (frames.count(frame_number))
390  {
391  // Create a scoped lock, to protect the cache from multiple threads
392  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
393 
394  // Loop through frame numbers
395  std::deque<int64_t>::iterator itr;
396  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
397  {
398  if (*itr == frame_number)
399  {
400  // erase frame number
401  frame_numbers.erase(itr);
402 
403  // add frame number to 'front' of queue
404  frame_numbers.push_front(frame_number);
405  break;
406  }
407  }
408  }
409 }
410 
411 // Clear the cache of all frames
413 {
414  // Create a scoped lock, to protect the cache from multiple threads
415  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
416 
417  // Clear all containers
418  frames.clear();
419  frame_numbers.clear();
420  ordered_frame_numbers.clear();
421  needs_range_processing = true;
422  frame_size_bytes = 0;
423 
424  // Delete cache directory, and recreate it
425  QString current_path = path.path();
426  path.removeRecursively();
427 
428  // Re-init folder
429  InitPath(current_path.toStdString());
430 }
431 
432 // Count the frames in the queue
434 {
435  // Create a scoped lock, to protect the cache from multiple threads
436  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
437 
438  // Return the number of frames in the cache
439  return frames.size();
440 }
441 
442 // Clean up cached frames that exceed the number in our max_bytes variable
443 void CacheDisk::CleanUp()
444 {
445  // Do we auto clean up?
446  if (max_bytes > 0)
447  {
448  // Create a scoped lock, to protect the cache from multiple threads
449  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
450 
451  while (GetBytes() > max_bytes && frame_numbers.size() > 20)
452  {
453  // Get the oldest frame number.
454  int64_t frame_to_remove = frame_numbers.back();
455 
456  // Remove frame_number and frame
457  Remove(frame_to_remove);
458  }
459  }
460 }
461 
462 // Generate JSON string of this object
463 std::string CacheDisk::Json() {
464 
465  // Return formatted string
466  return JsonValue().toStyledString();
467 }
468 
469 // Generate Json::Value for this object
470 Json::Value CacheDisk::JsonValue() {
471 
472  // Process range data (if anything has changed)
473  CalculateRanges();
474 
475  // Create root json object
476  Json::Value root = CacheBase::JsonValue(); // get parent properties
477  root["type"] = cache_type;
478  root["path"] = path.path().toStdString();
479 
480  Json::Value version;
481  std::stringstream range_version_str;
482  range_version_str << range_version;
483  root["version"] = range_version_str.str();
484 
485  // Parse and append range data (if any)
486  // Parse and append range data (if any)
487  try {
488  const Json::Value ranges = openshot::stringToJson(json_ranges);
489  root["ranges"] = ranges;
490  } catch (...) { }
491 
492  // return JsonValue
493  return root;
494 }
495 
496 // Load JSON string into this object
497 void CacheDisk::SetJson(const std::string value) {
498 
499  // Parse JSON string into JSON objects
500  try
501  {
502  const Json::Value root = openshot::stringToJson(value);
503  // Set all values that match
504  SetJsonValue(root);
505  }
506  catch (const std::exception& e)
507  {
508  // Error parsing JSON (or missing keys)
509  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
510  }
511 }
512 
513 // Load Json::Value into this object
514 void CacheDisk::SetJsonValue(const Json::Value root) {
515 
516  // Close timeline before we do anything (this also removes all open and closing clips)
517  Clear();
518 
519  // Set parent data
521 
522  if (!root["type"].isNull())
523  cache_type = root["type"].asString();
524  if (!root["path"].isNull())
525  // Update duration of timeline
526  InitPath(root["path"].asString());
527 }
juce::CriticalSection * cacheCriticalSection
Section lock for multiple threads.
Definition: CacheBase.h:55
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheDisk.cpp:164
int64_t Count()
Count the frames in the queue.
Definition: CacheDisk.cpp:433
STL namespace.
int64_t GetBytes()
Gets the maximum bytes value.
Definition: CacheDisk.cpp:315
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
std::string Json()
Generate JSON string of this object.
Definition: CacheDisk.cpp:463
Header file for all Exception classes.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheDisk.cpp:223
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheDisk.cpp:331
Json::Value JsonValue()
Generate Json::Value for this object.
Definition: CacheDisk.cpp:470
void MoveToFront(int64_t frame_number)
Move frame to front of queue (so it lasts longer)
Definition: CacheDisk.cpp:386
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CacheDisk.cpp:514
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:48
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::Value for this object.
Definition: CacheBase.cpp:57
void Clear()
Clear the cache of all frames.
Definition: CacheDisk.cpp:412
Header file for QtUtilities (compatibiity overlay)
std::string cache_type
This is a friendly type name of the derived cache instance.
Definition: CacheBase.h:51
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: CacheBase.cpp:70
Exception for invalid JSON.
Definition: Exceptions.h:205
Header file for CacheDisk class.
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: CacheDisk.cpp:293
int64_t max_bytes
This is the max number of bytes to cache (0 = no limit)
Definition: CacheBase.h:52
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CacheDisk.cpp:497
CacheDisk(std::string cache_path, std::string format, float quality, float scale)
Default constructor, no max bytes.
Definition: CacheDisk.cpp:42