OpenShot Library | libopenshot  0.2.7
Frame.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for Frame class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "Frame.h"
32 #include <OpenShotAudio.h>
33 
34 #include <QApplication>
35 #include <QImage>
36 #include <QPixmap>
37 #include <QBitmap>
38 #include <QColor>
39 #include <QString>
40 #include <QVector>
41 #include <QPainter>
42 #include <QHBoxLayout>
43 #include <QWidget>
44 #include <QLabel>
45 #include <QPointF>
46 #include <QWidget>
47 
48 #include <thread> // for std::this_thread::sleep_for
49 #include <chrono> // for std::chrono::milliseconds
50 
51 using namespace std;
52 using namespace openshot;
53 
54 // Constructor - image & audio
55 Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
56  : audio(std::make_shared<juce::AudioSampleBuffer>(channels, samples)),
57  number(number), width(width), height(height),
58  pixel_ratio(1,1), color(color), qbuffer(NULL),
59  channels(channels), channel_layout(LAYOUT_STEREO),
60  sample_rate(44100),
61  has_audio_data(false), has_image_data(false),
62  max_audio_sample(0)
63 {
64  // zero (fill with silence) the audio buffer
65  audio->clear();
66 }
67 
68 // Delegating Constructor - blank frame
69 Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {};
70 
71 // Delegating Constructor - image only
72 Frame::Frame(int64_t number, int width, int height, std::string color)
73  : Frame::Frame(number, width, height, color, 0, 2) {};
74 
75 // Delegating Constructor - audio only
76 Frame::Frame(int64_t number, int samples, int channels)
77  : Frame::Frame(number, 1, 1, "#000000", samples, channels) {};
78 
79 
80 // Copy constructor
81 Frame::Frame ( const Frame &other )
82 {
83  // copy pointers and data
84  DeepCopy(other);
85 }
86 
87 // Assignment operator
89 {
90  // copy pointers and data
91  DeepCopy(other);
92 
93  return *this;
94 }
95 
96 // Copy data and pointers from another Frame instance
97 void Frame::DeepCopy(const Frame& other)
98 {
99  number = other.number;
100  channels = other.channels;
101  width = other.width;
102  height = other.height;
103  channel_layout = other.channel_layout;
106  sample_rate = other.sample_rate;
107  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
108  color = other.color;
109  max_audio_sample = other.max_audio_sample;
110 
111  if (other.image)
112  image = std::make_shared<QImage>(*(other.image));
113  if (other.audio)
114  audio = std::make_shared<juce::AudioSampleBuffer>(*(other.audio));
115  if (other.wave_image)
116  wave_image = std::make_shared<QImage>(*(other.wave_image));
117 }
118 
119 // Destructor
121  // Clear all pointers
122  image.reset();
123  audio.reset();
124  #ifdef USE_OPENCV
125  imagecv.release();
126  #endif
127 }
128 
129 // Display the frame image to the screen (primarily used for debugging reasons)
131 {
132  if (!QApplication::instance()) {
133  // Only create the QApplication once
134  static int argc = 1;
135  static char* argv[1] = {NULL};
136  previewApp = std::make_shared<QApplication>(argc, argv);
137  }
138 
139  // Get preview image
140  std::shared_ptr<QImage> previewImage = GetImage();
141 
142  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
143  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
144  {
145  // Calculate correct DAR (display aspect ratio)
146  int new_width = previewImage->size().width();
147  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
148 
149  // Resize to fix DAR
150  previewImage = std::make_shared<QImage>(previewImage->scaled(
151  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
152  }
153 
154  // Create window
155  QWidget previewWindow;
156  previewWindow.setStyleSheet("background-color: #000000;");
157  QHBoxLayout layout;
158 
159  // Create label with current frame's image
160  QLabel previewLabel;
161  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
162  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
163  layout.addWidget(&previewLabel);
164 
165  // Show the window
166  previewWindow.setLayout(&layout);
167  previewWindow.show();
168  previewApp->exec();
169 }
170 
171 // Get an audio waveform image
172 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
173 {
174  // Clear any existing waveform image
175  ClearWaveform();
176 
177  // Init a list of lines
178  QVector<QPointF> lines;
179  QVector<QPointF> labels;
180 
181  // Calculate width of an image based on the # of samples
182  int total_samples = GetAudioSamplesCount();
183  if (total_samples > 0)
184  {
185  // If samples are present...
186  int new_height = 200 * audio->getNumChannels();
187  int height_padding = 20 * (audio->getNumChannels() - 1);
188  int total_height = new_height + height_padding;
189  int total_width = 0;
190 
191  // Loop through each audio channel
192  float Y = 100.0;
193  for (int channel = 0; channel < audio->getNumChannels(); channel++)
194  {
195  float X = 0.0;
196 
197  // Get audio for this channel
198  const float *samples = audio->getReadPointer(channel);
199 
200  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
201  {
202  // Sample value (scaled to -100 to 100)
203  float value = samples[sample] * 100.0;
204 
205  // Append a line segment for each sample
206  lines.push_back(QPointF(X,Y+1.0));
207  lines.push_back(QPointF(X,(Y-value)+1.0));
208  }
209 
210  // Add Channel Label Coordinate
211  labels.push_back(QPointF(5.0, Y - 5.0));
212 
213  // Increment Y
214  Y += (200 + height_padding);
215  total_width = X;
216  }
217 
218  // Create blank image
219  wave_image = std::make_shared<QImage>(
220  total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
221  wave_image->fill(QColor(0,0,0,0));
222 
223  // Load QPainter with wave_image device
224  QPainter painter(wave_image.get());
225 
226  // Set pen color
227  QPen pen;
228  pen.setColor(QColor(Red, Green, Blue, Alpha));
229  pen.setWidthF(1.0);
230  pen.setStyle(Qt::SolidLine);
231  painter.setPen(pen);
232 
233  // Draw the waveform
234  painter.drawLines(lines);
235  painter.end();
236 
237  // Resize Image (if requested)
238  if (width != total_width || height != total_height) {
239  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
240  wave_image = std::make_shared<QImage>(scaled_wave_image);
241  }
242  }
243  else
244  {
245  // No audio samples present
246  wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
247  wave_image->fill(QColor(QString::fromStdString("#000000")));
248  }
249 
250  // Return new image
251  return wave_image;
252 }
253 
254 // Clear the waveform image (and deallocate its memory)
256 {
257  if (wave_image)
258  wave_image.reset();
259 }
260 
261 // Get an audio waveform image pixels
262 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
263 {
264  // Get audio wave form image
265  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
266 
267  // Return array of pixel packets
268  return wave_image->constBits();
269 }
270 
271 // Display the wave form
273 {
274  // Get audio wave form image
275  GetWaveform(720, 480, 0, 123, 255, 255);
276 
277  if (!QApplication::instance()) {
278  // Only create the QApplication once
279  static int argc = 1;
280  static char* argv[1] = {NULL};
281  previewApp = std::make_shared<QApplication>(argc, argv);
282  }
283 
284  // Create window
285  QWidget previewWindow;
286  previewWindow.setStyleSheet("background-color: #000000;");
287  QHBoxLayout layout;
288 
289  // Create label with current frame's waveform image
290  QLabel previewLabel;
291  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
292  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
293  layout.addWidget(&previewLabel);
294 
295  // Show the window
296  previewWindow.setLayout(&layout);
297  previewWindow.show();
298  previewApp->exec();
299 
300  // Deallocate waveform image
301  ClearWaveform();
302 }
303 
304 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
305 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
306 {
307  if (channel > 0) {
308  // return average magnitude for a specific channel/sample range
309  return audio->getMagnitude(channel, sample, magnitude_range);
310 
311  } else {
312  // Return average magnitude for all channels
313  return audio->getMagnitude(sample, magnitude_range);
314  }
315 }
316 
317 // Get an array of sample data
318 float* Frame::GetAudioSamples(int channel)
319 {
320  // return JUCE audio data for this channel
321  return audio->getWritePointer(channel);
322 }
323 
324 // Get a planar array of sample data, using any sample rate
325 float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
326 {
327  float *output = NULL;
328  juce::AudioSampleBuffer *buffer(audio.get());
329  int num_of_channels = audio->getNumChannels();
330  int num_of_samples = GetAudioSamplesCount();
331 
332  // Resample to new sample rate (if needed)
333  if (new_sample_rate != sample_rate)
334  {
335  // YES, RESAMPLE AUDIO
336  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
337 
338  // Resample data, and return new buffer pointer
339  buffer = resampler->GetResampledBuffer();
340 
341  // Update num_of_samples
342  num_of_samples = buffer->getNumSamples();
343  }
344 
345  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
346  output = new float[num_of_channels * num_of_samples];
347  int position = 0;
348 
349  // Loop through samples in each channel (combining them)
350  for (int channel = 0; channel < num_of_channels; channel++)
351  {
352  for (int sample = 0; sample < num_of_samples; sample++)
353  {
354  // Add sample to output array
355  output[position] = buffer->getReadPointer(channel)[sample];
356 
357  // increment position
358  position++;
359  }
360  }
361 
362  // Update sample count (since it might have changed due to resampling)
363  *sample_count = num_of_samples;
364 
365  // return combined array
366  return output;
367 }
368 
369 
370 // Get an array of sample data (all channels interleaved together), using any sample rate
371 float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
372 {
373  float *output = NULL;
374  juce::AudioSampleBuffer *buffer(audio.get());
375  int num_of_channels = audio->getNumChannels();
376  int num_of_samples = GetAudioSamplesCount();
377 
378  // Resample to new sample rate (if needed)
379  if (new_sample_rate != sample_rate && resampler)
380  {
381  // YES, RESAMPLE AUDIO
382  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
383 
384  // Resample data, and return new buffer pointer
385  buffer = resampler->GetResampledBuffer();
386 
387  // Update num_of_samples
388  num_of_samples = buffer->getNumSamples();
389  }
390 
391  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
392  output = new float[num_of_channels * num_of_samples];
393  int position = 0;
394 
395  // Loop through samples in each channel (combining them)
396  for (int sample = 0; sample < num_of_samples; sample++)
397  {
398  for (int channel = 0; channel < num_of_channels; channel++)
399  {
400  // Add sample to output array
401  output[position] = buffer->getReadPointer(channel)[sample];
402 
403  // increment position
404  position++;
405  }
406  }
407 
408  // Update sample count (since it might have changed due to resampling)
409  *sample_count = num_of_samples;
410 
411  // return combined array
412  return output;
413 }
414 
415 // Get number of audio channels
417 {
418  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
419  if (audio)
420  return audio->getNumChannels();
421  else
422  return 0;
423 }
424 
425 // Get number of audio samples
427 {
428  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
429  return max_audio_sample;
430 }
431 
432 juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer()
433 {
434  return audio.get();
435 }
436 
437 // Get the size in bytes of this frame (rough estimate)
439 {
440  int64_t total_bytes = 0;
441  if (image)
442  total_bytes += (width * height * sizeof(char) * 4);
443  if (audio) {
444  // approximate audio size (sample rate / 24 fps)
445  total_bytes += (sample_rate / 24.0) * sizeof(float);
446  }
447 
448  // return size of this frame
449  return total_bytes;
450 }
451 
452 // Get pixel data (as packets)
453 const unsigned char* Frame::GetPixels()
454 {
455  // Check for blank image
456  if (!image)
457  // Fill with black
458  AddColor(width, height, color);
459 
460  // Return array of pixel packets
461  return image->constBits();
462 }
463 
464 // Get pixel data (for only a single scan-line)
465 const unsigned char* Frame::GetPixels(int row)
466 {
467  // Check for blank image
468  if (!image)
469  // Fill with black
470  AddColor(width, height, color);
471 
472  // Return array of pixel packets
473  return image->constScanLine(row);
474 }
475 
476 // Check a specific pixel color value (returns True/False)
477 bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
478  int col_pos = col * 4; // Find column array position
479  if (!image || row < 0 || row >= (height - 1) ||
480  col_pos < 0 || col_pos >= (width - 1) ) {
481  // invalid row / col
482  return false;
483  }
484  // Check pixel color
485  const unsigned char* pixels = GetPixels(row);
486  if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
487  pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
488  pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
489  pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
490  // Pixel color matches successfully
491  return true;
492  } else {
493  // Pixel color does not match
494  return false;
495  }
496 }
497 
498 // Set Pixel Aspect Ratio
499 void Frame::SetPixelRatio(int num, int den)
500 {
501  pixel_ratio.num = num;
502  pixel_ratio.den = den;
503 }
504 
505 // Set frame number
506 void Frame::SetFrameNumber(int64_t new_number)
507 {
508  number = new_number;
509 }
510 
511 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
512 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
513 {
514  // Get the total # of samples for the previous frame, and the current frame (rounded)
515  double fps_rate = fps.Reciprocal().ToDouble();
516 
517  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
518  double previous_samples = (sample_rate * fps_rate) * (number - 1);
519  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
520  previous_samples -= previous_samples_remainder;
521 
522  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
523  double total_samples = (sample_rate * fps_rate) * number;
524  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
525  total_samples -= total_samples_remainder;
526 
527  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
528  // be evenly divided into frames, so each frame can have have different # of samples.
529  int samples_per_frame = round(total_samples - previous_samples);
530  if (samples_per_frame < 0)
531  samples_per_frame = 0;
532  return samples_per_frame;
533 }
534 
535 // Calculate the # of samples per video frame (for the current frame number)
536 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
537 {
538  return GetSamplesPerFrame(number, fps, sample_rate, channels);
539 }
540 
541 // Get height of image
543 {
544  return height;
545 }
546 
547 // Get height of image
549 {
550  return width;
551 }
552 
553 // Get the original sample rate of this frame's audio data
555 {
556  return sample_rate;
557 }
558 
559 // Get the original sample rate of this frame's audio data
561 {
562  return channel_layout;
563 }
564 
565 
566 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
567 void Frame::Save(std::string path, float scale, std::string format, int quality)
568 {
569  // Get preview image
570  std::shared_ptr<QImage> previewImage = GetImage();
571 
572  // scale image if needed
573  if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
574  {
575  int new_width = width;
576  int new_height = height;
577 
578  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
579  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
580  {
581  // Calculate correct DAR (display aspect ratio)
582  int new_width = previewImage->size().width();
583  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
584 
585  // Resize to fix DAR
586  previewImage = std::make_shared<QImage>(previewImage->scaled(
587  new_width, new_height,
588  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
589  }
590 
591  // Resize image
592  previewImage = std::make_shared<QImage>(previewImage->scaled(
593  new_width * scale, new_height * scale,
594  Qt::KeepAspectRatio, Qt::SmoothTransformation));
595  }
596 
597  // Save image
598  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
599 }
600 
601 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
602 void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
603  std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
604 
605  // Create blank thumbnail image & fill background color
606  auto thumbnail = std::make_shared<QImage>(
607  new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
608  thumbnail->fill(QColor(QString::fromStdString(background_color)));
609 
610  // Create painter
611  QPainter painter(thumbnail.get());
612  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
613 
614  // Get preview image
615  std::shared_ptr<QImage> previewImage = GetImage();
616 
617  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
618  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
619  {
620  // Calculate correct DAR (display aspect ratio)
621  int aspect_width = previewImage->size().width();
622  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
623 
624  // Resize to fix DAR
625  previewImage = std::make_shared<QImage>(previewImage->scaled(
626  aspect_width, aspect_height,
627  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
628  }
629 
630  // Resize frame image
631  if (ignore_aspect)
632  // Ignore aspect ratio
633  previewImage = std::make_shared<QImage>(previewImage->scaled(
634  new_width, new_height,
635  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
636  else
637  // Maintain aspect ratio
638  previewImage = std::make_shared<QImage>(previewImage->scaled(
639  new_width, new_height,
640  Qt::KeepAspectRatio, Qt::SmoothTransformation));
641 
642  // Composite frame image onto background (centered)
643  int x = (new_width - previewImage->size().width()) / 2.0; // center
644  int y = (new_height - previewImage->size().height()) / 2.0; // center
645  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
646 
647 
648  // Create transform and rotate (if needed)
649  QTransform transform;
650  float origin_x = previewImage->width() / 2.0;
651  float origin_y = previewImage->height() / 2.0;
652  transform.translate(origin_x, origin_y);
653  transform.rotate(rotate);
654  transform.translate(-origin_x,-origin_y);
655  painter.setTransform(transform);
656 
657  // Draw image onto QImage
658  painter.drawImage(x, y, *previewImage);
659 
660 
661  // Overlay Image (if any)
662  if (overlay_path != "") {
663  // Open overlay
664  auto overlay = std::make_shared<QImage>();
665  overlay->load(QString::fromStdString(overlay_path));
666 
667  // Set pixel format
668  overlay = std::make_shared<QImage>(
669  overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
670 
671  // Resize to fit
672  overlay = std::make_shared<QImage>(overlay->scaled(
673  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
674 
675  // Composite onto thumbnail
676  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
677  painter.drawImage(0, 0, *overlay);
678  }
679 
680 
681  // Mask Image (if any)
682  if (mask_path != "") {
683  // Open mask
684  auto mask = std::make_shared<QImage>();
685  mask->load(QString::fromStdString(mask_path));
686 
687  // Set pixel format
688  mask = std::make_shared<QImage>(
689  mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
690 
691  // Resize to fit
692  mask = std::make_shared<QImage>(mask->scaled(
693  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
694 
695  // Negate mask
696  mask->invertPixels();
697 
698  // Get pixels
699  unsigned char *pixels = (unsigned char *) thumbnail->bits();
700  const unsigned char *mask_pixels = (const unsigned char *) mask->constBits();
701 
702  // Convert the mask image to grayscale
703  // Loop through pixels
704  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
705  {
706  // Get the RGB values from the pixel
707  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
708  int Frame_Alpha = pixels[byte_index + 3];
709  int Mask_Value = constrain(Frame_Alpha - gray_value);
710 
711  // Set all alpha pixels to gray value
712  pixels[byte_index + 3] = Mask_Value;
713  }
714  }
715 
716 
717  // End painter
718  painter.end();
719 
720  // Save image
721  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
722 }
723 
724 // Constrain a color value from 0 to 255
725 int Frame::constrain(int color_value)
726 {
727  // Constrain new color from 0 to 255
728  if (color_value < 0)
729  color_value = 0;
730  else if (color_value > 255)
731  color_value = 255;
732 
733  return color_value;
734 }
735 
736 // Add (or replace) pixel data to the frame (based on a solid color)
737 void Frame::AddColor(int new_width, int new_height, std::string new_color)
738 {
739  // Set color
740  color = new_color;
741 
742  // Create new image object, and fill with pixel data
743  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
744  image = std::make_shared<QImage>(new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
745 
746  // Fill with solid color
747  image->fill(QColor(QString::fromStdString(color)));
748 
749  // Update height and width
750  width = image->width();
751  height = image->height();
752  has_image_data = true;
753 }
754 
755 // Add (or replace) pixel data to the frame
757  int new_width, int new_height, int bytes_per_pixel,
758  QImage::Format type, const unsigned char *pixels_)
759 {
760  // Create new buffer
761  {
762  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
763  qbuffer = pixels_;
764  } // Release addingImageSection lock
765 
766  // Create new image object from pixel data
767  auto new_image = std::make_shared<QImage>(
768  qbuffer,
769  new_width, new_height,
770  new_width * bytes_per_pixel,
771  type,
772  (QImageCleanupFunction) &openshot::Frame::cleanUpBuffer,
773  (void*) qbuffer
774  );
775  AddImage(new_image);
776 }
777 
778 // Add (or replace) pixel data to the frame
779 void Frame::AddImage(std::shared_ptr<QImage> new_image)
780 {
781  // Ignore blank images
782  if (!new_image)
783  return;
784 
785  // assign image data
786  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
787  image = new_image;
788 
789  // Always convert to Format_RGBA8888_Premultiplied (if different)
790  if (image->format() != QImage::Format_RGBA8888_Premultiplied)
791  *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
792 
793  // Update height and width
794  width = image->width();
795  height = image->height();
796  has_image_data = true;
797 }
798 
799 // Add (or replace) pixel data to the frame (for only the odd or even lines)
800 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
801 {
802  // Ignore blank new_image
803  if (!new_image)
804  return;
805 
806  // Check for blank source image
807  if (!image) {
808  // Replace the blank source image
809  AddImage(new_image);
810 
811  } else {
812  // Ignore image of different sizes or formats
813  bool ret=false;
814  if (image == new_image || image->size() != new_image->size()) {
815  ret = true;
816  }
817  else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
818  new_image = std::make_shared<QImage>(
819  new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
820  }
821  if (ret) {
822  return;
823  }
824 
825  // Get the frame's image
826  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
827  unsigned char *pixels = image->bits();
828  const unsigned char *new_pixels = new_image->constBits();
829 
830  // Loop through the scanlines of the image (even or odd)
831  int start = 0;
832  if (only_odd_lines)
833  start = 1;
834 
835  for (int row = start; row < image->height(); row += 2) {
836  int offset = row * image->bytesPerLine();
837  memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
838  }
839 
840  // Update height and width
841  height = image->height();
842  width = image->width();
843  has_image_data = true;
844  }
845 }
846 
847 
848 // Resize audio container to hold more (or less) samples and channels
849 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
850 {
851  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
852 
853  // Resize JUCE audio buffer
854  audio->setSize(channels, length, true, true, false);
855  channel_layout = layout;
856  sample_rate = rate;
857 
858  // Calculate max audio sample added
859  max_audio_sample = length;
860 }
861 
862 // Add audio samples to a specific channel
863 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
864  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
865 
866  // Clamp starting sample to 0
867  int destStartSampleAdjusted = max(destStartSample, 0);
868 
869  // Extend audio container to hold more (or less) samples and channels.. if needed
870  int new_length = destStartSampleAdjusted + numSamples;
871  int new_channel_length = audio->getNumChannels();
872  if (destChannel >= new_channel_length)
873  new_channel_length = destChannel + 1;
874  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
875  audio->setSize(new_channel_length, new_length, true, true, false);
876 
877  // Clear the range of samples first (if needed)
878  if (replaceSamples)
879  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
880 
881  // Add samples to frame's audio buffer
882  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
883  has_audio_data = true;
884 
885  // Calculate max audio sample added
886  if (new_length > max_audio_sample)
887  max_audio_sample = new_length;
888 }
889 
890 // Apply gain ramp (i.e. fading volume)
891 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
892 {
893  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
894 
895  // Apply gain ramp
896  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
897 }
898 
899 // Get pointer to Magick++ image object
900 std::shared_ptr<QImage> Frame::GetImage()
901 {
902  // Check for blank image
903  if (!image)
904  // Fill with black
905  AddColor(width, height, color);
906 
907  return image;
908 }
909 
910 #ifdef USE_OPENCV
911 
912 // Convert Qimage to Mat
913 cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
914 
915  cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
916  cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
917  int from_to[] = { 0,0, 1,1, 2,2 };
918  cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
919  cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
920  return mat2;
921 };
922 
923 // Get pointer to OpenCV image object
925 {
926  // Check for blank image
927  if (!image)
928  // Fill with black
929  AddColor(width, height, color);
930 
931  // if (imagecv.empty())
932  // Convert Qimage to Mat
933  imagecv = Qimage2mat(image);
934 
935  return imagecv;
936 }
937 
938 std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
939  cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
940  QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
941 
942  std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
943 
944  // Always convert to RGBA8888 (if different)
945  if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
946  *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
947 
948  return imgIn;
949 }
950 
951 // Set pointer to OpenCV image object
952 void Frame::SetImageCV(cv::Mat _image)
953 {
954  imagecv = _image;
955  image = Mat2Qimage(_image);
956 }
957 #endif
958 
959 #ifdef USE_IMAGEMAGICK
960 // Get pointer to ImageMagick image object
961 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
962 {
963  // Check for blank image
964  if (!image)
965  // Fill with black
966  AddColor(width, height, "#000000");
967 
968  // Get the pixels from the frame image
969  const QRgb *tmpBits = (const QRgb*)image->constBits();
970 
971  // Create new image object, and fill with pixel data
972  auto magick_image = std::make_shared<Magick::Image>(
973  image->width(), image->height(),"RGBA", Magick::CharPixel, tmpBits);
974 
975  // Give image a transparent background color
976  magick_image->backgroundColor(Magick::Color("none"));
977  magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
978  MAGICK_IMAGE_ALPHA(magick_image, true);
979 
980  return magick_image;
981 }
982 #endif
983 
984 #ifdef USE_IMAGEMAGICK
985 // Get pointer to QImage of frame
986 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
987 {
988  const int BPP = 4;
989  const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
990 
991  /// Use realloc for fast memory allocation.
992  /// TODO: consider locking the buffer for mt safety
993  //qbuffer = reinterpret_cast<unsigned char*>(realloc(qbuffer, bufferSize));
994  qbuffer = new unsigned char[bufferSize]();
995  unsigned char *buffer = (unsigned char*)qbuffer;
996 
997  MagickCore::ExceptionInfo exception;
998  // TODO: Actually do something, if we get an exception here
999  MagickCore::ExportImagePixels(new_image->constImage(), 0, 0, new_image->columns(), new_image->rows(), "RGBA", Magick::CharPixel, buffer, &exception);
1000 
1001  // Create QImage of frame data
1002  image = std::make_shared<QImage>(
1003  qbuffer, width, height, width * BPP, QImage::Format_RGBA8888_Premultiplied,
1004  (QImageCleanupFunction) &cleanUpBuffer, (void*) qbuffer);
1005 
1006  // Update height and width
1007  width = image->width();
1008  height = image->height();
1009  has_image_data = true;
1010 }
1011 #endif
1012 
1013 // Play audio samples for this frame
1015 {
1016  // Check if samples are present
1017  if (!GetAudioSamplesCount())
1018  return;
1019 
1020  juce::AudioDeviceManager deviceManager;
1021  juce::String error = deviceManager.initialise (
1022  0, /* number of input channels */
1023  2, /* number of output channels */
1024  0, /* no XML settings.. */
1025  true /* select default device on failure */);
1026 
1027  // Output error (if any)
1028  if (error.isNotEmpty()) {
1029  cout << "Error on initialise(): " << error << endl;
1030  }
1031 
1032  juce::AudioSourcePlayer audioSourcePlayer;
1033  deviceManager.addAudioCallback (&audioSourcePlayer);
1034 
1035  std::unique_ptr<AudioBufferSource> my_source;
1036  my_source.reset (new AudioBufferSource (audio.get()));
1037 
1038  // Create TimeSliceThread for audio buffering
1039  juce::TimeSliceThread my_thread("Audio buffer thread");
1040 
1041  // Start thread
1042  my_thread.startThread();
1043 
1044  juce::AudioTransportSource transport1;
1045  transport1.setSource (my_source.get(),
1046  5000, // tells it to buffer this many samples ahead
1047  &my_thread,
1048  (double) sample_rate,
1049  audio->getNumChannels()); // sample rate of source
1050  transport1.setPosition (0);
1051  transport1.setGain(1.0);
1052 
1053 
1054  // Create MIXER
1055  juce::MixerAudioSource mixer;
1056  mixer.addInputSource(&transport1, false);
1057  audioSourcePlayer.setSource (&mixer);
1058 
1059  // Start transports
1060  transport1.start();
1061 
1062  while (transport1.isPlaying())
1063  {
1064  cout << "playing" << endl;
1065  std::this_thread::sleep_for(std::chrono::seconds(1));
1066  }
1067 
1068  cout << "DONE!!!" << endl;
1069 
1070  transport1.stop();
1071  transport1.setSource (0);
1072  audioSourcePlayer.setSource (0);
1073  my_thread.stopThread(500);
1074  deviceManager.removeAudioCallback (&audioSourcePlayer);
1075  deviceManager.closeAudioDevice();
1076  deviceManager.removeAllChangeListeners();
1077  deviceManager.dispatchPendingMessages();
1078 
1079  cout << "End of Play()" << endl;
1080 
1081 
1082 }
1083 
1084 // Clean up buffer after QImage is deleted
1085 void Frame::cleanUpBuffer(void *info)
1086 {
1087  if (info)
1088  {
1089  // Remove buffer since QImage tells us to
1090  unsigned char* ptr_to_qbuffer = (unsigned char*) info;
1091  delete[] ptr_to_qbuffer;
1092  }
1093 }
1094 
1095 // Add audio silence
1096 void Frame::AddAudioSilence(int numSamples)
1097 {
1098  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
1099 
1100  // Resize audio container
1101  audio->setSize(channels, numSamples, false, true, false);
1102  audio->clear();
1103  has_audio_data = true;
1104 
1105  // Calculate max audio sample added
1106  if (numSamples > max_audio_sample)
1107  max_audio_sample = numSamples;
1108 }
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
Definition: Frame.cpp:986
int GetWidth()
Get height of image.
Definition: Frame.cpp:548
int num
Numerator for the fraction.
Definition: Fraction.h:50
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:426
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:737
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:602
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:849
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:938
#define MAGICK_IMAGE_ALPHA(im, a)
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:107
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
Definition: Frame.cpp:961
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:262
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:432
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:453
STL namespace.
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:59
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:1014
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:97
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:863
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:130
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:135
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:88
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:536
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:255
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:325
Header file for Frame class.
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:318
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:506
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:924
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1096
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:136
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
Definition: Frame.cpp:371
This class represents a fraction.
Definition: Fraction.h:48
void SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1085
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:94
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:756
Frame()
Constructor - blank frame.
Definition: Frame.cpp:69
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:891
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:272
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:913
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:499
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:416
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:560
std::shared_ptr< juce::AudioSampleBuffer > audio
Definition: Frame.h:134
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:900
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:172
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:438
virtual ~Frame()
Destructor.
Definition: Frame.cpp:120
juce::AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
int den
Denominator for the fraction.
Definition: Fraction.h:51
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
Definition: Frame.cpp:305
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:952
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:137
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
Definition: Frame.cpp:567
int GetHeight()
Get height of image.
Definition: Frame.cpp:542
int SampleRate()
Get the original sample rate of this frame&#39;s audio data.
Definition: Frame.cpp:554
This class is used to resample audio data for many sequential frames.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:477