OpenShot Library | libopenshot  0.2.7
FrameMapper.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for the FrameMapper class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "FrameMapper.h"
32 #include "Exceptions.h"
33 #include "Clip.h"
34 
35 using namespace std;
36 using namespace openshot;
37 
38 FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
39  reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0)
40 {
41  // Set the original frame rate from the reader
42  original = Fraction(reader->info.fps.num, reader->info.fps.den);
43 
44  // Set all info struct members equal to the internal reader
45  info = reader->info;
46  info.fps.num = target.num;
47  info.fps.den = target.den;
48  info.video_timebase.num = target.den;
49  info.video_timebase.den = target.num;
51  info.sample_rate = target_sample_rate;
52  info.channels = target_channels;
53  info.channel_layout = target_channel_layout;
54  info.width = reader->info.width;
55  info.height = reader->info.height;
56 
57  // Used to toggle odd / even fields
58  field_toggle = true;
59 
60  // Adjust cache size based on size of frame and audio
62 }
63 
64 // Destructor
66 
67  // Auto Close if not already
68  Close();
69 
70  reader = NULL;
71 }
72 
73 /// Get the current reader
75 {
76  if (reader)
77  return reader;
78  else
79  // Throw error if reader not initialized
80  throw ReaderClosed("No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
81 }
82 
83 void FrameMapper::AddField(int64_t frame)
84 {
85  // Add a field, and toggle the odd / even field
86  AddField(Field(frame, field_toggle));
87 }
88 
89 void FrameMapper::AddField(Field field)
90 {
91  // Add a field to the end of the field list
92  fields.push_back(field);
93 
94  // toggle the odd / even flag
95  field_toggle = (field_toggle ? false : true);
96 }
97 
98 // Use the original and target frame rates and a pull-down technique to create
99 // a mapping between the original fields and frames or a video to a new frame rate.
100 // This might repeat or skip fields and frames of the original video, depending on
101 // whether the frame rate is increasing or decreasing.
102 void FrameMapper::Init()
103 {
104  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)");
105 
106  // Do not initialize anything if just a picture with no audio
108  // Skip initialization
109  return;
110 
111  // Clear the fields & frames lists
112  fields.clear();
113  frames.clear();
114 
115  // Find parent position (if any)
116  Clip *parent = (Clip *) ParentClip();
117  if (parent) {
118  parent_position = parent->Position();
119  parent_start = parent->Start();
120  } else {
121  parent_position = 0.0;
122  parent_start = 0.0;
123  }
124 
125  // Mark as not dirty
126  is_dirty = false;
127 
128  // Clear cache
129  final_cache.Clear();
130 
131  // Some framerates are handled special, and some use a generic Keyframe curve to
132  // map the framerates. These are the special framerates:
133  if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
134  (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
135 
136  // Get the difference (in frames) between the original and target frame rates
137  float difference = target.ToInt() - original.ToInt();
138 
139  // Find the number (i.e. interval) of fields that need to be skipped or repeated
140  int field_interval = 0;
141  int frame_interval = 0;
142 
143  if (difference != 0)
144  {
145  field_interval = round(fabs(original.ToInt() / difference));
146 
147  // Get frame interval (2 fields per frame)
148  frame_interval = field_interval * 2.0f;
149  }
150 
151 
152  // Calculate # of fields to map
153  int64_t frame = 1;
154  int64_t number_of_fields = reader->info.video_length * 2;
155 
156  // Loop through all fields in the original video file
157  for (int64_t field = 1; field <= number_of_fields; field++)
158  {
159 
160  if (difference == 0) // Same frame rate, NO pull-down or special techniques required
161  {
162  // Add fields
163  AddField(frame);
164  }
165  else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
166  {
167  // Add current field
168  AddField(frame);
169 
170  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
171  {
172  // Add extra field for each 'field interval
173  AddField(frame);
174  }
175  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
176  {
177  // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
178  AddField(frame); // add field for current frame
179 
180  if (frame + 1 <= info.video_length)
181  // add field for next frame (if the next frame exists)
182  AddField(Field(frame + 1, field_toggle));
183  }
184  else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
185  {
186  // No pull-down technique needed, just repeat this frame
187  AddField(frame);
188  AddField(frame);
189  }
190  }
191  else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
192  {
193 
194  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
195  {
196  // skip current field and toggle the odd/even flag
197  field_toggle = (field_toggle ? false : true);
198  }
199  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
200  {
201  // skip this field, plus the next field
202  field++;
203  }
204  else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
205  {
206  // skip this field, plus the next one
207  field++;
208  }
209  else
210  {
211  // No skipping needed, so add the field
212  AddField(frame);
213  }
214  }
215 
216  // increment frame number (if field is divisible by 2)
217  if (field % 2 == 0 && field > 0)
218  frame++;
219  }
220 
221  } else {
222  // Map the remaining framerates using a linear algorithm
223  double rate_diff = target.ToDouble() / original.ToDouble();
224  int64_t new_length = reader->info.video_length * rate_diff;
225 
226  // Calculate the value difference
227  double value_increment = (reader->info.video_length + 1) / (double) (new_length);
228 
229  // Loop through curve, and build list of frames
230  double original_frame_num = 1.0f;
231  for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
232  {
233  // Add 2 fields per frame
234  AddField(round(original_frame_num));
235  AddField(round(original_frame_num));
236 
237  // Increment original frame number
238  original_frame_num += value_increment;
239  }
240  }
241 
242  // Loop through the target frames again (combining fields into frames)
243  Field Odd(0, true); // temp field used to track the ODD field
244  Field Even(0, true); // temp field used to track the EVEN field
245 
246  // Variables used to remap audio samples
247  int64_t start_samples_frame = 1;
248  int start_samples_position = 0;
249 
250  for (std::vector<Field>::size_type field = 1; field <= fields.size(); field++)
251  {
252  // Get the current field
253  Field f = fields[field - 1];
254 
255  // Is field divisible by 2?
256  if (field % 2 == 0 && field > 0)
257  {
258  // New frame number
259  int64_t frame_number = field / 2;
260 
261  // Set the bottom frame
262  if (f.isOdd)
263  Odd = f;
264  else
265  Even = f;
266 
267  // Determine the range of samples (from the original rate). Resampling happens in real-time when
268  // calling the GetFrame() method. So this method only needs to redistribute the original samples with
269  // the original sample rate.
270  int64_t end_samples_frame = start_samples_frame;
271  int end_samples_position = start_samples_position;
272  int remaining_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels);
273 
274  while (remaining_samples > 0)
275  {
276  // Get original samples (with NO framerate adjustments)
277  // This is the original reader's frame numbers
278  int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
279 
280  // Enough samples
281  if (original_samples >= remaining_samples)
282  {
283  // Take all that we need, and break loop
284  end_samples_position += remaining_samples - 1;
285  remaining_samples = 0;
286  } else
287  {
288  // Not enough samples (take them all, and keep looping)
289  end_samples_frame += 1; // next frame
290  end_samples_position = 0; // next frame, starting on 1st sample
291  remaining_samples -= original_samples; // reduce the remaining amount
292  }
293  }
294 
295 
296 
297  // Create the sample mapping struct
298  SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels)};
299 
300  // Reset the audio variables
301  start_samples_frame = end_samples_frame;
302  start_samples_position = end_samples_position + 1;
303  if (start_samples_position >= Frame::GetSamplesPerFrame(AdjustFrameNumber(start_samples_frame), original, reader->info.sample_rate, reader->info.channels))
304  {
305  start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
306  start_samples_position = 0; // reset to 0, since we wrapped
307  }
308 
309  // Create a frame and ADD it to the frames collection
310  MappedFrame frame = {Odd, Even, Samples};
311  frames.push_back(frame);
312  }
313  else
314  {
315  // Set the top field
316  if (f.isOdd)
317  Odd = f;
318  else
319  Even = f;
320  }
321  }
322 
323  // Clear the internal fields list (no longer needed)
324  fields.clear();
325 }
326 
327 MappedFrame FrameMapper::GetMappedFrame(int64_t TargetFrameNumber)
328 {
329  // Check if mappings are dirty (and need to be recalculated)
330  if (is_dirty)
331  // Recalculate mappings
332  Init();
333 
334  // Ignore mapping on single image readers
336  // Return the same number
337  MappedFrame frame;
338  frame.Even.Frame = TargetFrameNumber;
339  frame.Odd.Frame = TargetFrameNumber;
340  frame.Samples.frame_start = 0;
341  frame.Samples.frame_end = 0;
342  frame.Samples.sample_start = 0;
343  frame.Samples.sample_end = 0;
344  frame.Samples.total = 0;
345  return frame;
346  }
347 
348  // Check if frame number is valid
349  if(TargetFrameNumber < 1 || frames.size() == 0)
350  // frame too small, return error
351  throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
352 
353  else if (TargetFrameNumber > (int64_t)frames.size())
354  // frame too large, set to end frame
355  TargetFrameNumber = frames.size();
356 
357  // Debug output
358  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetMappedFrame", "TargetFrameNumber", TargetFrameNumber, "frames.size()", frames.size(), "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame, "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame);
359 
360  // Return frame
361  return frames[TargetFrameNumber - 1];
362 }
363 
364 // Get or generate a blank frame
365 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
366 {
367  std::shared_ptr<Frame> new_frame;
368 
369  // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
370  int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(number), target, reader->info.sample_rate, reader->info.channels);
371 
372  try {
373  // Debug output
374  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (from reader)", "number", number, "samples_in_frame", samples_in_frame);
375 
376  // Attempt to get a frame (but this could fail if a reader has just been closed)
377  new_frame = reader->GetFrame(number);
378 
379  // Return real frame
380  return new_frame;
381 
382  } catch (const ReaderClosed & e) {
383  // ...
384  } catch (const OutOfBoundsFrame & e) {
385  // ...
386  }
387 
388  // Debug output
389  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (create blank)", "number", number, "samples_in_frame", samples_in_frame);
390 
391  // Create blank frame
392  new_frame = std::make_shared<Frame>(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels);
393  new_frame->SampleRate(reader->info.sample_rate);
394  new_frame->ChannelsLayout(info.channel_layout);
395  new_frame->AddAudioSilence(samples_in_frame);
396  return new_frame;
397 }
398 
399 // Get an openshot::Frame object for a specific frame number of this reader.
400 std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
401 {
402  // Check final cache, and just return the frame (if it's available)
403  std::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
404  if (final_frame) return final_frame;
405 
406  // Create a scoped lock, allowing only a single thread to run the following code at one time
407  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
408 
409  // Find parent properties (if any)
410  Clip *parent = (Clip *) ParentClip();
411  if (parent) {
412  float position = parent->Position();
413  float start = parent->Start();
414  if (parent_position != position || parent_start != start) {
415  // Force dirty if parent clip has moved or been trimmed
416  // since this heavily affects frame #s and audio mappings
417  is_dirty = true;
418  }
419  }
420 
421  // Check if mappings are dirty (and need to be recalculated)
422  if (is_dirty)
423  Init();
424 
425  // Check final cache a 2nd time (due to potential lock already generating this frame)
426  final_frame = final_cache.GetFrame(requested_frame);
427  if (final_frame) return final_frame;
428 
429  // Minimum number of frames to process (for performance reasons)
430  // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes
431  int minimum_frames = 1;
432 
433  // Debug output
434  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (Loop through frames)", "requested_frame", requested_frame, "minimum_frames", minimum_frames);
435 
436  // Loop through all requested frames
437  for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
438  {
439 
440  // Debug output
441  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (inside omp for loop)", "frame_number", frame_number, "minimum_frames", minimum_frames, "requested_frame", requested_frame);
442 
443  // Get the mapped frame
444  MappedFrame mapped = GetMappedFrame(frame_number);
445  std::shared_ptr<Frame> mapped_frame;
446 
447  // Get the mapped frame (keeping the sample rate and channels the same as the original... for the moment)
448  mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
449 
450  // Get # of channels in the actual frame
451  int channels_in_frame = mapped_frame->GetAudioChannelsCount();
452  int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
453 
454  // Determine if mapped frame is identical to source frame
455  // including audio sample distribution according to mapped.Samples,
456  // and frame_number. In some cases such as end of stream, the reader
457  // will return a frame with a different frame number. In these cases,
458  // we cannot use the frame as is, nor can we modify the frame number,
459  // otherwise the reader's cache object internals become invalid.
460  if (info.sample_rate == mapped_frame->SampleRate() &&
461  info.channels == mapped_frame->GetAudioChannelsCount() &&
462  info.channel_layout == mapped_frame->ChannelsLayout() &&
463  mapped.Samples.total == mapped_frame->GetAudioSamplesCount() &&
464  mapped.Samples.frame_start == mapped.Odd.Frame &&
465  mapped.Samples.sample_start == 0 &&
466  mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream)
467  info.fps.num == reader->info.fps.num &&
468  info.fps.den == reader->info.fps.den) {
469  // Add original frame to cache, and skip the rest (for performance reasons)
470  final_cache.Add(mapped_frame);
471  continue;
472  }
473 
474  // Create a new frame
475  auto frame = std::make_shared<Frame>(
476  frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame);
477  frame->SampleRate(mapped_frame->SampleRate());
478  frame->ChannelsLayout(mapped_frame->ChannelsLayout());
479 
480 
481  // Copy the image from the odd field
482  std::shared_ptr<Frame> odd_frame;
483  odd_frame = GetOrCreateFrame(mapped.Odd.Frame);
484 
485  if (odd_frame)
486  frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()), true);
487  if (mapped.Odd.Frame != mapped.Even.Frame) {
488  // Add even lines (if different than the previous image)
489  std::shared_ptr<Frame> even_frame;
490  even_frame = GetOrCreateFrame(mapped.Even.Frame);
491  if (even_frame)
492  frame->AddImage(
493  std::make_shared<QImage>(*even_frame->GetImage()), false);
494  }
495 
496  // Resample audio on frame (if needed)
497  bool need_resampling = false;
498  if (info.has_audio &&
499  (info.sample_rate != frame->SampleRate() ||
500  info.channels != frame->GetAudioChannelsCount() ||
501  info.channel_layout != frame->ChannelsLayout()))
502  // Resample audio and correct # of channels if needed
503  need_resampling = true;
504 
505  // create a copy of mapped.Samples that will be used by copy loop
506  SampleRange copy_samples = mapped.Samples;
507 
508  if (need_resampling)
509  {
510  // Resampling needed, modify copy of SampleRange object that
511  // includes some additional input samples on first iteration,
512  // and continues the offset to ensure that the sample rate
513  // converter isn't input limited.
514  const int EXTRA_INPUT_SAMPLES = 100;
515 
516  // Extend end sample count by an additional EXTRA_INPUT_SAMPLES samples
517  copy_samples.sample_end += EXTRA_INPUT_SAMPLES;
518  int samples_per_end_frame =
519  Frame::GetSamplesPerFrame(copy_samples.frame_end, original,
520  reader->info.sample_rate, reader->info.channels);
521  if (copy_samples.sample_end >= samples_per_end_frame)
522  {
523  // check for wrapping
524  copy_samples.frame_end++;
525  copy_samples.sample_end -= samples_per_end_frame;
526  }
527  copy_samples.total += EXTRA_INPUT_SAMPLES;
528 
529  if (avr) {
530  // Sample rate conversion has been allocated on this clip, so
531  // this is not the first iteration. Extend start position by
532  // EXTRA_INPUT_SAMPLES to keep step with previous frame
533  copy_samples.sample_start += EXTRA_INPUT_SAMPLES;
534  int samples_per_start_frame =
535  Frame::GetSamplesPerFrame(copy_samples.frame_start, original,
536  reader->info.sample_rate, reader->info.channels);
537  if (copy_samples.sample_start >= samples_per_start_frame)
538  {
539  // check for wrapping
540  copy_samples.frame_start++;
541  copy_samples.sample_start -= samples_per_start_frame;
542  }
543  copy_samples.total -= EXTRA_INPUT_SAMPLES;
544  }
545  }
546 
547  // Copy the samples
548  int samples_copied = 0;
549  int64_t starting_frame = copy_samples.frame_start;
550  while (info.has_audio && samples_copied < copy_samples.total)
551  {
552  // Init number of samples to copy this iteration
553  int remaining_samples = copy_samples.total - samples_copied;
554  int number_to_copy = 0;
555 
556  // number of original samples on this frame
557  std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
558  int original_samples = original_frame->GetAudioSamplesCount();
559 
560  // Loop through each channel
561  for (int channel = 0; channel < channels_in_frame; channel++)
562  {
563  if (starting_frame == copy_samples.frame_start)
564  {
565  // Starting frame (take the ending samples)
566  number_to_copy = original_samples - copy_samples.sample_start;
567  if (number_to_copy > remaining_samples)
568  number_to_copy = remaining_samples;
569 
570  // Add samples to new frame
571  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0);
572  }
573  else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end)
574  {
575  // Middle frame (take all samples)
576  number_to_copy = original_samples;
577  if (number_to_copy > remaining_samples)
578  number_to_copy = remaining_samples;
579 
580  // Add samples to new frame
581  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
582  }
583  else
584  {
585  // Ending frame (take the beginning samples)
586  number_to_copy = copy_samples.sample_end + 1;
587  if (number_to_copy > remaining_samples)
588  number_to_copy = remaining_samples;
589 
590  // Add samples to new frame
591  frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
592  }
593  }
594 
595  // increment frame
596  samples_copied += number_to_copy;
597  starting_frame++;
598  }
599 
600  // Resample audio on frame (if needed)
601  if (need_resampling)
602  // Resample audio and correct # of channels if needed
603  ResampleMappedAudio(frame, mapped.Odd.Frame);
604 
605  // Add frame to final cache
606  final_cache.Add(frame);
607 
608  } // for loop
609 
610  // Return processed openshot::Frame
611  return final_cache.GetFrame(requested_frame);
612 }
613 
615 {
616  // Check if mappings are dirty (and need to be recalculated)
617  if (is_dirty)
618  // Recalculate mappings
619  Init();
620 
621  // Loop through frame mappings
622  for (float map = 1; map <= frames.size(); map++)
623  {
624  MappedFrame frame = frames[map - 1];
625  cout << "Target frame #: " << map << " mapped to original frame #:\t(" << frame.Odd.Frame << " odd, " << frame.Even.Frame << " even)" << endl;
626  cout << " - Audio samples mapped to frame " << frame.Samples.frame_start << ":" << frame.Samples.sample_start << " to frame " << frame.Samples.frame_end << ":" << frame.Samples.sample_end << endl;
627  }
628 
629 }
630 
631 // Determine if reader is open or closed
633  if (reader)
634  return reader->IsOpen();
635  else
636  return false;
637 }
638 
639 // Open the internal reader
641 {
642  if (reader)
643  {
644  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open");
645 
646  // Open the reader
647  reader->Open();
648  }
649 }
650 
651 // Close the internal reader
653 {
654  if (reader)
655  {
656  // Create a scoped lock, allowing only a single thread to run the following code at one time
657  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
658 
659  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close");
660 
661  // Close internal reader
662  reader->Close();
663 
664  // Clear the fields & frames lists
665  fields.clear();
666  frames.clear();
667 
668  // Mark as dirty
669  is_dirty = true;
670 
671  // Clear cache
672  final_cache.Clear();
673 
674  // Deallocate resample buffer
675  if (avr) {
676  SWR_CLOSE(avr);
677  SWR_FREE(&avr);
678  avr = NULL;
679  }
680  }
681 }
682 
683 
684 // Generate JSON string of this object
685 std::string FrameMapper::Json() const {
686 
687  // Return formatted string
688  return JsonValue().toStyledString();
689 }
690 
691 // Generate Json::Value for this object
692 Json::Value FrameMapper::JsonValue() const {
693 
694  // Create root json object
695  Json::Value root = ReaderBase::JsonValue(); // get parent properties
696  root["type"] = "FrameMapper";
697 
698  // return JsonValue
699  return root;
700 }
701 
702 // Load JSON string into this object
703 void FrameMapper::SetJson(const std::string value) {
704 
705  // Parse JSON string into JSON objects
706  try
707  {
708  const Json::Value root = openshot::stringToJson(value);
709  // Set all values that match
710  SetJsonValue(root);
711  }
712  catch (const std::exception& e)
713  {
714  // Error parsing JSON (or missing keys)
715  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
716  }
717 }
718 
719 // Load Json::Value into this object
720 void FrameMapper::SetJsonValue(const Json::Value root) {
721 
722  // Set parent data
724 
725  // Re-Open path, and re-init everything (if needed)
726  if (reader) {
727 
728  Close();
729  Open();
730  }
731 }
732 
733 // Change frame rate or audio mapping details
734 void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
735 {
736  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ChangeMapping", "target_fps.num", target_fps.num, "target_fps.den", target_fps.den, "target_pulldown", target_pulldown, "target_sample_rate", target_sample_rate, "target_channels", target_channels, "target_channel_layout", target_channel_layout);
737 
738  // Mark as dirty
739  is_dirty = true;
740 
741  // Update mapping details
742  target.num = target_fps.num;
743  target.den = target_fps.den;
744  info.fps.num = target_fps.num;
745  info.fps.den = target_fps.den;
746  info.video_timebase.num = target_fps.den;
747  info.video_timebase.den = target_fps.num;
748  pulldown = target_pulldown;
749  info.sample_rate = target_sample_rate;
750  info.channels = target_channels;
751  info.channel_layout = target_channel_layout;
752 
753  // Clear cache
754  final_cache.Clear();
755 
756  // Adjust cache size based on size of frame and audio
758 
759  // Deallocate resample buffer
760  if (avr) {
761  SWR_CLOSE(avr);
762  SWR_FREE(&avr);
763  avr = NULL;
764  }
765 }
766 
767 // Resample audio and map channels (if needed)
768 void FrameMapper::ResampleMappedAudio(std::shared_ptr<Frame> frame, int64_t original_frame_number)
769 {
770  // Check if mappings are dirty (and need to be recalculated)
771  if (is_dirty)
772  // Recalculate mappings
773  Init();
774 
775  // Init audio buffers / variables
776  int total_frame_samples = 0;
777  int channels_in_frame = frame->GetAudioChannelsCount();
778  int sample_rate_in_frame = frame->SampleRate();
779  int samples_in_frame = frame->GetAudioSamplesCount();
780  ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
781 
782  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio", "frame->number", frame->number, "original_frame_number", original_frame_number, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "sample_rate_in_frame", sample_rate_in_frame);
783 
784  // Get audio sample array
785  float* frame_samples_float = NULL;
786  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
787  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
788 
789  // Calculate total samples
790  total_frame_samples = samples_in_frame * channels_in_frame;
791 
792  // Create a new array (to hold all S16 audio samples for the current queued frames)
793  int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
794 
795  // Translate audio sample values back to 16 bit integers with saturation
796  float valF;
797  int16_t conv;
798  const int16_t max16 = 32767;
799  const int16_t min16 = -32768;
800  for (int s = 0; s < total_frame_samples; s++) {
801  valF = frame_samples_float[s] * (1 << 15);
802  if (valF > max16)
803  conv = max16;
804  else if (valF < min16)
805  conv = min16;
806  else
807  conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
808 
809  // Copy into buffer
810  frame_samples[s] = conv;
811  }
812 
813 
814  // Deallocate float array
815  delete[] frame_samples_float;
816  frame_samples_float = NULL;
817 
818  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (got sample data from frame)", "frame->number", frame->number, "total_frame_samples", total_frame_samples, "target channels", info.channels, "channels_in_frame", channels_in_frame, "target sample_rate", info.sample_rate, "samples_in_frame", samples_in_frame);
819 
820 
821  // Create input frame (and allocate arrays)
822  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
823  AV_RESET_FRAME(audio_frame);
824  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
825 
826  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
827  audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
828 
829  if (error_code < 0)
830  {
832  "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) + "]",
833  "error_code", error_code);
834  throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
835  }
836 
837  // Update total samples & input frame size (due to bigger or smaller data types)
838  total_frame_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame->number), target, info.sample_rate, info.channels);
839 
840  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number);
841 
842  // Create output frame (and allocate arrays)
843  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
844  AV_RESET_FRAME(audio_converted);
845  audio_converted->nb_samples = total_frame_samples;
846  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
847 
848  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (preparing for resample)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", AV_SAMPLE_FMT_S16, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
849 
850  int nb_samples = 0;
851 
852  // setup resample context
853  if (!avr) {
854  avr = SWR_ALLOC();
855  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
856  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
857  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
858  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
859  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
860  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
861  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
862  av_opt_set_int(avr, "out_channels", info.channels, 0);
863  SWR_INIT(avr);
864  }
865 
866  // Convert audio samples
867  nb_samples = SWR_CONVERT(avr, // audio resample context
868  audio_converted->data, // output data pointers
869  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
870  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
871  audio_frame->data, // input data pointers
872  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
873  audio_frame->nb_samples); // number of input samples to convert
874 
875  // Create a new array (to hold all resampled S16 audio samples)
876  int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
877 
878  // Copy audio samples over original samples
879  memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
880 
881  // Free frames
882  av_freep(&audio_frame->data[0]);
883  AV_FREE_FRAME(&audio_frame);
884  av_freep(&audio_converted->data[0]);
885  AV_FREE_FRAME(&audio_converted);
886  frame_samples = NULL;
887 
888  // Resize the frame to hold the right # of channels and samples
889  int channel_buffer_size = nb_samples;
890  frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
891 
892  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Audio successfully resampled)", "nb_samples", nb_samples, "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "channels_in_frame", channels_in_frame, "info.channels", info.channels, "info.channel_layout", info.channel_layout);
893 
894  // Array of floats (to hold samples for each channel)
895  float *channel_buffer = new float[channel_buffer_size];
896 
897  // Divide audio into channels. Loop through each channel
898  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
899  {
900  // Init array
901  for (int z = 0; z < channel_buffer_size; z++)
902  channel_buffer[z] = 0.0f;
903 
904  // Loop through all samples and add them to our Frame based on channel.
905  // Toggle through each channel number, since channel data is stored like (left right left right)
906  int channel = 0;
907  int position = 0;
908  for (int sample = 0; sample < (nb_samples * info.channels); sample++)
909  {
910  // Only add samples for current channel
911  if (channel_filter == channel)
912  {
913  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
914  channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
915 
916  // Increment audio position
917  position++;
918  }
919 
920  // increment channel (if needed)
921  if ((channel + 1) < info.channels)
922  // move to next channel
923  channel ++;
924  else
925  // reset channel
926  channel = 0;
927  }
928 
929  // Add samples to frame for this channel
930  frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
931 
932  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Add audio to channel)", "number of samples", position, "channel_filter", channel_filter);
933  }
934 
935  // Update frame's audio meta data
936  frame->SampleRate(info.sample_rate);
937  frame->ChannelsLayout(info.channel_layout);
938 
939  // clear channel buffer
940  delete[] channel_buffer;
941  channel_buffer = NULL;
942 
943  // Delete arrays
944  delete[] resampled_samples;
945  resampled_samples = NULL;
946 }
947 
948 // Adjust frame number for Clip position and start (which can result in a different number)
949 int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
950 
951  // Get clip position from parent clip (if any)
952  float position = 0.0;
953  float start = 0.0;
954  Clip *parent = (Clip *) ParentClip();
955  if (parent) {
956  position = parent->Position();
957  start = parent->Start();
958  }
959 
960  // Adjust start frame and position based on parent clip. This prevents ensures the same
961  // frame # is used by mapped readers and clips, when calculating samples per frame. Thus,
962  // this prevents gaps and mismatches in # of samples.
963  int64_t clip_start_frame = (start * info.fps.ToDouble()) + 1;
964  int64_t clip_start_position = round(position * info.fps.ToDouble()) + 1;
965  int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
966 
967  return frame_number;
968 }
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
Definition: FrameMapper.h:61
#define AV_FREE_FRAME(av_frame)
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
int num
Numerator for the fraction.
Definition: Fraction.h:50
ReaderBase * Reader()
Get the current reader.
Definition: FrameMapper.cpp:74
#define SWR_INIT(ctx)
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:110
std::string Json() const override
Generate JSON string of this object.
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:68
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:54
virtual ~FrameMapper()
Destructor.
Definition: FrameMapper.cpp:65
float duration
Length of time (in seconds)
Definition: ReaderBase.h:65
std::vector< MappedFrame > frames
Definition: FrameMapper.h:170
void Close() override
Close the openshot::FrameMapper and internal reader.
STL namespace.
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:59
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:49
void Open() override
Open the internal reader.
std::vector< Field > fields
Definition: FrameMapper.h:169
#define SWR_CLOSE(ctx)
#define SWR_FREE(ctx)
virtual void Close()=0
Close the reader (and any resources it was consuming)
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:97
void SetJson(const std::string value) override
Load JSON string into this object.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:337
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:62
This struct holds a single field (half a frame).
Definition: FrameMapper.h:72
Exception when encoding audio packet.
Definition: Exceptions.h:125
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:190
This struct holds a the range of samples needed by this frame.
Definition: FrameMapper.h:92
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:536
Header file for all Exception classes.
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:63
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:116
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:75
int height
The height of the video (in pixels)
Definition: ReaderBase.h:67
#define AV_ALLOCATE_FRAME()
Json::Value JsonValue() const override
Generate Json::Value for this object.
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:77
Header file for Clip class.
This class represents a fraction.
Definition: Fraction.h:48
Header file for the FrameMapper class.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:84
juce::CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:101
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:64
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:64
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:171
This struct holds two fields which together make up a complete video frame.
Definition: FrameMapper.h:109
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:254
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
Exception for frames that are out of bounds.
Definition: Exceptions.h:285
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:52
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:46
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition: FrameMapper.h:63
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
Definition: Exceptions.h:205
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
#define SWR_ALLOC()
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:108
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition: FrameMapper.h:59
int den
Denominator for the fraction.
Definition: Fraction.h:51
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:83
bool IsOpen() override
Determine if reader is open or closed.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:70
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition: FrameMapper.h:62
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:82
virtual bool IsOpen()=0
Determine if reader is open or closed.