1 /**
2 * @file
3 * @brief Source file for the FrameMapper class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 *
6 * @ref License
7 */
8
9 /* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC
12 * <http://www.openshotstudios.com/>. This file is part of
13 * OpenShot Library (libopenshot), an open-source project dedicated to
14 * delivering high quality video editing and animation solutions to the
15 * world. For more information visit <http://www.openshot.org/>.
16 *
17 * OpenShot Library (libopenshot) is free software: you can redistribute it
18 * and/or modify it under the terms of the GNU Lesser General Public License
19 * as published by the Free Software Foundation, either version 3 of the
20 * License, or (at your option) any later version.
21 *
22 * OpenShot Library (libopenshot) is distributed in the hope that it will be
23 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU Lesser General Public License for more details.
26 *
27 * You should have received a copy of the GNU Lesser General Public License
28 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29 */
30
31 #include "FrameMapper.h"
32 #include "Exceptions.h"
33 #include "Clip.h"
34
35 using namespace std;
36 using namespace openshot;
37
FrameMapper(ReaderBase * reader,Fraction target,PulldownType target_pulldown,int target_sample_rate,int target_channels,ChannelLayout target_channel_layout)38 FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
39 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0)
40 {
41 // Set the original frame rate from the reader
42 original = Fraction(reader->info.fps.num, reader->info.fps.den);
43
44 // Set all info struct members equal to the internal reader
45 info = reader->info;
46 info.fps.num = target.num;
47 info.fps.den = target.den;
48 info.video_timebase.num = target.den;
49 info.video_timebase.den = target.num;
50 info.video_length = round(info.duration * info.fps.ToDouble());
51 info.sample_rate = target_sample_rate;
52 info.channels = target_channels;
53 info.channel_layout = target_channel_layout;
54 info.width = reader->info.width;
55 info.height = reader->info.height;
56
57 // Used to toggle odd / even fields
58 field_toggle = true;
59
60 // Adjust cache size based on size of frame and audio
61 final_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
62 }
63
64 // Destructor
~FrameMapper()65 FrameMapper::~FrameMapper() {
66
67 // Auto Close if not already
68 Close();
69
70 reader = NULL;
71 }
72
73 /// Get the current reader
Reader()74 ReaderBase* FrameMapper::Reader()
75 {
76 if (reader)
77 return reader;
78 else
79 // Throw error if reader not initialized
80 throw ReaderClosed("No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
81 }
82
AddField(int64_t frame)83 void FrameMapper::AddField(int64_t frame)
84 {
85 // Add a field, and toggle the odd / even field
86 AddField(Field(frame, field_toggle));
87 }
88
AddField(Field field)89 void FrameMapper::AddField(Field field)
90 {
91 // Add a field to the end of the field list
92 fields.push_back(field);
93
94 // toggle the odd / even flag
95 field_toggle = (field_toggle ? false : true);
96 }
97
98 // Use the original and target frame rates and a pull-down technique to create
99 // a mapping between the original fields and frames or a video to a new frame rate.
100 // This might repeat or skip fields and frames of the original video, depending on
101 // whether the frame rate is increasing or decreasing.
Init()102 void FrameMapper::Init()
103 {
104 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)");
105
106 // Do not initialize anything if just a picture with no audio
107 if (info.has_video and !info.has_audio and info.has_single_image)
108 // Skip initialization
109 return;
110
111 // Clear the fields & frames lists
112 fields.clear();
113 frames.clear();
114
115 // Find parent position (if any)
116 Clip *parent = (Clip *) ParentClip();
117 if (parent) {
118 parent_position = parent->Position();
119 parent_start = parent->Start();
120 } else {
121 parent_position = 0.0;
122 parent_start = 0.0;
123 }
124
125 // Mark as not dirty
126 is_dirty = false;
127
128 // Clear cache
129 final_cache.Clear();
130
131 // Some framerates are handled special, and some use a generic Keyframe curve to
132 // map the framerates. These are the special framerates:
133 if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
134 (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
135
136 // Get the difference (in frames) between the original and target frame rates
137 float difference = target.ToInt() - original.ToInt();
138
139 // Find the number (i.e. interval) of fields that need to be skipped or repeated
140 int field_interval = 0;
141 int frame_interval = 0;
142
143 if (difference != 0)
144 {
145 field_interval = round(fabs(original.ToInt() / difference));
146
147 // Get frame interval (2 fields per frame)
148 frame_interval = field_interval * 2.0f;
149 }
150
151
152 // Calculate # of fields to map
153 int64_t frame = 1;
154 int64_t number_of_fields = reader->info.video_length * 2;
155
156 // Loop through all fields in the original video file
157 for (int64_t field = 1; field <= number_of_fields; field++)
158 {
159
160 if (difference == 0) // Same frame rate, NO pull-down or special techniques required
161 {
162 // Add fields
163 AddField(frame);
164 }
165 else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
166 {
167 // Add current field
168 AddField(frame);
169
170 if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
171 {
172 // Add extra field for each 'field interval
173 AddField(frame);
174 }
175 else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
176 {
177 // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
178 AddField(frame); // add field for current frame
179
180 if (frame + 1 <= info.video_length)
181 // add field for next frame (if the next frame exists)
182 AddField(Field(frame + 1, field_toggle));
183 }
184 else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
185 {
186 // No pull-down technique needed, just repeat this frame
187 AddField(frame);
188 AddField(frame);
189 }
190 }
191 else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
192 {
193
194 if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
195 {
196 // skip current field and toggle the odd/even flag
197 field_toggle = (field_toggle ? false : true);
198 }
199 else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
200 {
201 // skip this field, plus the next field
202 field++;
203 }
204 else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
205 {
206 // skip this field, plus the next one
207 field++;
208 }
209 else
210 {
211 // No skipping needed, so add the field
212 AddField(frame);
213 }
214 }
215
216 // increment frame number (if field is divisible by 2)
217 if (field % 2 == 0 && field > 0)
218 frame++;
219 }
220
221 } else {
222 // Map the remaining framerates using a linear algorithm
223 double rate_diff = target.ToDouble() / original.ToDouble();
224 int64_t new_length = reader->info.video_length * rate_diff;
225
226 // Calculate the value difference
227 double value_increment = (reader->info.video_length + 1) / (double) (new_length);
228
229 // Loop through curve, and build list of frames
230 double original_frame_num = 1.0f;
231 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
232 {
233 // Add 2 fields per frame
234 AddField(round(original_frame_num));
235 AddField(round(original_frame_num));
236
237 // Increment original frame number
238 original_frame_num += value_increment;
239 }
240 }
241
242 // Loop through the target frames again (combining fields into frames)
243 Field Odd(0, true); // temp field used to track the ODD field
244 Field Even(0, true); // temp field used to track the EVEN field
245
246 // Variables used to remap audio samples
247 int64_t start_samples_frame = 1;
248 int start_samples_position = 0;
249
250 for (std::vector<Field>::size_type field = 1; field <= fields.size(); field++)
251 {
252 // Get the current field
253 Field f = fields[field - 1];
254
255 // Is field divisible by 2?
256 if (field % 2 == 0 && field > 0)
257 {
258 // New frame number
259 int64_t frame_number = field / 2;
260
261 // Set the bottom frame
262 if (f.isOdd)
263 Odd = f;
264 else
265 Even = f;
266
267 // Determine the range of samples (from the original rate). Resampling happens in real-time when
268 // calling the GetFrame() method. So this method only needs to redistribute the original samples with
269 // the original sample rate.
270 int64_t end_samples_frame = start_samples_frame;
271 int end_samples_position = start_samples_position;
272 int remaining_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels);
273
274 while (remaining_samples > 0)
275 {
276 // Get original samples (with NO framerate adjustments)
277 // This is the original reader's frame numbers
278 int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
279
280 // Enough samples
281 if (original_samples >= remaining_samples)
282 {
283 // Take all that we need, and break loop
284 end_samples_position += remaining_samples - 1;
285 remaining_samples = 0;
286 } else
287 {
288 // Not enough samples (take them all, and keep looping)
289 end_samples_frame += 1; // next frame
290 end_samples_position = 0; // next frame, starting on 1st sample
291 remaining_samples -= original_samples; // reduce the remaining amount
292 }
293 }
294
295
296
297 // Create the sample mapping struct
298 SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels)};
299
300 // Reset the audio variables
301 start_samples_frame = end_samples_frame;
302 start_samples_position = end_samples_position + 1;
303 if (start_samples_position >= Frame::GetSamplesPerFrame(AdjustFrameNumber(start_samples_frame), original, reader->info.sample_rate, reader->info.channels))
304 {
305 start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
306 start_samples_position = 0; // reset to 0, since we wrapped
307 }
308
309 // Create a frame and ADD it to the frames collection
310 MappedFrame frame = {Odd, Even, Samples};
311 frames.push_back(frame);
312 }
313 else
314 {
315 // Set the top field
316 if (f.isOdd)
317 Odd = f;
318 else
319 Even = f;
320 }
321 }
322
323 // Clear the internal fields list (no longer needed)
324 fields.clear();
325 }
326
GetMappedFrame(int64_t TargetFrameNumber)327 MappedFrame FrameMapper::GetMappedFrame(int64_t TargetFrameNumber)
328 {
329 // Check if mappings are dirty (and need to be recalculated)
330 if (is_dirty)
331 // Recalculate mappings
332 Init();
333
334 // Ignore mapping on single image readers
335 if (info.has_video and !info.has_audio and info.has_single_image) {
336 // Return the same number
337 MappedFrame frame;
338 frame.Even.Frame = TargetFrameNumber;
339 frame.Odd.Frame = TargetFrameNumber;
340 frame.Samples.frame_start = 0;
341 frame.Samples.frame_end = 0;
342 frame.Samples.sample_start = 0;
343 frame.Samples.sample_end = 0;
344 frame.Samples.total = 0;
345 return frame;
346 }
347
348 // Check if frame number is valid
349 if(TargetFrameNumber < 1 || frames.size() == 0)
350 // frame too small, return error
351 throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
352
353 else if (TargetFrameNumber > (int64_t)frames.size())
354 // frame too large, set to end frame
355 TargetFrameNumber = frames.size();
356
357 // Debug output
358 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetMappedFrame", "TargetFrameNumber", TargetFrameNumber, "frames.size()", frames.size(), "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame, "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame);
359
360 // Return frame
361 return frames[TargetFrameNumber - 1];
362 }
363
364 // Get or generate a blank frame
GetOrCreateFrame(int64_t number)365 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
366 {
367 std::shared_ptr<Frame> new_frame;
368
369 // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
370 int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(number), target, reader->info.sample_rate, reader->info.channels);
371
372 try {
373 // Debug output
374 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (from reader)", "number", number, "samples_in_frame", samples_in_frame);
375
376 // Attempt to get a frame (but this could fail if a reader has just been closed)
377 new_frame = reader->GetFrame(number);
378
379 // Return real frame
380 return new_frame;
381
382 } catch (const ReaderClosed & e) {
383 // ...
384 } catch (const OutOfBoundsFrame & e) {
385 // ...
386 }
387
388 // Debug output
389 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (create blank)", "number", number, "samples_in_frame", samples_in_frame);
390
391 // Create blank frame
392 new_frame = std::make_shared<Frame>(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels);
393 new_frame->SampleRate(reader->info.sample_rate);
394 new_frame->ChannelsLayout(info.channel_layout);
395 new_frame->AddAudioSilence(samples_in_frame);
396 return new_frame;
397 }
398
399 // Get an openshot::Frame object for a specific frame number of this reader.
GetFrame(int64_t requested_frame)400 std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
401 {
402 // Check final cache, and just return the frame (if it's available)
403 std::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
404 if (final_frame) return final_frame;
405
406 // Create a scoped lock, allowing only a single thread to run the following code at one time
407 const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
408
409 // Find parent properties (if any)
410 Clip *parent = (Clip *) ParentClip();
411 if (parent) {
412 float position = parent->Position();
413 float start = parent->Start();
414 if (parent_position != position || parent_start != start) {
415 // Force dirty if parent clip has moved or been trimmed
416 // since this heavily affects frame #s and audio mappings
417 is_dirty = true;
418 }
419 }
420
421 // Check if mappings are dirty (and need to be recalculated)
422 if (is_dirty)
423 Init();
424
425 // Check final cache a 2nd time (due to potential lock already generating this frame)
426 final_frame = final_cache.GetFrame(requested_frame);
427 if (final_frame) return final_frame;
428
429 // Minimum number of frames to process (for performance reasons)
430 // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes
431 int minimum_frames = 1;
432
433 // Debug output
434 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (Loop through frames)", "requested_frame", requested_frame, "minimum_frames", minimum_frames);
435
436 // Loop through all requested frames
437 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
438 {
439
440 // Debug output
441 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (inside omp for loop)", "frame_number", frame_number, "minimum_frames", minimum_frames, "requested_frame", requested_frame);
442
443 // Get the mapped frame
444 MappedFrame mapped = GetMappedFrame(frame_number);
445 std::shared_ptr<Frame> mapped_frame;
446
447 // Get the mapped frame (keeping the sample rate and channels the same as the original... for the moment)
448 mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
449
450 // Get # of channels in the actual frame
451 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
452 int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
453
454 // Determine if mapped frame is identical to source frame
455 // including audio sample distribution according to mapped.Samples,
456 // and frame_number. In some cases such as end of stream, the reader
457 // will return a frame with a different frame number. In these cases,
458 // we cannot use the frame as is, nor can we modify the frame number,
459 // otherwise the reader's cache object internals become invalid.
460 if (info.sample_rate == mapped_frame->SampleRate() &&
461 info.channels == mapped_frame->GetAudioChannelsCount() &&
462 info.channel_layout == mapped_frame->ChannelsLayout() &&
463 mapped.Samples.total == mapped_frame->GetAudioSamplesCount() &&
464 mapped.Samples.frame_start == mapped.Odd.Frame &&
465 mapped.Samples.sample_start == 0 &&
466 mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream)
467 info.fps.num == reader->info.fps.num &&
468 info.fps.den == reader->info.fps.den) {
469 // Add original frame to cache, and skip the rest (for performance reasons)
470 final_cache.Add(mapped_frame);
471 continue;
472 }
473
474 // Create a new frame
475 auto frame = std::make_shared<Frame>(
476 frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame);
477 frame->SampleRate(mapped_frame->SampleRate());
478 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
479
480
481 // Copy the image from the odd field
482 std::shared_ptr<Frame> odd_frame;
483 odd_frame = GetOrCreateFrame(mapped.Odd.Frame);
484
485 if (odd_frame)
486 frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()), true);
487 if (mapped.Odd.Frame != mapped.Even.Frame) {
488 // Add even lines (if different than the previous image)
489 std::shared_ptr<Frame> even_frame;
490 even_frame = GetOrCreateFrame(mapped.Even.Frame);
491 if (even_frame)
492 frame->AddImage(
493 std::make_shared<QImage>(*even_frame->GetImage()), false);
494 }
495
496 // Resample audio on frame (if needed)
497 bool need_resampling = false;
498 if (info.has_audio &&
499 (info.sample_rate != frame->SampleRate() ||
500 info.channels != frame->GetAudioChannelsCount() ||
501 info.channel_layout != frame->ChannelsLayout()))
502 // Resample audio and correct # of channels if needed
503 need_resampling = true;
504
505 // create a copy of mapped.Samples that will be used by copy loop
506 SampleRange copy_samples = mapped.Samples;
507
508 if (need_resampling)
509 {
510 // Resampling needed, modify copy of SampleRange object that
511 // includes some additional input samples on first iteration,
512 // and continues the offset to ensure that the sample rate
513 // converter isn't input limited.
514 const int EXTRA_INPUT_SAMPLES = 100;
515
516 // Extend end sample count by an additional EXTRA_INPUT_SAMPLES samples
517 copy_samples.sample_end += EXTRA_INPUT_SAMPLES;
518 int samples_per_end_frame =
519 Frame::GetSamplesPerFrame(copy_samples.frame_end, original,
520 reader->info.sample_rate, reader->info.channels);
521 if (copy_samples.sample_end >= samples_per_end_frame)
522 {
523 // check for wrapping
524 copy_samples.frame_end++;
525 copy_samples.sample_end -= samples_per_end_frame;
526 }
527 copy_samples.total += EXTRA_INPUT_SAMPLES;
528
529 if (avr) {
530 // Sample rate conversion has been allocated on this clip, so
531 // this is not the first iteration. Extend start position by
532 // EXTRA_INPUT_SAMPLES to keep step with previous frame
533 copy_samples.sample_start += EXTRA_INPUT_SAMPLES;
534 int samples_per_start_frame =
535 Frame::GetSamplesPerFrame(copy_samples.frame_start, original,
536 reader->info.sample_rate, reader->info.channels);
537 if (copy_samples.sample_start >= samples_per_start_frame)
538 {
539 // check for wrapping
540 copy_samples.frame_start++;
541 copy_samples.sample_start -= samples_per_start_frame;
542 }
543 copy_samples.total -= EXTRA_INPUT_SAMPLES;
544 }
545 }
546
547 // Copy the samples
548 int samples_copied = 0;
549 int64_t starting_frame = copy_samples.frame_start;
550 while (info.has_audio && samples_copied < copy_samples.total)
551 {
552 // Init number of samples to copy this iteration
553 int remaining_samples = copy_samples.total - samples_copied;
554 int number_to_copy = 0;
555
556 // number of original samples on this frame
557 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
558 int original_samples = original_frame->GetAudioSamplesCount();
559
560 // Loop through each channel
561 for (int channel = 0; channel < channels_in_frame; channel++)
562 {
563 if (starting_frame == copy_samples.frame_start)
564 {
565 // Starting frame (take the ending samples)
566 number_to_copy = original_samples - copy_samples.sample_start;
567 if (number_to_copy > remaining_samples)
568 number_to_copy = remaining_samples;
569
570 // Add samples to new frame
571 frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0);
572 }
573 else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end)
574 {
575 // Middle frame (take all samples)
576 number_to_copy = original_samples;
577 if (number_to_copy > remaining_samples)
578 number_to_copy = remaining_samples;
579
580 // Add samples to new frame
581 frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
582 }
583 else
584 {
585 // Ending frame (take the beginning samples)
586 number_to_copy = copy_samples.sample_end + 1;
587 if (number_to_copy > remaining_samples)
588 number_to_copy = remaining_samples;
589
590 // Add samples to new frame
591 frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
592 }
593 }
594
595 // increment frame
596 samples_copied += number_to_copy;
597 starting_frame++;
598 }
599
600 // Resample audio on frame (if needed)
601 if (need_resampling)
602 // Resample audio and correct # of channels if needed
603 ResampleMappedAudio(frame, mapped.Odd.Frame);
604
605 // Add frame to final cache
606 final_cache.Add(frame);
607
608 } // for loop
609
610 // Return processed openshot::Frame
611 return final_cache.GetFrame(requested_frame);
612 }
613
PrintMapping()614 void FrameMapper::PrintMapping()
615 {
616 // Check if mappings are dirty (and need to be recalculated)
617 if (is_dirty)
618 // Recalculate mappings
619 Init();
620
621 // Loop through frame mappings
622 for (float map = 1; map <= frames.size(); map++)
623 {
624 MappedFrame frame = frames[map - 1];
625 cout << "Target frame #: " << map << " mapped to original frame #:\t(" << frame.Odd.Frame << " odd, " << frame.Even.Frame << " even)" << endl;
626 cout << " - Audio samples mapped to frame " << frame.Samples.frame_start << ":" << frame.Samples.sample_start << " to frame " << frame.Samples.frame_end << ":" << frame.Samples.sample_end << endl;
627 }
628
629 }
630
631 // Determine if reader is open or closed
IsOpen()632 bool FrameMapper::IsOpen() {
633 if (reader)
634 return reader->IsOpen();
635 else
636 return false;
637 }
638
639 // Open the internal reader
Open()640 void FrameMapper::Open()
641 {
642 if (reader)
643 {
644 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open");
645
646 // Open the reader
647 reader->Open();
648 }
649 }
650
651 // Close the internal reader
Close()652 void FrameMapper::Close()
653 {
654 if (reader)
655 {
656 // Create a scoped lock, allowing only a single thread to run the following code at one time
657 const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
658
659 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close");
660
661 // Close internal reader
662 reader->Close();
663
664 // Clear the fields & frames lists
665 fields.clear();
666 frames.clear();
667
668 // Mark as dirty
669 is_dirty = true;
670
671 // Clear cache
672 final_cache.Clear();
673
674 // Deallocate resample buffer
675 if (avr) {
676 SWR_CLOSE(avr);
677 SWR_FREE(&avr);
678 avr = NULL;
679 }
680 }
681 }
682
683
684 // Generate JSON string of this object
Json() const685 std::string FrameMapper::Json() const {
686
687 // Return formatted string
688 return JsonValue().toStyledString();
689 }
690
691 // Generate Json::Value for this object
JsonValue() const692 Json::Value FrameMapper::JsonValue() const {
693
694 // Create root json object
695 Json::Value root = ReaderBase::JsonValue(); // get parent properties
696 root["type"] = "FrameMapper";
697
698 // return JsonValue
699 return root;
700 }
701
702 // Load JSON string into this object
SetJson(const std::string value)703 void FrameMapper::SetJson(const std::string value) {
704
705 // Parse JSON string into JSON objects
706 try
707 {
708 const Json::Value root = openshot::stringToJson(value);
709 // Set all values that match
710 SetJsonValue(root);
711 }
712 catch (const std::exception& e)
713 {
714 // Error parsing JSON (or missing keys)
715 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
716 }
717 }
718
719 // Load Json::Value into this object
SetJsonValue(const Json::Value root)720 void FrameMapper::SetJsonValue(const Json::Value root) {
721
722 // Set parent data
723 ReaderBase::SetJsonValue(root);
724
725 // Re-Open path, and re-init everything (if needed)
726 if (reader) {
727
728 Close();
729 Open();
730 }
731 }
732
733 // Change frame rate or audio mapping details
ChangeMapping(Fraction target_fps,PulldownType target_pulldown,int target_sample_rate,int target_channels,ChannelLayout target_channel_layout)734 void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
735 {
736 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ChangeMapping", "target_fps.num", target_fps.num, "target_fps.den", target_fps.den, "target_pulldown", target_pulldown, "target_sample_rate", target_sample_rate, "target_channels", target_channels, "target_channel_layout", target_channel_layout);
737
738 // Mark as dirty
739 is_dirty = true;
740
741 // Update mapping details
742 target.num = target_fps.num;
743 target.den = target_fps.den;
744 info.fps.num = target_fps.num;
745 info.fps.den = target_fps.den;
746 info.video_timebase.num = target_fps.den;
747 info.video_timebase.den = target_fps.num;
748 pulldown = target_pulldown;
749 info.sample_rate = target_sample_rate;
750 info.channels = target_channels;
751 info.channel_layout = target_channel_layout;
752
753 // Clear cache
754 final_cache.Clear();
755
756 // Adjust cache size based on size of frame and audio
757 final_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
758
759 // Deallocate resample buffer
760 if (avr) {
761 SWR_CLOSE(avr);
762 SWR_FREE(&avr);
763 avr = NULL;
764 }
765 }
766
767 // Resample audio and map channels (if needed)
ResampleMappedAudio(std::shared_ptr<Frame> frame,int64_t original_frame_number)768 void FrameMapper::ResampleMappedAudio(std::shared_ptr<Frame> frame, int64_t original_frame_number)
769 {
770 // Check if mappings are dirty (and need to be recalculated)
771 if (is_dirty)
772 // Recalculate mappings
773 Init();
774
775 // Init audio buffers / variables
776 int total_frame_samples = 0;
777 int channels_in_frame = frame->GetAudioChannelsCount();
778 int sample_rate_in_frame = frame->SampleRate();
779 int samples_in_frame = frame->GetAudioSamplesCount();
780 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
781
782 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio", "frame->number", frame->number, "original_frame_number", original_frame_number, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "sample_rate_in_frame", sample_rate_in_frame);
783
784 // Get audio sample array
785 float* frame_samples_float = NULL;
786 // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
787 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
788
789 // Calculate total samples
790 total_frame_samples = samples_in_frame * channels_in_frame;
791
792 // Create a new array (to hold all S16 audio samples for the current queued frames)
793 int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
794
795 // Translate audio sample values back to 16 bit integers with saturation
796 float valF;
797 int16_t conv;
798 const int16_t max16 = 32767;
799 const int16_t min16 = -32768;
800 for (int s = 0; s < total_frame_samples; s++) {
801 valF = frame_samples_float[s] * (1 << 15);
802 if (valF > max16)
803 conv = max16;
804 else if (valF < min16)
805 conv = min16;
806 else
807 conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
808
809 // Copy into buffer
810 frame_samples[s] = conv;
811 }
812
813
814 // Deallocate float array
815 delete[] frame_samples_float;
816 frame_samples_float = NULL;
817
818 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (got sample data from frame)", "frame->number", frame->number, "total_frame_samples", total_frame_samples, "target channels", info.channels, "channels_in_frame", channels_in_frame, "target sample_rate", info.sample_rate, "samples_in_frame", samples_in_frame);
819
820
821 // Create input frame (and allocate arrays)
822 AVFrame *audio_frame = AV_ALLOCATE_FRAME();
823 AV_RESET_FRAME(audio_frame);
824 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
825
826 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
827 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
828
829 if (error_code < 0)
830 {
831 ZmqLogger::Instance()->AppendDebugMethod(
832 "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) + "]",
833 "error_code", error_code);
834 throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
835 }
836
837 // Update total samples & input frame size (due to bigger or smaller data types)
838 total_frame_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame->number), target, info.sample_rate, info.channels);
839
840 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number);
841
842 // Create output frame (and allocate arrays)
843 AVFrame *audio_converted = AV_ALLOCATE_FRAME();
844 AV_RESET_FRAME(audio_converted);
845 audio_converted->nb_samples = total_frame_samples;
846 av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
847
848 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (preparing for resample)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", AV_SAMPLE_FMT_S16, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
849
850 int nb_samples = 0;
851
852 // setup resample context
853 if (!avr) {
854 avr = SWR_ALLOC();
855 av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
856 av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
857 av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
858 av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
859 av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
860 av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
861 av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
862 av_opt_set_int(avr, "out_channels", info.channels, 0);
863 SWR_INIT(avr);
864 }
865
866 // Convert audio samples
867 nb_samples = SWR_CONVERT(avr, // audio resample context
868 audio_converted->data, // output data pointers
869 audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
870 audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
871 audio_frame->data, // input data pointers
872 audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
873 audio_frame->nb_samples); // number of input samples to convert
874
875 // Create a new array (to hold all resampled S16 audio samples)
876 int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
877
878 // Copy audio samples over original samples
879 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
880
881 // Free frames
882 av_freep(&audio_frame->data[0]);
883 AV_FREE_FRAME(&audio_frame);
884 av_freep(&audio_converted->data[0]);
885 AV_FREE_FRAME(&audio_converted);
886 frame_samples = NULL;
887
888 // Resize the frame to hold the right # of channels and samples
889 int channel_buffer_size = nb_samples;
890 frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
891
892 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Audio successfully resampled)", "nb_samples", nb_samples, "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "channels_in_frame", channels_in_frame, "info.channels", info.channels, "info.channel_layout", info.channel_layout);
893
894 // Array of floats (to hold samples for each channel)
895 float *channel_buffer = new float[channel_buffer_size];
896
897 // Divide audio into channels. Loop through each channel
898 for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
899 {
900 // Init array
901 for (int z = 0; z < channel_buffer_size; z++)
902 channel_buffer[z] = 0.0f;
903
904 // Loop through all samples and add them to our Frame based on channel.
905 // Toggle through each channel number, since channel data is stored like (left right left right)
906 int channel = 0;
907 int position = 0;
908 for (int sample = 0; sample < (nb_samples * info.channels); sample++)
909 {
910 // Only add samples for current channel
911 if (channel_filter == channel)
912 {
913 // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
914 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
915
916 // Increment audio position
917 position++;
918 }
919
920 // increment channel (if needed)
921 if ((channel + 1) < info.channels)
922 // move to next channel
923 channel ++;
924 else
925 // reset channel
926 channel = 0;
927 }
928
929 // Add samples to frame for this channel
930 frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
931
932 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Add audio to channel)", "number of samples", position, "channel_filter", channel_filter);
933 }
934
935 // Update frame's audio meta data
936 frame->SampleRate(info.sample_rate);
937 frame->ChannelsLayout(info.channel_layout);
938
939 // clear channel buffer
940 delete[] channel_buffer;
941 channel_buffer = NULL;
942
943 // Delete arrays
944 delete[] resampled_samples;
945 resampled_samples = NULL;
946 }
947
948 // Adjust frame number for Clip position and start (which can result in a different number)
AdjustFrameNumber(int64_t clip_frame_number)949 int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
950
951 // Get clip position from parent clip (if any)
952 float position = 0.0;
953 float start = 0.0;
954 Clip *parent = (Clip *) ParentClip();
955 if (parent) {
956 position = parent->Position();
957 start = parent->Start();
958 }
959
960 // Adjust start frame and position based on parent clip. This prevents ensures the same
961 // frame # is used by mapped readers and clips, when calculating samples per frame. Thus,
962 // this prevents gaps and mismatches in # of samples.
963 int64_t clip_start_frame = (start * info.fps.ToDouble()) + 1;
964 int64_t clip_start_position = round(position * info.fps.ToDouble()) + 1;
965 int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
966
967 return frame_number;
968 }
969