1 /**
2 * @file
3 * @brief Source file for Frame class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 *
6 * @ref License
7 */
8
9 /* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC
12 * <http://www.openshotstudios.com/>. This file is part of
13 * OpenShot Library (libopenshot), an open-source project dedicated to
14 * delivering high quality video editing and animation solutions to the
15 * world. For more information visit <http://www.openshot.org/>.
16 *
17 * OpenShot Library (libopenshot) is free software: you can redistribute it
18 * and/or modify it under the terms of the GNU Lesser General Public License
19 * as published by the Free Software Foundation, either version 3 of the
20 * License, or (at your option) any later version.
21 *
22 * OpenShot Library (libopenshot) is distributed in the hope that it will be
23 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU Lesser General Public License for more details.
26 *
27 * You should have received a copy of the GNU Lesser General Public License
28 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29 */
30
31 #include "Frame.h"
32 #include <OpenShotAudio.h>
33
34 #include <QApplication>
35 #include <QImage>
36 #include <QPixmap>
37 #include <QBitmap>
38 #include <QColor>
39 #include <QString>
40 #include <QVector>
41 #include <QPainter>
42 #include <QHBoxLayout>
43 #include <QWidget>
44 #include <QLabel>
45 #include <QPointF>
46 #include <QWidget>
47
48 #include <thread> // for std::this_thread::sleep_for
49 #include <chrono> // for std::chrono::milliseconds
50
51 using namespace std;
52 using namespace openshot;
53
54 // Constructor - image & audio
Frame(int64_t number,int width,int height,std::string color,int samples,int channels)55 Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
56 : audio(std::make_shared<juce::AudioSampleBuffer>(channels, samples)),
57 number(number), width(width), height(height),
58 pixel_ratio(1,1), color(color), qbuffer(NULL),
59 channels(channels), channel_layout(LAYOUT_STEREO),
60 sample_rate(44100),
61 has_audio_data(false), has_image_data(false),
62 max_audio_sample(0)
63 {
64 // zero (fill with silence) the audio buffer
65 audio->clear();
66 }
67
68 // Delegating Constructor - blank frame
Frame()69 Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {};
70
71 // Delegating Constructor - image only
Frame(int64_t number,int width,int height,std::string color)72 Frame::Frame(int64_t number, int width, int height, std::string color)
73 : Frame::Frame(number, width, height, color, 0, 2) {};
74
75 // Delegating Constructor - audio only
Frame(int64_t number,int samples,int channels)76 Frame::Frame(int64_t number, int samples, int channels)
77 : Frame::Frame(number, 1, 1, "#000000", samples, channels) {};
78
79
80 // Copy constructor
Frame(const Frame & other)81 Frame::Frame ( const Frame &other )
82 {
83 // copy pointers and data
84 DeepCopy(other);
85 }
86
87 // Assignment operator
operator =(const Frame & other)88 Frame& Frame::operator= (const Frame& other)
89 {
90 // copy pointers and data
91 DeepCopy(other);
92
93 return *this;
94 }
95
96 // Copy data and pointers from another Frame instance
DeepCopy(const Frame & other)97 void Frame::DeepCopy(const Frame& other)
98 {
99 number = other.number;
100 channels = other.channels;
101 width = other.width;
102 height = other.height;
103 channel_layout = other.channel_layout;
104 has_audio_data = other.has_audio_data;
105 has_image_data = other.has_image_data;
106 sample_rate = other.sample_rate;
107 pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
108 color = other.color;
109 max_audio_sample = other.max_audio_sample;
110
111 if (other.image)
112 image = std::make_shared<QImage>(*(other.image));
113 if (other.audio)
114 audio = std::make_shared<juce::AudioSampleBuffer>(*(other.audio));
115 if (other.wave_image)
116 wave_image = std::make_shared<QImage>(*(other.wave_image));
117 }
118
119 // Destructor
~Frame()120 Frame::~Frame() {
121 // Clear all pointers
122 image.reset();
123 audio.reset();
124 #ifdef USE_OPENCV
125 imagecv.release();
126 #endif
127 }
128
129 // Display the frame image to the screen (primarily used for debugging reasons)
Display()130 void Frame::Display()
131 {
132 if (!QApplication::instance()) {
133 // Only create the QApplication once
134 static int argc = 1;
135 static char* argv[1] = {NULL};
136 previewApp = std::make_shared<QApplication>(argc, argv);
137 }
138
139 // Get preview image
140 std::shared_ptr<QImage> previewImage = GetImage();
141
142 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
143 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
144 {
145 // Calculate correct DAR (display aspect ratio)
146 int new_width = previewImage->size().width();
147 int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
148
149 // Resize to fix DAR
150 previewImage = std::make_shared<QImage>(previewImage->scaled(
151 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
152 }
153
154 // Create window
155 QWidget previewWindow;
156 previewWindow.setStyleSheet("background-color: #000000;");
157 QHBoxLayout layout;
158
159 // Create label with current frame's image
160 QLabel previewLabel;
161 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
162 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
163 layout.addWidget(&previewLabel);
164
165 // Show the window
166 previewWindow.setLayout(&layout);
167 previewWindow.show();
168 previewApp->exec();
169 }
170
171 // Get an audio waveform image
GetWaveform(int width,int height,int Red,int Green,int Blue,int Alpha)172 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
173 {
174 // Clear any existing waveform image
175 ClearWaveform();
176
177 // Init a list of lines
178 QVector<QPointF> lines;
179 QVector<QPointF> labels;
180
181 // Calculate width of an image based on the # of samples
182 int total_samples = GetAudioSamplesCount();
183 if (total_samples > 0)
184 {
185 // If samples are present...
186 int new_height = 200 * audio->getNumChannels();
187 int height_padding = 20 * (audio->getNumChannels() - 1);
188 int total_height = new_height + height_padding;
189 int total_width = 0;
190
191 // Loop through each audio channel
192 float Y = 100.0;
193 for (int channel = 0; channel < audio->getNumChannels(); channel++)
194 {
195 float X = 0.0;
196
197 // Get audio for this channel
198 const float *samples = audio->getReadPointer(channel);
199
200 for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
201 {
202 // Sample value (scaled to -100 to 100)
203 float value = samples[sample] * 100.0;
204
205 // Append a line segment for each sample
206 lines.push_back(QPointF(X,Y+1.0));
207 lines.push_back(QPointF(X,(Y-value)+1.0));
208 }
209
210 // Add Channel Label Coordinate
211 labels.push_back(QPointF(5.0, Y - 5.0));
212
213 // Increment Y
214 Y += (200 + height_padding);
215 total_width = X;
216 }
217
218 // Create blank image
219 wave_image = std::make_shared<QImage>(
220 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
221 wave_image->fill(QColor(0,0,0,0));
222
223 // Load QPainter with wave_image device
224 QPainter painter(wave_image.get());
225
226 // Set pen color
227 QPen pen;
228 pen.setColor(QColor(Red, Green, Blue, Alpha));
229 pen.setWidthF(1.0);
230 pen.setStyle(Qt::SolidLine);
231 painter.setPen(pen);
232
233 // Draw the waveform
234 painter.drawLines(lines);
235 painter.end();
236
237 // Resize Image (if requested)
238 if (width != total_width || height != total_height) {
239 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
240 wave_image = std::make_shared<QImage>(scaled_wave_image);
241 }
242 }
243 else
244 {
245 // No audio samples present
246 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
247 wave_image->fill(QColor(QString::fromStdString("#000000")));
248 }
249
250 // Return new image
251 return wave_image;
252 }
253
254 // Clear the waveform image (and deallocate its memory)
ClearWaveform()255 void Frame::ClearWaveform()
256 {
257 if (wave_image)
258 wave_image.reset();
259 }
260
261 // Get an audio waveform image pixels
GetWaveformPixels(int width,int height,int Red,int Green,int Blue,int Alpha)262 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
263 {
264 // Get audio wave form image
265 wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
266
267 // Return array of pixel packets
268 return wave_image->constBits();
269 }
270
271 // Display the wave form
DisplayWaveform()272 void Frame::DisplayWaveform()
273 {
274 // Get audio wave form image
275 GetWaveform(720, 480, 0, 123, 255, 255);
276
277 if (!QApplication::instance()) {
278 // Only create the QApplication once
279 static int argc = 1;
280 static char* argv[1] = {NULL};
281 previewApp = std::make_shared<QApplication>(argc, argv);
282 }
283
284 // Create window
285 QWidget previewWindow;
286 previewWindow.setStyleSheet("background-color: #000000;");
287 QHBoxLayout layout;
288
289 // Create label with current frame's waveform image
290 QLabel previewLabel;
291 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
292 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
293 layout.addWidget(&previewLabel);
294
295 // Show the window
296 previewWindow.setLayout(&layout);
297 previewWindow.show();
298 previewApp->exec();
299
300 // Deallocate waveform image
301 ClearWaveform();
302 }
303
304 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
GetAudioSample(int channel,int sample,int magnitude_range)305 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
306 {
307 if (channel > 0) {
308 // return average magnitude for a specific channel/sample range
309 return audio->getMagnitude(channel, sample, magnitude_range);
310
311 } else {
312 // Return average magnitude for all channels
313 return audio->getMagnitude(sample, magnitude_range);
314 }
315 }
316
317 // Get an array of sample data
GetAudioSamples(int channel)318 float* Frame::GetAudioSamples(int channel)
319 {
320 // return JUCE audio data for this channel
321 return audio->getWritePointer(channel);
322 }
323
324 // Get a planar array of sample data, using any sample rate
GetPlanarAudioSamples(int new_sample_rate,AudioResampler * resampler,int * sample_count)325 float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
326 {
327 float *output = NULL;
328 juce::AudioSampleBuffer *buffer(audio.get());
329 int num_of_channels = audio->getNumChannels();
330 int num_of_samples = GetAudioSamplesCount();
331
332 // Resample to new sample rate (if needed)
333 if (new_sample_rate != sample_rate)
334 {
335 // YES, RESAMPLE AUDIO
336 resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
337
338 // Resample data, and return new buffer pointer
339 buffer = resampler->GetResampledBuffer();
340
341 // Update num_of_samples
342 num_of_samples = buffer->getNumSamples();
343 }
344
345 // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
346 output = new float[num_of_channels * num_of_samples];
347 int position = 0;
348
349 // Loop through samples in each channel (combining them)
350 for (int channel = 0; channel < num_of_channels; channel++)
351 {
352 for (int sample = 0; sample < num_of_samples; sample++)
353 {
354 // Add sample to output array
355 output[position] = buffer->getReadPointer(channel)[sample];
356
357 // increment position
358 position++;
359 }
360 }
361
362 // Update sample count (since it might have changed due to resampling)
363 *sample_count = num_of_samples;
364
365 // return combined array
366 return output;
367 }
368
369
370 // Get an array of sample data (all channels interleaved together), using any sample rate
GetInterleavedAudioSamples(int new_sample_rate,AudioResampler * resampler,int * sample_count)371 float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
372 {
373 float *output = NULL;
374 juce::AudioSampleBuffer *buffer(audio.get());
375 int num_of_channels = audio->getNumChannels();
376 int num_of_samples = GetAudioSamplesCount();
377
378 // Resample to new sample rate (if needed)
379 if (new_sample_rate != sample_rate && resampler)
380 {
381 // YES, RESAMPLE AUDIO
382 resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
383
384 // Resample data, and return new buffer pointer
385 buffer = resampler->GetResampledBuffer();
386
387 // Update num_of_samples
388 num_of_samples = buffer->getNumSamples();
389 }
390
391 // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
392 output = new float[num_of_channels * num_of_samples];
393 int position = 0;
394
395 // Loop through samples in each channel (combining them)
396 for (int sample = 0; sample < num_of_samples; sample++)
397 {
398 for (int channel = 0; channel < num_of_channels; channel++)
399 {
400 // Add sample to output array
401 output[position] = buffer->getReadPointer(channel)[sample];
402
403 // increment position
404 position++;
405 }
406 }
407
408 // Update sample count (since it might have changed due to resampling)
409 *sample_count = num_of_samples;
410
411 // return combined array
412 return output;
413 }
414
415 // Get number of audio channels
GetAudioChannelsCount()416 int Frame::GetAudioChannelsCount()
417 {
418 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
419 if (audio)
420 return audio->getNumChannels();
421 else
422 return 0;
423 }
424
425 // Get number of audio samples
GetAudioSamplesCount()426 int Frame::GetAudioSamplesCount()
427 {
428 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
429 return max_audio_sample;
430 }
431
GetAudioSampleBuffer()432 juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer()
433 {
434 return audio.get();
435 }
436
437 // Get the size in bytes of this frame (rough estimate)
GetBytes()438 int64_t Frame::GetBytes()
439 {
440 int64_t total_bytes = 0;
441 if (image)
442 total_bytes += (width * height * sizeof(char) * 4);
443 if (audio) {
444 // approximate audio size (sample rate / 24 fps)
445 total_bytes += (sample_rate / 24.0) * sizeof(float);
446 }
447
448 // return size of this frame
449 return total_bytes;
450 }
451
452 // Get pixel data (as packets)
GetPixels()453 const unsigned char* Frame::GetPixels()
454 {
455 // Check for blank image
456 if (!image)
457 // Fill with black
458 AddColor(width, height, color);
459
460 // Return array of pixel packets
461 return image->constBits();
462 }
463
464 // Get pixel data (for only a single scan-line)
GetPixels(int row)465 const unsigned char* Frame::GetPixels(int row)
466 {
467 // Check for blank image
468 if (!image)
469 // Fill with black
470 AddColor(width, height, color);
471
472 // Return array of pixel packets
473 return image->constScanLine(row);
474 }
475
476 // Check a specific pixel color value (returns True/False)
CheckPixel(int row,int col,int red,int green,int blue,int alpha,int threshold)477 bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
478 int col_pos = col * 4; // Find column array position
479 if (!image || row < 0 || row >= (height - 1) ||
480 col_pos < 0 || col_pos >= (width - 1) ) {
481 // invalid row / col
482 return false;
483 }
484 // Check pixel color
485 const unsigned char* pixels = GetPixels(row);
486 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
487 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
488 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
489 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
490 // Pixel color matches successfully
491 return true;
492 } else {
493 // Pixel color does not match
494 return false;
495 }
496 }
497
498 // Set Pixel Aspect Ratio
SetPixelRatio(int num,int den)499 void Frame::SetPixelRatio(int num, int den)
500 {
501 pixel_ratio.num = num;
502 pixel_ratio.den = den;
503 }
504
505 // Set frame number
SetFrameNumber(int64_t new_number)506 void Frame::SetFrameNumber(int64_t new_number)
507 {
508 number = new_number;
509 }
510
511 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
GetSamplesPerFrame(int64_t number,Fraction fps,int sample_rate,int channels)512 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
513 {
514 // Get the total # of samples for the previous frame, and the current frame (rounded)
515 double fps_rate = fps.Reciprocal().ToDouble();
516
517 // Determine previous samples total, and make sure it's evenly divisible by the # of channels
518 double previous_samples = (sample_rate * fps_rate) * (number - 1);
519 double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
520 previous_samples -= previous_samples_remainder;
521
522 // Determine the current samples total, and make sure it's evenly divisible by the # of channels
523 double total_samples = (sample_rate * fps_rate) * number;
524 double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
525 total_samples -= total_samples_remainder;
526
527 // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
528 // be evenly divided into frames, so each frame can have have different # of samples.
529 int samples_per_frame = round(total_samples - previous_samples);
530 if (samples_per_frame < 0)
531 samples_per_frame = 0;
532 return samples_per_frame;
533 }
534
535 // Calculate the # of samples per video frame (for the current frame number)
GetSamplesPerFrame(Fraction fps,int sample_rate,int channels)536 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
537 {
538 return GetSamplesPerFrame(number, fps, sample_rate, channels);
539 }
540
541 // Get height of image
GetHeight()542 int Frame::GetHeight()
543 {
544 return height;
545 }
546
547 // Get height of image
GetWidth()548 int Frame::GetWidth()
549 {
550 return width;
551 }
552
553 // Get the original sample rate of this frame's audio data
SampleRate()554 int Frame::SampleRate()
555 {
556 return sample_rate;
557 }
558
559 // Get the original sample rate of this frame's audio data
ChannelsLayout()560 ChannelLayout Frame::ChannelsLayout()
561 {
562 return channel_layout;
563 }
564
565
566 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
Save(std::string path,float scale,std::string format,int quality)567 void Frame::Save(std::string path, float scale, std::string format, int quality)
568 {
569 // Get preview image
570 std::shared_ptr<QImage> previewImage = GetImage();
571
572 // scale image if needed
573 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
574 {
575 int new_width = width;
576 int new_height = height;
577
578 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
579 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
580 {
581 // Calculate correct DAR (display aspect ratio)
582 int new_width = previewImage->size().width();
583 int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
584
585 // Resize to fix DAR
586 previewImage = std::make_shared<QImage>(previewImage->scaled(
587 new_width, new_height,
588 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
589 }
590
591 // Resize image
592 previewImage = std::make_shared<QImage>(previewImage->scaled(
593 new_width * scale, new_height * scale,
594 Qt::KeepAspectRatio, Qt::SmoothTransformation));
595 }
596
597 // Save image
598 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
599 }
600
601 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
Thumbnail(std::string path,int new_width,int new_height,std::string mask_path,std::string overlay_path,std::string background_color,bool ignore_aspect,std::string format,int quality,float rotate)602 void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
603 std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
604
605 // Create blank thumbnail image & fill background color
606 auto thumbnail = std::make_shared<QImage>(
607 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
608 thumbnail->fill(QColor(QString::fromStdString(background_color)));
609
610 // Create painter
611 QPainter painter(thumbnail.get());
612 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
613
614 // Get preview image
615 std::shared_ptr<QImage> previewImage = GetImage();
616
617 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
618 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
619 {
620 // Calculate correct DAR (display aspect ratio)
621 int aspect_width = previewImage->size().width();
622 int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
623
624 // Resize to fix DAR
625 previewImage = std::make_shared<QImage>(previewImage->scaled(
626 aspect_width, aspect_height,
627 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
628 }
629
630 // Resize frame image
631 if (ignore_aspect)
632 // Ignore aspect ratio
633 previewImage = std::make_shared<QImage>(previewImage->scaled(
634 new_width, new_height,
635 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
636 else
637 // Maintain aspect ratio
638 previewImage = std::make_shared<QImage>(previewImage->scaled(
639 new_width, new_height,
640 Qt::KeepAspectRatio, Qt::SmoothTransformation));
641
642 // Composite frame image onto background (centered)
643 int x = (new_width - previewImage->size().width()) / 2.0; // center
644 int y = (new_height - previewImage->size().height()) / 2.0; // center
645 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
646
647
648 // Create transform and rotate (if needed)
649 QTransform transform;
650 float origin_x = previewImage->width() / 2.0;
651 float origin_y = previewImage->height() / 2.0;
652 transform.translate(origin_x, origin_y);
653 transform.rotate(rotate);
654 transform.translate(-origin_x,-origin_y);
655 painter.setTransform(transform);
656
657 // Draw image onto QImage
658 painter.drawImage(x, y, *previewImage);
659
660
661 // Overlay Image (if any)
662 if (overlay_path != "") {
663 // Open overlay
664 auto overlay = std::make_shared<QImage>();
665 overlay->load(QString::fromStdString(overlay_path));
666
667 // Set pixel format
668 overlay = std::make_shared<QImage>(
669 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
670
671 // Resize to fit
672 overlay = std::make_shared<QImage>(overlay->scaled(
673 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
674
675 // Composite onto thumbnail
676 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
677 painter.drawImage(0, 0, *overlay);
678 }
679
680
681 // Mask Image (if any)
682 if (mask_path != "") {
683 // Open mask
684 auto mask = std::make_shared<QImage>();
685 mask->load(QString::fromStdString(mask_path));
686
687 // Set pixel format
688 mask = std::make_shared<QImage>(
689 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
690
691 // Resize to fit
692 mask = std::make_shared<QImage>(mask->scaled(
693 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
694
695 // Negate mask
696 mask->invertPixels();
697
698 // Get pixels
699 unsigned char *pixels = (unsigned char *) thumbnail->bits();
700 const unsigned char *mask_pixels = (const unsigned char *) mask->constBits();
701
702 // Convert the mask image to grayscale
703 // Loop through pixels
704 for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
705 {
706 // Get the RGB values from the pixel
707 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
708 int Frame_Alpha = pixels[byte_index + 3];
709 int Mask_Value = constrain(Frame_Alpha - gray_value);
710
711 // Set all alpha pixels to gray value
712 pixels[byte_index + 3] = Mask_Value;
713 }
714 }
715
716
717 // End painter
718 painter.end();
719
720 // Save image
721 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
722 }
723
724 // Constrain a color value from 0 to 255
constrain(int color_value)725 int Frame::constrain(int color_value)
726 {
727 // Constrain new color from 0 to 255
728 if (color_value < 0)
729 color_value = 0;
730 else if (color_value > 255)
731 color_value = 255;
732
733 return color_value;
734 }
735
736 // Add (or replace) pixel data to the frame (based on a solid color)
AddColor(int new_width,int new_height,std::string new_color)737 void Frame::AddColor(int new_width, int new_height, std::string new_color)
738 {
739 // Set color
740 color = new_color;
741
742 // Create new image object, and fill with pixel data
743 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
744 image = std::make_shared<QImage>(new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
745
746 // Fill with solid color
747 image->fill(QColor(QString::fromStdString(color)));
748
749 // Update height and width
750 width = image->width();
751 height = image->height();
752 has_image_data = true;
753 }
754
755 // Add (or replace) pixel data to the frame
AddImage(int new_width,int new_height,int bytes_per_pixel,QImage::Format type,const unsigned char * pixels_)756 void Frame::AddImage(
757 int new_width, int new_height, int bytes_per_pixel,
758 QImage::Format type, const unsigned char *pixels_)
759 {
760 // Create new buffer
761 {
762 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
763 qbuffer = pixels_;
764 } // Release addingImageSection lock
765
766 // Create new image object from pixel data
767 auto new_image = std::make_shared<QImage>(
768 qbuffer,
769 new_width, new_height,
770 new_width * bytes_per_pixel,
771 type,
772 (QImageCleanupFunction) &openshot::Frame::cleanUpBuffer,
773 (void*) qbuffer
774 );
775 AddImage(new_image);
776 }
777
778 // Add (or replace) pixel data to the frame
AddImage(std::shared_ptr<QImage> new_image)779 void Frame::AddImage(std::shared_ptr<QImage> new_image)
780 {
781 // Ignore blank images
782 if (!new_image)
783 return;
784
785 // assign image data
786 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
787 image = new_image;
788
789 // Always convert to Format_RGBA8888_Premultiplied (if different)
790 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
791 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
792
793 // Update height and width
794 width = image->width();
795 height = image->height();
796 has_image_data = true;
797 }
798
799 // Add (or replace) pixel data to the frame (for only the odd or even lines)
AddImage(std::shared_ptr<QImage> new_image,bool only_odd_lines)800 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
801 {
802 // Ignore blank new_image
803 if (!new_image)
804 return;
805
806 // Check for blank source image
807 if (!image) {
808 // Replace the blank source image
809 AddImage(new_image);
810
811 } else {
812 // Ignore image of different sizes or formats
813 bool ret=false;
814 if (image == new_image || image->size() != new_image->size()) {
815 ret = true;
816 }
817 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
818 new_image = std::make_shared<QImage>(
819 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
820 }
821 if (ret) {
822 return;
823 }
824
825 // Get the frame's image
826 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
827 unsigned char *pixels = image->bits();
828 const unsigned char *new_pixels = new_image->constBits();
829
830 // Loop through the scanlines of the image (even or odd)
831 int start = 0;
832 if (only_odd_lines)
833 start = 1;
834
835 for (int row = start; row < image->height(); row += 2) {
836 int offset = row * image->bytesPerLine();
837 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
838 }
839
840 // Update height and width
841 height = image->height();
842 width = image->width();
843 has_image_data = true;
844 }
845 }
846
847
848 // Resize audio container to hold more (or less) samples and channels
ResizeAudio(int channels,int length,int rate,ChannelLayout layout)849 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
850 {
851 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
852
853 // Resize JUCE audio buffer
854 audio->setSize(channels, length, true, true, false);
855 channel_layout = layout;
856 sample_rate = rate;
857
858 // Calculate max audio sample added
859 max_audio_sample = length;
860 }
861
862 // Add audio samples to a specific channel
AddAudio(bool replaceSamples,int destChannel,int destStartSample,const float * source,int numSamples,float gainToApplyToSource=1.0f)863 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
864 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
865
866 // Clamp starting sample to 0
867 int destStartSampleAdjusted = max(destStartSample, 0);
868
869 // Extend audio container to hold more (or less) samples and channels.. if needed
870 int new_length = destStartSampleAdjusted + numSamples;
871 int new_channel_length = audio->getNumChannels();
872 if (destChannel >= new_channel_length)
873 new_channel_length = destChannel + 1;
874 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
875 audio->setSize(new_channel_length, new_length, true, true, false);
876
877 // Clear the range of samples first (if needed)
878 if (replaceSamples)
879 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
880
881 // Add samples to frame's audio buffer
882 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
883 has_audio_data = true;
884
885 // Calculate max audio sample added
886 if (new_length > max_audio_sample)
887 max_audio_sample = new_length;
888 }
889
890 // Apply gain ramp (i.e. fading volume)
ApplyGainRamp(int destChannel,int destStartSample,int numSamples,float initial_gain=0.0f,float final_gain=1.0f)891 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
892 {
893 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
894
895 // Apply gain ramp
896 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
897 }
898
899 // Get pointer to Magick++ image object
GetImage()900 std::shared_ptr<QImage> Frame::GetImage()
901 {
902 // Check for blank image
903 if (!image)
904 // Fill with black
905 AddColor(width, height, color);
906
907 return image;
908 }
909
910 #ifdef USE_OPENCV
911
912 // Convert Qimage to Mat
Qimage2mat(std::shared_ptr<QImage> & qimage)913 cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
914
915 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
916 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
917 int from_to[] = { 0,0, 1,1, 2,2 };
918 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
919 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
920 return mat2;
921 };
922
923 // Get pointer to OpenCV image object
GetImageCV()924 cv::Mat Frame::GetImageCV()
925 {
926 // Check for blank image
927 if (!image)
928 // Fill with black
929 AddColor(width, height, color);
930
931 // if (imagecv.empty())
932 // Convert Qimage to Mat
933 imagecv = Qimage2mat(image);
934
935 return imagecv;
936 }
937
Mat2Qimage(cv::Mat img)938 std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
939 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
940 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
941
942 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
943
944 // Always convert to RGBA8888 (if different)
945 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
946 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
947
948 return imgIn;
949 }
950
951 // Set pointer to OpenCV image object
SetImageCV(cv::Mat _image)952 void Frame::SetImageCV(cv::Mat _image)
953 {
954 imagecv = _image;
955 image = Mat2Qimage(_image);
956 }
957 #endif
958
959 #ifdef USE_IMAGEMAGICK
960 // Get pointer to ImageMagick image object
GetMagickImage()961 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
962 {
963 // Check for blank image
964 if (!image)
965 // Fill with black
966 AddColor(width, height, "#000000");
967
968 // Get the pixels from the frame image
969 const QRgb *tmpBits = (const QRgb*)image->constBits();
970
971 // Create new image object, and fill with pixel data
972 auto magick_image = std::make_shared<Magick::Image>(
973 image->width(), image->height(),"RGBA", Magick::CharPixel, tmpBits);
974
975 // Give image a transparent background color
976 magick_image->backgroundColor(Magick::Color("none"));
977 magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
978 MAGICK_IMAGE_ALPHA(magick_image, true);
979
980 return magick_image;
981 }
982 #endif
983
984 #ifdef USE_IMAGEMAGICK
985 // Get pointer to QImage of frame
AddMagickImage(std::shared_ptr<Magick::Image> new_image)986 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
987 {
988 const int BPP = 4;
989 const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
990
991 /// Use realloc for fast memory allocation.
992 /// TODO: consider locking the buffer for mt safety
993 //qbuffer = reinterpret_cast<unsigned char*>(realloc(qbuffer, bufferSize));
994 qbuffer = new unsigned char[bufferSize]();
995 unsigned char *buffer = (unsigned char*)qbuffer;
996
997 MagickCore::ExceptionInfo exception;
998 // TODO: Actually do something, if we get an exception here
999 MagickCore::ExportImagePixels(new_image->constImage(), 0, 0, new_image->columns(), new_image->rows(), "RGBA", Magick::CharPixel, buffer, &exception);
1000
1001 // Create QImage of frame data
1002 image = std::make_shared<QImage>(
1003 qbuffer, width, height, width * BPP, QImage::Format_RGBA8888_Premultiplied,
1004 (QImageCleanupFunction) &cleanUpBuffer, (void*) qbuffer);
1005
1006 // Update height and width
1007 width = image->width();
1008 height = image->height();
1009 has_image_data = true;
1010 }
1011 #endif
1012
1013 // Play audio samples for this frame
Play()1014 void Frame::Play()
1015 {
1016 // Check if samples are present
1017 if (!GetAudioSamplesCount())
1018 return;
1019
1020 juce::AudioDeviceManager deviceManager;
1021 juce::String error = deviceManager.initialise (
1022 0, /* number of input channels */
1023 2, /* number of output channels */
1024 0, /* no XML settings.. */
1025 true /* select default device on failure */);
1026
1027 // Output error (if any)
1028 if (error.isNotEmpty()) {
1029 cout << "Error on initialise(): " << error << endl;
1030 }
1031
1032 juce::AudioSourcePlayer audioSourcePlayer;
1033 deviceManager.addAudioCallback (&audioSourcePlayer);
1034
1035 std::unique_ptr<AudioBufferSource> my_source;
1036 my_source.reset (new AudioBufferSource (audio.get()));
1037
1038 // Create TimeSliceThread for audio buffering
1039 juce::TimeSliceThread my_thread("Audio buffer thread");
1040
1041 // Start thread
1042 my_thread.startThread();
1043
1044 juce::AudioTransportSource transport1;
1045 transport1.setSource (my_source.get(),
1046 5000, // tells it to buffer this many samples ahead
1047 &my_thread,
1048 (double) sample_rate,
1049 audio->getNumChannels()); // sample rate of source
1050 transport1.setPosition (0);
1051 transport1.setGain(1.0);
1052
1053
1054 // Create MIXER
1055 juce::MixerAudioSource mixer;
1056 mixer.addInputSource(&transport1, false);
1057 audioSourcePlayer.setSource (&mixer);
1058
1059 // Start transports
1060 transport1.start();
1061
1062 while (transport1.isPlaying())
1063 {
1064 cout << "playing" << endl;
1065 std::this_thread::sleep_for(std::chrono::seconds(1));
1066 }
1067
1068 cout << "DONE!!!" << endl;
1069
1070 transport1.stop();
1071 transport1.setSource (0);
1072 audioSourcePlayer.setSource (0);
1073 my_thread.stopThread(500);
1074 deviceManager.removeAudioCallback (&audioSourcePlayer);
1075 deviceManager.closeAudioDevice();
1076 deviceManager.removeAllChangeListeners();
1077 deviceManager.dispatchPendingMessages();
1078
1079 cout << "End of Play()" << endl;
1080
1081
1082 }
1083
1084 // Clean up buffer after QImage is deleted
cleanUpBuffer(void * info)1085 void Frame::cleanUpBuffer(void *info)
1086 {
1087 if (info)
1088 {
1089 // Remove buffer since QImage tells us to
1090 unsigned char* ptr_to_qbuffer = (unsigned char*) info;
1091 delete[] ptr_to_qbuffer;
1092 }
1093 }
1094
1095 // Add audio silence
AddAudioSilence(int numSamples)1096 void Frame::AddAudioSilence(int numSamples)
1097 {
1098 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
1099
1100 // Resize audio container
1101 audio->setSize(channels, numSamples, false, true, false);
1102 audio->clear();
1103 has_audio_data = true;
1104
1105 // Calculate max audio sample added
1106 if (numSamples > max_audio_sample)
1107 max_audio_sample = numSamples;
1108 }
1109