OpenShot Library | libopenshot 0.3.2
Loading...
Searching...
No Matches
Frame.cpp
Go to the documentation of this file.
1
9// Copyright (c) 2008-2019 OpenShot Studios, LLC
10//
11// SPDX-License-Identifier: LGPL-3.0-or-later
12
13#include <thread> // for std::this_thread::sleep_for
14#include <chrono> // for std::chrono::milliseconds
15#include <sstream>
16#include <iomanip>
17
18#include "Frame.h"
19#include "AudioBufferSource.h"
20#include "AudioResampler.h"
21#include "QtUtilities.h"
22
23#include <AppConfig.h>
24#include <juce_audio_basics/juce_audio_basics.h>
25#include <juce_audio_devices/juce_audio_devices.h>
26
27#include <QApplication>
28#include <QImage>
29#include <QPixmap>
30#include <QBitmap>
31#include <QColor>
32#include <QString>
33#include <QVector>
34#include <QPainter>
35#include <QHBoxLayout>
36#include <QWidget>
37#include <QLabel>
38#include <QPointF>
39#include <QWidget>
40
41using namespace std;
42using namespace openshot;
43
44// Constructor - image & audio
45Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
46 : audio(std::make_shared<juce::AudioBuffer<float>>(channels, samples)),
47 number(number), width(width), height(height),
48 pixel_ratio(1,1), color(color),
49 channels(channels), channel_layout(LAYOUT_STEREO),
50 sample_rate(44100),
51 has_audio_data(false), has_image_data(false),
52 max_audio_sample(0)
53{
54 // zero (fill with silence) the audio buffer
55 audio->clear();
56}
57
58// Delegating Constructor - blank frame
59Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {}
60
61// Delegating Constructor - image only
62Frame::Frame(int64_t number, int width, int height, std::string color)
63 : Frame::Frame(number, width, height, color, 0, 2) {}
64
65// Delegating Constructor - audio only
66Frame::Frame(int64_t number, int samples, int channels)
67 : Frame::Frame(number, 1, 1, "#000000", samples, channels) {}
68
69
70// Copy constructor
71Frame::Frame ( const Frame &other )
72{
73 // copy pointers and data
74 DeepCopy(other);
75}
76
77// Assignment operator
79{
80 // copy pointers and data
81 DeepCopy(other);
82
83 return *this;
84}
85
86// Copy data and pointers from another Frame instance
87void Frame::DeepCopy(const Frame& other)
88{
89 number = other.number;
90 channels = other.channels;
91 width = other.width;
92 height = other.height;
93 channel_layout = other.channel_layout;
96 sample_rate = other.sample_rate;
97 pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
98 color = other.color;
99 max_audio_sample = other.max_audio_sample;
100
101 if (other.image)
102 image = std::make_shared<QImage>(*(other.image));
103 if (other.audio)
104 audio = std::make_shared<juce::AudioBuffer<float>>(*(other.audio));
105 if (other.wave_image)
106 wave_image = std::make_shared<QImage>(*(other.wave_image));
107}
108
109// Destructor
111 // Clear all pointers
112 image.reset();
113 audio.reset();
114 #ifdef USE_OPENCV
115 imagecv.release();
116 #endif
117}
118
119// Display the frame image to the screen (primarily used for debugging reasons)
121{
122 if (!QApplication::instance()) {
123 // Only create the QApplication once
124 static int argc = 1;
125 static char* argv[1] = {NULL};
126 previewApp = std::make_shared<QApplication>(argc, argv);
127 }
128
129 // Get preview image
130 std::shared_ptr<QImage> previewImage = GetImage();
131
132 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
133 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
134 {
135 // Resize to fix DAR
136 previewImage = std::make_shared<QImage>(previewImage->scaled(
137 previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
138 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
139 }
140
141 // Create window
142 QWidget previewWindow;
143 previewWindow.setStyleSheet("background-color: #000000;");
144 QHBoxLayout layout;
145
146 // Create label with current frame's image
147 QLabel previewLabel;
148 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150 layout.addWidget(&previewLabel);
151
152 // Show the window
153 previewWindow.setLayout(&layout);
154 previewWindow.show();
155 previewApp->exec();
156}
157
158// Get an audio waveform image
159std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
160{
161 // Clear any existing waveform image
163
164 // Init a list of lines
165 QVector<QPointF> lines;
166 QVector<QPointF> labels;
167
168 // Calculate width of an image based on the # of samples
169 int total_samples = GetAudioSamplesCount();
170 if (total_samples > 0)
171 {
172 // If samples are present...
173 int new_height = 200 * audio->getNumChannels();
174 int height_padding = 20 * (audio->getNumChannels() - 1);
175 int total_height = new_height + height_padding;
176 int total_width = 0;
177 float zero_height = 1.0; // Used to clamp near-zero vales to this value to prevent gaps
178
179 // Loop through each audio channel
180 float Y = 100.0;
181 for (int channel = 0; channel < audio->getNumChannels(); channel++)
182 {
183 float X = 0.0;
184
185 // Get audio for this channel
186 const float *samples = audio->getReadPointer(channel);
187
188 for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
189 {
190 // Sample value (scaled to -100 to 100)
191 float value = samples[sample] * 100.0;
192
193 // Set threshold near zero (so we don't allow near-zero values)
194 // This prevents empty gaps from appearing in the waveform
195 if (value > -zero_height && value < 0.0) {
196 value = -zero_height;
197 } else if (value > 0.0 && value < zero_height) {
198 value = zero_height;
199 }
200
201 // Append a line segment for each sample
202 lines.push_back(QPointF(X, Y));
203 lines.push_back(QPointF(X, Y - value));
204 }
205
206 // Add Channel Label Coordinate
207 labels.push_back(QPointF(5.0, Y - 5.0));
208
209 // Increment Y
210 Y += (200 + height_padding);
211 total_width = X;
212 }
213
214 // Create blank image
215 wave_image = std::make_shared<QImage>(
216 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217 wave_image->fill(QColor(0,0,0,0));
218
219 // Load QPainter with wave_image device
220 QPainter painter(wave_image.get());
221
222 // Set pen color
223 QPen pen;
224 pen.setColor(QColor(Red, Green, Blue, Alpha));
225 pen.setWidthF(1.0);
226 pen.setStyle(Qt::SolidLine);
227 painter.setPen(pen);
228
229 // Draw the waveform
230 painter.drawLines(lines);
231 painter.end();
232 }
233 else
234 {
235 // No audio samples present
236 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237 wave_image->fill(QColor(QString::fromStdString("#000000")));
238 }
239
240 // Resize Image (if needed)
241 if (wave_image->width() != width || wave_image->height() != height) {
242 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243 wave_image = std::make_shared<QImage>(scaled_wave_image);
244 }
245
246 // Return new image
247 return wave_image;
248}
249
250// Clear the waveform image (and deallocate its memory)
252{
253 if (wave_image)
254 wave_image.reset();
255}
256
257// Get an audio waveform image pixels
258const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
259{
260 // Get audio wave form image
261 wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
262
263 // Return array of pixel packets
264 return wave_image->constBits();
265}
266
267// Display the wave form
269{
270 // Get audio wave form image
271 GetWaveform(720, 480, 0, 123, 255, 255);
272
273 if (!QApplication::instance()) {
274 // Only create the QApplication once
275 static int argc = 1;
276 static char* argv[1] = {NULL};
277 previewApp = std::make_shared<QApplication>(argc, argv);
278 }
279
280 // Create window
281 QWidget previewWindow;
282 previewWindow.setStyleSheet("background-color: #000000;");
283 QHBoxLayout layout;
284
285 // Create label with current frame's waveform image
286 QLabel previewLabel;
287 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289 layout.addWidget(&previewLabel);
290
291 // Show the window
292 previewWindow.setLayout(&layout);
293 previewWindow.show();
294 previewApp->exec();
295
296 // Deallocate waveform image
298}
299
300// Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
301float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
302{
303 if (channel > 0) {
304 // return average magnitude for a specific channel/sample range
305 return audio->getMagnitude(channel, sample, magnitude_range);
306
307 } else {
308 // Return average magnitude for all channels
309 return audio->getMagnitude(sample, magnitude_range);
310 }
311}
312
313// Get an array of sample data (and optional reverse the sample values)
314float* Frame::GetAudioSamples(int channel) {
315
316 // Copy audio data
317 juce::AudioBuffer<float> *buffer(audio.get());
318
319 // return JUCE audio data for this channel
320 return buffer->getWritePointer(channel);
321}
322
323// Get an array of sample data (all channels interleaved together), using any sample rate
324float* Frame::GetInterleavedAudioSamples(int* sample_count)
325{
326 // Copy audio data
327 juce::AudioBuffer<float> *buffer(audio.get());
328
329 float *output = NULL;
330 int num_of_channels = audio->getNumChannels();
331 int num_of_samples = GetAudioSamplesCount();
332
333 // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
334 output = new float[num_of_channels * num_of_samples];
335 int position = 0;
336
337 // Loop through samples in each channel (combining them)
338 for (int sample = 0; sample < num_of_samples; sample++)
339 {
340 for (int channel = 0; channel < num_of_channels; channel++)
341 {
342 // Add sample to output array
343 output[position] = buffer->getReadPointer(channel)[sample];
344
345 // increment position
346 position++;
347 }
348 }
349
350 // Update sample count (since it might have changed due to resampling)
351 *sample_count = num_of_samples;
352
353 // return combined array
354 return output;
355}
356
357// Get number of audio channels
359{
360 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
361 if (audio)
362 return audio->getNumChannels();
363 else
364 return 0;
365}
366
367// Get number of audio samples
369{
370 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
371 return max_audio_sample;
372}
373
375{
376 return audio.get();
377}
378
379// Get the size in bytes of this frame (rough estimate)
381{
382 int64_t total_bytes = 0;
383 if (image) {
384 total_bytes += static_cast<int64_t>(
385 width * height * sizeof(char) * 4);
386 }
387 if (audio) {
388 // approximate audio size (sample rate / 24 fps)
389 total_bytes += (sample_rate / 24.0) * sizeof(float);
390 }
391
392 // return size of this frame
393 return total_bytes;
394}
395
396// Get pixel data (as packets)
397const unsigned char* Frame::GetPixels()
398{
399 // Check for blank image
400 if (!image)
401 // Fill with black
402 AddColor(width, height, color);
403
404 // Return array of pixel packets
405 return image->constBits();
406}
407
408// Get pixel data (for only a single scan-line)
409const unsigned char* Frame::GetPixels(int row)
410{
411 // Check for blank image
412 if (!image)
413 // Fill with black
414 AddColor(width, height, color);
415
416 // Return array of pixel packets
417 return image->constScanLine(row);
418}
419
420// Check a specific pixel color value (returns True/False)
421bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
422 int col_pos = col * 4; // Find column array position
423 if (!image || row < 0 || row >= (height - 1) ||
424 col_pos < 0 || col_pos >= (width - 1) ) {
425 // invalid row / col
426 return false;
427 }
428 // Check pixel color
429 const unsigned char* pixels = GetPixels(row);
430 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
431 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
432 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
433 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
434 // Pixel color matches successfully
435 return true;
436 } else {
437 // Pixel color does not match
438 return false;
439 }
440}
441
442// Set Pixel Aspect Ratio
443void Frame::SetPixelRatio(int num, int den)
444{
445 pixel_ratio.num = num;
446 pixel_ratio.den = den;
447}
448
449// Set frame number
450void Frame::SetFrameNumber(int64_t new_number)
451{
452 number = new_number;
453}
454
455// Calculate the # of samples per video frame (for a specific frame number and frame rate)
456int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
457{
458 // Get the total # of samples for the previous frame, and the current frame (rounded)
459 double fps_rate = fps.Reciprocal().ToDouble();
460
461 // Determine previous samples total, and make sure it's evenly divisible by the # of channels
462 double previous_samples = (sample_rate * fps_rate) * (number - 1);
463 double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
464 previous_samples -= previous_samples_remainder;
465
466 // Determine the current samples total, and make sure it's evenly divisible by the # of channels
467 double total_samples = (sample_rate * fps_rate) * number;
468 double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
469 total_samples -= total_samples_remainder;
470
471 // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
472 // be evenly divided into frames, so each frame can have have different # of samples.
473 int samples_per_frame = round(total_samples - previous_samples);
474 if (samples_per_frame < 0)
475 samples_per_frame = 0;
476 return samples_per_frame;
477}
478
479// Calculate the # of samples per video frame (for the current frame number)
480int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
481{
482 return GetSamplesPerFrame(number, fps, sample_rate, channels);
483}
484
485// Get height of image
487{
488 return height;
489}
490
491// Get height of image
493{
494 return width;
495}
496
497// Get the original sample rate of this frame's audio data
499{
500 return sample_rate;
501}
502
503// Get the original sample rate of this frame's audio data
505{
506 return channel_layout;
507}
508
509
510// Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
511void Frame::Save(std::string path, float scale, std::string format, int quality)
512{
513 // Get preview image
514 std::shared_ptr<QImage> previewImage = GetImage();
515
516 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
517 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
518 {
519 // Resize to fix DAR
520 previewImage = std::make_shared<QImage>(previewImage->scaled(
521 previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
522 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
523 }
524
525 // scale image if needed
526 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
527 {
528 // Resize image
529 previewImage = std::make_shared<QImage>(previewImage->scaled(
530 previewImage->size().width() * scale, previewImage->size().height() * scale,
531 Qt::KeepAspectRatio, Qt::SmoothTransformation));
532 }
533
534 // Save image
535 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
536}
537
538// Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
539void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
540 std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
541
542 // Create blank thumbnail image & fill background color
543 auto thumbnail = std::make_shared<QImage>(
544 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
545 thumbnail->fill(QColor(QString::fromStdString(background_color)));
546
547 // Create painter
548 QPainter painter(thumbnail.get());
549 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
550
551 // Get preview image
552 std::shared_ptr<QImage> previewImage = GetImage();
553
554 // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
555 if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
556 {
557 // Calculate correct DAR (display aspect ratio)
558 int aspect_width = previewImage->size().width();
559 int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
560
561 // Resize to fix DAR
562 previewImage = std::make_shared<QImage>(previewImage->scaled(
563 aspect_width, aspect_height,
564 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
565 }
566
567 // Resize frame image
568 if (ignore_aspect)
569 // Ignore aspect ratio
570 previewImage = std::make_shared<QImage>(previewImage->scaled(
571 new_width, new_height,
572 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
573 else
574 // Maintain aspect ratio
575 previewImage = std::make_shared<QImage>(previewImage->scaled(
576 new_width, new_height,
577 Qt::KeepAspectRatio, Qt::SmoothTransformation));
578
579 // Composite frame image onto background (centered)
580 int x = (new_width - previewImage->size().width()) / 2.0; // center
581 int y = (new_height - previewImage->size().height()) / 2.0; // center
582 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
583
584
585 // Create transform and rotate (if needed)
586 QTransform transform;
587 float origin_x = previewImage->width() / 2.0;
588 float origin_y = previewImage->height() / 2.0;
589 transform.translate(origin_x, origin_y);
590 transform.rotate(rotate);
591 transform.translate(-origin_x,-origin_y);
592 painter.setTransform(transform);
593
594 // Draw image onto QImage
595 painter.drawImage(x, y, *previewImage);
596
597
598 // Overlay Image (if any)
599 if (overlay_path != "") {
600 // Open overlay
601 auto overlay = std::make_shared<QImage>();
602 overlay->load(QString::fromStdString(overlay_path));
603
604 // Set pixel format
605 overlay = std::make_shared<QImage>(
606 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
607
608 // Resize to fit
609 overlay = std::make_shared<QImage>(overlay->scaled(
610 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
611
612 // Composite onto thumbnail
613 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
614 painter.drawImage(0, 0, *overlay);
615 }
616
617
618 // Mask Image (if any)
619 if (mask_path != "") {
620 // Open mask
621 auto mask = std::make_shared<QImage>();
622 mask->load(QString::fromStdString(mask_path));
623
624 // Set pixel format
625 mask = std::make_shared<QImage>(
626 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
627
628 // Resize to fit
629 mask = std::make_shared<QImage>(mask->scaled(
630 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
631
632 // Negate mask
633 mask->invertPixels();
634
635 // Get pixels
636 unsigned char *pixels = static_cast<unsigned char *>(thumbnail->bits());
637 const unsigned char *mask_pixels = static_cast<const unsigned char *>(mask->constBits());
638
639 // Convert the mask image to grayscale
640 // Loop through pixels
641 for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
642 {
643 // Get the RGB values from the pixel
644 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
645 int Frame_Alpha = pixels[byte_index + 3];
646 int Mask_Value = constrain(Frame_Alpha - gray_value);
647
648 // Set all alpha pixels to gray value
649 pixels[byte_index + 3] = Mask_Value;
650 }
651 }
652
653
654 // End painter
655 painter.end();
656
657 // Save image
658 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
659}
660
661// Constrain a color value from 0 to 255
662int Frame::constrain(int color_value)
663{
664 // Constrain new color from 0 to 255
665 if (color_value < 0)
666 color_value = 0;
667 else if (color_value > 255)
668 color_value = 255;
669
670 return color_value;
671}
672
673void Frame::AddColor(int new_width, int new_height, std::string new_color)
674{
675 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
676 // Update parameters
677 width = new_width;
678 height = new_height;
679 color = new_color;
680 AddColor(QColor(QString::fromStdString(new_color)));
681}
682
683// Add (or replace) pixel data to the frame (based on a solid color)
684void Frame::AddColor(const QColor& new_color)
685{
686 // Create new image object, and fill with pixel data
687 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
688 image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
689
690 // Fill with solid color
691 image->fill(new_color);
692 has_image_data = true;
693}
694
695// Add (or replace) pixel data to the frame
697 int new_width, int new_height, int bytes_per_pixel,
698 QImage::Format type, const unsigned char *pixels_)
699{
700 if (has_image_data) {
701 // Delete the previous QImage
702 image.reset();
703 }
704
705 // Create new image object from pixel data
706 auto new_image = std::make_shared<QImage>(
707 pixels_,
708 new_width, new_height,
709 new_width * bytes_per_pixel,
710 type,
711 (QImageCleanupFunction) &openshot::cleanUpBuffer,
712 (void*) pixels_
713 );
714 AddImage(new_image);
715}
716
717// Add (or replace) pixel data to the frame
718void Frame::AddImage(std::shared_ptr<QImage> new_image)
719{
720 // Ignore blank images
721 if (!new_image)
722 return;
723
724 // assign image data
725 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
726 image = new_image;
727
728 // Always convert to Format_RGBA8888_Premultiplied (if different)
729 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
730 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
731
732 // Update height and width
733 width = image->width();
734 height = image->height();
735 has_image_data = true;
736}
737
738// Add (or replace) pixel data to the frame (for only the odd or even lines)
739void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
740{
741 // Ignore blank new_image
742 if (!new_image)
743 return;
744
745 // Check for blank source image
746 if (!image) {
747 // Replace the blank source image
748 AddImage(new_image);
749
750 } else {
751 // Ignore image of different sizes or formats
752 bool ret=false;
753 if (image == new_image || image->size() != new_image->size()) {
754 ret = true;
755 }
756 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
757 new_image = std::make_shared<QImage>(
758 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
759 }
760 if (ret) {
761 return;
762 }
763
764 // Get the frame's image
765 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
766 unsigned char *pixels = image->bits();
767 const unsigned char *new_pixels = new_image->constBits();
768
769 // Loop through the scanlines of the image (even or odd)
770 int start = 0;
771 if (only_odd_lines)
772 start = 1;
773
774 for (int row = start; row < image->height(); row += 2) {
775 int offset = row * image->bytesPerLine();
776 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
777 }
778
779 // Update height and width
780 height = image->height();
781 width = image->width();
782 has_image_data = true;
783 }
784}
785
786
787// Resize audio container to hold more (or less) samples and channels
788void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
789{
790 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
791
792 // Resize JUCE audio buffer
793 audio->setSize(channels, length, true, true, false);
794 channel_layout = layout;
795 sample_rate = rate;
796
797 // Calculate max audio sample added
798 max_audio_sample = length;
799}
800
801// Reverse the audio buffer of this frame (will only reverse a single time, regardless of how many times
802// you invoke this method)
804 if (audio && !audio_reversed) {
805 // Reverse audio buffer
806 audio->reverse(0, audio->getNumSamples());
807 audio_reversed = true;
808 }
809}
810
811// Add audio samples to a specific channel
812void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
813 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
814
815 // Clamp starting sample to 0
816 int destStartSampleAdjusted = max(destStartSample, 0);
817
818 // Extend audio container to hold more (or less) samples and channels.. if needed
819 int new_length = destStartSampleAdjusted + numSamples;
820 int new_channel_length = audio->getNumChannels();
821 if (destChannel >= new_channel_length)
822 new_channel_length = destChannel + 1;
823 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
824 audio->setSize(new_channel_length, new_length, true, true, false);
825
826 // Clear the range of samples first (if needed)
827 if (replaceSamples)
828 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
829
830 // Add samples to frame's audio buffer
831 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
832 has_audio_data = true;
833
834 // Calculate max audio sample added
835 if (new_length > max_audio_sample)
836 max_audio_sample = new_length;
837
838 // Reset audio reverse flag
839 audio_reversed = false;
840}
841
842// Apply gain ramp (i.e. fading volume)
843void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
844{
845 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
846
847 // Apply gain ramp
848 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
849}
850
851// Get pointer to Magick++ image object
852std::shared_ptr<QImage> Frame::GetImage()
853{
854 // Check for blank image
855 if (!image)
856 // Fill with black
857 AddColor(width, height, color);
858
859 return image;
860}
861
862#ifdef USE_OPENCV
863
864// Convert Qimage to Mat
865cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
866
867 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
868 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
869 int from_to[] = { 0,0, 1,1, 2,2 };
870 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
871 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
872 return mat2;
873}
874
875// Get pointer to OpenCV image object
877{
878 // Check for blank image
879 if (!image)
880 // Fill with black
881 AddColor(width, height, color);
882
883 // if (imagecv.empty())
884 // Convert Qimage to Mat
885 imagecv = Qimage2mat(image);
886
887 return imagecv;
888}
889
890std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
891 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
892 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
893
894 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
895
896 // Always convert to RGBA8888 (if different)
897 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
898 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
899
900 return imgIn;
901}
902
903// Set pointer to OpenCV image object
904void Frame::SetImageCV(cv::Mat _image)
905{
906 imagecv = _image;
907 image = Mat2Qimage(_image);
908}
909#endif
910
911// Play audio samples for this frame
913{
914 // Check if samples are present
916 return;
917
918 juce::AudioDeviceManager deviceManager;
919 juce::String error = deviceManager.initialise (
920 0, /* number of input channels */
921 2, /* number of output channels */
922 0, /* no XML settings.. */
923 true /* select default device on failure */);
924
925 // Output error (if any)
926 if (error.isNotEmpty()) {
927 cout << "Error on initialise(): " << error << endl;
928 }
929
930 juce::AudioSourcePlayer audioSourcePlayer;
931 deviceManager.addAudioCallback (&audioSourcePlayer);
932
933 std::unique_ptr<AudioBufferSource> my_source;
934 my_source.reset (new AudioBufferSource (audio.get()));
935
936 // Create TimeSliceThread for audio buffering
937 juce::TimeSliceThread my_thread("Audio buffer thread");
938
939 // Start thread
940 my_thread.startThread();
941
942 juce::AudioTransportSource transport1;
943 transport1.setSource (my_source.get(),
944 5000, // tells it to buffer this many samples ahead
945 &my_thread,
946 (double) sample_rate,
947 audio->getNumChannels()); // sample rate of source
948 transport1.setPosition (0);
949 transport1.setGain(1.0);
950
951
952 // Create MIXER
953 juce::MixerAudioSource mixer;
954 mixer.addInputSource(&transport1, false);
955 audioSourcePlayer.setSource (&mixer);
956
957 // Start transports
958 transport1.start();
959
960 while (transport1.isPlaying())
961 {
962 cout << "playing" << endl;
963 std::this_thread::sleep_for(std::chrono::seconds(1));
964 }
965
966 cout << "DONE!!!" << endl;
967
968 transport1.stop();
969 transport1.setSource (0);
970 audioSourcePlayer.setSource (0);
971 my_thread.stopThread(500);
972 deviceManager.removeAudioCallback (&audioSourcePlayer);
973 deviceManager.closeAudioDevice();
974 deviceManager.removeAllChangeListeners();
975 deviceManager.dispatchPendingMessages();
976
977 cout << "End of Play()" << endl;
978
979
980}
981
982// Add audio silence
983void Frame::AddAudioSilence(int numSamples)
984{
985 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
986
987 // Resize audio container
988 audio->setSize(channels, numSamples, false, true, false);
989 audio->clear();
990 has_audio_data = true;
991
992 // Calculate max audio sample added
993 max_audio_sample = numSamples;
994
995 // Reset audio reverse flag
996 audio_reversed = false;
997}
Header file for AudioBufferSource class.
Header file for AudioResampler class.
Header file for Frame class.
Header file for QtUtilities (compatibiity overlay)
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
This class represents a fraction.
Definition: Fraction.h:30
int num
Numerator for the fraction.
Definition: Fraction.h:32
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
int den
Denominator for the fraction.
Definition: Fraction.h:33
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:91
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:78
std::shared_ptr< juce::AudioBuffer< float > > audio
Definition: Frame.h:117
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:890
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:673
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:397
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:159
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:511
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:120
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:358
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:865
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:120
int GetWidth()
Get height of image.
Definition: Frame.cpp:492
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:498
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:788
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:87
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:876
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:251
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:912
float * GetInterleavedAudioSamples(int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:324
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:119
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:380
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:504
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:983
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:539
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:268
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:421
float * GetAudioSamples(int channel)
Get an array of sample data (and optional reverse the sample values)
Definition: Frame.cpp:314
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:301
virtual ~Frame()
Destructor.
Definition: Frame.cpp:110
void ReverseAudio()
Definition: Frame.cpp:803
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:480
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:852
Frame()
Constructor - blank frame.
Definition: Frame.cpp:59
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:696
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:843
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:368
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:812
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:258
int GetHeight()
Get height of image.
Definition: Frame.cpp:486
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:443
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:450
juce::AudioBuffer< float > * GetAudioSampleBuffer()
Definition: Frame.cpp:374
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:118
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:904
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...