31 #include "../include/Frame.h" 37 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1), color(
"#000000"),
38 channel_layout(
LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
42 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
50 : number(number), pixel_ratio(1,1), channels(2), width(width), height(height), color(color),
55 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
63 number(number), pixel_ratio(1,1), channels(channels), width(1), height(1), color(
"#000000"),
68 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
75 Frame::Frame(int64_t
number,
int width,
int height, std::string color,
int samples,
int channels)
76 : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height), color(color),
81 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
108 channels = other.channels;
110 height = other.height;
111 channel_layout = other.channel_layout;
114 sample_rate = other.sample_rate;
115 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
117 max_audio_sample = other.max_audio_sample;
120 image = std::shared_ptr<QImage>(
new QImage(*(other.image)));
122 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(*(other.audio)));
123 if (other.wave_image)
124 wave_image = std::shared_ptr<QImage>(
new QImage(*(other.wave_image)));
137 if (!QApplication::instance()) {
140 static char* argv[1] = {NULL};
141 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
145 std::shared_ptr<QImage> previewImage =
GetImage();
148 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
151 int new_width = previewImage->size().width();
155 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
159 QWidget previewWindow;
160 previewWindow.setStyleSheet(
"background-color: #000000;");
165 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
166 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
167 layout.addWidget(&previewLabel);
170 previewWindow.setLayout(&layout);
171 previewWindow.show();
176 std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
182 QVector<QPointF> lines;
183 QVector<QPointF> labels;
187 if (total_samples > 0)
190 int new_height = 200 * audio->getNumChannels();
191 int height_padding = 20 * (audio->getNumChannels() - 1);
192 int total_height = new_height + height_padding;
197 for (
int channel = 0; channel < audio->getNumChannels(); channel++)
202 const float *samples = audio->getReadPointer(channel);
207 float value = samples[sample] * 100;
212 lines.push_back(QPointF(X,Y));
213 lines.push_back(QPointF(X,Y-value));
217 lines.push_back(QPointF(X,Y));
218 lines.push_back(QPointF(X,Y));
223 labels.push_back(QPointF(5, Y - 5));
226 Y += (200 + height_padding);
231 wave_image = std::shared_ptr<QImage>(
new QImage(total_width, total_height, QImage::Format_RGBA8888));
232 wave_image->fill(QColor(0,0,0,0));
235 QPainter painter(wave_image.get());
238 painter.setPen(QColor(Red, Green, Blue, Alpha));
241 painter.drawLines(lines);
254 if (width != total_width || height != total_height) {
255 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
256 wave_image = std::shared_ptr<QImage>(
new QImage(scaled_wave_image));
262 wave_image = std::shared_ptr<QImage>(
new QImage(width, height, QImage::Format_RGBA8888));
263 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
281 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
284 return wave_image->constBits();
293 if (!QApplication::instance()) {
296 static char* argv[1] = {NULL};
297 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
301 QWidget previewWindow;
302 previewWindow.setStyleSheet(
"background-color: #000000;");
307 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
308 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
309 layout.addWidget(&previewLabel);
312 previewWindow.setLayout(&layout);
313 previewWindow.show();
325 return audio->getMagnitude(channel, sample, magnitude_range);
329 return audio->getMagnitude(sample, magnitude_range);
337 return audio->getWritePointer(channel);
343 float *output = NULL;
344 juce::AudioSampleBuffer *buffer(audio.get());
345 int num_of_channels = audio->getNumChannels();
349 if (new_sample_rate != sample_rate)
352 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
358 num_of_samples = buffer->getNumSamples();
362 output =
new float[num_of_channels * num_of_samples];
366 for (
int channel = 0; channel < num_of_channels; channel++)
368 for (
int sample = 0; sample < num_of_samples; sample++)
371 output[position] = buffer->getReadPointer(channel)[sample];
379 *sample_count = num_of_samples;
389 float *output = NULL;
390 juce::AudioSampleBuffer *buffer(audio.get());
391 int num_of_channels = audio->getNumChannels();
395 if (new_sample_rate != sample_rate && resampler)
398 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
404 num_of_samples = buffer->getNumSamples();
408 output =
new float[num_of_channels * num_of_samples];
412 for (
int sample = 0; sample < num_of_samples; sample++)
414 for (
int channel = 0; channel < num_of_channels; channel++)
417 output[position] = buffer->getReadPointer(channel)[sample];
425 *sample_count = num_of_samples;
434 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
436 return audio->getNumChannels();
444 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
445 return max_audio_sample;
456 int64_t total_bytes = 0;
458 total_bytes += (width * height *
sizeof(char) * 4);
461 total_bytes += (sample_rate / 24.0) *
sizeof(float);
477 return image->constBits();
484 return image->constScanLine(row);
488 bool Frame::CheckPixel(
int row,
int col,
int red,
int green,
int blue,
int alpha,
int threshold) {
489 int col_pos = col * 4;
490 if (!image || row < 0 || row >= (height - 1) ||
491 col_pos < 0 || col_pos >= (width - 1) ) {
496 const unsigned char* pixels =
GetPixels(row);
497 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
498 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
499 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
500 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
512 pixel_ratio.
num = num;
513 pixel_ratio.
den = den;
529 double previous_samples = (sample_rate * fps_rate) * (number - 1);
530 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
531 previous_samples -= previous_samples_remainder;
534 double total_samples = (sample_rate * fps_rate) * number;
535 double total_samples_remainder = fmod(total_samples, (
double)channels);
536 total_samples -= total_samples_remainder;
540 int samples_per_frame = round(total_samples - previous_samples);
541 if (samples_per_frame < 0)
542 samples_per_frame = 0;
543 return samples_per_frame;
573 return channel_layout;
581 std::shared_ptr<QImage> previewImage =
GetImage();
584 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
586 int new_width = width;
587 int new_height = height;
590 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
593 int new_width = previewImage->size().width();
597 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
601 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
605 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
609 void Frame::Thumbnail(std::string
path,
int new_width,
int new_height, std::string mask_path, std::string overlay_path,
610 std::string background_color,
bool ignore_aspect, std::string format,
int quality,
float rotate) {
613 std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
614 thumbnail->fill(QColor(QString::fromStdString(background_color)));
617 QPainter painter(thumbnail.get());
618 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
621 std::shared_ptr<QImage> previewImage =
GetImage();
624 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
627 int aspect_width = previewImage->size().width();
628 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
631 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
637 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
640 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
643 int x = (new_width - previewImage->size().width()) / 2.0;
644 int y = (new_height - previewImage->size().height()) / 2.0;
645 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
649 QTransform transform;
650 float origin_x = previewImage->width() / 2.0;
651 float origin_y = previewImage->height() / 2.0;
652 transform.translate(origin_x, origin_y);
653 transform.rotate(rotate);
654 transform.translate(-origin_x,-origin_y);
655 painter.setTransform(transform);
658 painter.drawImage(x, y, *previewImage);
662 if (overlay_path !=
"") {
664 std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(
new QImage());
665 overlay->load(QString::fromStdString(overlay_path));
668 overlay = std::shared_ptr<QImage>(
new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
671 overlay = std::shared_ptr<QImage>(
new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
674 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
675 painter.drawImage(0, 0, *overlay);
680 if (mask_path !=
"") {
682 std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(
new QImage());
683 mask->load(QString::fromStdString(mask_path));
686 mask = std::shared_ptr<QImage>(
new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
689 mask = std::shared_ptr<QImage>(
new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
692 mask->invertPixels();
695 unsigned char *pixels = (
unsigned char *) thumbnail->bits();
696 const unsigned char *mask_pixels = (
const unsigned char *) mask->constBits();
700 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
703 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
704 int Frame_Alpha = pixels[byte_index + 3];
705 int Mask_Value = constrain(Frame_Alpha - gray_value);
708 pixels[byte_index + 3] = Mask_Value;
717 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
721 int Frame::constrain(
int color_value)
726 else if (color_value > 255)
739 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
740 #pragma omp critical (AddImage) 742 image = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
745 image->fill(QColor(QString::fromStdString(color)));
748 width = image->width();
749 height = image->height();
754 void Frame::AddImage(
int new_width,
int new_height,
int bytes_per_pixel, QImage::Format type,
const unsigned char *pixels_)
757 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
758 int buffer_size = new_width * new_height * bytes_per_pixel;
759 qbuffer =
new unsigned char[buffer_size]();
762 memcpy((
unsigned char*)qbuffer, pixels_, buffer_size);
765 #pragma omp critical (AddImage) 767 image = std::shared_ptr<QImage>(
new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &
openshot::Frame::cleanUpBuffer, (
void*) qbuffer));
770 if (image->format() != QImage::Format_RGBA8888)
771 *image = image->convertToFormat(QImage::Format_RGBA8888);
774 width = image->width();
775 height = image->height();
788 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
789 #pragma omp critical (AddImage) 794 if (image->format() != QImage::Format_RGBA8888)
795 *image = image->convertToFormat(QImage::Format_RGBA8888);
798 width = image->width();
799 height = image->height();
820 #pragma omp critical (AddImage) 822 if (image == new_image || image->size() != new_image->size()) {
825 else if (new_image->format() != image->format()) {
826 new_image = std::shared_ptr<QImage>(
new QImage(new_image->convertToFormat(image->format())));
834 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
835 #pragma omp critical (AddImage) 837 unsigned char *pixels = image->bits();
838 const unsigned char *new_pixels = new_image->constBits();
845 for (
int row = start; row < image->height(); row += 2) {
846 int offset = row * image->bytesPerLine();
847 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
851 height = image->height();
852 width = image->width();
862 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
865 audio->setSize(channels, length,
true,
true,
false);
866 channel_layout = layout;
870 max_audio_sample = length;
874 void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
875 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
876 #pragma omp critical (adding_audio) 879 int destStartSampleAdjusted = max(destStartSample, 0);
882 int new_length = destStartSampleAdjusted + numSamples;
883 int new_channel_length = audio->getNumChannels();
884 if (destChannel >= new_channel_length)
885 new_channel_length = destChannel + 1;
886 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
887 audio->setSize(new_channel_length, new_length,
true,
true,
false);
891 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
894 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
898 if (new_length > max_audio_sample)
899 max_audio_sample = new_length;
904 void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
906 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
909 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
923 #ifdef USE_IMAGEMAGICK 933 const QRgb *tmpBits = (
const QRgb*)image->constBits();
936 std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(
new Magick::Image(image->width(), image->height(),
"RGBA", Magick::CharPixel, tmpBits));
939 magick_image->backgroundColor(Magick::Color(
"none"));
940 magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
947 #ifdef USE_IMAGEMAGICK 952 const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
957 qbuffer =
new unsigned char[bufferSize]();
958 unsigned char *buffer = (
unsigned char*)qbuffer;
960 MagickCore::ExceptionInfo exception;
962 MagickCore::ExportImagePixels(new_image->constImage(), 0, 0, new_image->columns(), new_image->rows(),
"RGBA", Magick::CharPixel, buffer, &exception);
965 image = std::shared_ptr<QImage>(
new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &
cleanUpBuffer, (
void*) qbuffer));
968 width = image->width();
969 height = image->height();
981 juce::AudioDeviceManager deviceManager;
982 String error = deviceManager.initialise (0,
988 if (error.isNotEmpty()) {
989 cout <<
"Error on initialise(): " << error.toStdString() << endl;
992 juce::AudioSourcePlayer audioSourcePlayer;
993 deviceManager.addAudioCallback (&audioSourcePlayer);
995 ScopedPointer<AudioBufferSource> my_source;
999 juce::TimeSliceThread my_thread(
"Audio buffer thread");
1002 my_thread.startThread();
1004 AudioTransportSource transport1;
1005 transport1.setSource (my_source,
1008 (
double) sample_rate,
1009 audio->getNumChannels());
1010 transport1.setPosition (0);
1011 transport1.setGain(1.0);
1015 juce::MixerAudioSource mixer;
1016 mixer.addInputSource(&transport1,
false);
1017 audioSourcePlayer.setSource (&mixer);
1022 while (transport1.isPlaying())
1024 cout <<
"playing" << endl;
1028 cout <<
"DONE!!!" << endl;
1031 transport1.setSource (0);
1032 audioSourcePlayer.setSource (0);
1033 my_thread.stopThread(500);
1034 deviceManager.removeAudioCallback (&audioSourcePlayer);
1035 deviceManager.closeAudioDevice();
1036 deviceManager.removeAllChangeListeners();
1037 deviceManager.dispatchPendingMessages();
1039 cout <<
"End of Play()" << endl;
1050 unsigned char* ptr_to_qbuffer = (
unsigned char*) info;
1051 delete[] ptr_to_qbuffer;
1058 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
1061 audio->setSize(channels, numSamples,
false,
true,
false);
1066 if (numSamples > max_audio_sample)
1067 max_audio_sample = numSamples;
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
int GetWidth()
Get height of image.
int num
Numerator for the fraction.
int GetAudioSamplesCount()
Get number of audio samples.
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
#define MAGICK_IMAGE_ALPHA(im, a)
This class represents a single frame of video (i.e. image & audio data)
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
juce::AudioSampleBuffer * GetAudioSampleBuffer()
const unsigned char * GetPixels()
Get pixel data (as packets)
void Play()
Play audio samples for this frame.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Fraction Reciprocal()
Return the reciprocal as a Fraction.
int64_t number
This is the frame number (starting at 1)
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
float * GetAudioSamples(int channel)
Get an array of sample data.
void SetFrameNumber(int64_t number)
Set frame number.
void AddAudioSilence(int numSamples)
Add audio silence.
bool has_audio_data
This frame has been loaded with audio data.
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
This class represents a fraction.
void SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
void DisplayWaveform()
Display the wave form.
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
int GetAudioChannelsCount()
Get number of audio channels.
This namespace is the default namespace for all code in the openshot library.
openshot::ChannelLayout ChannelsLayout()
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
virtual ~Frame()
Destructor.
juce::AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
int den
Denominator for the fraction.
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
bool has_image_data
This frame has been loaded with pixel data.
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
int GetHeight()
Get height of image.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int SampleRate()
Get the original sample rate of this frame's audio data.
This class is used to resample audio data for many sequential frames.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)