31 #include "../include/FrameMapper.h" 37 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
78 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
81 void FrameMapper::AddField(int64_t frame)
84 AddField(
Field(frame, field_toggle));
87 void FrameMapper::AddField(
Field field)
93 field_toggle = (field_toggle ? false :
true);
100 void FrameMapper::Init()
121 if ((fabs(original.
ToFloat() - 24.0) < 1e-7 || fabs(original.
ToFloat() - 25.0) < 1e-7 || fabs(original.
ToFloat() - 30.0) < 1e-7) &&
122 (fabs(target.
ToFloat() - 24.0) < 1e-7 || fabs(target.
ToFloat() - 25.0) < 1e-7 || fabs(target.
ToFloat() - 30.0) < 1e-7)) {
125 float difference = target.
ToInt() - original.
ToInt();
128 int field_interval = 0;
129 int frame_interval = 0;
133 field_interval = round(fabs(original.
ToInt() / difference));
136 frame_interval = field_interval * 2.0f;
145 for (int64_t field = 1; field <= number_of_fields; field++)
153 else if (difference > 0)
163 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
170 AddField(
Field(frame + 1, field_toggle));
172 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
179 else if (difference < 0)
185 field_toggle = (field_toggle ? false :
true);
187 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
192 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
205 if (field % 2 == 0 && field > 0)
215 double value_increment = (reader->
info.
video_length + 1) / (
double) (new_length);
218 double original_frame_num = 1.0f;
219 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
222 AddField(round(original_frame_num));
223 AddField(round(original_frame_num));
226 original_frame_num += value_increment;
235 int64_t start_samples_frame = 1;
236 int start_samples_position = 0;
238 for (std::vector<Field>::size_type field = 1; field <=
fields.size(); field++)
244 if (field % 2 == 0 && field > 0)
247 int64_t frame_number = field / 2;
258 int64_t end_samples_frame = start_samples_frame;
259 int end_samples_position = start_samples_position;
262 while (remaining_samples > 0)
268 if (original_samples >= remaining_samples)
271 end_samples_position += remaining_samples - 1;
272 remaining_samples = 0;
276 end_samples_frame += 1;
277 end_samples_position = 0;
278 remaining_samples -= original_samples;
288 start_samples_frame = end_samples_frame;
289 start_samples_position = end_samples_position + 1;
292 start_samples_frame += 1;
293 start_samples_position = 0;
326 frame.
Odd.
Frame = TargetFrameNumber;
336 if(TargetFrameNumber < 1 ||
frames.size() == 0)
340 else if (TargetFrameNumber > (int64_t)
frames.size())
342 TargetFrameNumber =
frames.size();
345 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame);
348 return frames[TargetFrameNumber - 1];
352 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
354 std::shared_ptr<Frame> new_frame;
364 new_frame = reader->
GetFrame(number);
384 new_frame->AddAudioSilence(samples_in_frame);
392 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
393 if (final_frame)
return final_frame;
404 final_frame = final_cache.
GetFrame(requested_frame);
405 if (final_frame)
return final_frame;
409 int minimum_frames = 1;
415 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
419 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame);
423 std::shared_ptr<Frame> mapped_frame;
426 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
429 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
439 info.
channels == mapped_frame->GetAudioChannelsCount() &&
441 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
444 mapped_frame->number == frame_number &&
448 final_cache.
Add(mapped_frame);
453 std::shared_ptr<Frame> frame = std::make_shared<Frame>(frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
454 frame->SampleRate(mapped_frame->SampleRate());
455 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
459 std::shared_ptr<Frame> odd_frame;
460 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
463 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*odd_frame->GetImage())),
true);
466 std::shared_ptr<Frame> even_frame;
467 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
469 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*even_frame->GetImage())),
false);
473 bool need_resampling =
false;
479 need_resampling =
true;
490 const int EXTRA_INPUT_SAMPLES = 20;
493 copy_samples.
sample_end += EXTRA_INPUT_SAMPLES;
494 int samples_per_end_frame =
497 if (copy_samples.
sample_end >= samples_per_end_frame)
501 copy_samples.
sample_end -= samples_per_end_frame;
503 copy_samples.
total += EXTRA_INPUT_SAMPLES;
510 int samples_per_start_frame =
513 if (copy_samples.
sample_start >= samples_per_start_frame)
519 copy_samples.
total -= EXTRA_INPUT_SAMPLES;
524 int samples_copied = 0;
529 int remaining_samples = copy_samples.
total - samples_copied;
530 int number_to_copy = 0;
533 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
534 int original_samples = original_frame->GetAudioSamplesCount();
537 for (
int channel = 0; channel < channels_in_frame; channel++)
542 number_to_copy = original_samples - copy_samples.
sample_start;
543 if (number_to_copy > remaining_samples)
544 number_to_copy = remaining_samples;
547 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
552 number_to_copy = original_samples;
553 if (number_to_copy > remaining_samples)
554 number_to_copy = remaining_samples;
557 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
563 if (number_to_copy > remaining_samples)
564 number_to_copy = remaining_samples;
567 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
572 samples_copied += number_to_copy;
582 final_cache.
Add(frame);
587 return final_cache.
GetFrame(requested_frame);
598 float difference = target.
ToInt() - original.
ToInt();
600 int field_interval = 0;
601 int frame_interval = 0;
606 field_interval = round(fabs(original.
ToInt() / difference));
609 frame_interval = field_interval * 2.0f;
613 for (
float map = 1; map <=
frames.size(); map++)
616 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
689 root[
"type"] =
"FrameMapper";
705 catch (
const std::exception& e)
708 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
729 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
den,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
735 target.
num = target_fps.
num;
736 target.
den = target_fps.
den;
741 pulldown = target_pulldown;
769 int total_frame_samples = 0;
770 int channels_in_frame = frame->GetAudioChannelsCount();
771 int sample_rate_in_frame = frame->SampleRate();
772 int samples_in_frame = frame->GetAudioSamplesCount();
773 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
775 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame);
778 float* frame_samples_float = NULL;
780 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
783 total_frame_samples = samples_in_frame * channels_in_frame;
786 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
789 for (
int s = 0; s < total_frame_samples; s++)
791 frame_samples[s] =
int(frame_samples_float[s] * (1 << 15));
795 delete[] frame_samples_float;
796 frame_samples_float = NULL;
798 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
804 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
806 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
807 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
812 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
818 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
823 audio_converted->nb_samples = total_frame_samples;
824 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
826 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
833 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
835 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
836 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
837 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
839 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
846 audio_converted->data,
847 audio_converted->linesize[0],
848 audio_converted->nb_samples,
850 audio_frame->linesize[0],
851 audio_frame->nb_samples);
854 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
857 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
860 av_freep(&audio_frame->data[0]);
862 av_freep(&audio_converted->data[0]);
864 frame_samples = NULL;
867 int channel_buffer_size = nb_samples;
870 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
873 float *channel_buffer =
new float[channel_buffer_size];
876 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
879 for (
int z = 0; z < channel_buffer_size; z++)
880 channel_buffer[z] = 0.0f;
886 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
889 if (channel_filter == channel)
892 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
908 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
918 delete[] channel_buffer;
919 channel_buffer = NULL;
922 delete[] resampled_samples;
923 resampled_samples = NULL;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
ReaderBase * Reader()
Get the current reader.
std::string Json() const override
Get and Set JSON methods.
int width
The width of the video (in pixesl)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
virtual ~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
std::vector< MappedFrame > frames
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
std::vector< Field > fields
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
const Json::Value stringToJson(const std::string value)
This abstract class is the base class, used by all readers in libopenshot.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
This struct holds a single field (half a frame).
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
Exception when encoding audio packet.
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
This struct holds a the range of samples needed by this frame.
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
bool has_audio
Determines if this file has an audio stream.
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
Json::Value JsonValue() const override
Generate Json::Value for this object.
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
bool IsOpen()
Determine if reader is open or closed.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
This class represents a fraction.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
juce::CriticalSection getFrameCriticalSection
Section lock for multiple threads.
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
#define av_err2str(errnum)
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
This struct holds two fields which together make up a complete video frame.
openshot::ReaderInfo info
Information about the current media file.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
This namespace is the default namespace for all code in the openshot library.
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Do not apply pull-down techniques, just repeat or skip entire frames.
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
void SetJson(const std::string value)
Load JSON string into this object.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Advanced 2:3:3:2 pull-down (minimal dirty frames)
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.
virtual bool IsOpen()=0
Determine if reader is open or closed.