Skip to content

Commit

Permalink
Remove queuing and spooling support from FFmpegWriter to reduce compl…
Browse files Browse the repository at this point in the history 
…exity. Most codecs handle the multi-tasking themselves, making this code mostly useless.
  • Loading branch information
@jonoomph
jonoomph committed Jul 18, 2023
1 parent 9aa3713 commit 48f11e9
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 157 deletions.
197 changes: 61 additions & 136 deletions src/FFmpegWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6
FFmpegWriter::FFmpegWriter(const std::string& path) :
path(path), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL),
audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(1), num_of_rescalers(1),
initial_audio_input_frame_size(0), img_convert_ctx(NULL), num_of_rescalers(1),
rescaler_position(0), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0),
original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) {
Expand Down Expand Up @@ -679,118 +679,56 @@ void FFmpegWriter::WriteFrame(std::shared_ptr<openshot::Frame> frame) {
if (!is_open)
throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);

// Add frame pointer to "queue", waiting to be processed the next
// time the WriteFrames() method is called.
if (info.has_video && video_st)
spooled_video_frames.push_back(frame);

if (info.has_audio && audio_st)
spooled_audio_frames.push_back(frame);

ZmqLogger::Instance()->AppendDebugMethod(
"FFmpegWriter::WriteFrame",
"frame->number", frame->number,
"spooled_video_frames.size()", spooled_video_frames.size(),
"spooled_audio_frames.size()", spooled_audio_frames.size(),
"cache_size", cache_size,
"is_writing", is_writing);

// Write the frames once it reaches the correct cache size
if ((int)spooled_video_frames.size() == cache_size || (int)spooled_audio_frames.size() == cache_size) {
// Write frames to video file
write_queued_frames();
}
// Write frames to video file
write_frame(frame);

// Keep track of the last frame added
last_frame = frame;
}

// Write all frames in the queue to the video file.
void FFmpegWriter::write_queued_frames() {
ZmqLogger::Instance()->AppendDebugMethod(
"FFmpegWriter::write_queued_frames",
"spooled_video_frames.size()", spooled_video_frames.size(),
"spooled_audio_frames.size()", spooled_audio_frames.size());

void FFmpegWriter::write_frame(std::shared_ptr<Frame> frame) {
// Flip writing flag
is_writing = true;

// Transfer spool to queue
queued_video_frames = spooled_video_frames;
queued_audio_frames = spooled_audio_frames;

// Empty spool
spooled_video_frames.clear();
spooled_audio_frames.clear();

// Create blank exception
bool has_error_encoding_video = false;

// Process all audio frames (in a separate thread)
if (info.has_audio && audio_st && !queued_audio_frames.empty())
write_audio_packets(false);

// Loop through each queued image frame
while (!queued_video_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = queued_video_frames.front();

// Add to processed queue
processed_frames.push_back(frame);

// Encode and add the frame to the output file
if (info.has_video && video_st)
process_video_packet(frame);

// Remove front item
queued_video_frames.pop_front();

} // end while


// Loop back through the frames (in order), and write them to the video file
while (!processed_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = processed_frames.front();

if (info.has_video && video_st) {
// Add to deallocate queue (so we can remove the AVFrames when we are done)
deallocate_frames.push_back(frame);

// Does this frame's AVFrame still exist
if (av_frames.count(frame)) {
// Get AVFrame
AVFrame *frame_final = av_frames[frame];

// Write frame to video file
bool success = write_video_packet(frame, frame_final);
if (!success)
has_error_encoding_video = true;
}
}

// Remove front item
processed_frames.pop_front();
}
if (info.has_audio && audio_st)
write_audio_packets(false, frame);

// Loop through, and deallocate AVFrames
while (!deallocate_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = deallocate_frames.front();
// Encode and add the frame to the output file
if (info.has_video && video_st)
process_video_packet(frame);

if (info.has_video && video_st) {
// Does this frame's AVFrame still exist
if (av_frames.count(frame)) {
// Get AVFrame
AVFrame *av_frame = av_frames[frame];
AVFrame *frame_final = av_frames[frame];

// Deallocate buffer and AVFrame
av_freep(&(av_frame->data[0]));
AV_FREE_FRAME(&av_frame);
av_frames.erase(frame);
// Write frame to video file
bool success = write_video_packet(frame, frame_final);
if (!success)
has_error_encoding_video = true;

Check warning on line 719 in src/FFmpegWriter.cpp

View check run for this annotation

Codecov / codecov/patch

src/FFmpegWriter.cpp#L719 

Added line #L719 was not covered by tests
}
}

// Remove front item
deallocate_frames.pop_front();
// Does this frame's AVFrame still exist
if (av_frames.count(frame)) {
// Get AVFrame
AVFrame *av_frame = av_frames[frame];

// Deallocate buffer and AVFrame
av_freep(&(av_frame->data[0]));
AV_FREE_FRAME(&av_frame);
av_frames.erase(frame);
}

// Done writing
Expand Down Expand Up @@ -820,13 +758,6 @@ void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length)

// Write the file trailer (after all frames are written)
void FFmpegWriter::WriteTrailer() {
// Write any remaining queued frames to video file
write_queued_frames();

// Process final audio frame (if any)
if (info.has_audio && audio_st)
write_audio_packets(true);

// Flush encoders (who sometimes hold on to frames)
flush_encoders();

Expand Down Expand Up @@ -1598,7 +1529,10 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
}

// write all queued frames' audio to the video file
void FFmpegWriter::write_audio_packets(bool is_final) {
void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr<openshot::Frame> frame) {
if (!frame)
return;

Check warning on line 1534 in src/FFmpegWriter.cpp

View check run for this annotation

Codecov / codecov/patch

src/FFmpegWriter.cpp#L1534 

Added line #L1534 was not covered by tests

// Init audio buffers / variables
int total_frame_samples = 0;
int frame_position = 0;
Expand All @@ -1608,56 +1542,47 @@ void FFmpegWriter::write_audio_packets(bool is_final) {
ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout

// Create a new array (to hold all S16 audio samples, for the current queued frames
unsigned int all_queued_samples_size = sizeof(int16_t) * (queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE);
unsigned int all_queued_samples_size = sizeof(int16_t) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size);
int16_t *all_resampled_samples = NULL;
int16_t *final_samples_planar = NULL;
int16_t *final_samples = NULL;

// Loop through each queued audio frame
while (!queued_audio_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = queued_audio_frames.front();

// Get the audio details from this frame
sample_rate_in_frame = frame->SampleRate();
samples_in_frame = frame->GetAudioSamplesCount();
channels_in_frame = frame->GetAudioChannelsCount();
channel_layout_in_frame = frame->ChannelsLayout();

// Get audio sample array
float *frame_samples_float = NULL;
// Get samples interleaved together (c1 c2 c1 c2 c1 c2)
frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);

// Calculate total samples
total_frame_samples = samples_in_frame * channels_in_frame;

// Translate audio sample values back to 16 bit integers with saturation
const int16_t max16 = 32767;
const int16_t min16 = -32768;
for (int s = 0; s < total_frame_samples; s++, frame_position++) {
float valF = frame_samples_float[s] * (1 << 15);
int16_t conv;
if (valF > max16) {
conv = max16;
} else if (valF < min16) {
conv = min16;
} else {
conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
}

// Copy into buffer
all_queued_samples[frame_position] = conv;
// Get the audio details from this frame
sample_rate_in_frame = frame->SampleRate();
samples_in_frame = frame->GetAudioSamplesCount();
channels_in_frame = frame->GetAudioChannelsCount();
channel_layout_in_frame = frame->ChannelsLayout();

// Get audio sample array
float *frame_samples_float = NULL;
// Get samples interleaved together (c1 c2 c1 c2 c1 c2)
frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);

// Calculate total samples
total_frame_samples = samples_in_frame * channels_in_frame;

// Translate audio sample values back to 16 bit integers with saturation
const int16_t max16 = 32767;
const int16_t min16 = -32768;
for (int s = 0; s < total_frame_samples; s++, frame_position++) {
float valF = frame_samples_float[s] * (1 << 15);
int16_t conv;
if (valF > max16) {
conv = max16;

Check warning on line 1573 in src/FFmpegWriter.cpp

View check run for this annotation

Codecov / codecov/patch

src/FFmpegWriter.cpp#L1573 

Added line #L1573 was not covered by tests
} else if (valF < min16) {
conv = min16;

Check warning on line 1575 in src/FFmpegWriter.cpp

View check run for this annotation

Codecov / codecov/patch

src/FFmpegWriter.cpp#L1575 

Added line #L1575 was not covered by tests
} else {
conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
}

// Deallocate float array
delete[] frame_samples_float;

// Remove front item
queued_audio_frames.pop_front();
// Copy into buffer
all_queued_samples[frame_position] = conv;
}

} // end while
// Deallocate float array
delete[] frame_samples_float;


// Update total samples (since we've combined all queued frames)
Expand Down
25 changes: 4 additions & 21 deletions src/FFmpegWriter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ namespace openshot {

/// This enumeration designates the type of stream when encoding (video or audio)
enum StreamType {
VIDEO_STREAM, ///< A video stream (used to determine which type of stream)
AUDIO_STREAM ///< An audio stream (used to determine which type of stream)
VIDEO_STREAM, ///< A video stream (used to determine which type of stream)
AUDIO_STREAM ///< An audio stream (used to determine which type of stream)
};

/**
Expand Down Expand Up @@ -116,7 +116,6 @@ namespace openshot {
class FFmpegWriter : public WriterBase {
private:
std::string path;
int cache_size;
bool is_writing;
bool is_open;
int64_t video_timestamp;
Expand Down Expand Up @@ -152,15 +151,6 @@ namespace openshot {
int original_channels;

std::shared_ptr<openshot::Frame> last_frame;
std::deque<std::shared_ptr<openshot::Frame> > spooled_audio_frames;
std::deque<std::shared_ptr<openshot::Frame> > spooled_video_frames;

std::deque<std::shared_ptr<openshot::Frame> > queued_audio_frames;
std::deque<std::shared_ptr<openshot::Frame> > queued_video_frames;

std::deque<std::shared_ptr<openshot::Frame> > processed_frames;
std::deque<std::shared_ptr<openshot::Frame> > deallocate_frames;

std::map<std::shared_ptr<openshot::Frame>, AVFrame *> av_frames;

/// Add an AVFrame to the cache
Expand Down Expand Up @@ -205,13 +195,13 @@ namespace openshot {
void process_video_packet(std::shared_ptr<openshot::Frame> frame);

/// write all queued frames' audio to the video file
void write_audio_packets(bool is_final);
void write_audio_packets(bool is_final, std::shared_ptr<openshot::Frame> frame);

/// write video frame
bool write_video_packet(std::shared_ptr<openshot::Frame> frame, AVFrame *frame_final);

/// write all queued frames
void write_queued_frames();
void write_frame(std::shared_ptr<Frame> frame);

public:

Expand All @@ -224,9 +214,6 @@ namespace openshot {
/// Close the writer
void Close();

/// Get the cache size (number of frames to queue before writing)
int GetCacheSize() { return cache_size; };

/// Determine if writer is open or closed
bool IsOpen() { return is_open; };

Expand Down Expand Up @@ -273,10 +260,6 @@ namespace openshot {
/// \note This is an overloaded function.
void SetAudioOptions(std::string codec, int sample_rate, int bit_rate);

/// @brief Set the cache size
/// @param new_size The number of frames to queue before writing to the file
void SetCacheSize(int new_size) { cache_size = new_size; };

/// @brief Set video export options
/// @param has_video Does this file need a video stream
/// @param codec The codec used to encode the images in this video
Expand Down

0 comments on commit 48f11e9

Please sign in to comment.