From 48f11e92f0a9fedf986c0c5db9b0713e5aa36464 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 18 Jul 2023 18:21:57 -0500 Subject: [PATCH] Remove queuing and spooling support from FFmpegWriter to reduce complexity. Most codecs handle the multi-tasking themselves, making this code mostly useless. --- src/FFmpegWriter.cpp | 197 ++++++++++++++----------------------------- src/FFmpegWriter.h | 25 +----- 2 files changed, 65 insertions(+), 157 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 273afdb4f..a18c31701 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -75,7 +75,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6 FFmpegWriter::FFmpegWriter(const std::string& path) : path(path), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL), audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0), - initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(1), num_of_rescalers(1), + initial_audio_input_frame_size(0), img_convert_ctx(NULL), num_of_rescalers(1), rescaler_position(0), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0), original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false), write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) { @@ -679,118 +679,56 @@ void FFmpegWriter::WriteFrame(std::shared_ptr frame) { if (!is_open) throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path); - // Add frame pointer to "queue", waiting to be processed the next - // time the WriteFrames() method is called. - if (info.has_video && video_st) - spooled_video_frames.push_back(frame); - - if (info.has_audio && audio_st) - spooled_audio_frames.push_back(frame); - ZmqLogger::Instance()->AppendDebugMethod( "FFmpegWriter::WriteFrame", "frame->number", frame->number, - "spooled_video_frames.size()", spooled_video_frames.size(), - "spooled_audio_frames.size()", spooled_audio_frames.size(), - "cache_size", cache_size, "is_writing", is_writing); - // Write the frames once it reaches the correct cache size - if ((int)spooled_video_frames.size() == cache_size || (int)spooled_audio_frames.size() == cache_size) { - // Write frames to video file - write_queued_frames(); - } + // Write frames to video file + write_frame(frame); // Keep track of the last frame added last_frame = frame; } // Write all frames in the queue to the video file. -void FFmpegWriter::write_queued_frames() { - ZmqLogger::Instance()->AppendDebugMethod( - "FFmpegWriter::write_queued_frames", - "spooled_video_frames.size()", spooled_video_frames.size(), - "spooled_audio_frames.size()", spooled_audio_frames.size()); - +void FFmpegWriter::write_frame(std::shared_ptr frame) { // Flip writing flag is_writing = true; - // Transfer spool to queue - queued_video_frames = spooled_video_frames; - queued_audio_frames = spooled_audio_frames; - - // Empty spool - spooled_video_frames.clear(); - spooled_audio_frames.clear(); - // Create blank exception bool has_error_encoding_video = false; // Process all audio frames (in a separate thread) - if (info.has_audio && audio_st && !queued_audio_frames.empty()) - write_audio_packets(false); - - // Loop through each queued image frame - while (!queued_video_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = queued_video_frames.front(); - - // Add to processed queue - processed_frames.push_back(frame); - - // Encode and add the frame to the output file - if (info.has_video && video_st) - process_video_packet(frame); - - // Remove front item - queued_video_frames.pop_front(); - - } // end while - - - // Loop back through the frames (in order), and write them to the video file - while (!processed_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = processed_frames.front(); - - if (info.has_video && video_st) { - // Add to deallocate queue (so we can remove the AVFrames when we are done) - deallocate_frames.push_back(frame); - - // Does this frame's AVFrame still exist - if (av_frames.count(frame)) { - // Get AVFrame - AVFrame *frame_final = av_frames[frame]; - - // Write frame to video file - bool success = write_video_packet(frame, frame_final); - if (!success) - has_error_encoding_video = true; - } - } - - // Remove front item - processed_frames.pop_front(); - } + if (info.has_audio && audio_st) + write_audio_packets(false, frame); - // Loop through, and deallocate AVFrames - while (!deallocate_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = deallocate_frames.front(); + // Encode and add the frame to the output file + if (info.has_video && video_st) + process_video_packet(frame); + if (info.has_video && video_st) { // Does this frame's AVFrame still exist if (av_frames.count(frame)) { // Get AVFrame - AVFrame *av_frame = av_frames[frame]; + AVFrame *frame_final = av_frames[frame]; - // Deallocate buffer and AVFrame - av_freep(&(av_frame->data[0])); - AV_FREE_FRAME(&av_frame); - av_frames.erase(frame); + // Write frame to video file + bool success = write_video_packet(frame, frame_final); + if (!success) + has_error_encoding_video = true; } + } - // Remove front item - deallocate_frames.pop_front(); + // Does this frame's AVFrame still exist + if (av_frames.count(frame)) { + // Get AVFrame + AVFrame *av_frame = av_frames[frame]; + + // Deallocate buffer and AVFrame + av_freep(&(av_frame->data[0])); + AV_FREE_FRAME(&av_frame); + av_frames.erase(frame); } // Done writing @@ -820,13 +758,6 @@ void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length) // Write the file trailer (after all frames are written) void FFmpegWriter::WriteTrailer() { - // Write any remaining queued frames to video file - write_queued_frames(); - - // Process final audio frame (if any) - if (info.has_audio && audio_st) - write_audio_packets(true); - // Flush encoders (who sometimes hold on to frames) flush_encoders(); @@ -1598,7 +1529,10 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { } // write all queued frames' audio to the video file -void FFmpegWriter::write_audio_packets(bool is_final) { +void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr frame) { + if (!frame) + return; + // Init audio buffers / variables int total_frame_samples = 0; int frame_position = 0; @@ -1608,56 +1542,47 @@ void FFmpegWriter::write_audio_packets(bool is_final) { ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout // Create a new array (to hold all S16 audio samples, for the current queued frames - unsigned int all_queued_samples_size = sizeof(int16_t) * (queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE); + unsigned int all_queued_samples_size = sizeof(int16_t) * AVCODEC_MAX_AUDIO_FRAME_SIZE; int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size); int16_t *all_resampled_samples = NULL; int16_t *final_samples_planar = NULL; int16_t *final_samples = NULL; - // Loop through each queued audio frame - while (!queued_audio_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = queued_audio_frames.front(); - - // Get the audio details from this frame - sample_rate_in_frame = frame->SampleRate(); - samples_in_frame = frame->GetAudioSamplesCount(); - channels_in_frame = frame->GetAudioChannelsCount(); - channel_layout_in_frame = frame->ChannelsLayout(); - - // Get audio sample array - float *frame_samples_float = NULL; - // Get samples interleaved together (c1 c2 c1 c2 c1 c2) - frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame); - - // Calculate total samples - total_frame_samples = samples_in_frame * channels_in_frame; - - // Translate audio sample values back to 16 bit integers with saturation - const int16_t max16 = 32767; - const int16_t min16 = -32768; - for (int s = 0; s < total_frame_samples; s++, frame_position++) { - float valF = frame_samples_float[s] * (1 << 15); - int16_t conv; - if (valF > max16) { - conv = max16; - } else if (valF < min16) { - conv = min16; - } else { - conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding - } - // Copy into buffer - all_queued_samples[frame_position] = conv; + // Get the audio details from this frame + sample_rate_in_frame = frame->SampleRate(); + samples_in_frame = frame->GetAudioSamplesCount(); + channels_in_frame = frame->GetAudioChannelsCount(); + channel_layout_in_frame = frame->ChannelsLayout(); + + // Get audio sample array + float *frame_samples_float = NULL; + // Get samples interleaved together (c1 c2 c1 c2 c1 c2) + frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame); + + // Calculate total samples + total_frame_samples = samples_in_frame * channels_in_frame; + + // Translate audio sample values back to 16 bit integers with saturation + const int16_t max16 = 32767; + const int16_t min16 = -32768; + for (int s = 0; s < total_frame_samples; s++, frame_position++) { + float valF = frame_samples_float[s] * (1 << 15); + int16_t conv; + if (valF > max16) { + conv = max16; + } else if (valF < min16) { + conv = min16; + } else { + conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding } - // Deallocate float array - delete[] frame_samples_float; - - // Remove front item - queued_audio_frames.pop_front(); + // Copy into buffer + all_queued_samples[frame_position] = conv; + } - } // end while + // Deallocate float array + delete[] frame_samples_float; // Update total samples (since we've combined all queued frames) diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 639b19e62..4aa895506 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -26,8 +26,8 @@ namespace openshot { /// This enumeration designates the type of stream when encoding (video or audio) enum StreamType { - VIDEO_STREAM, ///< A video stream (used to determine which type of stream) - AUDIO_STREAM ///< An audio stream (used to determine which type of stream) + VIDEO_STREAM, ///< A video stream (used to determine which type of stream) + AUDIO_STREAM ///< An audio stream (used to determine which type of stream) }; /** @@ -116,7 +116,6 @@ namespace openshot { class FFmpegWriter : public WriterBase { private: std::string path; - int cache_size; bool is_writing; bool is_open; int64_t video_timestamp; @@ -152,15 +151,6 @@ namespace openshot { int original_channels; std::shared_ptr last_frame; - std::deque > spooled_audio_frames; - std::deque > spooled_video_frames; - - std::deque > queued_audio_frames; - std::deque > queued_video_frames; - - std::deque > processed_frames; - std::deque > deallocate_frames; - std::map, AVFrame *> av_frames; /// Add an AVFrame to the cache @@ -205,13 +195,13 @@ namespace openshot { void process_video_packet(std::shared_ptr frame); /// write all queued frames' audio to the video file - void write_audio_packets(bool is_final); + void write_audio_packets(bool is_final, std::shared_ptr frame); /// write video frame bool write_video_packet(std::shared_ptr frame, AVFrame *frame_final); /// write all queued frames - void write_queued_frames(); + void write_frame(std::shared_ptr frame); public: @@ -224,9 +214,6 @@ namespace openshot { /// Close the writer void Close(); - /// Get the cache size (number of frames to queue before writing) - int GetCacheSize() { return cache_size; }; - /// Determine if writer is open or closed bool IsOpen() { return is_open; }; @@ -273,10 +260,6 @@ namespace openshot { /// \note This is an overloaded function. void SetAudioOptions(std::string codec, int sample_rate, int bit_rate); - /// @brief Set the cache size - /// @param new_size The number of frames to queue before writing to the file - void SetCacheSize(int new_size) { cache_size = new_size; }; - /// @brief Set video export options /// @param has_video Does this file need a video stream /// @param codec The codec used to encode the images in this video