From 81cee38f2e8ef5ca1f87d171dd6fc6d4f3ef0f48 Mon Sep 17 00:00:00 2001
From: Jens Koenen <koenen@vr.rwth-aachen.de>
Date: Wed, 6 Jul 2022 17:12:30 +0200
Subject: [PATCH] Implemented encoder rate control.

---
 src/headset/remote_headset.cpp |  32 ++--
 src/headset/remote_headset.hpp |   1 -
 src/utility/encoder.cpp        | 284 +++++++++++++++++++++++++++------
 src/utility/encoder.hpp        |  54 +++++--
 4 files changed, 292 insertions(+), 79 deletions(-)

diff --git a/src/headset/remote_headset.cpp b/src/headset/remote_headset.cpp
index d6cc6373..251a3f6a 100644
--- a/src/headset/remote_headset.cpp
+++ b/src/headset/remote_headset.cpp
@@ -106,36 +106,30 @@ void RemoteHeadset::submit_frame(VkCommandBuffer command_buffer, VkImageLayout f
 
     lava::renderer& renderer = this->get_application()->get_renderer();
     
-    Encoder::Ptr encoder = this->encoders[frame_id];
-    bool encoder_config = this->encoder_configs[frame_id];
-
+    Encoder::Ptr encoder = this->encoders[frame_id];    
     lava::image::ptr framebuffer = this->framebuffers[frame_id];
     uint32_t frame_number = this->frame_number;
     uint32_t transform_id = this->transform_id;
 
-    if (!encoder_config)
+    bool result = encoder->encode_frame(command_buffer, renderer, framebuffer, frame_layout, [this, frame_number, frame_id, transform_id](const std::span<uint8_t>& content, EncoderFrameType type)
     {
-        bool result = encoder->encode_config(command_buffer, renderer, [this, frame_id](const std::span<uint8_t>& content)
+        switch (type)
         {
+        case ENCODER_FRAME_TYPE_CONFIG:
             this->send_packet_frame_config_nal(frame_id, content);
-        });
-
-        if (!result)
-        {
-            lava::log()->error("Remote Headset: Error during encode config submission!");
+            break;
+        case ENCODER_FRAME_TYPE_FRAME:
+            this->send_packet_frame_nal(frame_number, frame_id, transform_id, content);
+            break;
+        case ENCODER_FRAME_TYPE_CONTROL:
+        default:
+            break;
         }
-
-        this->encoder_configs[frame_id] = true;
-    }
-
-    bool result = encoder->encode_frame(command_buffer, renderer, framebuffer, frame_layout, [this, frame_number, frame_id, transform_id](const std::span<uint8_t>& content)
-    {
-        this->send_packet_frame_nal(frame_number, frame_id, transform_id, content);
     });
 
     if (!result)
     {
-        lava::log()->error("Remote Headset: Error during encode frame submission!");
+        lava::log()->error("Remote Headset: Error during encode submission!");
     }
 }
 
@@ -251,7 +245,6 @@ void RemoteHeadset::destroy_framebuffers()
 bool RemoteHeadset::create_encoders()
 {
     this->encoders.resize(this->frame_id_count);
-    this->encoder_configs.resize(this->frame_id_count);
 
     for (uint32_t index = 0; index < this->encoders.size(); index++)
     {
@@ -267,7 +260,6 @@ bool RemoteHeadset::create_encoders()
         }
 
         this->encoders[index] = encoder;
-        this->encoder_configs[index] = false;
     }
 
     return true;
diff --git a/src/headset/remote_headset.hpp b/src/headset/remote_headset.hpp
index e4f1360d..1995200f 100644
--- a/src/headset/remote_headset.hpp
+++ b/src/headset/remote_headset.hpp
@@ -105,7 +105,6 @@ private:
 
     std::vector<lava::image::ptr> framebuffers;
     std::vector<Encoder::Ptr> encoders;
-    std::vector<bool> encoder_configs;
 
     LatencyStatistic::Ptr latency_statistic;
 
diff --git a/src/utility/encoder.cpp b/src/utility/encoder.cpp
index 1857b8aa..978f8ee3 100644
--- a/src/utility/encoder.cpp
+++ b/src/utility/encoder.cpp
@@ -175,36 +175,42 @@ void Encoder::destroy()
     this->destroy_session(device);
 }
 
-bool Encoder::encode_config(VkCommandBuffer command_buffer, lava::renderer& renderer, EncoderCallback callback)
+void Encoder::set_mode(EncoderMode mode)
 {
-    lava::device_ptr device = this->descriptor_layout->get_device();
-    EncoderFrame::Ptr frame = nullptr;
+    this->setting_mode = mode;
+    this->control_change = true;
+}
 
-    //Get an unused image form the frame pool
-    if (!this->aquire_frame(frame))
-    {
-        return false;
-    }
+void Encoder::set_bitrate(uint32_t bitrate)
+{
+    this->setting_bitrate = bitrate;
+    this->control_change = true;
+}
 
-    frame->callback = std::move(callback);
+void Encoder::set_frame_rate(uint32_t frame_rate)
+{
+    this->setting_frame_rate = frame_rate;
+    this->control_change = true;
+}
 
-    //Create the encode command buffer.
-    if (!this->record_encoding(device, frame, true))
+void Encoder::set_quality(uint32_t quality)
+{
+    this->setting_quality = glm::clamp(quality, 0u, 51u);
+    this->control_change = true;
+}
+
+bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& renderer, lava::image::ptr image, VkImageLayout image_layout, EncoderCallback callback)
+{
+    if (!this->encode_control(renderer, callback))
     {
         return false;
     }
-
-    //Submit the encode command buffer at the end of the frame and async wait for the completion of the frame.
-    if (!this->submit_frame(device, frame, renderer))
+    
+    if (!this->encode_config(renderer, callback))
     {
         return false;
     }
 
-    return true;
-}
-
-bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& renderer, lava::image::ptr image, VkImageLayout image_layout, EncoderCallback callback)
-{
     lava::device_ptr device = image->get_device();
     EncoderFrame::Ptr frame = nullptr;
 
@@ -214,6 +220,7 @@ bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& rende
         return false;
     }
 
+    frame->type = ENCODER_FRAME_TYPE_FRAME;
     frame->callback = std::move(callback);
 
     //Check if the input image has the right layout. If not change the layout.
@@ -295,7 +302,7 @@ bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& rende
     lava::insert_image_memory_barrier(device, command_buffer, frame->input_combined, 0, 0, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, image_range);
 
     //Create the encode command buffer.
-    if (!this->record_encoding(device, frame, false))
+    if (!this->record_encoding(device, frame))
     {
         return false;
     }
@@ -311,6 +318,78 @@ bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& rende
     return true;
 }
 
+bool Encoder::encode_control(lava::renderer& renderer, EncoderCallback callback)
+{
+    if (!this->control_change)
+    {
+        return true;
+    }
+
+    lava::device_ptr device = this->descriptor_layout->get_device();
+    EncoderFrame::Ptr frame = nullptr;
+
+    //Get an unused image form the frame pool
+    if (!this->aquire_frame(frame))
+    {
+        return false;
+    }
+
+    frame->type = ENCODER_FRAME_TYPE_CONTROL;
+    frame->callback = std::move(callback);
+
+    //Create the encode command buffer.
+    if (!this->record_encoding(device, frame))
+    {
+        return false;
+    }
+
+    //Submit the encode command buffer at the end of the frame and async wait for the completion of the frame.
+    if (!this->submit_frame(device, frame, renderer))
+    {
+        return false;
+    }
+
+    this->control_change = false;
+
+    return true;
+}
+
+bool Encoder::encode_config(lava::renderer& renderer, EncoderCallback callback)
+{
+    if (!this->config_change)
+    {
+        return true;
+    }
+
+    lava::device_ptr device = this->descriptor_layout->get_device();
+    EncoderFrame::Ptr frame = nullptr;
+
+    //Get an unused image form the frame pool
+    if (!this->aquire_frame(frame))
+    {
+        return false;
+    }
+
+    frame->type = ENCODER_FRAME_TYPE_CONFIG;
+    frame->callback = std::move(callback);
+
+    //Create the encode command buffer.
+    if (!this->record_encoding(device, frame))
+    {
+        return false;
+    }
+
+    //Submit the encode command buffer at the end of the frame and async wait for the completion of the frame.
+    if (!this->submit_frame(device, frame, renderer))
+    {
+        return false;
+    }
+
+    this->config_change = false;
+
+    return true;
+}
+
 bool Encoder::aquire_frame(EncoderFrame::Ptr& frame)
 {
     std::unique_lock<std::mutex> lock(this->frame_mutex);
@@ -346,25 +425,35 @@ bool Encoder::submit_frame(lava::device_ptr device, EncoderFrame::Ptr frame, lav
     });
 
     lava::frame_submission submission;
-    submission.semaphore = frame->encode_semaphore;
+    submission.semaphore = frame->render_semaphore;
     submission.callback = [=]()
     {
         VkPipelineStageFlags stage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
 
+        std::vector<VkSemaphore> wait_semaphores;
+        wait_semaphores.push_back(frame->render_semaphore);
+
+        if (this->previous_semaphore != VK_NULL_HANDLE)
+        {
+            wait_semaphores.push_back(this->previous_semaphore);
+        }
+
         VkSubmitInfo submit_info;
         submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
         submit_info.pNext = nullptr;
-        submit_info.waitSemaphoreCount = 1;
-        submit_info.pWaitSemaphores = &frame->encode_semaphore;
+        submit_info.waitSemaphoreCount = wait_semaphores.size();
+        submit_info.pWaitSemaphores = wait_semaphores.data();
         submit_info.pWaitDstStageMask = &stage;
         submit_info.commandBufferCount = 1;
         submit_info.pCommandBuffers = &frame->encode_command_buffer;
-        submit_info.signalSemaphoreCount = 0;
-        submit_info.pSignalSemaphores = nullptr;
+        submit_info.signalSemaphoreCount = 1;
+        submit_info.pSignalSemaphores = &frame->encode_semaphore;
 
-        vkDeviceWaitIdle(device->get()); //TODO: Try to remove this wait idle
+        vkDeviceWaitIdle(device->get()); //TODO: Check if this wait can be removed.
 
         vkQueueSubmit(this->encode_queue->vk_queue, 1, &submit_info, frame->encode_fence->get()); //TODO: Return value check
+
+        this->previous_semaphore = frame->encode_semaphore;
     };
     
     renderer.add_submission(submission);
@@ -394,7 +483,7 @@ void Encoder::write_image(lava::device_ptr device, EncoderFrame::Ptr frame, lava
     vkUpdateDescriptorSets(device->get(), 1, &write_info, 0, nullptr);
 }
 
-bool Encoder::record_encoding(lava::device_ptr device, EncoderFrame::Ptr frame, bool config)
+bool Encoder::record_encoding(lava::device_ptr device, EncoderFrame::Ptr frame)
 {
     VkFence fence = frame->encode_fence->get();
 
@@ -419,14 +508,19 @@ bool Encoder::record_encoding(lava::device_ptr device, EncoderFrame::Ptr frame,
         return false;
     }
 
-    if (config)
+    switch (frame->type)
     {
+    case ENCODER_FRAME_TYPE_CONTROL:
+        this->encode_pass_control_setup(frame->encode_command_buffer, frame);
+        break;
+    case ENCODER_FRAME_TYPE_CONFIG:
         this->encode_pass_config_setup(frame->encode_command_buffer, frame);
-    }
-
-    else
-    {
+        break;
+    case ENCODER_FRAME_TYPE_FRAME:
         this->encode_pass_frame_setup(frame->encode_command_buffer, frame);
+        break;
+    default:
+        return false;
     }
 
     if (vkEndCommandBuffer(frame->encode_command_buffer) != VK_SUCCESS)
@@ -439,29 +533,37 @@ bool Encoder::record_encoding(lava::device_ptr device, EncoderFrame::Ptr frame,
 
 void Encoder::retrive_encoding(lava::device_ptr device, EncoderFrame::Ptr frame)
 {
-    std::array<uint32_t, 3> buffer_range;
-    memset(buffer_range.data(), 0, sizeof(uint32_t) * buffer_range.size());
-
-    if (vkGetQueryPoolResults(device->get(), this->query_pool, frame->output_query_index, 1, sizeof(uint32_t) * buffer_range.size(), buffer_range.data(), 0, VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_WITH_STATUS_BIT_KHR) != VK_SUCCESS)
+    if (frame->type == ENCODER_FRAME_TYPE_CONTROL)
     {
-        return;
+        frame->callback(std::span<uint8_t>(), frame->type);
     }
 
-    uint32_t range_offset = buffer_range[0];
-    uint32_t range_size = buffer_range[1];
+    else
+    {
+        std::array<uint32_t, 3> buffer_range;
+        memset(buffer_range.data(), 0, sizeof(uint32_t) * buffer_range.size());
+
+        if (vkGetQueryPoolResults(device->get(), this->query_pool, frame->output_query_index, 1, sizeof(uint32_t) * buffer_range.size(), buffer_range.data(), 0, VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_WITH_STATUS_BIT_KHR) != VK_SUCCESS)
+        {
+            return;
+        }
+
+        uint32_t range_offset = buffer_range[0];
+        uint32_t range_size = buffer_range[1];
 
-    vmaInvalidateAllocation(device->alloc(), frame->output_memory, range_offset, range_size);
+        vmaInvalidateAllocation(device->alloc(), frame->output_memory, range_offset, range_size);
 
-    uint8_t* range_pointer = nullptr;
+        uint8_t* range_pointer = nullptr;
 
-    if (vmaMapMemory(device->alloc(), frame->output_memory, (void**) (&range_pointer)))
-    {
-        return;
-    }
+        if (vmaMapMemory(device->alloc(), frame->output_memory, (void**) (&range_pointer)))
+        {
+            return;
+        }
 
-    frame->callback(std::span(range_pointer + range_offset, range_size));
+        frame->callback(std::span(range_pointer + range_offset, range_size), frame->type);
 
-    vmaUnmapMemory(device->alloc(), frame->output_memory);
+        vmaUnmapMemory(device->alloc(), frame->output_memory);
+    }
 }
 
 bool Encoder::create_profiles(lava::device_ptr device)
@@ -1478,6 +1580,11 @@ bool Encoder::create_encode_resources(lava::device_ptr device, asio::executor ex
     semaphore_create_info.pNext = nullptr;
     semaphore_create_info.flags = 0;
 
+    if (vkCreateSemaphore(device->get(), &semaphore_create_info, lava::memory::alloc(), &frame->render_semaphore) != VK_SUCCESS)
+    {
+        return false;
+    }
+
     if (vkCreateSemaphore(device->get(), &semaphore_create_info, lava::memory::alloc(), &frame->encode_semaphore) != VK_SUCCESS)
     {
         return false;
@@ -1556,6 +1663,7 @@ void Encoder::destroy_frame(lava::device_ptr device, EncoderFrame::Ptr frame)
 
     vkFreeCommandBuffers(device->get(), this->command_pool, 1, &frame->encode_command_buffer);
 
+    vkDestroySemaphore(device->get(), frame->render_semaphore, lava::memory::alloc());
     vkDestroySemaphore(device->get(), frame->encode_semaphore, lava::memory::alloc());
     frame->encode_fence->destroy(device);
 }
@@ -1644,6 +1752,88 @@ void Encoder::subsample_pass(VkCommandBuffer command_buffer)
     vkCmdDraw(command_buffer, 4, 1, 0, 0); //Draw fullscreen quad
 }
 
+void Encoder::encode_pass_control_setup(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame)
+{
+    VkVideoBeginCodingInfoKHR begin_info;
+    begin_info.sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR;
+    begin_info.pNext = nullptr;
+    begin_info.flags = 0;
+    begin_info.codecQualityPreset = VK_VIDEO_CODING_QUALITY_PRESET_NORMAL_BIT_KHR;
+    begin_info.videoSession = this->video_session;
+    begin_info.videoSessionParameters = this->video_session_paremeters;
+    begin_info.referenceSlotCount = 0;
+    begin_info.pReferenceSlots = nullptr;
+
+    vkCmdBeginVideoCodingKHR(command_buffer, &begin_info);
+
+    this->encode_pass_control_command(command_buffer, frame);
+
+    VkVideoEndCodingInfoKHR end_info;
+    end_info.sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR;
+    end_info.pNext = nullptr;
+    end_info.flags = 0;
+
+    vkCmdEndVideoCodingKHR(command_buffer, &end_info);
+}
+
+void Encoder::encode_pass_control_command(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame)
+{
+    VkVideoEncodeH264RateControlLayerInfoEXT codec_layer_info;
+    codec_layer_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_RATE_CONTROL_LAYER_INFO_EXT;
+    codec_layer_info.pNext = nullptr;
+    codec_layer_info.temporalLayerId = 0;
+    codec_layer_info.useInitialRcQp = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? VK_FALSE : VK_TRUE;
+    codec_layer_info.initialRcQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.initialRcQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.initialRcQp.qpB = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.useMinQp = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? VK_FALSE : VK_TRUE;
+    codec_layer_info.minQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.minQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.minQp.qpB = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.useMaxQp = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? VK_FALSE : VK_TRUE;
+    codec_layer_info.maxQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.maxQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.maxQp.qpB = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+    codec_layer_info.useMaxFrameSize = VK_FALSE;
+    codec_layer_info.maxFrameSize.frameISize = 0;
+    codec_layer_info.maxFrameSize.framePSize = 0;
+    codec_layer_info.maxFrameSize.frameBSize = 0;
+
+    VkVideoEncodeRateControlLayerInfoKHR layer_info;
+    layer_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_LAYER_INFO_KHR;
+    layer_info.pNext = &codec_layer_info;
+    layer_info.averageBitrate = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? this->setting_bitrate : 0;
+    layer_info.maxBitrate = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? this->setting_bitrate : 0;
+    layer_info.frameRateNumerator = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 1 : 0;
+    layer_info.frameRateDenominator = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? this->setting_frame_rate : 0;
+    layer_info.virtualBufferSizeInMs = 0;
+    layer_info.initialVirtualBufferSizeInMs = 0;
+
+    VkVideoEncodeH264RateControlInfoEXT codec_rate_info;
+    codec_rate_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_RATE_CONTROL_INFO_EXT;
+    codec_rate_info.pNext = nullptr;
+    codec_rate_info.gopFrameCount = 0;
+    codec_rate_info.idrPeriod = 0;
+    codec_rate_info.consecutiveBFrameCount = 0;
+    codec_rate_info.rateControlStructure = VK_VIDEO_ENCODE_H264_RATE_CONTROL_STRUCTURE_UNKNOWN_EXT;
+    codec_rate_info.temporalLayerCount = 0;
+
+    VkVideoEncodeRateControlInfoKHR rate_info;
+    rate_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_INFO_KHR;
+    rate_info.pNext = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? &codec_rate_info : nullptr;
+    rate_info.flags = 0;
+    rate_info.rateControlMode = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR : VK_VIDEO_ENCODE_RATE_CONTROL_MODE_NONE_BIT_KHR;
+    rate_info.layerCount = 1;
+    rate_info.pLayerConfigs = &layer_info;
+
+    VkVideoCodingControlInfoKHR control_info;
+    control_info.sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR;
+    control_info.pNext = &rate_info;
+    control_info.flags = VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR;
+
+    vkCmdControlVideoCodingKHR(command_buffer, &control_info);
+}
+
 void Encoder::encode_pass_config_setup(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame)
 {
     VkVideoBeginCodingInfoKHR begin_info;
diff --git a/src/utility/encoder.hpp b/src/utility/encoder.hpp
index 241dd418..6038ce37 100644
--- a/src/utility/encoder.hpp
+++ b/src/utility/encoder.hpp
@@ -2,7 +2,7 @@
   The encoder can be used to create a h264 stream out of a sequence of images.
   In order to be able to use the encoder, it is neccessary to call the function setup_encoder(...) during the setup of the vulkan device.
   Besides that, the requested vulkan version has to be set to Vulkan 1.1 during the setup of the vulkan instance.
-  Before submitting any image to the encoder by calling the function encode_frame(...) it is neccessary to call te function encode_config(...) once.
+  A frame can be submitted for encoding using the function encode_frame(...).
   After the completion of an encode task, a callback function is executed to which the resulting ouput of the task is passed.
   Example:
 
@@ -17,12 +17,7 @@
 
    encoder->create(...);
 
-   encoder->encoder_config(..., [](const std::span<uint8_t>& content) //Only once at the beginning
-   {
-        //Use content
-   });
-
-   encoder->encoder_frame(..., image, ..., [](const std::span<uint8_t>& content)
+   encoder->encoder_frame(..., image, ..., [](const std::span<uint8_t>& content, EncoderFrameType type)
    {
         //Use content
    });
@@ -40,7 +35,20 @@
 
 #include "extern_fence.hpp"
 
-typedef std::function<void(const std::span<uint8_t>& content)> EncoderCallback;
+enum EncoderMode
+{
+    ENCODER_MODE_CONSTANT_BITRATE,
+    ENCODER_MODE_CONSTANT_QUALITY
+};
+
+enum EncoderFrameType
+{
+    ENCODER_FRAME_TYPE_CONTROL,
+    ENCODER_FRAME_TYPE_CONFIG,
+    ENCODER_FRAME_TYPE_FRAME
+};
+
+typedef std::function<void(const std::span<uint8_t>& content, EncoderFrameType type)> EncoderCallback;
 
 struct EncoderFrame
 {
@@ -81,10 +89,12 @@ public:
     lava::graphics_pipeline::ptr subsample_pipeline;
     lava::render_pass::ptr subsample_pass;
 
+    VkSemaphore render_semaphore = nullptr;
     VkSemaphore encode_semaphore = nullptr;
     VkCommandBuffer encode_command_buffer = nullptr;
     ExternFence::Ptr encode_fence;
 
+    EncoderFrameType type;
     EncoderCallback callback;
 };
 
@@ -100,17 +110,26 @@ public:
     //NOTE: Assume all async operations has been completed.
     void destroy();
 
-    //Note: Before calling the function encode_frame, the function encode_config has to be called once.
-    bool encode_config(VkCommandBuffer command_buffer, lava::renderer& renderer, EncoderCallback callback);
+    void set_mode(EncoderMode mode);
+    //NOTE: The bitrate in bit per second.
+    void set_bitrate(uint32_t bitrate);
+    //NOTE: The bitrate in frames per second.
+    void set_frame_rate(uint32_t frame_rate);
+    //NOTE: The quality in range from 0 to 51, where 0 means a lowest compression and 51 means highest compression.
+    void set_quality(uint32_t quality);
+
     bool encode_frame(VkCommandBuffer command_buffer, lava::renderer& renderer, lava::image::ptr image, VkImageLayout image_layout, EncoderCallback callback);
 
 private:
+    bool encode_control(lava::renderer& renderer, EncoderCallback callback);
+    bool encode_config(lava::renderer& renderer, EncoderCallback callback);
+
     bool aquire_frame(EncoderFrame::Ptr& frame);
     void release_frame(EncoderFrame::Ptr frame);
     bool submit_frame(lava::device_ptr device, EncoderFrame::Ptr frame, lava::renderer& renderer);
 
     void write_image(lava::device_ptr device, EncoderFrame::Ptr frame, lava::image::ptr image);
-    bool record_encoding(lava::device_ptr device, EncoderFrame::Ptr frame, bool config);
+    bool record_encoding(lava::device_ptr device, EncoderFrame::Ptr frame);
     void retrive_encoding(lava::device_ptr device, EncoderFrame::Ptr frame);
 
     bool create_profiles(lava::device_ptr device);
@@ -146,6 +165,9 @@ private:
 
     void convert_pass(VkCommandBuffer command_buffer);
     void subsample_pass(VkCommandBuffer command_buffer);
+
+    void encode_pass_control_setup(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame);
+    void encode_pass_control_command(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame);
     void encode_pass_config_setup(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame);
     void encode_pass_config_command(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame);
     void encode_pass_frame_setup(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame);
@@ -155,6 +177,14 @@ private:
     uint32_t frame_count = 0;
     uint32_t frame_index = 0;
 
+    bool config_change = true;
+    bool control_change = true;
+
+    EncoderMode setting_mode = ENCODER_MODE_CONSTANT_QUALITY;
+    uint32_t setting_bitrate = 6000000;
+    uint32_t setting_frame_rate = 60;
+    uint32_t setting_quality = 1;
+
     glm::uvec2 setting_block_count;
     glm::uvec2 setting_frame_crop;
     glm::uvec2 setting_image_size;
@@ -169,6 +199,8 @@ private:
     lava::descriptor::ptr descriptor_layout;
     lava::pipeline_layout::ptr pipeline_layout;
 
+    VkSemaphore previous_semaphore = VK_NULL_HANDLE;
+
     std::mutex frame_mutex;
     std::vector<EncoderFrame::Ptr> frame_queue; //NOTE: Protected by frame_mutex
     std::vector<EncoderFrame::Ptr> frame_list;
-- 
GitLab