From d07b6650c36dbea19a15a9e1aab1bff3f5b6004f Mon Sep 17 00:00:00 2001
From: Jens Koenen <koenen@vr.rwth-aachen.de>
Date: Thu, 13 Oct 2022 16:52:33 +0200
Subject: [PATCH] The encoder nw runs on the latest version of the vulkan video
extension. Besides that fixed some bugs and added sliders for i-frame and
p-frame quality.
---
src/headset/remote_headset.cpp | 27 ++--
src/headset/remote_headset.hpp | 7 +-
src/utility/encoder.cpp | 251 ++++++++++++++++++++++++---------
src/utility/encoder.hpp | 9 +-
src/utility/pass_timer.cpp | 34 ++++-
src/utility/pass_timer.hpp | 4 +-
6 files changed, 247 insertions(+), 85 deletions(-)
diff --git a/src/headset/remote_headset.cpp b/src/headset/remote_headset.cpp
index ac7d972d..61e57da8 100644
--- a/src/headset/remote_headset.cpp
+++ b/src/headset/remote_headset.cpp
@@ -2,6 +2,8 @@
#include "vr_application.hpp"
#include <imgui.h>
+#include <glm/gtx/string_cast.hpp>
+
RemoteHeadset::RemoteHeadset()
{
this->controller_states.fill(CONTROLLER_STATE_DETACHED);
@@ -129,9 +131,9 @@ bool RemoteHeadset::on_interface()
}
}
- ImGui::SliderInt("Input-Rate (Fps)", (int32_t*) &this->encoder_input_rate, 1, 90);
+ ImGui::SliderInt("Input-Rate (Fps)", (int32_t*) &this->encoder_input_rate, 1, 180);
- if (ImGui::SliderInt("Key-Rate (Kps)", (int32_t*)&this->encoder_key_rate, 90, 900))
+ if (ImGui::SliderInt("Key-Rate (Frames)", (int32_t*)&this->encoder_key_rate, 1, 180))
{
for (Encoder::Ptr encoder : this->encoders)
{
@@ -149,7 +151,7 @@ bool RemoteHeadset::on_interface()
}
}
- if (ImGui::SliderInt("Frame-Rate (Fps)", (int32_t*)&this->encoder_frame_rate, 10, 120))
+ if (ImGui::SliderInt("Frame-Rate (Fps)", (int32_t*)&this->encoder_frame_rate, 1, 180))
{
for (Encoder::Ptr encoder : this->encoders)
{
@@ -160,11 +162,19 @@ bool RemoteHeadset::on_interface()
else if (this->encoder_mode == ENCODER_MODE_CONSTANT_QUALITY)
{
- if (ImGui::SliderInt("Quality", (int32_t*)&this->encoder_quality, 1, 51))
+ if (ImGui::SliderInt("Quality I-Frame", (int32_t*)&this->encoder_quality_iframe, 1, 51))
+ {
+ for (Encoder::Ptr encoder : this->encoders)
+ {
+ encoder->set_quality_iframe(this->encoder_quality_iframe);
+ }
+ }
+
+ if (ImGui::SliderInt("Quality P-Frame", (int32_t*)&this->encoder_quality_pframe, 1, 51))
{
for (Encoder::Ptr encoder : this->encoders)
{
- encoder->set_quality(this->encoder_quality);
+ encoder->set_quality_pframe(this->encoder_quality_pframe);
}
}
}
@@ -212,12 +222,12 @@ bool RemoteHeadset::on_update(lava::delta delta_time)
this->update_bitrates();
this->update_stage(delta_time);
- this->frame_number++;
this->encoder_last_submit += delta_time;
this->encoder_enable_submit = false;
if (this->encoder_last_submit > (1.0 / (float)this->encoder_input_rate))
{
+ this->frame_number++;
this->encoder_enable_submit = true;
this->encoder_last_submit = 0.0f;
}
@@ -451,7 +461,8 @@ bool RemoteHeadset::create_encoders()
encoder->set_key_rate(this->encoder_key_rate);
encoder->set_bit_rate(this->encoder_bit_rate);
encoder->set_frame_rate(this->encoder_frame_rate);
- encoder->set_quality(this->encoder_quality);
+ encoder->set_quality_iframe(this->encoder_quality_iframe);
+ encoder->set_quality_pframe(this->encoder_quality_pframe);
this->encoders[index] = encoder;
}
@@ -554,7 +565,7 @@ void RemoteHeadset::on_head_transform(PacketTransformId transform_id, const glm:
void RemoteHeadset::on_controller_transform(PacketControllerId controller_id, const glm::mat4& controller_transform)
{
std::unique_lock<std::mutex> lock(this->transport_mutex);
- this->controller_matrices[controller_id] = controller_transform;
+ this->controller_transforms[controller_id] = controller_transform;
lock.unlock();
}
diff --git a/src/headset/remote_headset.hpp b/src/headset/remote_headset.hpp
index a3ee731d..0719de7f 100644
--- a/src/headset/remote_headset.hpp
+++ b/src/headset/remote_headset.hpp
@@ -114,10 +114,11 @@ private:
std::vector<Encoder::Ptr> encoders;
uint32_t encoder_mode = ENCODER_MODE_CONSTANT_QUALITY;
uint32_t encoder_input_rate = 90;
- uint32_t encoder_key_rate = 120;
+ uint32_t encoder_key_rate = 90;
uint32_t encoder_frame_rate = 90;
- uint32_t encoder_quality = 51;
- float encoder_bit_rate = 6.0; //NOTE: Bitrate in Mbit per seconds
+ uint32_t encoder_quality_iframe = 45;
+ uint32_t encoder_quality_pframe = 45;
+ float encoder_bit_rate = 8.0; //NOTE: Bitrate in Mbit per seconds
float encoder_last_submit = 0.0f;
bool encoder_enable_submit = true;
diff --git a/src/utility/encoder.cpp b/src/utility/encoder.cpp
index bcfd4000..f4c69032 100644
--- a/src/utility/encoder.cpp
+++ b/src/utility/encoder.cpp
@@ -201,12 +201,42 @@ void Encoder::set_frame_rate(uint32_t frame_rate)
this->control_change = true;
}
-void Encoder::set_quality(uint32_t quality)
+void Encoder::set_quality_iframe(uint32_t quality)
{
- this->setting_quality = glm::clamp(quality, 0u, 51u);
+ this->setting_quality_iframe = glm::clamp(quality, 0u, 51u);
this->control_change = true;
}
+void Encoder::set_quality_pframe(uint32_t quality)
+{
+ this->setting_quality_pframe = glm::clamp(quality, 0u, 51u);
+ this->control_change = true;
+}
+
+void full_barrier(VkCommandBuffer command_buffer)
+{
+ VkMemoryBarrier2 memoryBarrier;
+ memoryBarrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
+ memoryBarrier.pNext = nullptr;
+ memoryBarrier.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT_KHR;
+ memoryBarrier.srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT_KHR | VK_ACCESS_2_MEMORY_WRITE_BIT_KHR;
+ memoryBarrier.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT_KHR;
+ memoryBarrier.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT_KHR | VK_ACCESS_2_MEMORY_WRITE_BIT_KHR;
+
+ VkDependencyInfo dependency;
+ dependency.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
+ dependency.pNext = nullptr;
+ dependency.dependencyFlags = 0;
+ dependency.memoryBarrierCount = 1;
+ dependency.pMemoryBarriers = &memoryBarrier;
+ dependency.bufferMemoryBarrierCount = 0;
+ dependency.pBufferMemoryBarriers = nullptr;
+ dependency.imageMemoryBarrierCount = 0;
+ dependency.pImageMemoryBarriers = nullptr;
+
+ vkCmdPipelineBarrier2KHR(command_buffer, &dependency);
+}
+
bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& renderer, lava::image::ptr image, VkImageLayout image_layout, EncoderCallback callback)
{
if (!this->encode_control(renderer, callback))
@@ -262,10 +292,8 @@ bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& rende
image_range.baseArrayLayer = 0;
image_range.layerCount = 1;
- lava::insert_image_memory_barrier(device, command_buffer, frame->input_combined, 0, 0, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, image_range);
- lava::insert_image_memory_barrier(device, command_buffer, frame->input_luma, 0, 0, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, image_range);
- lava::insert_image_memory_barrier(device, command_buffer, frame->input_chroma_subsampled, 0, 0, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, image_range);
-
+ lava::insert_image_memory_barrier(device, command_buffer, frame->input_combined, 0, VK_ACCESS_TRANSFER_WRITE_BIT, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, image_range);
+
VkImageCopy copy_luma;
copy_luma.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
copy_luma.srcSubresource.baseArrayLayer = 0;
@@ -308,8 +336,6 @@ bool Encoder::encode_frame(VkCommandBuffer command_buffer, lava::renderer& rende
vkCmdCopyImage(command_buffer, frame->input_luma, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, frame->input_combined, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©_luma);
vkCmdCopyImage(command_buffer, frame->input_chroma_subsampled, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, frame->input_combined, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©_chroma);
- lava::insert_image_memory_barrier(device, command_buffer, frame->input_combined, 0, 0, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, image_range);
-
//Create the encode command buffer.
if (!this->record_encoding(device, frame))
{
@@ -405,7 +431,7 @@ bool Encoder::aquire_frame(EncoderFrame::Ptr& frame)
{
std::unique_lock<std::mutex> lock(this->frame_mutex);
- if (this->frame_queue.size() >= 2) //TODO: Check why the encoder chrashed if the last frame of the queue is used. This problem only appears if I- and P-Frames are used.
+ if (this->frame_queue.size() >= 1)
{
frame = this->frame_queue.front();
this->frame_queue.erase(this->frame_queue.begin());
@@ -439,28 +465,55 @@ bool Encoder::submit_frame(lava::device_ptr device, EncoderFrame::Ptr frame, lav
submission.semaphore = frame->render_semaphore;
submission.callback = [=]()
{
- VkPipelineStageFlags stage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
-
- std::vector<VkSemaphore> wait_semaphores;
- wait_semaphores.push_back(frame->render_semaphore);
+ std::vector<VkSemaphoreSubmitInfo> wait_semaphores;
+
+ VkSemaphoreSubmitInfo wait_render;
+ wait_render.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
+ wait_render.pNext = nullptr;
+ wait_render.semaphore = frame->render_semaphore;
+ wait_render.value = 0;
+ wait_render.stageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT;
+ wait_render.deviceIndex = 0;
+ wait_semaphores.push_back(wait_render);
if (this->previous_semaphore != VK_NULL_HANDLE)
{
- wait_semaphores.push_back(this->previous_semaphore);
+ VkSemaphoreSubmitInfo wait_encode;
+ wait_encode.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
+ wait_encode.pNext = nullptr;
+ wait_encode.semaphore = this->previous_semaphore;
+ wait_encode.value = 0;
+ wait_encode.stageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR;
+ wait_encode.deviceIndex = 0;
+ wait_semaphores.push_back(wait_encode);
}
- VkSubmitInfo submit_info;
+ VkCommandBufferSubmitInfo command_info;
+ command_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO;
+ command_info.pNext = nullptr;
+ command_info.commandBuffer = frame->encode_command_buffer;
+ command_info.deviceMask = 0;
+
+ VkSemaphoreSubmitInfo signal_info;
+ signal_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
+ signal_info.pNext = nullptr;
+ signal_info.semaphore = frame->encode_semaphore;
+ signal_info.value = 0;
+ signal_info.stageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR;
+ signal_info.deviceIndex = 0;
+
+ VkSubmitInfo2 submit_info;
submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info.pNext = nullptr;
- submit_info.waitSemaphoreCount = wait_semaphores.size();
- submit_info.pWaitSemaphores = wait_semaphores.data();
- submit_info.pWaitDstStageMask = &stage;
- submit_info.commandBufferCount = 1;
- submit_info.pCommandBuffers = &frame->encode_command_buffer;
- submit_info.signalSemaphoreCount = 1;
- submit_info.pSignalSemaphores = &frame->encode_semaphore;
+ submit_info.flags = 0;
+ submit_info.waitSemaphoreInfoCount = wait_semaphores.size();
+ submit_info.pWaitSemaphoreInfos = wait_semaphores.data();
+ submit_info.commandBufferInfoCount = 1;
+ submit_info.pCommandBufferInfos = &command_info;
+ submit_info.signalSemaphoreInfoCount = 1;
+ submit_info.pSignalSemaphoreInfos = &signal_info;
- vkQueueSubmit(this->encode_queue->vk_queue, 1, &submit_info, frame->encode_fence->get()); //TODO: Return value check
+ vkQueueSubmit2KHR(this->encode_queue->vk_queue, 1, &submit_info, frame->encode_fence->get()); //TODO: Return value check
this->previous_semaphore = frame->encode_semaphore;
};
@@ -581,6 +634,7 @@ bool Encoder::create_profiles(lava::device_ptr device)
this->encode_profile.pNext = nullptr;
this->encode_profile.stdProfileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH;
+ //NOTE: Not used since it leads to an error during the initialisation.
this->usage_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_USAGE_INFO_KHR;
this->usage_info.pNext = &this->encode_profile;
this->usage_info.videoUsageHints = VK_VIDEO_ENCODE_USAGE_STREAMING_BIT_KHR;
@@ -588,7 +642,7 @@ bool Encoder::create_profiles(lava::device_ptr device)
this->usage_info.tuningMode = VK_VIDEO_ENCODE_TUNING_MODE_ULTRA_LOW_LATENCY_KHR;
this->video_profile.sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
- this->video_profile.pNext = &this->usage_info;
+ this->video_profile.pNext = &this->encode_profile;
this->video_profile.videoCodecOperation = VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_EXT;
this->video_profile.chromaSubsampling = VK_VIDEO_CHROMA_SUBSAMPLING_420_BIT_KHR;
this->video_profile.lumaBitDepth = VK_VIDEO_COMPONENT_BIT_DEPTH_8_BIT_KHR;
@@ -1343,7 +1397,7 @@ bool Encoder::create_convert_pass(lava::device_ptr device, EncoderFrame::Ptr fra
lava::attachment::ptr luma_attachment = lava::make_attachment(VK_FORMAT_R8_UNORM);
luma_attachment->set_op(VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE);
luma_attachment->set_stencil_op(VK_ATTACHMENT_LOAD_OP_DONT_CARE, VK_ATTACHMENT_STORE_OP_DONT_CARE);
- luma_attachment->set_layouts(VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL);
+ luma_attachment->set_layouts(VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
lava::attachment::ptr chroma_attachment = lava::make_attachment(VK_FORMAT_R8G8_UNORM);
chroma_attachment->set_op(VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE);
@@ -1370,8 +1424,8 @@ bool Encoder::create_convert_pass(lava::device_ptr device, EncoderFrame::Ptr fra
subpass_begin_dependency->set_access_mask(0, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT);
lava::subpass_dependency::ptr subpass_end_dependency = lava::make_subpass_dependency(0, VK_SUBPASS_EXTERNAL);
- subpass_end_dependency->set_stage_mask(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
- subpass_end_dependency->set_access_mask(VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, 0);
+ subpass_end_dependency->set_stage_mask(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT);
+ subpass_end_dependency->set_access_mask(VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT);
frame->convert_pass = lava::make_render_pass(device);
frame->convert_pass->add(subpass);
@@ -1470,7 +1524,7 @@ bool Encoder::create_subsample_pass(lava::device_ptr device, EncoderFrame::Ptr f
lava::attachment::ptr chroma_attachment = lava::make_attachment(VK_FORMAT_R8G8_UNORM);
chroma_attachment->set_op(VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE);
chroma_attachment->set_stencil_op(VK_ATTACHMENT_LOAD_OP_DONT_CARE, VK_ATTACHMENT_STORE_OP_DONT_CARE);
- chroma_attachment->set_layouts(VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL);
+ chroma_attachment->set_layouts(VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
VkAttachmentReference chroma_attachment_reference;
chroma_attachment_reference.attachment = 0;
@@ -1483,12 +1537,12 @@ bool Encoder::create_subsample_pass(lava::device_ptr device, EncoderFrame::Ptr f
});
lava::subpass_dependency::ptr subpass_begin_dependency = lava::make_subpass_dependency(VK_SUBPASS_EXTERNAL, 0);
- subpass_begin_dependency->set_stage_mask(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
- subpass_begin_dependency->set_access_mask(0, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT);
+ subpass_begin_dependency->set_stage_mask(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
+ subpass_begin_dependency->set_access_mask(0, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT);
lava::subpass_dependency::ptr subpass_end_dependency = lava::make_subpass_dependency(0, VK_SUBPASS_EXTERNAL);
- subpass_end_dependency->set_stage_mask(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
- subpass_end_dependency->set_access_mask(VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, 0);
+ subpass_end_dependency->set_stage_mask(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
+ subpass_end_dependency->set_access_mask(VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
frame->subsample_pass = lava::make_render_pass(device);
frame->subsample_pass->add(subpass);
@@ -1795,18 +1849,18 @@ void Encoder::encode_pass_control_command(VkCommandBuffer command_buffer, Encode
codec_layer_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_RATE_CONTROL_LAYER_INFO_EXT;
codec_layer_info.pNext = nullptr;
codec_layer_info.temporalLayerId = 0;
- codec_layer_info.useInitialRcQp = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? VK_FALSE : VK_TRUE;
- codec_layer_info.initialRcQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
- codec_layer_info.initialRcQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
- codec_layer_info.initialRcQp.qpB = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+ codec_layer_info.useInitialRcQp = VK_FALSE;
+ codec_layer_info.initialRcQp.qpI = 0;
+ codec_layer_info.initialRcQp.qpP = 0;
+ codec_layer_info.initialRcQp.qpB = 0;
codec_layer_info.useMinQp = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? VK_FALSE : VK_TRUE;
- codec_layer_info.minQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
- codec_layer_info.minQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
- codec_layer_info.minQp.qpB = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+ codec_layer_info.minQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality_iframe;
+ codec_layer_info.minQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality_pframe;
+ codec_layer_info.minQp.qpB = 51;
codec_layer_info.useMaxQp = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? VK_FALSE : VK_TRUE;
- codec_layer_info.maxQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
- codec_layer_info.maxQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
- codec_layer_info.maxQp.qpB = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality;
+ codec_layer_info.maxQp.qpI = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality_iframe;
+ codec_layer_info.maxQp.qpP = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 0 : this->setting_quality_pframe;
+ codec_layer_info.maxQp.qpB = 51;
codec_layer_info.useMaxFrameSize = VK_FALSE;
codec_layer_info.maxFrameSize.frameISize = 0;
codec_layer_info.maxFrameSize.framePSize = 0;
@@ -1819,8 +1873,8 @@ void Encoder::encode_pass_control_command(VkCommandBuffer command_buffer, Encode
layer_info.maxBitrate = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? (uint32_t)(this->setting_bit_rate * 1000000.0) : 0;
layer_info.frameRateNumerator = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 1 : 0;
layer_info.frameRateDenominator = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? this->setting_frame_rate : 0;
- layer_info.virtualBufferSizeInMs = 0;
- layer_info.initialVirtualBufferSizeInMs = 0;
+ layer_info.virtualBufferSizeInMs = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 10 : 0;
+ layer_info.initialVirtualBufferSizeInMs = (this->setting_mode == ENCODER_MODE_CONSTANT_BITRATE) ? 10 : 0;
VkVideoEncodeH264RateControlInfoEXT codec_rate_info; //Currently not used
codec_rate_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_RATE_CONTROL_INFO_EXT;
@@ -1829,7 +1883,7 @@ void Encoder::encode_pass_control_command(VkCommandBuffer command_buffer, Encode
codec_rate_info.idrPeriod = this->setting_key_rate;
codec_rate_info.consecutiveBFrameCount = 0;
codec_rate_info.rateControlStructure = VK_VIDEO_ENCODE_H264_RATE_CONTROL_STRUCTURE_UNKNOWN_EXT;
- codec_rate_info.temporalLayerCount = 0;
+ codec_rate_info.temporalLayerCount = 1;
VkVideoEncodeRateControlInfoKHR rate_info; //Currently not used
rate_info.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_INFO_KHR;
@@ -1841,10 +1895,12 @@ void Encoder::encode_pass_control_command(VkCommandBuffer command_buffer, Encode
VkVideoCodingControlInfoKHR control_info;
control_info.sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR;
- control_info.pNext = &rate_info;
- control_info.flags = VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR | VK_VIDEO_CODING_CONTROL_ENCODE_RATE_CONTROL_BIT_KHR | VK_VIDEO_CODING_CONTROL_ENCODE_RATE_CONTROL_LAYER_BIT_KHR;
+ control_info.pNext = &layer_info;
+ control_info.flags = VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR | VK_VIDEO_CODING_CONTROL_ENCODE_RATE_CONTROL_LAYER_BIT_KHR;
vkCmdControlVideoCodingKHR(command_buffer, &control_info);
+
+ this->invalidate_slots();
}
void Encoder::encode_pass_config_setup(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame)
@@ -1924,29 +1980,59 @@ void Encoder::encode_pass_frame_setup(VkCommandBuffer command_buffer, EncoderFra
image_range.baseArrayLayer = 0;
image_range.layerCount = 1;
- VkImageMemoryBarrier slot_barrier;
- slot_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
- slot_barrier.pNext = nullptr;
- slot_barrier.srcAccessMask = 0;
- slot_barrier.dstAccessMask = 0;
- slot_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
- slot_barrier.newLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
- slot_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- slot_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- slot_barrier.image = frame->slot_image;
- slot_barrier.subresourceRange = image_range;
-
- vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, &slot_barrier);
-
+ std::vector<VkImageMemoryBarrier2> slot_barriers;
std::vector<VkVideoPictureResourceInfoKHR> slot_resources;
std::vector<VkVideoReferenceSlotInfoKHR> slot_infos;
+ slot_barriers.resize(reference_slots.size() + 2);
slot_resources.resize(reference_slots.size());
slot_infos.resize(reference_slots.size());
+ VkImageMemoryBarrier2& read_input_barrier = slot_barriers[0];
+ read_input_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2;
+ read_input_barrier.pNext = nullptr;
+ read_input_barrier.srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT;
+ read_input_barrier.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
+ read_input_barrier.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR;
+ read_input_barrier.dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_READ_BIT_KHR;
+ read_input_barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+ read_input_barrier.newLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR;
+ read_input_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ read_input_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ read_input_barrier.image = frame->input_combined;
+ read_input_barrier.subresourceRange = image_range;
+
+ VkImageMemoryBarrier2& write_slot_barrier = slot_barriers[1];
+ write_slot_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
+ write_slot_barrier.pNext = nullptr;
+ write_slot_barrier.srcStageMask = VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT;
+ write_slot_barrier.srcAccessMask = 0;
+ write_slot_barrier.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR;
+ write_slot_barrier.dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_WRITE_BIT_KHR;
+ write_slot_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+ write_slot_barrier.newLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
+ write_slot_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ write_slot_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ write_slot_barrier.image = frame->slot_image;
+ write_slot_barrier.subresourceRange = image_range;
+
for (uint32_t index = 0; index < reference_slots.size(); index++)
{
EncoderFrame::Ptr reference_slot = reference_slots[index];
+ VkImageMemoryBarrier2& slot_barrier = slot_barriers[index + 2];
+ slot_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
+ slot_barrier.pNext = nullptr;
+ slot_barrier.srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR;
+ slot_barrier.srcAccessMask = VK_ACCESS_2_VIDEO_ENCODE_WRITE_BIT_KHR;
+ slot_barrier.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR;
+ slot_barrier.dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_READ_BIT_KHR;
+ slot_barrier.oldLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
+ slot_barrier.newLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
+ slot_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ slot_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ slot_barrier.image = frame->slot_image;
+ slot_barrier.subresourceRange = image_range;
+
VkVideoPictureResourceInfoKHR& slot_resource = slot_resources[index];
slot_resource.sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR;
slot_resource.pNext = nullptr;
@@ -1964,6 +2050,19 @@ void Encoder::encode_pass_frame_setup(VkCommandBuffer command_buffer, EncoderFra
slot_info.pPictureResource = &slot_resource;
}
+ VkDependencyInfo slot_dependency;
+ slot_dependency.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
+ slot_dependency.pNext = nullptr;
+ slot_dependency.dependencyFlags = 0;
+ slot_dependency.memoryBarrierCount = 0;
+ slot_dependency.pMemoryBarriers = nullptr;
+ slot_dependency.bufferMemoryBarrierCount = 0;
+ slot_dependency.pBufferMemoryBarriers = nullptr;
+ slot_dependency.imageMemoryBarrierCount = slot_barriers.size();
+ slot_dependency.pImageMemoryBarriers = slot_barriers.data();
+
+ vkCmdPipelineBarrier2KHR(command_buffer, &slot_dependency);
+
VkVideoBeginCodingInfoKHR begin_info;
begin_info.sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR;
begin_info.pNext = nullptr;
@@ -1989,16 +2088,31 @@ void Encoder::encode_pass_frame_setup(VkCommandBuffer command_buffer, EncoderFra
vkCmdEndVideoCodingKHR(command_buffer, &end_info);
- VkBufferMemoryBarrier buffer_barrier;
- buffer_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
+ VkBufferMemoryBarrier2 buffer_barrier;
+ buffer_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2;
buffer_barrier.pNext = nullptr;
- buffer_barrier.srcAccessMask = 0;
- buffer_barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
+ buffer_barrier.srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR;
+ buffer_barrier.srcAccessMask = VK_ACCESS_2_VIDEO_ENCODE_WRITE_BIT_KHR;
+ buffer_barrier.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT;
+ buffer_barrier.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT;
buffer_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
buffer_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
buffer_barrier.buffer = frame->output_buffer;
buffer_barrier.offset = 0;
buffer_barrier.size = VK_WHOLE_SIZE;
+
+ VkDependencyInfo buffer_dependency;
+ buffer_dependency.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
+ buffer_dependency.pNext = nullptr;
+ buffer_dependency.dependencyFlags = 0;
+ buffer_dependency.memoryBarrierCount = 0;
+ buffer_dependency.pMemoryBarriers = nullptr;
+ buffer_dependency.bufferMemoryBarrierCount = 1;
+ buffer_dependency.pBufferMemoryBarriers = &buffer_barrier;
+ buffer_dependency.imageMemoryBarrierCount = 0;
+ buffer_dependency.pImageMemoryBarriers = nullptr;
+
+ vkCmdPipelineBarrier2KHR(command_buffer, &buffer_dependency);
}
void Encoder::encode_pass_frame_command(VkCommandBuffer command_buffer, EncoderFrame::Ptr frame, const std::vector<EncoderFrame::Ptr>& reference_slots)
@@ -2238,6 +2352,8 @@ bool Encoder::is_key_frame() const
return (this->frame_index % this->setting_key_rate) == 0;
}
+VkPhysicalDeviceSynchronization2Features sync_feature;
+
bool setup_encoder(lava::device::create_param& parameters)
{
parameters.extensions.push_back(VK_KHR_VIDEO_QUEUE_EXTENSION_NAME);
@@ -2249,6 +2365,11 @@ bool setup_encoder(lava::device::create_param& parameters)
parameters.add_queue(VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
+ sync_feature.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SYNCHRONIZATION_2_FEATURES;
+ sync_feature.pNext = (void*)parameters.next;
+ sync_feature.synchronization2 = VK_TRUE;
+ parameters.next = &sync_feature;
+
return true;
}
diff --git a/src/utility/encoder.hpp b/src/utility/encoder.hpp
index 77c3f5ec..4e221f85 100644
--- a/src/utility/encoder.hpp
+++ b/src/utility/encoder.hpp
@@ -120,8 +120,10 @@ public:
void set_bit_rate(double bit_rate);
//NOTE: The frame rate in frames per second.
void set_frame_rate(uint32_t frame_rate);
- //NOTE: The quality in range from 0 to 51, where 0 means a lowest compression and 51 means highest compression.
- void set_quality(uint32_t quality);
+ //NOTE: The quality of i-frames in range from 0 to 51, where 0 means a lowest compression and 51 means highest compression.
+ void set_quality_iframe(uint32_t quality);
+ //NOTE: The quality of p-frames in range from 0 to 51, where 0 means a lowest compression and 51 means highest compression.
+ void set_quality_pframe(uint32_t quality);
bool encode_frame(VkCommandBuffer command_buffer, lava::renderer& renderer, lava::image::ptr image, VkImageLayout image_layout, EncoderCallback callback);
@@ -195,7 +197,8 @@ private:
uint32_t setting_key_rate = 120;
double setting_bit_rate = 2.0; //NOTE: Bitrate in Mbit per seconds
uint32_t setting_frame_rate = 90;
- uint32_t setting_quality = 51;
+ uint32_t setting_quality_iframe = 51;
+ uint32_t setting_quality_pframe = 51;
uint32_t setting_reference_frames = 1;
diff --git a/src/utility/pass_timer.cpp b/src/utility/pass_timer.cpp
index e1584b25..ad154c2a 100644
--- a/src/utility/pass_timer.cpp
+++ b/src/utility/pass_timer.cpp
@@ -231,13 +231,37 @@ bool PassTimer::evaluate_wating_passes(lava::device_ptr device, lava::index fram
if (!found)
{
- this->pass_statistcs.erase(this->pass_statistcs.begin() + index);
- }
+ bool inactive = true;
- else
- {
- index++;
+ for (int32_t index = 1; index < glm::min((uint32_t)PASS_TIMER_STATISTIC_INACTIVE_FRAMES, (uint32_t)this->frame_statistics.size()); index++)
+ {
+ uint32_t old_index = (this->frame_statistics.size() - 1) - index;
+ FrameStatistic old_statistic = this->frame_statistics[old_index];
+
+ for (const PassStatistic& pass_statistic : old_statistic)
+ {
+ if (statistic->get_name() == pass_statistic.name)
+ {
+ inactive = false;
+ break;
+ }
+ }
+
+ if (!inactive)
+ {
+ break;
+ }
+ }
+
+ if (inactive)
+ {
+ this->pass_statistcs.erase(this->pass_statistcs.begin() + index);
+
+ continue;
+ }
}
+
+ index++;
}
for (const PassStatistic& pass_statistic : frame_statistic)
diff --git a/src/utility/pass_timer.hpp b/src/utility/pass_timer.hpp
index 20e34933..eb048da1 100644
--- a/src/utility/pass_timer.hpp
+++ b/src/utility/pass_timer.hpp
@@ -20,6 +20,8 @@
#include "utility/statistic.hpp"
+#define PASS_TIMER_STATISTIC_INACTIVE_FRAMES 128
+
struct PassWaitInfo
{
std::string name;
@@ -48,7 +50,7 @@ public:
bool create(lava::device_ptr device, uint32_t frame_count, uint32_t pass_count);
void destroy(lava::device_ptr device);
void interface();
-
+
bool write_to_file(lava::device_ptr device);
void next_frame(VkCommandBuffer command_buffer, lava::index frame, lava::device_ptr device);
--
GitLab