From 17eca070eff29ae60b2fec2d7da0de4bca64bd30 Mon Sep 17 00:00:00 2001
From: Jens Koenen <koenen@vr.rwth-aachen.de>
Date: Wed, 23 Aug 2023 16:07:41 +0200
Subject: [PATCH] Moved the computation of the quad tree to the GPU
---
CMakeLists.txt | 1 +
.../mesh_based_reprojection_defines.inc | 1 +
.../mesh_based_reprojection_quad_tree.comp | 43 ++
src/strategy/mesh_based_reprojection.cpp | 531 +++++++++++++-----
src/strategy/mesh_based_reprojection.hpp | 29 +-
5 files changed, 462 insertions(+), 143 deletions(-)
create mode 100644 res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_quad_tree.comp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 139c9bcb..572b1932 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -791,6 +791,7 @@ message("=======================================================================
strategy/mesh_based_reprojection/mesh_based_reprojection_indirect.frag
strategy/mesh_based_reprojection/mesh_based_reprojection_indirect_shadow.frag
strategy/mesh_based_reprojection/mesh_based_reprojection_edge_detection.comp
+ strategy/mesh_based_reprojection/mesh_based_reprojection_quad_tree.comp
strategy/mesh_based_reprojection/mesh_based_reprojection_overlay.vert
strategy/mesh_based_reprojection/mesh_based_reprojection_overlay.frag
strategy/mesh_based_reprojection/mesh_based_reprojection_mesh_overlay.vert
diff --git a/res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_defines.inc b/res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_defines.inc
index 148a999f..1223ee0c 100644
--- a/res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_defines.inc
+++ b/res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_defines.inc
@@ -2,5 +2,6 @@
#define SHADER_INCLUDE_MESH_BASED_REPROJECTION_DEFINES
#define MESH_BASED_REPROJECTION_EDGE_DETECTION_GROUP_SIZE 16
+#define MESH_BASED_REPROJECTION_QUAD_TREE_GROUP_SIZE 16
#endif
\ No newline at end of file
diff --git a/res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_quad_tree.comp b/res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_quad_tree.comp
new file mode 100644
index 00000000..18476a62
--- /dev/null
+++ b/res/dpr/strategy/mesh_based_reprojection/mesh_based_reprojection_quad_tree.comp
@@ -0,0 +1,43 @@
+#version 450 core
+#extension GL_GOOGLE_include_directive : require
+
+#include "mesh_based_reprojection_defines.inc"
+
+layout(local_size_x = MESH_BASED_REPROJECTION_QUAD_TREE_GROUP_SIZE, local_size_y = MESH_BASED_REPROJECTION_QUAD_TREE_GROUP_SIZE, local_size_z = 1) in;
+
+layout(set = 0, binding = 0, r32f) uniform image2D image_layer_edge_src;
+layout(set = 0, binding = 1, r32f) uniform image2D image_layer_edge_dst;
+
+void main()
+{
+ ivec2 src_resolution = imageSize(image_layer_edge_src);
+ ivec2 dst_resolution = imageSize(image_layer_edge_dst);
+
+ ivec2 dst_coord = ivec2(gl_GlobalInvocationID);
+
+ if(any(greaterThanEqual(dst_coord, dst_resolution)))
+ {
+ return;
+ }
+
+ float dst_value = -1.0;
+
+ for(uint offset_y = 0; offset_y < 2; offset_y++)
+ {
+ for(uint offset_x = 0; offset_x < 2; offset_x++)
+ {
+ ivec2 src_coord = dst_coord * 2 + ivec2(offset_x, offset_y);
+
+ if(any(greaterThanEqual(src_coord, src_resolution)))
+ {
+ return;
+ }
+
+ float src_value = imageLoad(image_layer_edge_src, src_coord).x;
+
+ dst_value = max(dst_value, src_value);
+ }
+ }
+
+ imageStore(image_layer_edge_dst, dst_coord, vec4(dst_value));
+}
\ No newline at end of file
diff --git a/src/strategy/mesh_based_reprojection.cpp b/src/strategy/mesh_based_reprojection.cpp
index 698ebd90..97211fab 100644
--- a/src/strategy/mesh_based_reprojection.cpp
+++ b/src/strategy/mesh_based_reprojection.cpp
@@ -22,22 +22,24 @@ namespace glsl
MeshBasedReprojectionQuadTreeLevel::MeshBasedReprojectionQuadTreeLevel(const glm::uvec2& resolution) : resolution(resolution)
{
- this->buffer.resize(resolution.x * resolution.y);
+
}
-bool MeshBasedReprojectionQuadTreeLevel::fill(float* data, uint32_t size)
+bool MeshBasedReprojectionQuadTreeLevel::fill(lava::buffer::ptr buffer)
{
- if (this->buffer.size() * sizeof(float) != size)
+ uint32_t buffer_size = this->resolution.x * this->resolution.y * sizeof(float);
+
+ if (buffer->get_size() != buffer_size)
{
return false;
}
- memcpy(this->buffer.data(), data, size);
+ this->buffer = (float*)buffer->get_mapped_data();
return true;
}
-bool MeshBasedReprojectionQuadTreeLevel::set_pixel(const glm::ivec2& coord, float value)
+inline bool MeshBasedReprojectionQuadTreeLevel::set_pixel(const glm::ivec2& coord, float value)
{
if (coord.x < 0 || coord.x >= this->resolution.x)
{
@@ -55,7 +57,7 @@ bool MeshBasedReprojectionQuadTreeLevel::set_pixel(const glm::ivec2& coord, floa
return true;
}
-bool MeshBasedReprojectionQuadTreeLevel::get_pixel(const glm::ivec2& coord, float& value) const
+inline bool MeshBasedReprojectionQuadTreeLevel::get_pixel(const glm::ivec2& coord, float& value) const
{
if (coord.x < 0 || coord.x >= this->resolution.x)
{
@@ -104,49 +106,18 @@ MeshBasedReprojectionQuadTree::MeshBasedReprojectionQuadTree(const glm::uvec2& r
this->levels.push_back(make_mesh_based_reprojection_quad_tree_level(level_resolution));
}
-bool MeshBasedReprojectionQuadTree::fill(lava::buffer::ptr buffer)
+bool MeshBasedReprojectionQuadTree::fill(const std::vector<lava::buffer::ptr>& buffers)
{
- MeshBasedReprojectionQuadTreeLevel::Ptr base_level = this->levels.front();
-
- if (!base_level->fill((float*) buffer->get_mapped_data(), buffer->get_size()))
+ if (buffers.size() != this->levels.size())
{
return false;
}
- for (uint32_t index = 1; index < this->levels.size(); index++)
+ for (uint32_t index = 0; index < this->levels.size(); index++)
{
- MeshBasedReprojectionQuadTreeLevel::Ptr src_level = this->levels[index - 1];
- MeshBasedReprojectionQuadTreeLevel::Ptr dst_level = this->levels[index];
- glm::uvec2 dst_resolution = dst_level->get_resolution();
-
- for (uint32_t dst_y = 0; dst_y < dst_resolution.y; dst_y++)
+ if (!this->levels[index]->fill(buffers[index]))
{
- for (uint32_t dst_x = 0; dst_x < dst_resolution.x; dst_x++)
- {
- glm::ivec2 dst_coord = glm::ivec2(dst_x, dst_y);
- float dst_value = -1.0f;
-
- for (uint32_t offset_y = 0; offset_y < 2; offset_y++)
- {
- for (uint32_t offset_x = 0; offset_x < 2; offset_x++)
- {
- glm::ivec2 src_coord = dst_coord * 2 + glm::ivec2(offset_x, offset_y);
- float src_value = 0.0f;
-
- if (!src_level->get_pixel(src_coord, src_value))
- {
- continue;
- }
-
- dst_value = glm::max(dst_value, src_value);
- }
- }
-
- if(!dst_level->set_pixel(dst_coord, dst_value))
- {
- return false;
- }
- }
+ return false;
}
}
@@ -424,6 +395,11 @@ bool MeshBasedReprojection::on_create()
return false;
}
+ if (!this->create_quad_tree_pipeline())
+ {
+ return false;
+ }
+
if (!this->create_overlay_pass())
{
return false;
@@ -444,10 +420,16 @@ bool MeshBasedReprojection::on_create()
return false;
}
- this->statistic_edge_buffer_time = make_statistic("edge_buffer_time", UNIT_MILLISECONDS);
+ this->statistic_line_trace_time = make_statistic("line_trace_time", UNIT_MILLISECONDS);
+ this->statistic_triangulate_time = make_statistic("triangulate_time", UNIT_MILLISECONDS);
+ this->statistic_depth_sample_time = make_statistic("depth_sample_time", UNIT_MILLISECONDS);
+ this->statistic_metadata_bitrate = make_statistic("metadata_bitrate", UNIT_MBITS_PER_SECOND);
StatisticLog::Ptr statistic_log = this->get_application()->get_statistic_log();
- statistic_log->add_statistic(this->statistic_edge_buffer_time);
+ statistic_log->add_statistic(this->statistic_line_trace_time);
+ statistic_log->add_statistic(this->statistic_triangulate_time);
+ statistic_log->add_statistic(this->statistic_depth_sample_time);
+ statistic_log->add_statistic(this->statistic_metadata_bitrate);
return true;
}
@@ -498,7 +480,10 @@ bool MeshBasedReprojection::on_interface()
ImGui::Separator();
- this->statistic_edge_buffer_time->interface(128);
+ this->statistic_line_trace_time->interface(128);
+ this->statistic_triangulate_time->interface(128);
+ this->statistic_depth_sample_time->interface(128);
+ this->statistic_metadata_bitrate->interface(128);
return true;
}
@@ -508,12 +493,30 @@ bool MeshBasedReprojection::on_update(lava::delta delta_time)
std::unique_lock<std::mutex> lock(this->edge_buffer_mutex);
if (this->edge_buffer_process_count > 0)
{
- double edge_buffer_process_time = (double)this->edge_buffer_process_time_sum / this->edge_buffer_process_count;
+ double line_trace_time = this->line_trace_time_sum / (double) this->edge_buffer_process_count;
+ double triangulate_time = this->triangulate_time_sum / (double) this->edge_buffer_process_count;
+ double depth_sample_time = this->depth_sample_time_sum / (double) this->edge_buffer_process_count;
- this->statistic_edge_buffer_time->add_sample(edge_buffer_process_time);
+ this->statistic_line_trace_time->add_sample(line_trace_time);
+ this->statistic_triangulate_time->add_sample(triangulate_time);
+ this->statistic_depth_sample_time->add_sample(depth_sample_time);
this->edge_buffer_process_count = 0;
- this->edge_buffer_process_time_sum = 0.0f;
+ this->line_trace_time_sum = 0.0;
+ this->triangulate_time_sum = 0.0;
+ this->depth_sample_time_sum = 0.0;
+ }
+
+ this->metadata_time += delta_time;
+
+ if (this->metadata_time > 1.0)
+ {
+ double metadata_bitrate = ((double) this->metadata_bits / (1000.0 * 1000.0)) / this->metadata_time;
+
+ this->statistic_metadata_bitrate->add_sample(metadata_bitrate);
+
+ this->metadata_time = 0.0;
+ this->metadata_bits = 0.0;
}
lock.unlock();
@@ -589,6 +592,10 @@ bool MeshBasedReprojection::on_render(VkCommandBuffer command_buffer, lava::inde
this->process_edge_detection_pass(command_buffer);
this->get_pass_timer()->end_pass(command_buffer);
+ this->get_pass_timer()->begin_pass(command_buffer, "quad_tree");
+ this->process_quad_tree_pass(command_buffer);
+ this->get_pass_timer()->end_pass(command_buffer);
+
if (frame_number != FRAME_NUMBER_DROPPED)
{
if (!this->process_edge_buffer_copy(command_buffer, frame_number))
@@ -705,8 +712,8 @@ bool MeshBasedReprojection::on_render(VkCommandBuffer command_buffer, lava::inde
this->get_companion_window()->submit_image(command_buffer, EYE_RIGHT, this->layer_normal_buffers[1], VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
break;
case MESH_BASED_REPROJECTION_OVERLAY_EDGE_DETECTION:
- this->get_companion_window()->submit_image(command_buffer, EYE_LEFT, this->layer_edge_buffers[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
- this->get_companion_window()->submit_image(command_buffer, EYE_RIGHT, this->layer_edge_buffers[1], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+ this->get_companion_window()->submit_image(command_buffer, EYE_LEFT, this->layer_edge_buffers[0][0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+ this->get_companion_window()->submit_image(command_buffer, EYE_RIGHT, this->layer_edge_buffers[1][0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
break;
case MESH_BASED_REPROJECTION_OVERLAY_EDGE:
case MESH_BASED_REPROJECTION_OVERLAY_TRIANGLE:
@@ -744,7 +751,6 @@ bool MeshBasedReprojection::create_buffers()
{
glm::uvec2 resolution = this->get_headset()->get_framebuffer_resolution();
uint32_t depth_image_buffer_size = resolution.x * resolution.y * sizeof(float);
- uint32_t edge_image_buffer_size = resolution.x * resolution.y * sizeof(float);
uint32_t edge_geometry_buffer_size = MESH_BASED_REPROJECTION_EDGE_GEOMETRY_BUFFER_SIZE * 2 * sizeof(glm::vec2);
uint32_t triangle_geometry_buffer_size = MESH_BASED_REPROJECTION_TRIANGLE_GEOMETRY_BUFFER_SIZE * 3 * sizeof(glm::vec3);
@@ -834,17 +840,37 @@ bool MeshBasedReprojection::create_buffers()
for (uint32_t layer = 0; layer < this->layer_edge_buffers.size(); layer++)
{
- lava::image::ptr edge_buffer = lava::make_image(VK_FORMAT_R32_SFLOAT);
- edge_buffer->set_usage(VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT);
+ this->layer_edge_buffers[layer].resize(this->layer_quad_tree_levels);
+ glm::uvec2 level_resolution = resolution;
- if (!edge_buffer->create(this->get_device(), resolution))
+ for (uint32_t level = 0; level < this->layer_quad_tree_levels; level++)
{
- lava::log()->error("Mesh Based Reprojection: Can't create layer edge buffer!");
+ lava::image::ptr edge_buffer = lava::make_image(VK_FORMAT_R32_SFLOAT);
+ edge_buffer->set_usage(VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT);
- return false;
- }
+ if (!edge_buffer->create(this->get_device(), level_resolution))
+ {
+ lava::log()->error("Mesh Based Reprojection: Can't create layer edge buffer!");
+
+ return false;
+ }
+
+ this->layer_edge_buffers[layer][level] = edge_buffer;
+
+ glm::uvec2 next_resolution = level_resolution / 2u;
+
+ if (level_resolution.x % 2 != 0)
+ {
+ next_resolution.x += 1;
+ }
+
+ if (level_resolution.y % 2 != 0)
+ {
+ next_resolution.y += 1;
+ }
- this->layer_edge_buffers[layer] = edge_buffer;
+ level_resolution = next_resolution;
+ }
}
for (uint32_t layer = 0; layer < this->layer_overlay_color_buffers.size(); layer++)
@@ -906,7 +932,7 @@ bool MeshBasedReprojection::create_buffers()
}
std::array<lava::buffer::ptr, 2> depth_image_buffers;
- std::array<lava::buffer::ptr, 2> edge_image_buffers;
+ std::array<std::vector<lava::buffer::ptr>, 2> edge_image_buffers;
for (uint32_t layer = 0; layer < 2; layer++)
{
@@ -919,13 +945,35 @@ bool MeshBasedReprojection::create_buffers()
return false;
}
- edge_image_buffers[layer] = lava::make_buffer();
+ edge_image_buffers[layer].resize(this->layer_quad_tree_levels);
+ glm::uvec2 level_resolution = resolution;
- if (!edge_image_buffers[layer]->create_mapped(this->get_device(), nullptr, edge_image_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT, VMA_MEMORY_USAGE_GPU_TO_CPU))
+ for (uint32_t level = 0; level < this->layer_quad_tree_levels; level++)
{
- lava::log()->error("Mesh based Reprojection: Can't create edge image buffer!");
+ uint32_t edge_image_buffer_size = level_resolution.x * level_resolution.y * sizeof(float);
+
+ edge_image_buffers[layer][level] = lava::make_buffer();
- return false;
+ if (!edge_image_buffers[layer][level]->create_mapped(this->get_device(), nullptr, edge_image_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT, VMA_MEMORY_USAGE_GPU_TO_CPU))
+ {
+ lava::log()->error("Mesh based Reprojection: Can't create edge image buffer!");
+
+ return false;
+ }
+
+ glm::uvec2 next_resolution = level_resolution / 2u;
+
+ if (level_resolution.x % 2 != 0)
+ {
+ next_resolution.x += 1;
+ }
+
+ if (level_resolution.y % 2 != 0)
+ {
+ next_resolution.y += 1;
+ }
+
+ level_resolution = next_resolution;
}
}
@@ -982,13 +1030,13 @@ bool MeshBasedReprojection::create_descriptors()
lava::VkDescriptorPoolSizes descriptor_type_count =
{
{ VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4 },
- { VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2 },
+ { VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2 + 4 * (this->layer_quad_tree_levels - 1) },
{ VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, frame_count }
};
this->descriptor_pool = lava::make_descriptor_pool();
- if (!this->descriptor_pool->create(this->get_device(), descriptor_type_count, frame_count + 4 + 2))
+ if (!this->descriptor_pool->create(this->get_device(), descriptor_type_count, frame_count + 4 + 2 + 2 * (this->layer_quad_tree_levels - 1)))
{
lava::log()->error("Mesh Based Reprojection: Can't create descriptor pool!");
@@ -1026,8 +1074,22 @@ bool MeshBasedReprojection::create_descriptors()
return false;
}
- std::array<VkDescriptorImageInfo, 6> descriptor_image_infos;
+ this->quad_tree_descriptor = lava::make_descriptor();
+ this->quad_tree_descriptor->add_binding(0, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT);
+ this->quad_tree_descriptor->add_binding(1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT);
+
+ if (!this->quad_tree_descriptor->create(this->get_device()))
+ {
+ lava::log()->error("Mesh Based Reprojection: Can't create quad tree descriptor!");
+
+ return false;
+ }
+
std::vector<VkWriteDescriptorSet> descriptor_writes;
+ std::vector<VkDescriptorImageInfo> descriptor_image_infos;
+ descriptor_image_infos.resize(6 + 4 * (this->layer_quad_tree_levels - 1));
+
+ uint32_t descriptor_image_info_offset = 0;
this->transform_descriptor_sets.resize(frame_count, VK_NULL_HANDLE);
@@ -1052,16 +1114,20 @@ bool MeshBasedReprojection::create_descriptors()
{
this->layer_descriptor_sets[layer] = this->layer_descriptor->allocate(this->descriptor_pool->get());
- VkDescriptorImageInfo& depth_image_info = descriptor_image_infos[layer * 2];
+ VkDescriptorImageInfo& depth_image_info = descriptor_image_infos[descriptor_image_info_offset];
depth_image_info.sampler = this->nearest_sampler;
depth_image_info.imageView = this->layer_depth_buffers[layer]->get_view();
depth_image_info.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
- VkDescriptorImageInfo& normal_image_info = descriptor_image_infos[layer * 2 + 1];
+ descriptor_image_info_offset++;
+
+ VkDescriptorImageInfo& normal_image_info = descriptor_image_infos[descriptor_image_info_offset];
normal_image_info.sampler = this->nearest_sampler;
normal_image_info.imageView = this->layer_normal_buffers[layer]->get_view();
normal_image_info.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+ descriptor_image_info_offset++;
+
VkWriteDescriptorSet& descriptor_write_depth = descriptor_writes.emplace_back();
descriptor_write_depth.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
descriptor_write_depth.pNext = nullptr;
@@ -1091,11 +1157,13 @@ bool MeshBasedReprojection::create_descriptors()
{
this->edge_descriptor_sets[layer] = this->edge_descriptor->allocate(this->descriptor_pool->get());
- VkDescriptorImageInfo& descriptor_image_info = descriptor_image_infos[layer + 4];
+ VkDescriptorImageInfo& descriptor_image_info = descriptor_image_infos[descriptor_image_info_offset];
descriptor_image_info.sampler = VK_NULL_HANDLE;
- descriptor_image_info.imageView = this->layer_edge_buffers[layer]->get_view();
+ descriptor_image_info.imageView = this->layer_edge_buffers[layer][0]->get_view();
descriptor_image_info.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
+ descriptor_image_info_offset++;
+
VkWriteDescriptorSet& descriptor_write = descriptor_writes.emplace_back();
descriptor_write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
descriptor_write.pNext = nullptr;
@@ -1109,6 +1177,54 @@ bool MeshBasedReprojection::create_descriptors()
descriptor_write.pTexelBufferView = nullptr;
}
+ for (uint32_t layer = 0; layer < 2; layer++)
+ {
+ this->quad_tree_descriptor_sets[layer].resize(this->layer_quad_tree_levels - 1);
+
+ for (uint32_t level = 0; level < this->layer_quad_tree_levels - 1; level++)
+ {
+ this->quad_tree_descriptor_sets[layer][level] = this->quad_tree_descriptor->allocate(this->descriptor_pool->get());
+
+ VkDescriptorImageInfo& src_image_info = descriptor_image_infos[descriptor_image_info_offset];
+ src_image_info.sampler = VK_NULL_HANDLE;
+ src_image_info.imageView = this->layer_edge_buffers[layer][level]->get_view();
+ src_image_info.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
+
+ descriptor_image_info_offset++;
+
+ VkDescriptorImageInfo& dst_image_info = descriptor_image_infos[descriptor_image_info_offset];
+ dst_image_info.sampler = VK_NULL_HANDLE;
+ dst_image_info.imageView = this->layer_edge_buffers[layer][level + 1]->get_view();
+ dst_image_info.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
+
+ descriptor_image_info_offset++;
+
+ VkWriteDescriptorSet& descriptor_write_src = descriptor_writes.emplace_back();
+ descriptor_write_src.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
+ descriptor_write_src.pNext = nullptr;
+ descriptor_write_src.dstSet = this->quad_tree_descriptor_sets[layer][level];
+ descriptor_write_src.dstBinding = 0;
+ descriptor_write_src.dstArrayElement = 0;
+ descriptor_write_src.descriptorCount = 1;
+ descriptor_write_src.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
+ descriptor_write_src.pImageInfo = &src_image_info;
+ descriptor_write_src.pBufferInfo = nullptr;
+ descriptor_write_src.pTexelBufferView = nullptr;
+
+ VkWriteDescriptorSet& descriptor_write_dst = descriptor_writes.emplace_back();
+ descriptor_write_dst.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
+ descriptor_write_dst.pNext = nullptr;
+ descriptor_write_dst.dstSet = this->quad_tree_descriptor_sets[layer][level];
+ descriptor_write_dst.dstBinding = 1;
+ descriptor_write_dst.dstArrayElement = 0;
+ descriptor_write_dst.descriptorCount = 1;
+ descriptor_write_dst.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
+ descriptor_write_dst.pImageInfo = &dst_image_info;
+ descriptor_write_dst.pBufferInfo = nullptr;
+ descriptor_write_dst.pTexelBufferView = nullptr;
+ }
+ }
+
vkUpdateDescriptorSets(this->get_device()->get(), descriptor_writes.size(), descriptor_writes.data(), 0, nullptr);
return true;
@@ -1346,6 +1462,38 @@ bool MeshBasedReprojection::create_edge_detection_pipeline()
return true;
}
+bool MeshBasedReprojection::create_quad_tree_pipeline()
+{
+ this->quad_tree_pipeline_layout = lava::make_pipeline_layout();
+ this->quad_tree_pipeline_layout->add(this->quad_tree_descriptor);
+
+ if (!this->quad_tree_pipeline_layout->create(this->get_device()))
+ {
+ lava::log()->error("Mesh Based Reprojection: Can't create quad tree pipeline layout!");
+
+ return false;
+ }
+
+ this->quad_tree_pipeline = lava::make_compute_pipeline(this->get_device());
+ this->quad_tree_pipeline->set_layout(this->quad_tree_pipeline_layout);
+
+ if (!this->quad_tree_pipeline->set_shader_stage(lava::file_data("dpr/binary/mesh_based_reprojection_quad_tree_compute.spirv"), VK_SHADER_STAGE_COMPUTE_BIT))
+ {
+ lava::log()->error("Mesh Based Reprojection: Can't load compute shader for quad tree pipeline!");
+
+ return false;
+ }
+
+ if (!this->quad_tree_pipeline->create())
+ {
+ lava::log()->error("Mesh Based Reprojection: Can't create quad tree pipeline!");
+
+ return false;
+ }
+
+ return true;
+}
+
bool MeshBasedReprojection::create_overlay_pass()
{
lava::attachment::ptr color_attachment = lava::make_attachment(this->get_headset()->get_format());
@@ -1720,6 +1868,27 @@ bool MeshBasedReprojection::build_projection()
this->layer_head_to_eye_matrix = glm::translate(glm::mat4(1.0f), -center_point_combined);
this->layer_projection_matrix = glm::perspective(horizontal_angle_combined, aspect_ratio_combined, near_combined, far_combined);
this->layer_inv_projection_matrix = glm::inverse(this->layer_projection_matrix);
+ this->layer_quad_tree_levels = 1;
+
+ glm::uvec2 level_resolution = this->layer_resolution;
+
+ while (level_resolution.x > 1 && level_resolution.y > 1)
+ {
+ glm::uvec2 next_resolution = level_resolution / 2u;
+
+ if (level_resolution.x % 2 != 0)
+ {
+ next_resolution.x += 1;
+ }
+
+ if (level_resolution.y % 2 != 0)
+ {
+ next_resolution.y += 1;
+ }
+
+ level_resolution = next_resolution;
+ this->layer_quad_tree_levels++;
+ }
return true;
}
@@ -1880,24 +2049,29 @@ void MeshBasedReprojection::process_edge_detection_pass(VkCommandBuffer command_
work_group_count.y += 1;
}
- std::array<VkImageMemoryBarrier, 2> image_barriers;
+ std::vector<VkImageMemoryBarrier> image_barriers;
for (uint32_t layer = 0; layer < 2; layer++)
{
- image_barriers[layer].sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
- image_barriers[layer].pNext = nullptr;
- image_barriers[layer].srcAccessMask = 0;
- image_barriers[layer].dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
- image_barriers[layer].oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
- image_barriers[layer].newLayout = VK_IMAGE_LAYOUT_GENERAL;
- image_barriers[layer].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- image_barriers[layer].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- image_barriers[layer].image = this->layer_edge_buffers[layer]->get();
- image_barriers[layer].subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
- image_barriers[layer].subresourceRange.baseMipLevel = 0;
- image_barriers[layer].subresourceRange.levelCount = 1;
- image_barriers[layer].subresourceRange.baseArrayLayer = 0;
- image_barriers[layer].subresourceRange.layerCount = 1;
+ for (uint32_t level = 0; level < this->layer_quad_tree_levels; level++)
+ {
+ VkImageMemoryBarrier& image_barrier = image_barriers.emplace_back();
+
+ image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
+ image_barrier.pNext = nullptr;
+ image_barrier.srcAccessMask = 0;
+ image_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
+ image_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+ image_barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+ image_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ image_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ image_barrier.image = this->layer_edge_buffers[layer][level]->get();
+ image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+ image_barrier.subresourceRange.baseMipLevel = 0;
+ image_barrier.subresourceRange.levelCount = 1;
+ image_barrier.subresourceRange.baseArrayLayer = 0;
+ image_barrier.subresourceRange.layerCount = 1;
+ }
}
vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 0, nullptr, image_barriers.size(), image_barriers.data());
@@ -1921,15 +2095,65 @@ void MeshBasedReprojection::process_edge_detection_pass(VkCommandBuffer command_
}
}
+void MeshBasedReprojection::process_quad_tree_pass(VkCommandBuffer command_buffer)
+{
+ this->quad_tree_pipeline->bind(command_buffer);
+
+ for (uint32_t level = 0; level < this->layer_quad_tree_levels - 1; level++)
+ {
+ std::array<VkImageMemoryBarrier, 2> image_barriers;
+
+ for (uint32_t layer = 0; layer < 2; layer++)
+ {
+ image_barriers[layer].sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
+ image_barriers[layer].pNext = nullptr;
+ image_barriers[layer].srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
+ image_barriers[layer].dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
+ image_barriers[layer].oldLayout = VK_IMAGE_LAYOUT_GENERAL;
+ image_barriers[layer].newLayout = VK_IMAGE_LAYOUT_GENERAL;
+ image_barriers[layer].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ image_barriers[layer].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ image_barriers[layer].image = this->layer_edge_buffers[layer][level]->get();
+ image_barriers[layer].subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+ image_barriers[layer].subresourceRange.baseMipLevel = 0;
+ image_barriers[layer].subresourceRange.levelCount = 1;
+ image_barriers[layer].subresourceRange.baseArrayLayer = 0;
+ image_barriers[layer].subresourceRange.layerCount = 1;
+ }
+
+ vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 0, nullptr, image_barriers.size(), image_barriers.data());
+
+ glm::uvec2 level_resolution = this->layer_edge_buffers[0][level]->get_size();
+ glm::uvec2 work_group_count = level_resolution / glm::uvec2(MESH_BASED_REPROJECTION_QUAD_TREE_GROUP_SIZE);
+
+ if ((level_resolution.x % MESH_BASED_REPROJECTION_QUAD_TREE_GROUP_SIZE) != 0)
+ {
+ work_group_count.x += 1;
+ }
+
+ if ((level_resolution.y % MESH_BASED_REPROJECTION_QUAD_TREE_GROUP_SIZE) != 0)
+ {
+ work_group_count.y += 1;
+ }
+
+ for (uint32_t layer = 0; layer < 2; layer++)
+ {
+ vkCmdBindDescriptorSets(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, this->quad_tree_pipeline_layout->get(), 0, 1, &this->quad_tree_descriptor_sets[layer][level], 0, nullptr);
+
+ vkCmdDispatch(command_buffer, work_group_count.x, work_group_count.y, 1);
+ }
+ }
+}
+
bool MeshBasedReprojection::process_edge_buffer_copy(VkCommandBuffer command_buffer, FrameNumber frame_number)
{
glm::uvec2 resolution = this->get_headset()->get_framebuffer_resolution();
- std::array<VkImageMemoryBarrier, 4> image_barriers;
+ std::vector<VkImageMemoryBarrier> image_barriers;
for (uint32_t layer = 0; layer < 2; layer++)
{
- VkImageMemoryBarrier& depth_barrier = image_barriers[layer * 2];
+ VkImageMemoryBarrier& depth_barrier = image_barriers.emplace_back();
depth_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
depth_barrier.pNext = nullptr;
depth_barrier.srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
@@ -1945,21 +2169,24 @@ bool MeshBasedReprojection::process_edge_buffer_copy(VkCommandBuffer command_buf
depth_barrier.subresourceRange.baseArrayLayer = 0;
depth_barrier.subresourceRange.layerCount = 1;
- VkImageMemoryBarrier& edge_barrier = image_barriers[layer * 2 + 1];
- edge_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
- edge_barrier.pNext = nullptr;
- edge_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
- edge_barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
- edge_barrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
- edge_barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
- edge_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- edge_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- edge_barrier.image = this->layer_edge_buffers[layer]->get();
- edge_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
- edge_barrier.subresourceRange.baseMipLevel = 0;
- edge_barrier.subresourceRange.levelCount = 1;
- edge_barrier.subresourceRange.baseArrayLayer = 0;
- edge_barrier.subresourceRange.layerCount = 1;
+ for (uint32_t level = 0; level < this->layer_quad_tree_levels; level++)
+ {
+ VkImageMemoryBarrier& edge_barrier = image_barriers.emplace_back();
+ edge_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
+ edge_barrier.pNext = nullptr;
+ edge_barrier.srcAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
+ edge_barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+ edge_barrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
+ edge_barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+ edge_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ edge_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ edge_barrier.image = this->layer_edge_buffers[layer][level]->get();
+ edge_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+ edge_barrier.subresourceRange.baseMipLevel = 0;
+ edge_barrier.subresourceRange.levelCount = 1;
+ edge_barrier.subresourceRange.baseArrayLayer = 0;
+ edge_barrier.subresourceRange.layerCount = 1;
+ }
}
vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, image_barriers.size(), image_barriers.data());
@@ -1990,25 +2217,40 @@ bool MeshBasedReprojection::process_edge_buffer_copy(VkCommandBuffer command_buf
depth_copy.imageExtent.height = resolution.y;
depth_copy.imageExtent.depth = 1;
- VkBufferImageCopy edge_copy = depth_copy;
- edge_copy.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
-
vkCmdCopyImageToBuffer(command_buffer, this->layer_depth_buffers[layer]->get(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, edge_buffer->depth_image_buffers[layer]->get(), 1, &depth_copy);
- vkCmdCopyImageToBuffer(command_buffer, this->layer_edge_buffers[layer]->get(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, edge_buffer->edge_image_buffers[layer]->get(), 1, &edge_copy);
-
- std::array<VkBufferMemoryBarrier, 2> buffer_barriers;
- buffer_barriers[0].sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
- buffer_barriers[0].pNext = 0;
- buffer_barriers[0].srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
- buffer_barriers[0].dstAccessMask = VK_ACCESS_HOST_READ_BIT;
- buffer_barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- buffer_barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- buffer_barriers[0].buffer = edge_buffer->depth_image_buffers[layer]->get();
- buffer_barriers[0].offset = 0;
- buffer_barriers[0].size = VK_WHOLE_SIZE;
-
- buffer_barriers[1] = buffer_barriers[0];
- buffer_barriers[1].buffer = edge_buffer->edge_image_buffers[layer]->get();
+
+ for(uint32_t level = 0; level < this->layer_quad_tree_levels; level++)
+ {
+ lava::image::ptr src_image = this->layer_edge_buffers[layer][level];
+ lava::buffer::ptr dst_buffer = edge_buffer->edge_image_buffers[layer][level];
+
+ VkBufferImageCopy edge_copy = depth_copy;
+ edge_copy.imageExtent.width = src_image->get_size().x;
+ edge_copy.imageExtent.height = src_image->get_size().y;
+ edge_copy.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+
+ vkCmdCopyImageToBuffer(command_buffer, src_image->get(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst_buffer->get(), 1, &edge_copy);
+ }
+
+ std::vector<VkBufferMemoryBarrier> buffer_barriers;
+
+ VkBufferMemoryBarrier& depth_barrier = buffer_barriers.emplace_back();
+ depth_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
+ depth_barrier.pNext = 0;
+ depth_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+ depth_barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
+ depth_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ depth_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ depth_barrier.buffer = edge_buffer->depth_image_buffers[layer]->get();
+ depth_barrier.offset = 0;
+ depth_barrier.size = VK_WHOLE_SIZE;
+
+ for (uint32_t level = 0; level < this->layer_quad_tree_levels; level++)
+ {
+ VkBufferMemoryBarrier& edge_barrier = buffer_barriers.emplace_back();
+ edge_barrier = depth_barrier;
+ edge_barrier.buffer = edge_buffer->edge_image_buffers[layer][level]->get();
+ }
vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, buffer_barriers.size(), buffer_barriers.data(), 0, nullptr);
}
@@ -2144,10 +2386,14 @@ void MeshBasedReprojection::release_edge_buffer(MeshBasedReprojectionEdgeBuffer:
void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, FrameNumber frame_number)
{
- std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
+ double line_trace_time = 0.0;
+ double triangulate_time = 0.0;
+ double depth_sample_time = 0.0;
for (uint32_t layer = 0; layer < 2; layer++)
{
+ std::chrono::high_resolution_clock::time_point line_trace_start = std::chrono::high_resolution_clock::now();
+
edge_buffer->edges[layer].clear();
edge_buffer->triangles[layer].clear();
@@ -2220,6 +2466,11 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
}
}
+ std::chrono::high_resolution_clock::time_point line_trace_end = std::chrono::high_resolution_clock::now();
+ line_trace_time += std::chrono::duration_cast<std::chrono::duration<double, std::chrono::milliseconds::period>>(line_trace_end - line_trace_start).count();
+
+ std::chrono::high_resolution_clock::time_point triangulate_start = std::chrono::high_resolution_clock::now();
+
Point2 border_point1 = Point2(1.0, 1.0);
Point2 border_point2 = Point2(1.0, resolution.y-2);
Point2 border_point3 = Point2(resolution.x-2, resolution.y-2);
@@ -2239,6 +2490,11 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
triangulation.insert_constraint(edge_start, edge_end);
}
+ std::chrono::high_resolution_clock::time_point triangulate_end = std::chrono::high_resolution_clock::now();
+ triangulate_time += std::chrono::duration_cast<std::chrono::duration<double, std::chrono::milliseconds::period>>(triangulate_end - triangulate_start).count();
+
+ std::chrono::high_resolution_clock::time_point depth_sample_start = std::chrono::high_resolution_clock::now();
+
std::vector<MeshBasedReprojectionEdge> debug_edges;
for (FaceHandle face_handle : triangulation.finite_face_handles())
@@ -2343,7 +2599,7 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
sample_direction.x = this->triangle_depth_offset * glm::cos(center_angle);
sample_direction.y = this->triangle_depth_offset * glm::sin(center_angle);
- for (const MeshBasedReprojectionEdge& edge : edge_buffer->edges[layer])
+ /*for (const MeshBasedReprojectionEdge& edge : edge_buffer->edges[layer])
{
glm::vec2 edge_start = glm::vec2(edge.start) - sample_base;
glm::vec2 edge_end = glm::vec2(edge.end) - sample_base;
@@ -2383,7 +2639,7 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
{
sample_direction = edge_intersect / 2.0f;
}
- }
+ }*/
sample_offset = sample_direction;
}
@@ -2392,7 +2648,7 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
glm::ivec2 sample_point = glm::ivec2(sample_base) + sample_offset;
sample_point = glm::clamp(sample_point, glm::ivec2(0), glm::ivec2(resolution - 1u));
- MeshBasedReprojectionEdge line1;
+ /*MeshBasedReprojectionEdge line1;
line1.start = glm::ivec2(sample_point) + glm::ivec2(3, 3);
line1.end = glm::ivec2(sample_point) - glm::ivec2(2, 2);
@@ -2401,7 +2657,7 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
line2.end = glm::ivec2(sample_point) - glm::ivec2(2, -2);
debug_edges.push_back(line1);
- debug_edges.push_back(line2);
+ debug_edges.push_back(line2);*/
float sample_depth = depth_image_pointer[sample_point.y * resolution.x + sample_point.x];
sample_depth = glm::min(sample_depth, this->edge_detection_depth_threshold);
@@ -2409,7 +2665,7 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
points[index] = glm::vec3(vertex->point().x(), vertex->point().y(), sample_depth);
}
- glm::vec2 screen_coord1 = ((glm::vec2(points[0]) + glm::vec2(0.5f)) / glm::vec2(resolution)) * 2.0f - 1.0f;
+ /*glm::vec2 screen_coord1 = ((glm::vec2(points[0]) + glm::vec2(0.5f)) / glm::vec2(resolution)) * 2.0f - 1.0f;
glm::vec2 screen_coord2 = ((glm::vec2(points[1]) + glm::vec2(0.5f)) / glm::vec2(resolution)) * 2.0f - 1.0f;
glm::vec2 screen_coord3 = ((glm::vec2(points[2]) + glm::vec2(0.5f)) / glm::vec2(resolution)) * 2.0f - 1.0f;
@@ -2432,16 +2688,19 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
}
if (view_angle < this->triangle_angle)
- {
+ {*/
MeshBasedReprojectionTriangle triangle;
triangle.point1 = points[0];
triangle.point2 = points[1];
triangle.point3 = points[2];
edge_buffer->triangles[layer].push_back(triangle);
- }
+ //}
}
+ std::chrono::high_resolution_clock::time_point depth_sample_end = std::chrono::high_resolution_clock::now();
+ depth_sample_time += std::chrono::duration_cast<std::chrono::duration<double, std::chrono::milliseconds::period>>(depth_sample_end - depth_sample_start).count();
+
if (!this->overlay_geometry_locked)
{
std::unique_lock<std::mutex> geometry_lock(this->geometry_mutex);
@@ -2454,14 +2713,6 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
geometry_lock.unlock();
}
}
-
- std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
- double time = std::chrono::duration_cast<std::chrono::duration<double, std::chrono::milliseconds::period>>(end - start).count();
-
- std::unique_lock<std::mutex> edge_lock(this->edge_buffer_mutex);
- this->edge_buffer_process_count += 1;
- this->edge_buffer_process_time_sum += time;
- edge_lock.unlock();
if (!this->overlay_geometry_locked)
{
@@ -2497,6 +2748,14 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge
}
this->get_headset()->submit_metadata(frame_number, metadata);
+
+ std::unique_lock<std::mutex> edge_lock(this->edge_buffer_mutex);
+ this->edge_buffer_process_count += 1;
+ this->line_trace_time_sum += line_trace_time;
+ this->triangulate_time_sum += triangulate_time;
+ this->depth_sample_time_sum += depth_sample_time;
+ this->metadata_bits += 8 * metadata_size;
+ edge_lock.unlock();
}
bool MeshBasedReprojection::worker_traversal_lines(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, uint32_t layer, const glm::ivec2& start_coord, std::vector<glm::ivec2>& edge_coords)
diff --git a/src/strategy/mesh_based_reprojection.hpp b/src/strategy/mesh_based_reprojection.hpp
index 213d1e21..4afced17 100644
--- a/src/strategy/mesh_based_reprojection.hpp
+++ b/src/strategy/mesh_based_reprojection.hpp
@@ -66,7 +66,7 @@ public:
public:
MeshBasedReprojectionQuadTreeLevel(const glm::uvec2& resolution);
- bool fill(float* data, uint32_t size);
+ bool fill(lava::buffer::ptr buffer);
bool set_pixel(const glm::ivec2& coord, float value);
bool get_pixel(const glm::ivec2& coord, float& value) const;
@@ -75,7 +75,7 @@ public:
private:
glm::uvec2 resolution;
- std::vector<float> buffer;
+ float* buffer = nullptr;
};
class MeshBasedReprojectionQuadTree
@@ -86,7 +86,7 @@ public:
public:
MeshBasedReprojectionQuadTree(const glm::uvec2& resolution);
- bool fill(lava::buffer::ptr buffer);
+ bool fill(const std::vector<lava::buffer::ptr>& buffers);
bool remove(const glm::ivec2& coord);
bool find_global(float threshold, glm::ivec2& coord) const;
@@ -109,7 +109,7 @@ public:
ExternFence::Ptr fence;
std::array<lava::buffer::ptr, 2> depth_image_buffers;
- std::array<lava::buffer::ptr, 2> edge_image_buffers;
+ std::array<std::vector<lava::buffer::ptr>, 2> edge_image_buffers;
MeshBasedReprojectionQuadTree::Ptr quad_tree;
@@ -148,6 +148,7 @@ private:
bool create_layer_pass();
bool create_layer_pipeline();
bool create_edge_detection_pipeline();
+ bool create_quad_tree_pipeline();
bool create_overlay_pass();
bool create_overlay_pipeline();
bool create_mesh_overlay_pass();
@@ -160,6 +161,7 @@ private:
void process_layer_pass(VkCommandBuffer command_buffer);
void process_edge_detection_pass(VkCommandBuffer command_buffer);
+ void process_quad_tree_pass(VkCommandBuffer command_buffer);
bool process_edge_buffer_copy(VkCommandBuffer command_buffer, FrameNumber frame_number);
void process_edge_overlay_pass(VkCommandBuffer command_buffer);
void process_triangle_overlay_pass(VkCommandBuffer command_buffer);
@@ -202,13 +204,14 @@ private:
glm::mat4 layer_projection_matrix = glm::mat4(1.0f);
glm::mat4 layer_head_to_eye_matrix = glm::mat4(1.0f);
glm::mat4 layer_inv_projection_matrix = glm::mat4(1.0f);
+ uint32_t layer_quad_tree_levels = 0;
std::vector<lava::buffer::ptr> transform_buffers;
std::array<std::vector<lava::buffer::ptr>, 2> edge_geometry_buffers;
std::array<std::vector<lava::buffer::ptr>, 2> triangle_geometry_buffers;
std::array<lava::image::ptr, 2> layer_depth_buffers;
std::array<lava::image::ptr, 2> layer_normal_buffers;
- std::array<lava::image::ptr, 2> layer_edge_buffers;
+ std::array<std::vector<lava::image::ptr>, 2> layer_edge_buffers;
std::array<lava::image::ptr, 2> layer_overlay_color_buffers;
std::array<lava::image::ptr, 2> layer_overlay_depth_buffers;
@@ -216,7 +219,11 @@ private:
std::vector<MeshBasedReprojectionEdgeBuffer::Ptr> edge_buffers;
std::vector<MeshBasedReprojectionEdgeBuffer::Ptr> edge_buffer_pool; //NOTE: Protected by edge_buffer_mutex
uint32_t edge_buffer_process_count = 0; //NOTE: Protected by edge_buffer_mutex
- double edge_buffer_process_time_sum = 0.0; //NOTE: Protected by edge_buffer_mutex
+ double line_trace_time_sum = 0.0; //NOTE: Protected by edge_buffer_mutex
+ double triangulate_time_sum = 0.0; //NOTE: Protected by edge_buffer_mutex
+ double depth_sample_time_sum = 0.0; //NOTE: Protected by edge_buffer_mutex
+ double metadata_time = 0.0;
+ uint32_t metadata_bits = 0; //NOTE: Protected by edge_buffer_mutex
std::mutex geometry_mutex;
glm::mat4 geometry_view_matrix; //NOTE: Protected by geometry_mutex
@@ -229,9 +236,11 @@ private:
lava::descriptor::ptr transform_descriptor;
lava::descriptor::ptr layer_descriptor;
lava::descriptor::ptr edge_descriptor;
+ lava::descriptor::ptr quad_tree_descriptor;
std::array<VkDescriptorSet, 2> layer_descriptor_sets;
std::array<VkDescriptorSet, 2> edge_descriptor_sets;
std::vector<VkDescriptorSet> transform_descriptor_sets;
+ std::array<std::vector<VkDescriptorSet>, 2> quad_tree_descriptor_sets;
VkSampler nearest_sampler = VK_NULL_HANDLE;
@@ -242,6 +251,9 @@ private:
lava::pipeline_layout::ptr edge_detection_pipeline_layout;
lava::compute_pipeline::ptr edge_detection_pipeline;
+ lava::pipeline_layout::ptr quad_tree_pipeline_layout;
+ lava::compute_pipeline::ptr quad_tree_pipeline;
+
lava::render_pass::ptr overlay_pass;
lava::pipeline_layout::ptr overlay_pipeline_layout;
lava::graphics_pipeline::ptr edge_overlay_pipeline;
@@ -252,7 +264,10 @@ private:
lava::graphics_pipeline::ptr mesh_overlay_pipeline;
private:
- Statistic::Ptr statistic_edge_buffer_time;
+ Statistic::Ptr statistic_line_trace_time;
+ Statistic::Ptr statistic_triangulate_time;
+ Statistic::Ptr statistic_depth_sample_time;
+ Statistic::Ptr statistic_metadata_bitrate;
};
MeshBasedReprojection::Ptr make_mesh_based_reprojection();
--
GitLab