diff --git a/src/frame_capture.cpp b/src/frame_capture.cpp index 89bf4d0855779dc3d7da974bd90f22c7a43900e7..8797585a38636dbed2c512db0b0a1b2ab6aba122 100644 --- a/src/frame_capture.cpp +++ b/src/frame_capture.cpp @@ -1,44 +1,50 @@ #include "frame_capture.hpp" #include "stb_image_write.h" #include <algorithm> +#include <filesystem> -frame_capture::frame_capture(const std::string& directory) : directory(directory) +frame_capture::frame_capture(const std::string& base_directory) : base_directory(base_directory) { } +void frame_capture::create(uint32_t frame_count) +{ + this->frame_count = frame_count; +} + void frame_capture::destroy() { - //TODO: wait for all remaining captures that are still open (not neccessary since the function expects that vkDeviceWaitIdle was called before) - //TODO: write all remaining captures to files and destroy the staging buffer + for(uint32_t offset = 0; offset < this->frame_count; offset++) + { + uint32_t frame = (this->current_frame + offset) % this->frame_count; + + this->load_captures(frame); + } } -void frame_capture::next_frame(lava::device_ptr device, lava::index frame) +void frame_capture::next_frame(lava::index frame) { - this->load_captures(device, frame); + this->load_captures(frame); this->current_frame = frame; } -void frame_capture::capture_texture(VkCommandBuffer command_buffer, lava::device_ptr device, lava::texture::ptr texture, const std::string& texture_name) +void frame_capture::capture_image(VkCommandBuffer command_buffer, lava::image::ptr image, VkImageLayout image_layout, const std::string& image_name) { if(!this->enabled) { return; } - if(texture->get_type() != lava::texture_type::tex_2d) - { - lava::log()->error("can't capture texture '" + texture_name +"' because the texture type is not tex_2d"); - - return; - } + lava::device_ptr device = image->get_device(); + lava::uv2 image_size = image->get_size(); + VkFormat image_format = image->get_format(); - lava::uv2 texture_size = texture->get_size(); - uint32_t buffer_size = texture_size.x * texture_size.y * lava::format_block_size(texture->get_format()); + uint32_t staging_buffer_size = image_size.x * image_size.y * this->get_component_count(image_format) * this->get_component_size(image_format); lava::buffer::ptr staging_buffer = lava::make_buffer(); - staging_buffer->create(device, nullptr, buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT, false, VmaMemoryUsage::VMA_MEMORY_USAGE_GPU_TO_CPU); + staging_buffer->create(device, nullptr, staging_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT, false, VmaMemoryUsage::VMA_MEMORY_USAGE_GPU_TO_CPU); VkOffset3D image_offset; image_offset.x = 0; @@ -46,53 +52,107 @@ void frame_capture::capture_texture(VkCommandBuffer command_buffer, lava::device image_offset.z = 0; VkExtent3D image_extend; - image_extend.width = texture_size.x; - image_extend.height = texture_size.y; + image_extend.width = image_size.x; + image_extend.height = image_size.y; image_extend.depth = 1; - VkImageSubresourceLayers image_subresource; - image_subresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - image_subresource.baseArrayLayer = 0; - image_subresource.layerCount = 1; - image_subresource.mipLevel = 1; + VkImageSubresourceRange image_subresource_range; + image_subresource_range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_subresource_range.baseArrayLayer = 0; + image_subresource_range.baseMipLevel = 0; + image_subresource_range.layerCount = 1; + image_subresource_range.levelCount = 1; + + VkImageSubresourceLayers image_subresource_layers; + image_subresource_layers.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_subresource_layers.baseArrayLayer = 0; + image_subresource_layers.layerCount = 1; + image_subresource_layers.mipLevel = 1; + + VkImageMemoryBarrier begin_barrier; + begin_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + begin_barrier.pNext = nullptr; + begin_barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + begin_barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + begin_barrier.oldLayout = image_layout; + begin_barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + begin_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + begin_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + begin_barrier.image = image->get(); + begin_barrier.subresourceRange = image_subresource_range; + + vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &begin_barrier); VkBufferImageCopy region; region.bufferOffset = 0; region.bufferRowLength = 0; //Tightly packed region.bufferImageHeight = 0; - region.imageSubresource = image_subresource; + region.imageSubresource = image_subresource_layers; region.imageOffset = image_offset; region.imageExtent = image_extend; - vkCmdCopyImageToBuffer(command_buffer, texture->get_image()->get(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, staging_buffer->get(), 1, ®ion); + vkCmdCopyImageToBuffer(command_buffer, image->get(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, staging_buffer->get(), 1, ®ion); - texture_catpure capture; - capture.name = texture_name; - capture.frame_index = this->current_frame; + VkImageMemoryBarrier end_barrier; + end_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + end_barrier.pNext = nullptr; + end_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + end_barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + end_barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + end_barrier.newLayout = image_layout; + end_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + end_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + end_barrier.image = image->get(); + end_barrier.subresourceRange = image_subresource_range; + + vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 0, nullptr, 1, &begin_barrier); + + image_catpure capture; + capture.name = image_name; + capture.size = image_size; + capture.format = image_format; capture.staging_buffer = staging_buffer; + capture.frame_index = this->current_frame; this->captures.push_back(capture); } -void frame_capture::load_captures(lava::device_ptr device, lava::index frame) +void frame_capture::enable() +{ + this->enabled = true; +} + +void frame_capture::disable() +{ + this->enabled = false; +} + +bool frame_capture::is_enabled() const +{ + return this->enabled; +} + +void frame_capture::load_captures(lava::index frame) { - for(const texture_catpure& capture : this->captures) + for(const image_catpure& capture : this->captures) { if(capture.frame_index == frame) { lava::buffer::ptr staging_buffer = capture.staging_buffer; - staging_buffer->flush(); //Not sure if this is the right + lava::device_ptr device = staging_buffer->get_device(); + + vmaInvalidateAllocation(device->alloc(), staging_buffer->get_allocation(), 0, VK_WHOLE_SIZE); - uint8_t* texture_content = nullptr; + uint8_t* image_content = nullptr; - if(vmaMapMemory(device->alloc(), staging_buffer->get_allocation(), (void**)(&texture_content))) + if(vmaMapMemory(device->alloc(), staging_buffer->get_allocation(), (void**)(&image_content))) { - lava::log()->error("can't map the staging buffer for the frame capture of texture '" + capture.name + "'"); + lava::log()->error("can't map the staging buffer for the frame capture of image '" + capture.name + "'"); continue; } - this->write_to_file(capture.name, texture_content); + this->write_to_file(capture.name, image_content, capture.size, capture.format); vmaUnmapMemory(device->alloc(), staging_buffer->get_allocation()); @@ -100,34 +160,222 @@ void frame_capture::load_captures(lava::device_ptr device, lava::index frame) } } - this->captures.erase(std::remove_if(this->captures.begin(), this->captures.end(), [=](const texture_catpure& capture) + this->captures.erase(std::remove_if(this->captures.begin(), this->captures.end(), [=](const image_catpure& capture) { return capture.frame_index == frame; }), this->captures.end()); } -bool frame_capture::write_to_file(const std::string& texture_name, const uint8_t* texture_content) +bool frame_capture::write_to_file(const std::string& image_name, const uint8_t* image_content, const lava::uv2& image_size, VkFormat image_format) { - //TODO: convert the image format in a standard format. This is because stbi_write_png needs 8 bit per channel - //TODO: stbi_write_png() + if(!this->is_format_supported(image_format)) + { + lava::log()->error("the format of image '" + image_name + "' is not supported for frame capturing!"); + + return false; + } + + image_capture_directory directory; + auto directory_iter = this->directories.find(image_name); + + if(directory_iter == this->directories.end()) + { + directory.name = this->build_directory_name(image_name); + directory.index = 0; + + this->directories[image_name] = directory; + } + + else + { + directory_iter->second.index++; + directory = directory_iter->second; + } + + std::filesystem::path directory_path = this->base_directory + "/" + directory.name; + std::string file_name = this->base_directory + "/" + directory.name + "/frame_" + std::to_string(directory.index) + ".png"; + + if(!std::filesystem::exists(directory_path)) + { + std::filesystem::create_directory(directory_path); + } + + std::vector<uint8_t> converted_content = this->convert_content(image_content, image_size, image_format); + + if(stbi_write_png(file_name.c_str(), image_size.x, image_size.y, 4, converted_content.data(), image_size.x * 4 * sizeof(uint8_t)) == 0) + { + lava::log()->error("can't store image '" + image_name + "'"); + + return false; + } + + return true; } -std::string frame_capture::build_file_name(const std::string& texture_name) +std::string frame_capture::build_directory_name(const std::string& image_name) { - //TODO: build file name and create directory for each texture_name + time_t system_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + tm local_time = *localtime(&system_time); + + std::string name = image_name; + name += "_" + std::to_string(local_time.tm_mday); + name += "-" + std::to_string(local_time.tm_mon + 1); + name += "-" + std::to_string(local_time.tm_year + 1900); + name += "_" + std::to_string(local_time.tm_hour); + name += "-" + std::to_string(local_time.tm_min); + name += "-" + std::to_string(local_time.tm_sec); + + return name; } -void frame_capture::enable() +std::vector<uint8_t> frame_capture::convert_content(const uint8_t* image_content, const lava::uv2& image_size, VkFormat image_format) { - this->enabled = true; + std::vector<uint8_t> output_buffer; + output_buffer.resize(image_size.x * image_size.y * 4 * sizeof(uint8_t)); + + const uint8_t* input_pointer = image_content; + + for(uint32_t y = 0; y < image_size.y; y++) + { + for(uint32_t x = 0; x < image_size.x; x++) + { + uint32_t pixel_index = y * image_size.x + x; + + //initialize the pixel to black + output_buffer[pixel_index * 4 + 0] = 0; + output_buffer[pixel_index * 4 + 1] = 0; + output_buffer[pixel_index * 4 + 2] = 0; + output_buffer[pixel_index * 4 + 3] = 0xFF; + + for(uint32_t component = 0; component < this->get_component_count(image_format); component++) + { + uint32_t component_offset = this->get_component_offset(image_format, component); + uint32_t component_range = (1 << (this->get_component_size(image_format) << 8)) - 1; + uint32_t component_value = 0; + + for(int32_t byte = this->get_component_size(image_format) - 1; byte >= 0; byte--) + { + component_value |= *input_pointer << byte * 8; + input_pointer++; + } + + output_buffer[pixel_index * 4 + component_offset] = (uint8_t)((float)component_value / (float)component_range); + } + } + } + + return output_buffer; } -void frame_capture::disable() +uint32_t frame_capture::get_component_count(VkFormat format) { - this->enabled = false; + switch(format) + { + case VK_FORMAT_R8_UNORM: + case VK_FORMAT_R8_SRGB: + case VK_FORMAT_R16_UNORM: + return 1; + case VK_FORMAT_R8G8_UNORM: + case VK_FORMAT_R8G8_SRGB: + case VK_FORMAT_R16G16_UNORM: + return 2; + case VK_FORMAT_R8G8B8_UNORM: + case VK_FORMAT_R8G8B8_SRGB: + case VK_FORMAT_B8G8R8_UNORM: + case VK_FORMAT_B8G8R8_SRGB: + case VK_FORMAT_R16G16B16_UNORM: + return 3; + case VK_FORMAT_R8G8B8A8_UNORM: + case VK_FORMAT_R8G8B8A8_SRGB: + case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_B8G8R8A8_SRGB: + case VK_FORMAT_R16G16B16A16_UNORM: + return 4; + default: + break; + } + + return 0; } -bool frame_capture::is_enabled() const +uint32_t frame_capture::get_component_size(VkFormat format) { - return this->enabled; + switch(format) + { + case VK_FORMAT_R8_UNORM: + case VK_FORMAT_R8G8_UNORM: + case VK_FORMAT_R8G8B8_UNORM: + case VK_FORMAT_B8G8R8_UNORM: + case VK_FORMAT_R8G8B8A8_UNORM: + case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_R8_SRGB: + case VK_FORMAT_R8G8_SRGB: + case VK_FORMAT_R8G8B8_SRGB: + case VK_FORMAT_B8G8R8_SRGB: + case VK_FORMAT_R8G8B8A8_SRGB: + case VK_FORMAT_B8G8R8A8_SRGB: + return 1; + case VK_FORMAT_R16_UNORM: + case VK_FORMAT_R16G16_UNORM: + case VK_FORMAT_R16G16B16_UNORM: + case VK_FORMAT_R16G16B16A16_UNORM: + return 2; + default: + break; + } + + return 0; +} + +uint32_t frame_capture::get_component_offset(VkFormat format, uint32_t component) +{ + switch(format) + { + case VK_FORMAT_B8G8R8_UNORM: + case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_B8G8R8_SRGB: + case VK_FORMAT_B8G8R8A8_SRGB: + if(component == 0) + { + return 2; + } + if(component == 2) + { + return 0; + } + break; + default: + break; + } + + return component; +} + +bool frame_capture::is_format_supported(VkFormat format) +{ + switch(format) + { + case VK_FORMAT_R8_UNORM: + case VK_FORMAT_R8G8_UNORM: + case VK_FORMAT_R16_UNORM: + case VK_FORMAT_R8G8B8_UNORM: + case VK_FORMAT_B8G8R8_UNORM: + case VK_FORMAT_R8G8B8A8_UNORM: + case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_R16G16_UNORM: + case VK_FORMAT_R16G16B16_UNORM: + case VK_FORMAT_R16G16B16A16_UNORM: + return true; + case VK_FORMAT_R8_SRGB: + case VK_FORMAT_R8G8_SRGB: + case VK_FORMAT_R8G8B8_SRGB: + case VK_FORMAT_B8G8R8_SRGB: + case VK_FORMAT_R8G8B8A8_SRGB: + case VK_FORMAT_B8G8R8A8_SRGB: + return true; + default: + break; + } + + return false; } \ No newline at end of file diff --git a/src/frame_capture.hpp b/src/frame_capture.hpp index 1711f121c6ad9baa7900681daf722103760627d5..af961b5c1f3994309ade2fb97e4e3bce6e877285 100644 --- a/src/frame_capture.hpp +++ b/src/frame_capture.hpp @@ -2,43 +2,59 @@ #include <liblava/lava.hpp> #include <string> #include <vector> +#include <map> #include <cstdint> -struct texture_catpure +struct image_catpure { std::string name; + lava::uv2 size; + VkFormat format; lava::buffer::ptr staging_buffer; uint32_t frame_index; }; +struct image_capture_directory +{ + std::string name; + uint32_t index; +}; + class frame_capture { public: - frame_capture(const std::string& directory); + frame_capture(const std::string& base_directory); + void create(uint32_t frame_count); void destroy(); - void next_frame(lava::device_ptr device, lava::index frame); - //The function assumes that the texture has the image layout VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL. - //Besides that the function assumes that all writing operations have been completed (eg. vkCmdPiplineBarrier). - //Besdies that the function ony accepts color textures. - void capture_texture(VkCommandBuffer command_buffer, lava::device_ptr device, lava::texture::ptr texture, const std::string& texture_name); + void next_frame(lava::index frame); + void capture_image(VkCommandBuffer command_buffer, lava::image::ptr image, VkImageLayout image_layout, const std::string& image_name); void enable(); void disable(); bool is_enabled() const; private: - void load_captures(lava::device_ptr device, lava::index frame); - bool write_to_file(const std::string& texture_name, const uint8_t* texture_content); + void load_captures(lava::index frame); - std::string build_file_name(const std::string& texture_name); + bool write_to_file(const std::string& image_name, const uint8_t* image_content, const lava::uv2& image_size, VkFormat image_format); + std::string build_directory_name(const std::string& image_name); + + std::vector<uint8_t> convert_content(const uint8_t* image_content, const lava::uv2& image_size, VkFormat image_format); + uint32_t get_component_count(VkFormat format); + uint32_t get_component_size(VkFormat format); + uint32_t get_component_offset(VkFormat format, uint32_t component); + bool is_format_supported(VkFormat format); private: - std::string directory = ""; + std::string base_directory = ""; - bool enabled = false; + uint32_t frame_count = 0; uint32_t current_frame = 0; - std::vector<texture_catpure> captures; + bool enabled = false; + + std::vector<image_catpure> captures; + std::map<std::string, image_capture_directory> directories; }; \ No newline at end of file diff --git a/src/naive_stereo.cpp b/src/naive_stereo.cpp index c44e43a452fac699f8ebc4eceb34d218c560526c..f87903cf70b64d45104c9f349b8ef9ef97b2870d 100644 --- a/src/naive_stereo.cpp +++ b/src/naive_stereo.cpp @@ -33,6 +33,9 @@ void naive_stereo::render(VkCommandBuffer command_buffer, lava::index frame) { for (auto& eye_pass : render_passes_) { eye_pass.render_pass->process(command_buffer, 0); } + + frame_capture().capture_image(command_buffer, app()->left_eye_framebuffer()->color_image, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, "left_eye"); + frame_capture().capture_image(command_buffer, app()->right_eye_framebuffer()->color_image, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, "right_eye"); } naive_stereo::eye_render_pass naive_stereo::create_render_pass(const stereo_framebuffer::ptr& framebuffer) { diff --git a/src/stereo_strategy.cpp b/src/stereo_strategy.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4af58521e2d0624928090ab1e310857fd7e0322b 100644 --- a/src/stereo_strategy.cpp +++ b/src/stereo_strategy.cpp @@ -0,0 +1,7 @@ +#include "stereo_strategy.hpp" +#include "vr_app.hpp" + +frame_capture& stereo_strategy::frame_capture() const +{ + return app_->frame_capture(); +} \ No newline at end of file diff --git a/src/stereo_strategy.hpp b/src/stereo_strategy.hpp index d5ecc7031a4b8c39e8f635820813ba5e5890ecad..fd491832e428c2b1b5e4283182be7b1918d747a1 100644 --- a/src/stereo_strategy.hpp +++ b/src/stereo_strategy.hpp @@ -4,6 +4,7 @@ #include "stereo_framebuffer.hpp" class vr_app; +class frame_capture; class stereo_strategy { public: @@ -15,6 +16,7 @@ public: virtual void draw_imgui() {} inline vr_app* app() const { return app_; } + frame_capture& frame_capture() const; private: vr_app* app_; diff --git a/src/vr_app.cpp b/src/vr_app.cpp index 4693089ea0c7666c0f5f4edbb33741161241086c..7f3478ebdb76adae1c672f6f5c6d37b4424e6cf2 100644 --- a/src/vr_app.cpp +++ b/src/vr_app.cpp @@ -91,6 +91,9 @@ vr_app::vr_app(name name, argh::parser cmd_line) { app_->camera.movement_speed = 10.0f; app_->on_process = [this](VkCommandBuffer command_buffer, lava::index frame) { + + frame_capture_.next_frame(frame); + VkImageSubresourceRange const image_range{ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .levelCount = 1, @@ -164,6 +167,8 @@ bool vr_app::setup() { return false; } + frame_capture_.create(app_->target->get_frame_count()); + descriptor_pool_ = make_descriptor_pool(); if (!descriptor_pool_->create(app_->device, { { VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 2 }, // Stereo images @@ -222,6 +227,9 @@ bool vr_app::setup() { } app_->add_run_end([this]() { + + frame_capture_.destroy(); + framebuffers_[0]->destroy(); framebuffers_[1]->destroy(); test_texture_->destroy(); @@ -346,6 +354,22 @@ bool vr_app::setup() { } + if(ImGui::CollapsingHeader("Capture", ImGuiTreeNodeFlags_DefaultOpen)) + { + bool active = frame_capture_.is_enabled(); + ImGui::Checkbox("Active", &active); + + if(active) + { + frame_capture_.enable(); + } + + else + { + frame_capture_.disable(); + } + } + if (ImGui::CollapsingHeader("Timings", ImGuiTreeNodeFlags_DefaultOpen)) { // if (ImGui::InputInt("Frame Time Smoothing", &frame_time_smoothing_)) { // frame_time_smoothing_ = std::abs(frame_time_smoothing_); diff --git a/src/vr_app.hpp b/src/vr_app.hpp index a242f479828fb0b8d012ac85bd49147a28290516..cd7c5967ccd5ad2846dd8cdc29a08aa354e7b24c 100644 --- a/src/vr_app.hpp +++ b/src/vr_app.hpp @@ -7,6 +7,7 @@ #include "multiview_stereo.hpp" #include "depth_peeling_reprojection_stereo.hpp" #include "scene.hpp" +#include "frame_capture.hpp" @@ -18,6 +19,7 @@ public: auto& staging() { return app_->staging; } lava::camera& camera() { return app_->camera; } auto scene() { return scene_; } + auto& frame_capture() { return frame_capture_; } auto per_frame_descriptor() const { return per_frame_descriptor_; } @@ -59,6 +61,7 @@ private: std::string scene_path_; ::scene::ptr scene_; + ::frame_capture frame_capture_ = {"captures"}; bool setup_companion_window_pipeline(); stereo_framebuffer::ptr create_framebuffer(glm::uvec2 size, vr::EVREye eye);