diff --git a/src/headset/remote_headset.cpp b/src/headset/remote_headset.cpp index 45e5c216e0238c0e8adac81e5e2278dc6a0b72bf..d7ac5bbfc04e36e6bed9ceda36da6c43fd680047 100644 --- a/src/headset/remote_headset.cpp +++ b/src/headset/remote_headset.cpp @@ -65,11 +65,6 @@ bool RemoteHeadset::on_create() StereoStrategy::Ptr strategy = this->get_application()->get_stereo_strategy(); this->frame_id_count = strategy->get_remote_frame_id_count(); - - if (!convert_strategy(this->get_application()->get_stereo_strategy_type(), this->remote_strategy)) - { - return false; - } if (!this->create_transport()) { @@ -738,6 +733,7 @@ void RemoteHeadset::on_setup(const glm::u32vec2& display_resolution, const glm:: this->update_values(); StereoStrategy::Ptr strategy = this->get_application()->get_stereo_strategy(); + RemoteStrategy remote_strategy = strategy->get_remote_strategy(); if (!strategy->on_setup_remote()) { @@ -757,7 +753,7 @@ void RemoteHeadset::on_setup(const glm::u32vec2& display_resolution, const glm:: return; } - this->transport->send_setup_complete(this->remote_strategy, this->frame_id_count, encoder_codec, encode_resolution, framebuffer_resolution, this->near_plane, this->far_plane); + this->transport->send_setup_complete(remote_strategy, this->frame_id_count, encoder_codec, encode_resolution, framebuffer_resolution, this->near_plane, this->far_plane); lock.lock(); this->framebuffer_resolution = framebuffer_resolution; @@ -869,34 +865,4 @@ std::string RemoteHeadset::build_file_name(uint32_t frame_id) name += "-" + std::to_string(local_time.tm_sec); return name; -} - -bool RemoteHeadset::convert_strategy(StereoStrategyType strategy_type, RemoteStrategy& remote_strategy) -{ - switch (strategy_type) - { - case STEREO_STRATEGY_TYPE_NATIVE_FORWARD: - remote_strategy = REMOTE_STRATEGY_NATIVE; - break; - case STEREO_STRATEGY_TYPE_NATIVE_DEFERRED: - remote_strategy = REMOTE_STRATEGY_NATIVE; - break; - case STEREO_STRATEGY_TYPE_MULTI_VIEW: - remote_strategy = REMOTE_STRATEGY_NATIVE; - break; - case STEREO_STRATEGY_TYPE_DEPTH_PEELING_REPROJECTION: - remote_strategy = REMOTE_STRATEGY_NATIVE; - break; - case STEREO_STRATEGY_TYPE_DUAL_LAYER_REPROJECTION: - remote_strategy = REMOTE_STRATEGY_DUAL_LAYER_REPROJECTION; - break; - case STEREO_STRATEGY_TYPE_MESH_BASED_REPROJECTION: - remote_strategy = REMOTE_STRATEGY_NATIVE; - break; - default: - lava::log()->error("Remote Headset: Unkown strategy type!"); - return false; - } - - return true; } \ No newline at end of file diff --git a/src/headset/remote_headset.hpp b/src/headset/remote_headset.hpp index abf8aadf1aa6f6203c3a574a0fadf9712870538e..e5b3350de8957b0a8bc1c0ffc7a1d05b0f94b450 100644 --- a/src/headset/remote_headset.hpp +++ b/src/headset/remote_headset.hpp @@ -79,8 +79,7 @@ private: void on_encode_error(); static std::string build_file_name(uint32_t frame_id); - static bool convert_strategy(StereoStrategyType strategy_type, RemoteStrategy& remote_strategy); - + private: const uint32_t server_port = 4000; const std::string video_directory = "video_captures"; @@ -111,7 +110,6 @@ private: std::array<glm::vec2, 2> controller_thumbsticks; //NOTE: Protected by transport_mutex std::array<std::array<ButtonState, BUTTON_MAX_COUNT>, 2> controller_buttons; //NOTE: Protected by transport_mutex - RemoteStrategy remote_strategy = REMOTE_STRATEGY_NATIVE; uint32_t frame_number = 0; uint32_t frame_id_count = 0; uint32_t transform_id = 0; //NOTE: Protected by transport_mutex diff --git a/src/strategy/mesh_based_reprojection.cpp b/src/strategy/mesh_based_reprojection.cpp index 456c3e2029197eb71f0218d527c413f92558e095..2cd43c565723e3cc906bafa51d6aff7d4f0dc760 100644 --- a/src/strategy/mesh_based_reprojection.cpp +++ b/src/strategy/mesh_based_reprojection.cpp @@ -469,9 +469,6 @@ bool MeshBasedReprojection::on_render(VkCommandBuffer command_buffer, lava::inde { FrameNumber frame_number = this->get_headset()->get_frame_number(); - std::array<uint8_t, 1> metadata = { 0x00 }; - this->get_headset()->submit_metadata(frame_number, metadata); - if (!this->write_transform(frame)) { return false; @@ -537,9 +534,12 @@ bool MeshBasedReprojection::on_render(VkCommandBuffer command_buffer, lava::inde this->process_edge_detection_pass(command_buffer); this->get_pass_timer()->end_pass(command_buffer); - if (!this->process_edge_buffer_copy(command_buffer)) + if (frame_number != FRAME_NUMBER_DROPPED) { - return false; + if (!this->process_edge_buffer_copy(command_buffer, frame_number)) + { + return false; + } } if (this->get_companion_window()->is_enabled()) @@ -1826,7 +1826,7 @@ void MeshBasedReprojection::process_edge_detection_pass(VkCommandBuffer command_ } } -bool MeshBasedReprojection::process_edge_buffer_copy(VkCommandBuffer command_buffer) +bool MeshBasedReprojection::process_edge_buffer_copy(VkCommandBuffer command_buffer, FrameNumber frame_number) { glm::uvec2 resolution = this->get_headset()->get_framebuffer_resolution(); @@ -1949,7 +1949,7 @@ bool MeshBasedReprojection::process_edge_buffer_copy(VkCommandBuffer command_buf vkQueueSubmit(this->get_application()->get_renderer().get_queue().vk_queue, 1, &submit_info, edge_buffer->fence->get()); }; - edge_buffer->fence->async_wait([this, edge_buffer](const asio::error_code& error_code) + edge_buffer->fence->async_wait([this, edge_buffer, frame_number](const asio::error_code& error_code) { if (error_code) { @@ -1958,7 +1958,7 @@ bool MeshBasedReprojection::process_edge_buffer_copy(VkCommandBuffer command_buf else { - this->worker_process_edge_buffer(edge_buffer); + this->worker_process_edge_buffer(edge_buffer, frame_number); } this->release_edge_buffer(edge_buffer); @@ -2047,7 +2047,7 @@ void MeshBasedReprojection::release_edge_buffer(MeshBasedReprojectionEdgeBuffer: this->edge_buffer_pool.push_back(edge_buffer); } -void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer) +void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, FrameNumber frame_number) { std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now(); @@ -2374,6 +2374,34 @@ void MeshBasedReprojection::worker_process_edge_buffer(MeshBasedReprojectionEdge this->geometry_view_matrix = edge_buffer->view_matrix; geometry_lock.unlock(); } + + uint32_t metadata_size = sizeof(MeshBasedReprojectionMetadataHeader); + metadata_size += sizeof(MeshBasedReprojectionEncodedTriangle) * edge_buffer->triangles[0].size(); + metadata_size += sizeof(MeshBasedReprojectionEncodedTriangle) * edge_buffer->triangles[1].size(); + + std::vector<uint8_t> metadata(metadata_size); + uint8_t* metadata_pointer = metadata.data(); + + MeshBasedReprojectionMetadataHeader* metadata_header = (MeshBasedReprojectionMetadataHeader*)metadata_pointer; + metadata_header->layer_resolution = this->layer_resolution; + metadata_header->layer_head_eye_projection_matrix = this->layer_projection_matrix * this->layer_head_to_eye_matrix * edge_buffer->view_matrix; + metadata_header->triangle_count[0] = edge_buffer->triangles[0].size(); + metadata_header->triangle_count[1] = edge_buffer->triangles[1].size(); + metadata_pointer += sizeof(MeshBasedReprojectionMetadataHeader); + + for (uint32_t layer = 0; layer < 2; layer++) + { + for (const MeshBasedReprojectionTriangle& triangle : edge_buffer->triangles[layer]) + { + MeshBasedReprojectionEncodedTriangle* metadata_triangle = (MeshBasedReprojectionEncodedTriangle*)metadata_pointer; + metadata_triangle->point1 = glm::u16vec3(glm::vec2(triangle.point1), triangle.point1.z * 0xFFFF); + metadata_triangle->point2 = glm::u16vec3(glm::vec2(triangle.point2), triangle.point2.z * 0xFFFF); + metadata_triangle->point3 = glm::u16vec3(glm::vec2(triangle.point3), triangle.point3.z * 0xFFFF); + metadata_pointer += sizeof(MeshBasedReprojectionEncodedTriangle); + } + } + + this->get_headset()->submit_metadata(frame_number, metadata); } bool MeshBasedReprojection::worker_traversal_lines(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, uint32_t layer, const glm::ivec2& start_coord, std::vector<glm::ivec2>& edge_coords) diff --git a/src/strategy/mesh_based_reprojection.hpp b/src/strategy/mesh_based_reprojection.hpp index e1a46a21f6187735188dcf199f5c18f743c7fa18..dae2c6f6312bbeab77ccdcfb7f74a757e9a4b690 100644 --- a/src/strategy/mesh_based_reprojection.hpp +++ b/src/strategy/mesh_based_reprojection.hpp @@ -41,6 +41,23 @@ struct MeshBasedReprojectionTriangle glm::vec3 point3; }; +struct MeshBasedReprojectionEncodedTriangle +{ + glm::u16vec3 point1; + glm::u16vec3 point2; + glm::u16vec3 point3; +}; + +struct MeshBasedReprojectionMetadataHeader +{ + glm::uvec2 layer_resolution; + glm::mat4 layer_head_eye_projection_matrix; + + std::array<uint32_t, 2> triangle_count; + //Followed by MeshBasedReprojectionEncodedTriangle as many as defined by triangle_count[0] + //Followed by MeshBasedReprojectionEncodedTriangle as many as defined by triangle_count[1] +}; + class MeshBasedReprojectionQuadTreeLevel { public: @@ -141,7 +158,7 @@ private: void process_layer_pass(VkCommandBuffer command_buffer); void process_edge_detection_pass(VkCommandBuffer command_buffer); - bool process_edge_buffer_copy(VkCommandBuffer command_buffer); + bool process_edge_buffer_copy(VkCommandBuffer command_buffer, FrameNumber frame_number); void process_edge_overlay_pass(VkCommandBuffer command_buffer); void process_triangle_overlay_pass(VkCommandBuffer command_buffer); void process_mesh_overlay_pass(VkCommandBuffer command_buffer); @@ -149,7 +166,7 @@ private: bool acquire_edge_buffer(MeshBasedReprojectionEdgeBuffer::Ptr& edge_buffer); void release_edge_buffer(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer); - void worker_process_edge_buffer(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer); + void worker_process_edge_buffer(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, FrameNumber frame_number); bool worker_traversal_lines(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, uint32_t layer, const glm::ivec2& start_coord, std::vector<glm::ivec2>& edge_coords); void worker_create_lines(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, uint32_t layer, const std::vector<glm::ivec2>& edge_coords); void worker_close_loop(MeshBasedReprojectionEdgeBuffer::Ptr edge_buffer, uint32_t layer, uint32_t edge_index, uint32_t segment_index, glm::ivec2& edge_point, bool edge_is_end); diff --git a/src/transport/udp_transport.cpp b/src/transport/udp_transport.cpp index 3454a64eba3cd11fa962828435c43572929bb0ff..83586e95df44a36bd03503758fd23b12af893ef1 100644 --- a/src/transport/udp_transport.cpp +++ b/src/transport/udp_transport.cpp @@ -29,8 +29,6 @@ void UDPTransport::destroy() this->server_context.stop(); this->worker_thread.join(); - this->client_endpoint.reset(); - if (this->server_socket.has_value()) { this->server_socket->close(); @@ -43,6 +41,8 @@ void UDPTransport::destroy() this->server_timer.reset(); } + this->client_endpoint.reset(); + for (const Datagram& datagram : this->send_queue) { this->datagram_pool.release_datagram(datagram); @@ -53,14 +53,8 @@ void UDPTransport::destroy() this->datagram_pool.release_datagram(datagram); } - for (const Datagram& datagram : this->error_queue) - { - this->datagram_pool.release_datagram(datagram); - } - this->send_queue.clear(); this->receive_queue.clear(); - this->error_queue.clear(); this->bits_send = 0; this->bits_received = 0; @@ -653,7 +647,9 @@ void UDPTransport::receive_datagram() this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); + std::unique_lock<std::mutex> lock(this->worker_mutex); + this->datagram_pool.release_datagram(datagram); + lock.unlock(); return; } @@ -676,12 +672,15 @@ void UDPTransport::receive_datagram() this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); + std::unique_lock<std::mutex> lock(this->worker_mutex); + this->datagram_pool.release_datagram(datagram); + lock.unlock(); return; } const UDPPacketHeader* packet = (const UDPPacketHeader*)datagram.buffer; + bool packet_invalid = false; switch (packet->id) { @@ -697,51 +696,47 @@ void UDPTransport::receive_datagram() break; case UDP_PACKET_ID_SETUP_COMPLETE: lava::log()->error("UDP-Transport: Received setup complete packet!"); - this->on_transport_error(); - this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); - return; + packet_invalid = true; + break; case UDP_PACKET_ID_FRAME_CONFIG_NAL: lava::log()->error("UDP-Transport: Received frame config nal packet!"); - this->on_transport_error(); - this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); - return; + packet_invalid = true; + break; case UDP_PACKET_ID_FRAME_NAL: lava::log()->error("UDP-Transport: Received frame nal packet!"); - this->on_transport_error(); - this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); - return; + packet_invalid = true; + break; case UDP_PACKET_ID_FRAME_METADATA: lava::log()->error("UDP-Transport: Received frame metadata packet!"); - this->on_transport_error(); - this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); - return; + packet_invalid = true; + break; case UDP_PACKET_ID_OVERLAY_CONFIG: lava::log()->error("UDP-Transport: Received frame overlay config packet!"); - this->on_transport_error(); - this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); - return; + packet_invalid = true; + break; case UDP_PACKET_ID_OVERLAY_TEXT: lava::log()->error("UDP-Transport: Received frame overlay text packet!"); - this->on_transport_error(); - this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); - return; + packet_invalid = true; + break; case UDP_PACKET_ID_OVERLAY_GRAPH: lava::log()->error("UDP-Transport: Received frame overlay graph packet!"); - this->on_transport_error(); - this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); - return; + packet_invalid = true; + break; default: lava::log()->error("UDP-Transport: Received unknown packet!"); + packet_invalid = true; + break; + } + + if (packet_invalid) + { this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); + + std::unique_lock<std::mutex> lock(this->worker_mutex); + this->datagram_pool.release_datagram(datagram); + lock.unlock(); + return; } @@ -773,7 +768,10 @@ void UDPTransport::process_send_queue() this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); - this->error_queue.push_back(datagram); + std::unique_lock<std::mutex> lock(this->worker_mutex); + this->send_queue.erase(this->send_queue.begin()); + this->datagram_pool.release_datagram(datagram); + lock.unlock(); return; } @@ -856,11 +854,14 @@ void UDPTransport::process_receive_queue() this->set_state(TRANSPORT_STATE_ERROR); return; } + } - std::unique_lock<std::mutex> lock(this->worker_mutex); + std::unique_lock<std::mutex> lock(this->worker_mutex); + for (const Datagram& datagram : this->receive_queue) + { this->datagram_pool.release_datagram(datagram); - lock.unlock(); } + lock.unlock(); this->receive_queue.clear(); } diff --git a/src/transport/udp_transport.hpp b/src/transport/udp_transport.hpp index 86f1ba012d1fc978b12b90bf6ca0886d7bcbcda8..f13749e74b52a10808ac939cfec961ae81e0e52f 100644 --- a/src/transport/udp_transport.hpp +++ b/src/transport/udp_transport.hpp @@ -88,7 +88,6 @@ private: DatagramPool datagram_pool; //NOTE: Protected by worker_mutex std::vector<Datagram> send_queue; //NOTE: Protected by worker_mutex std::vector<Datagram> receive_queue; //NOTE: Owned by worker_thread and accessible after the thread has been stoped - std::vector<Datagram> error_queue; //NOTE: Owned by worker_thread and accessible after the thread has been stoped TransportState state = TRANSPORT_STATE_UNINITIALIZED; //NOTE: Protected by worker_mutex uint32_t bits_send = 0; //NOTE: Owned by worker_thread diff --git a/src/utility/datagram.cpp b/src/utility/datagram.cpp index 9611f86e3bdbf013e9ac112c9410b389f9df5515..32ebe2a9358b5023a962c9c42b9d163bad80d7b0 100644 --- a/src/utility/datagram.cpp +++ b/src/utility/datagram.cpp @@ -4,11 +4,12 @@ DatagramPool::~DatagramPool() { - for (const Datagram& datagam : this->pool) + for (const DatagramBlock& block : this->blocks) { - lava::free_data(datagam.buffer); + lava::free_data(block.buffer); } + this->blocks.clear(); this->pool.clear(); } @@ -20,6 +21,10 @@ Datagram DatagramPool::acquire_datagram() { datagram.size = DATAGRAM_SIZE; datagram.buffer = (uint8_t*)lava::alloc_data(datagram.size); + + DatagramBlock& block = this->blocks.emplace_back(); + block.count = 1; + block.buffer = datagram.buffer; } else @@ -49,6 +54,10 @@ void DatagramPool::acquire_datagrams(uint32_t datagram_count, std::vector<Datagr { uint8_t* datagram_buffer = (uint8_t*)lava::alloc_data(allocate_count * DATAGRAM_SIZE); + DatagramBlock& block = this->blocks.emplace_back(); + block.count = allocate_count; + block.buffer = datagram_buffer; + for (uint32_t index = 0; index < allocate_count; index++) { Datagram& datagram = datagrams.emplace_back(); diff --git a/src/utility/datagram.hpp b/src/utility/datagram.hpp index 01dddf10281167b4c8754a4ce4508bfe84ab7193..afa75304e905371e90e72685d1b62b22124337b0 100644 --- a/src/utility/datagram.hpp +++ b/src/utility/datagram.hpp @@ -10,10 +10,17 @@ struct Datagram uint8_t* buffer = nullptr; }; +struct DatagramBlock +{ + uint32_t count; + uint8_t* buffer = nullptr; +}; + class DatagramPool { private: std::vector<Datagram> pool; + std::vector<DatagramBlock> blocks; public: DatagramPool() = default;