diff --git a/src/headset/headset.hpp b/src/headset/headset.hpp index a6ee36bc1d5b367812357867c3b998048187e553..0c4825fd50c324c0c79f75caf9a7aac79a617585 100644 --- a/src/headset/headset.hpp +++ b/src/headset/headset.hpp @@ -78,6 +78,7 @@ public: virtual void submit_frame(VkCommandBuffer command_buffer, VkImageLayout frame_layout, FrameId frame_id) = 0; // This function can be used to send metadata to a remote headset and is only relevant in case of an remote headset. + // If the headset is an remote headset, this fuction has to be called every frame before submitting any frames. virtual void submit_metadata(const FrameMetadata& metadata); // Returns the image format of the framebuffer images. diff --git a/src/headset/remote_headset.cpp b/src/headset/remote_headset.cpp index 695f4f14b3ca018f77ff75878bf8f9d099ded3e2..0f274eccdc573a40b202e235b8f05dffc79648f7 100644 --- a/src/headset/remote_headset.cpp +++ b/src/headset/remote_headset.cpp @@ -62,13 +62,6 @@ bool RemoteHeadset::on_create() return false; } - this->file.open("video.h264", std::ios::out | std::ios::trunc | std::ios::binary); - - if (!this->file.good()) - { - return false; - } - return !this->check_error(); } @@ -109,6 +102,7 @@ void RemoteHeadset::submit_frame(VkCommandBuffer command_buffer, VkImageLayout f lava::image::ptr framebuffer = this->framebuffers[frame_id]; uint32_t frame_number = this->frame_number; + uint32_t transform_id = this->transform_id; if (!encoder_config) { @@ -125,9 +119,9 @@ void RemoteHeadset::submit_frame(VkCommandBuffer command_buffer, VkImageLayout f this->encoder_configs[frame_id] = true; } - bool result = encoder->encode_frame(command_buffer, renderer, framebuffer, frame_layout, [this, frame_number, frame_id](const std::span<uint8_t>& content) + bool result = encoder->encode_frame(command_buffer, renderer, framebuffer, frame_layout, [this, frame_number, frame_id, transform_id](const std::span<uint8_t>& content) { - this->send_packet_frame_nal(frame_number, frame_id, content); + this->send_packet_frame_nal(frame_number, frame_id, transform_id, content); }); if (!result) @@ -338,6 +332,7 @@ void RemoteHeadset::on_packet_head_transform(const std::span<uint8_t>& buffer) } std::unique_lock<std::mutex> lock(this->worker_mutex); + this->worker_transform_id = packet->transform_id; this->worker_view_matrix = packet->head_transform; lock.unlock(); } @@ -362,6 +357,21 @@ void RemoteHeadset::on_packet_projection_change(const std::span<uint8_t>& buffer lock.unlock(); } +void RemoteHeadset::on_packet_latency(const std::span<uint8_t>& buffer) +{ + PacketLatency* packet = (PacketLatency*) buffer.data(); + + if (packet->id != PACKET_ID_LATENCY) + { + lava::log()->error("Remote Headset: Wrong packet id for latency packet!"); + this->on_transport_error(); + + return; + } + + lava::log()->debug("Total latency (ms): {}", packet->timestamp_command_executed - packet->timestamp_transform_query); +} + void RemoteHeadset::on_transport_error() { lava::log()->error("Remote Headset: Transport error detected!"); @@ -381,6 +391,7 @@ void RemoteHeadset::update_values() { std::unique_lock<std::mutex> lock(this->worker_mutex); + this->transform_id = this->worker_transform_id; this->resolution = this->worker_resolution; this->view_matrix = this->worker_view_matrix; this->head_to_eye_matrices = this->worker_head_to_eye_matrices; @@ -487,6 +498,9 @@ void RemoteHeadset::receive_content(const PacketHeader& header) case PACKET_ID_PROJECTION_CHANGE: this->on_packet_projection_change(buffer); break; + case PACKET_ID_LATENCY: + this->on_packet_latency(buffer); + break; case PACKET_ID_FRAME_CONFIG_NAL: lava::log()->error("Remote Headset: Received frame config nal packet!"); this->on_transport_error(); @@ -549,11 +563,6 @@ void RemoteHeadset::send_packet_frame_config_nal(PacketFrameId frame_id, const s return; } - if (frame_id == 0) - { - //file.write((const char*)content.data(), content.size()); - } - uint32_t buffer_size = sizeof(PacketFrameConfigNal) + sizeof(uint8_t) * content.size(); uint8_t* buffer_ptr = new uint8_t[buffer_size]; @@ -576,18 +585,13 @@ void RemoteHeadset::send_packet_frame_config_nal(PacketFrameId frame_id, const s }); } -void RemoteHeadset::send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, const std::span<uint8_t>& content) +void RemoteHeadset::send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, const std::span<uint8_t>& content) { if (!this->client_socket.has_value()) { return; } - if (frame_id == 0) - { - //file.write((const char*)content.data(), content.size()); - } - uint32_t buffer_size = sizeof(PacketFrameNal) + sizeof(uint8_t) * content.size(); uint8_t* buffer_ptr = new uint8_t[buffer_size]; @@ -596,6 +600,7 @@ void RemoteHeadset::send_packet_frame_nal(PacketFrameNumber frame_number, Packet packet->size = buffer_size; packet->frame_number = frame_number; packet->frame_id = frame_id; + packet->transform_id = transform_id; memcpy(buffer_ptr + sizeof(PacketFrameNal), content.data(), sizeof(uint8_t) * content.size()); diff --git a/src/headset/remote_headset.hpp b/src/headset/remote_headset.hpp index f233bf82e063bd0f8fa53997b52a7395375e0b41..847e542955ae4d2d5c6cbd89ae068ef74bdc00b4 100644 --- a/src/headset/remote_headset.hpp +++ b/src/headset/remote_headset.hpp @@ -10,7 +10,6 @@ #include <span> #include <optional> #include <memory> - #include <fstream> #include "headset.hpp" @@ -65,6 +64,7 @@ private: void on_packet_setup(const std::span<uint8_t>& buffer); void on_packet_head_transform(const std::span<uint8_t>& buffer); void on_packet_projection_change(const std::span<uint8_t>& buffer); + void on_packet_latency(const std::span<uint8_t>& buffer); void on_transport_error(); void update_values(); @@ -78,13 +78,13 @@ private: void send_packet_setup_complete(PacketStereoStrategyId strategy_id, uint32_t max_frame_ids, float near_plane, float far_plane); void send_packet_frame_config_nal(PacketFrameId frame_id, const std::span<uint8_t>& content); - void send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, const std::span<uint8_t>& content); + void send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, const std::span<uint8_t>& content); void send_packet_frame_metadata(PacketFrameNumber frame_number, const std::span<uint8_t>& content); static bool convert_strategy(StereoStrategyType strategy_type, PacketStereoStrategyId& strategy_id); private: - const uint32_t server_port = 4000; + const uint32_t server_port = 8000; const float near_plane = 1.0f; const float far_plane = 10000.0f; @@ -94,6 +94,7 @@ private: std::optional<asio::ip::tcp::acceptor> server_acceptor; std::optional<asio::ip::tcp::socket> client_socket; + uint32_t transform_id; glm::uvec2 resolution; glm::mat4 view_matrix; std::array<glm::mat4, 2> head_to_eye_matrices; @@ -106,8 +107,6 @@ private: std::vector<Encoder::Ptr> encoders; std::vector<bool> encoder_configs; - std::fstream file; - private: std::thread worker_thread; std::mutex worker_mutex; @@ -116,6 +115,7 @@ private: bool worker_error = false; //NOTE: Protected by wroker_mutex bool worker_setup = false; //NOTE: Protected by wroker_mutex + uint32_t worker_transform_id; //NOTE: Protected by wroker_mutex glm::uvec2 worker_resolution; //NOTE: Protected by wroker_mutex glm::mat4 worker_view_matrix; //NOTE: Protected by worker_mutex std::array<glm::mat4, 2> worker_head_to_eye_matrices; //NOTE: Protected by worker_mutex diff --git a/src/stream/packet.hpp b/src/stream/packet.hpp index d3ad0029f3676cf436f27ef2abd2752152077718..d5678d1eb2c48179d24a6155a8382f1a30a957c0 100644 --- a/src/stream/packet.hpp +++ b/src/stream/packet.hpp @@ -35,6 +35,7 @@ struct PacketHeadTransform PacketId id = PACKET_ID_HEAD_TRANSFORM; uint32_t size = 0; + PacketTransformId transform_id; glm::mat4 head_transform; }; @@ -50,6 +51,22 @@ struct PacketProjectionChange glm::mat4 right_head_to_eye; }; +struct PacketLatency +{ + PacketId id = PACKET_ID_LATENCY; + uint32_t size = 0; + + PacketFrameNumber frame_number; + PacketFrameId frame_id; + PacketTransformId transform_id; + + double timestamp_transform_query = 0.0; + double timestamp_server_response = 0.0; + double timestamp_frame_decoded = 0.0; + double timestamp_command_construct = 0.0; + double timestamp_command_executed = 0.0; +}; + struct PacketFrameConfigNal { PacketId id = PACKET_ID_FRAME_NAL; @@ -67,6 +84,7 @@ struct PacketFrameNal PacketFrameNumber frame_number; PacketFrameId frame_id; + PacketTransformId transform_id; //Followed by: Nal }; diff --git a/src/stream/types.hpp b/src/stream/types.hpp index 8e35d5a7d8f6b1662c8dbab3164d7f96930424e7..afa0275799be13db0ff23614fc8285f195535bfe 100644 --- a/src/stream/types.hpp +++ b/src/stream/types.hpp @@ -7,9 +7,10 @@ enum PacketId : uint32_t PACKET_ID_SETUP_COMPLETE = 0x01, PACKET_ID_HEAD_TRANSFORM = 0x02, PACKET_ID_PROJECTION_CHANGE = 0x03, - PACKET_ID_FRAME_CONFIG_NAL = 0x04, - PACKET_ID_FRAME_NAL = 0x05, - PACKET_ID_FRAME_METADATA = 0x06 + PACKET_ID_LATENCY = 0x04, + PACKET_ID_FRAME_CONFIG_NAL = 0x05, + PACKET_ID_FRAME_NAL = 0x06, + PACKET_ID_FRAME_METADATA = 0x07 }; enum PacketStereoStrategyId : uint32_t @@ -20,4 +21,5 @@ enum PacketStereoStrategyId : uint32_t }; typedef uint32_t PacketFrameId; -typedef uint32_t PacketFrameNumber; \ No newline at end of file +typedef uint32_t PacketFrameNumber; +typedef uint32_t PacketTransformId; \ No newline at end of file