Implemented latency measurement.

src/headset/headset.hpp

@@ -78,6 +78,7 @@ public:
     virtual void submit_frame(VkCommandBuffer command_buffer, VkImageLayout frame_layout, FrameId frame_id) = 0;
 
     // This function can be used to send metadata to a remote headset and is only relevant in case of an remote headset.
+    // If the headset is an remote headset, this fuction has to be called every frame before submitting any frames.
     virtual void submit_metadata(const FrameMetadata& metadata);
 
     // Returns the image format of the framebuffer images.

src/headset/remote_headset.cpp

@@ -62,13 +62,6 @@ bool RemoteHeadset::on_create()
         return false;
     }
 
-    this->file.open("video.h264", std::ios::out | std::ios::trunc | std::ios::binary);
-
-    if (!this->file.good())
-    {
-        return false;
-    }
-
     return !this->check_error();
 }
 
@@ -109,6 +102,7 @@ void RemoteHeadset::submit_frame(VkCommandBuffer command_buffer, VkImageLayout f
 
     lava::image::ptr framebuffer = this->framebuffers[frame_id];
     uint32_t frame_number = this->frame_number;
+    uint32_t transform_id = this->transform_id;
 
     if (!encoder_config)
     {
@@ -125,9 +119,9 @@ void RemoteHeadset::submit_frame(VkCommandBuffer command_buffer, VkImageLayout f
         this->encoder_configs[frame_id] = true;
     }
 
-    bool result = encoder->encode_frame(command_buffer, renderer, framebuffer, frame_layout, [this, frame_number, frame_id](const std::span<uint8_t>& content)
+    bool result = encoder->encode_frame(command_buffer, renderer, framebuffer, frame_layout, [this, frame_number, frame_id, transform_id](const std::span<uint8_t>& content)
     {
-        this->send_packet_frame_nal(frame_number, frame_id, content);
+        this->send_packet_frame_nal(frame_number, frame_id, transform_id, content);
     });
 
     if (!result)
@@ -338,6 +332,7 @@ void RemoteHeadset::on_packet_head_transform(const std::span<uint8_t>& buffer)
     }
 
     std::unique_lock<std::mutex> lock(this->worker_mutex);
+    this->worker_transform_id = packet->transform_id;
     this->worker_view_matrix = packet->head_transform;
     lock.unlock();
 }
@@ -362,6 +357,21 @@ void RemoteHeadset::on_packet_projection_change(const std::span<uint8_t>& buffer
     lock.unlock();
 }
 
+void RemoteHeadset::on_packet_latency(const std::span<uint8_t>& buffer)
+{
+    PacketLatency* packet = (PacketLatency*) buffer.data();
+
+    if (packet->id != PACKET_ID_LATENCY)
+    {
+        lava::log()->error("Remote Headset: Wrong packet id for latency packet!");
+        this->on_transport_error();
+
+        return;
+    }
+
+    lava::log()->debug("Total latency (ms): {}", packet->timestamp_command_executed - packet->timestamp_transform_query);
+}
+
 void RemoteHeadset::on_transport_error()
 {
     lava::log()->error("Remote Headset: Transport error detected!");
@@ -381,6 +391,7 @@ void RemoteHeadset::update_values()
 {
     std::unique_lock<std::mutex> lock(this->worker_mutex);
 
+    this->transform_id = this->worker_transform_id;
     this->resolution = this->worker_resolution;
     this->view_matrix = this->worker_view_matrix;
     this->head_to_eye_matrices = this->worker_head_to_eye_matrices;
@@ -487,6 +498,9 @@ void RemoteHeadset::receive_content(const PacketHeader& header)
             case PACKET_ID_PROJECTION_CHANGE:
                 this->on_packet_projection_change(buffer);
                 break;
+            case PACKET_ID_LATENCY:
+                this->on_packet_latency(buffer);
+                break;
             case PACKET_ID_FRAME_CONFIG_NAL:
                 lava::log()->error("Remote Headset: Received frame config nal packet!");
                 this->on_transport_error();
@@ -549,11 +563,6 @@ void RemoteHeadset::send_packet_frame_config_nal(PacketFrameId frame_id, const s
         return;
     }
 
-    if (frame_id == 0)
-    {
-        //file.write((const char*)content.data(), content.size());
-    }
-
     uint32_t buffer_size = sizeof(PacketFrameConfigNal) + sizeof(uint8_t) * content.size();
     uint8_t* buffer_ptr = new uint8_t[buffer_size];
 
@@ -576,18 +585,13 @@ void RemoteHeadset::send_packet_frame_config_nal(PacketFrameId frame_id, const s
     });
 }
 
-void RemoteHeadset::send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, const std::span<uint8_t>& content)
+void RemoteHeadset::send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, const std::span<uint8_t>& content)
 {
     if (!this->client_socket.has_value())
     {
         return;
     }
 
-    if (frame_id == 0)
-    {
-        //file.write((const char*)content.data(), content.size());
-    }
-
     uint32_t buffer_size = sizeof(PacketFrameNal) + sizeof(uint8_t) * content.size();
     uint8_t* buffer_ptr = new uint8_t[buffer_size];
 
@@ -596,6 +600,7 @@ void RemoteHeadset::send_packet_frame_nal(PacketFrameNumber frame_number, Packet
     packet->size = buffer_size;
     packet->frame_number = frame_number;
 	packet->frame_id = frame_id;
+    packet->transform_id = transform_id;
 
     memcpy(buffer_ptr + sizeof(PacketFrameNal), content.data(), sizeof(uint8_t) * content.size());
 

src/headset/remote_headset.hpp

@@ -10,7 +10,6 @@
 #include <span>
 #include <optional>
 #include <memory>
-
 #include <fstream>
 
 #include "headset.hpp"
@@ -65,6 +64,7 @@ private:
     void on_packet_setup(const std::span<uint8_t>& buffer);
     void on_packet_head_transform(const std::span<uint8_t>& buffer);
     void on_packet_projection_change(const std::span<uint8_t>& buffer);
+    void on_packet_latency(const std::span<uint8_t>& buffer);
     void on_transport_error();
 
     void update_values();
@@ -78,13 +78,13 @@ private:
 
     void send_packet_setup_complete(PacketStereoStrategyId strategy_id, uint32_t max_frame_ids, float near_plane, float far_plane);
     void send_packet_frame_config_nal(PacketFrameId frame_id, const std::span<uint8_t>& content);
-    void send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, const std::span<uint8_t>& content);
+    void send_packet_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, const std::span<uint8_t>& content);
     void send_packet_frame_metadata(PacketFrameNumber frame_number, const std::span<uint8_t>& content);
 
     static bool convert_strategy(StereoStrategyType strategy_type, PacketStereoStrategyId& strategy_id);
 
 private:
-    const uint32_t server_port = 4000;
+    const uint32_t server_port = 8000;
 
     const float near_plane = 1.0f;
     const float far_plane = 10000.0f;
@@ -94,6 +94,7 @@ private:
     std::optional<asio::ip::tcp::acceptor> server_acceptor;
     std::optional<asio::ip::tcp::socket> client_socket;
 
+    uint32_t transform_id;
     glm::uvec2 resolution;
     glm::mat4 view_matrix;
     std::array<glm::mat4, 2> head_to_eye_matrices;
@@ -106,8 +107,6 @@ private:
     std::vector<Encoder::Ptr> encoders;
     std::vector<bool> encoder_configs;
 
-    std::fstream file;
-
 private:
     std::thread worker_thread;
     std::mutex worker_mutex;
@@ -116,6 +115,7 @@ private:
     bool worker_error = false;                                 //NOTE: Protected by wroker_mutex
     bool worker_setup = false;                                 //NOTE: Protected by wroker_mutex
     
+    uint32_t worker_transform_id;                              //NOTE: Protected by wroker_mutex
     glm::uvec2 worker_resolution;                              //NOTE: Protected by wroker_mutex
     glm::mat4 worker_view_matrix;                              //NOTE: Protected by worker_mutex
     std::array<glm::mat4, 2> worker_head_to_eye_matrices;      //NOTE: Protected by worker_mutex

src/stream/packet.hpp

@@ -35,6 +35,7 @@ struct PacketHeadTransform
     PacketId id = PACKET_ID_HEAD_TRANSFORM;
     uint32_t size = 0;
 
+    PacketTransformId transform_id;
     glm::mat4 head_transform;
 };
 
@@ -50,6 +51,22 @@ struct PacketProjectionChange
     glm::mat4 right_head_to_eye;
 };
 
+struct PacketLatency
+{
+    PacketId id = PACKET_ID_LATENCY;
+    uint32_t size = 0;
+
+    PacketFrameNumber frame_number;
+    PacketFrameId frame_id;
+    PacketTransformId transform_id;
+
+    double timestamp_transform_query = 0.0;
+    double timestamp_server_response = 0.0;
+    double timestamp_frame_decoded = 0.0;
+    double timestamp_command_construct = 0.0;
+    double timestamp_command_executed = 0.0;
+};
+
 struct PacketFrameConfigNal
 {
     PacketId id = PACKET_ID_FRAME_NAL;
@@ -67,6 +84,7 @@ struct PacketFrameNal
 
     PacketFrameNumber frame_number;
     PacketFrameId frame_id;
+    PacketTransformId transform_id;
 
     //Followed by: Nal
 };

src/stream/types.hpp

@@ -7,9 +7,10 @@ enum PacketId : uint32_t
     PACKET_ID_SETUP_COMPLETE    = 0x01,
     PACKET_ID_HEAD_TRANSFORM    = 0x02,
     PACKET_ID_PROJECTION_CHANGE = 0x03,
-    PACKET_ID_FRAME_CONFIG_NAL  = 0x04,
-    PACKET_ID_FRAME_NAL         = 0x05,
-    PACKET_ID_FRAME_METADATA    = 0x06
+    PACKET_ID_LATENCY           = 0x04,
+    PACKET_ID_FRAME_CONFIG_NAL  = 0x05,
+    PACKET_ID_FRAME_NAL         = 0x06,
+    PACKET_ID_FRAME_METADATA    = 0x07
 };
 
 enum PacketStereoStrategyId : uint32_t
@@ -21,3 +22,4 @@ enum PacketStereoStrategyId : uint32_t
 
 typedef uint32_t PacketFrameId;
 typedef uint32_t PacketFrameNumber;
+typedef uint32_t PacketTransformId;
\ No newline at end of file