From 78425610924a1d272a18d15d1f659a751624ddc0 Mon Sep 17 00:00:00 2001 From: Jens Koenen <koenen@vr.rwth-aachen.de> Date: Thu, 10 Nov 2022 17:22:33 +0100 Subject: [PATCH] Started to implement interface for transport --- CMakeLists.txt | 16 +- src/headset/headset.hpp | 29 +- src/headset/remote_headset.cpp | 69 ++-- src/headset/remote_headset.hpp | 26 +- src/stream/packet.hpp | 162 -------- src/stream/transport.hpp | 136 ------- src/stream/types.hpp | 60 --- src/transport/transport.cpp | 18 + src/transport/transport.hpp | 83 ++++ src/transport/udp_packet.cpp | 101 +++++ src/transport/udp_packet.hpp | 226 +++++++++++ .../udp_transport.cpp} | 383 ++++++++++-------- src/transport/udp_transport.hpp | 105 +++++ src/types.hpp | 54 +++ src/utility/command_parser.cpp | 32 ++ src/utility/command_parser.hpp | 4 + src/{stream => utility}/datagram.cpp | 0 src/{stream => utility}/datagram.hpp | 0 src/{stream => utility}/ring_buffer.cpp | 0 src/{stream => utility}/ring_buffer.hpp | 0 src/vr_application.cpp | 5 + src/vr_application.hpp | 2 + 22 files changed, 901 insertions(+), 610 deletions(-) delete mode 100644 src/stream/packet.hpp delete mode 100644 src/stream/transport.hpp delete mode 100644 src/stream/types.hpp create mode 100644 src/transport/transport.cpp create mode 100644 src/transport/transport.hpp create mode 100644 src/transport/udp_packet.cpp create mode 100644 src/transport/udp_packet.hpp rename src/{stream/transport.cpp => transport/udp_transport.cpp} (57%) create mode 100644 src/transport/udp_transport.hpp create mode 100644 src/types.hpp rename src/{stream => utility}/datagram.cpp (100%) rename src/{stream => utility}/datagram.hpp (100%) rename src/{stream => utility}/ring_buffer.cpp (100%) rename src/{stream => utility}/ring_buffer.hpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72a245b0..679f548d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -688,6 +688,7 @@ message("======================================================================= ${SRC_DIR}/main.cpp ${SRC_DIR}/vr_application.hpp ${SRC_DIR}/vr_application.cpp ${SRC_DIR}/scene.hpp ${SRC_DIR}/scene.cpp + ${SRC_DIR}/types.hpp #Headset ${SRC_DIR}/headset/headset.hpp ${SRC_DIR}/headset/headset.cpp @@ -703,16 +704,15 @@ message("======================================================================= ${SRC_DIR}/strategy/multi_view_stereo.hpp ${SRC_DIR}/strategy/multi_view_stereo.cpp ${SRC_DIR}/strategy/depth_peeling_reprojection_stereo.hpp ${SRC_DIR}/strategy/depth_peeling_reprojection_stereo.cpp - #Stream - ${SRC_DIR}/stream/datagram.hpp ${SRC_DIR}/stream/datagram.cpp - ${SRC_DIR}/stream/packet.hpp - ${SRC_DIR}/stream/ring_buffer.hpp ${SRC_DIR}/stream/ring_buffer.cpp - ${SRC_DIR}/stream/transport.hpp ${SRC_DIR}/stream/transport.cpp - ${SRC_DIR}/stream/types.hpp + #Transport + ${SRC_DIR}/transport/transport.hpp ${SRC_DIR}/transport/transport.cpp + ${SRC_DIR}/transport/udp_packet.hpp ${SRC_DIR}/transport/udp_packet.cpp + ${SRC_DIR}/transport/udp_transport.hpp ${SRC_DIR}/transport/udp_transport.cpp #Utility ${SRC_DIR}/utility/command_parser.hpp ${SRC_DIR}/utility/command_parser.cpp ${SRC_DIR}/utility/companion_window.hpp ${SRC_DIR}/utility/companion_window.cpp + ${SRC_DIR}/utility/datagram.hpp ${SRC_DIR}/utility/datagram.cpp ${SRC_DIR}/utility/encoder.hpp ${SRC_DIR}/utility/encoder.cpp ${SRC_DIR}/utility/extern_fence.hpp ${SRC_DIR}/utility/extern_fence.cpp ${SRC_DIR}/utility/frame_capture.hpp ${SRC_DIR}/utility/frame_capture.cpp @@ -723,6 +723,7 @@ message("======================================================================= ${SRC_DIR}/utility/shadow_cache.hpp ${SRC_DIR}/utility/shadow_cache.cpp ${SRC_DIR}/utility/statistic.hpp ${SRC_DIR}/utility/statistic.cpp ${SRC_DIR}/utility/stereo_transform.hpp ${SRC_DIR}/utility/stereo_transform.cpp + ${SRC_DIR}/utility/ring_buffer.hpp ${SRC_DIR}/utility/ring_buffer.cpp ) add_shaders( @@ -802,10 +803,11 @@ message("======================================================================= source_group("" FILES ${SRC_DIR}/vr_application.hpp) source_group("" FILES ${SRC_DIR}/scene.cpp) source_group("" FILES ${SRC_DIR}/scene.hpp) + source_group("" FILES ${SRC_DIR}/types.hpp) source_group("Headset" REGULAR_EXPRESSION ${SRC_DIR}/headset/*) source_group("Strategy" REGULAR_EXPRESSION ${SRC_DIR}/strategy/*) - source_group("Stream" REGULAR_EXPRESSION ${SRC_DIR}/stream/*) + source_group("Transport" REGULAR_EXPRESSION ${SRC_DIR}/transport/*) source_group("Utility" REGULAR_EXPRESSION ${SRC_DIR}/utility/*) endif() diff --git a/src/headset/headset.hpp b/src/headset/headset.hpp index 1ef99359..30af8d42 100644 --- a/src/headset/headset.hpp +++ b/src/headset/headset.hpp @@ -17,34 +17,7 @@ class VRApplication; -typedef uint32_t FrameId; -typedef std::span<uint8_t> FrameMetadata; - -// Defined the frame ids for the left and right eye in case of an non-remote headset. -enum Eye -{ - EYE_LEFT, - EYE_RIGHT -}; - -// Define left and right controller -enum Controller -{ - CONTROLLER_LEFT, - CONTROLLER_RIGHT -}; - -// Define buttons -enum Button -{ - BUTTON_A, - BUTTON_B, - BUTTON_X, - BUTTON_Y, - BUTTON_TRIGGER_LEFT, - BUTTON_TRIGGER_RIGHT, - BUTTON_MAX_COUNT -}; +#include "types.hpp" // When adding or removing a headset, the function make_headset(...) as well as // the function set_headset(...) of the CommandParser need to be adapted accordingly. diff --git a/src/headset/remote_headset.cpp b/src/headset/remote_headset.cpp index 61e57da8..d97e2f8c 100644 --- a/src/headset/remote_headset.cpp +++ b/src/headset/remote_headset.cpp @@ -9,7 +9,7 @@ RemoteHeadset::RemoteHeadset() this->controller_states.fill(CONTROLLER_STATE_DETACHED); this->transport_controller_states.fill(CONTROLLER_STATE_DETACHED); - for (std::array<PacketButtonState, BUTTON_ID_MAX_COUNT>& buttons : this->controller_buttons) + for (std::array<ButtonState, BUTTON_MAX_COUNT>& buttons : this->controller_buttons) { buttons.fill(BUTTON_STATE_RELEASED); } @@ -52,7 +52,7 @@ bool RemoteHeadset::on_create() StereoStrategy::Ptr strategy = this->get_application()->get_stereo_strategy(); this->frame_id_count = strategy->get_max_remote_frame_ids(); - if (!convert_strategy(this->get_application()->get_stereo_strategy_type(), this->strategy_id)) + if (!convert_strategy(this->get_application()->get_stereo_strategy_type(), this->remote_strategy)) { return false; } @@ -186,11 +186,6 @@ bool RemoteHeadset::on_interface() return false; } - /*if (ImGui::SliderFloat("Send-Rate Limit (Mbps)", &this->send_bit_rate_limit, 1.0, 20.0)) - { - this->transport->set_max_bitrate_send(this->send_bit_rate_limit); - }*/ - if (ImGui::SliderInt("Send-Queue Limit (KBytes)", (int32_t*)&this->send_queue_size_limit, 1, 1000)) { this->transport->set_max_send_queue_size(this->send_queue_size_limit * 1000); @@ -343,7 +338,7 @@ bool RemoteHeadset::is_remote() const bool RemoteHeadset::create_transport() { - this->transport = make_transport(); + this->transport = make_transport(this->get_application()->get_command_parser().get_transport()); this->transport->set_on_setup([this](const glm::u32vec2& resolution) { @@ -353,27 +348,27 @@ bool RemoteHeadset::create_transport() { this->on_projection_change(left_eye_projection, right_eye_projection, left_head_to_eye, right_head_to_eye); }); - this->transport->set_on_head_transform([this](PacketTransformId transform_id, const glm::mat4& head_transform) + this->transport->set_on_head_transform([this](TransformId transform_id, const glm::mat4& head_transform) { this->on_head_transform(transform_id, head_transform); }); - this->transport->set_on_controller_transform([this](PacketControllerId controller_id, const glm::mat4& controller_transform) + this->transport->set_on_controller_transform([this](Controller controller, const glm::mat4& controller_transform) { - this->on_controller_transform(controller_id, controller_transform); + this->on_controller_transform(controller, controller_transform); }); - this->transport->set_on_controller_event([this](PacketControllerId controller_id, PacketControllerState controller_state) + this->transport->set_on_controller_event([this](Controller controller, ControllerState controller_state) { - this->on_controller_event(controller_id, controller_state); + this->on_controller_event(controller, controller_state); }); - this->transport->set_on_controller_button([this](PacketControllerId controller_id, PacketButtonId button_id, PacketButtonState button_state) + this->transport->set_on_controller_button([this](Controller controller, Button button, ButtonState button_state) { - this->on_controller_button(controller_id, button_id, button_state); + this->on_controller_button(controller, button, button_state); }); - this->transport->set_on_controller_thumbstick([this](PacketControllerId controller_id, const glm::vec2& thumbstick) + this->transport->set_on_controller_thumbstick([this](Controller controller, const glm::vec2& thumbstick) { - this->on_controller_thumbstick(controller_id, thumbstick); + this->on_controller_thumbstick(controller, thumbstick); }); - this->transport->set_on_latency([this](PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, double timestamp_transfrom_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed) + this->transport->set_on_latency([this](FrameNumber frame_number, FrameId frame_id, TransformId transform_id, double timestamp_transfrom_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed) { this->on_latency(frame_number, frame_id, transform_id, timestamp_transfrom_query, timestamp_server_response, timestamp_frame_decoded, timestamp_command_construct, timestamp_command_executed); }); @@ -382,7 +377,6 @@ bool RemoteHeadset::create_transport() this->on_transport_error(); }); - //this->transport->set_max_bitrate_send(this->send_bit_rate_limit); this->transport->set_max_send_queue_size(this->send_queue_size_limit * 1000); if (!this->transport->create(this->server_port)) @@ -504,7 +498,7 @@ void RemoteHeadset::update_stage(lava::delta delta_time) for (uint32_t index = 0; index < this->controller_buttons.size(); index++) { - const std::array<PacketButtonState, BUTTON_ID_MAX_COUNT>& buttons = this->controller_buttons[index]; + const std::array<ButtonState, BUTTON_MAX_COUNT>& buttons = this->controller_buttons[index]; glm::vec2 thumbstick = this->controller_thumbsticks[index]; glm::mat4 controller_transform = this->controller_transforms[index]; @@ -513,7 +507,7 @@ void RemoteHeadset::update_stage(lava::delta delta_time) float strength = this->movement_speed * delta_time; - if (buttons[BUTTON_ID_TRIGGER] == BUTTON_STATE_PRESSED) + if (buttons[BUTTON_TRIGGER] == BUTTON_STATE_PRESSED) { strength *= 2.0f; } @@ -537,7 +531,7 @@ void RemoteHeadset::update_stage(lava::delta delta_time) void RemoteHeadset::on_setup(const glm::u32vec2& resolution) { - this->transport->send_setup_complete(this->strategy_id, this->frame_id_count, this->near_plane, this->far_plane); + this->transport->send_setup_complete(this->remote_strategy, this->frame_id_count, this->near_plane, this->far_plane); std::unique_lock<std::mutex> lock(this->transport_mutex); this->resolution = resolution; @@ -554,7 +548,7 @@ void RemoteHeadset::on_projection_change(const glm::mat4& left_eye_projection, c lock.unlock(); } -void RemoteHeadset::on_head_transform(PacketTransformId transform_id, const glm::mat4& head_transform) +void RemoteHeadset::on_head_transform(TransformId transform_id, const glm::mat4& head_transform) { std::unique_lock<std::mutex> lock(this->transport_mutex); this->transform_id = transform_id; @@ -562,35 +556,35 @@ void RemoteHeadset::on_head_transform(PacketTransformId transform_id, const glm: lock.unlock(); } -void RemoteHeadset::on_controller_transform(PacketControllerId controller_id, const glm::mat4& controller_transform) +void RemoteHeadset::on_controller_transform(Controller controller, const glm::mat4& controller_transform) { std::unique_lock<std::mutex> lock(this->transport_mutex); - this->controller_transforms[controller_id] = controller_transform; + this->controller_transforms[controller] = controller_transform; lock.unlock(); } -void RemoteHeadset::on_controller_event(PacketControllerId controller_id, PacketControllerState controller_state) +void RemoteHeadset::on_controller_event(Controller controller, ControllerState controller_state) { std::unique_lock<std::mutex> lock(this->transport_mutex); - this->transport_controller_states[controller_id] = controller_state; + this->transport_controller_states[controller] = controller_state; lock.unlock(); } -void RemoteHeadset::on_controller_button(PacketControllerId controller_id, PacketButtonId button_id, PacketButtonState button_state) +void RemoteHeadset::on_controller_button(Controller controller, Button button, ButtonState button_state) { std::unique_lock<std::mutex> lock(this->transport_mutex); - this->controller_buttons[controller_id][button_id] = button_state; + this->controller_buttons[controller][button] = button_state; lock.unlock(); } -void RemoteHeadset::on_controller_thumbstick(PacketControllerId controller_id, const glm::vec2& thumbstick) +void RemoteHeadset::on_controller_thumbstick(Controller controller, const glm::vec2& thumbstick) { std::unique_lock<std::mutex> lock(this->transport_mutex); - this->controller_thumbsticks[controller_id] = thumbstick; + this->controller_thumbsticks[controller] = thumbstick; lock.unlock(); } -void RemoteHeadset::on_latency(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, double timestamp_transform_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed) +void RemoteHeadset::on_latency(FrameNumber frame_number, FrameId frame_id, TransformId transform_id, double timestamp_transform_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed) { LatencyCapture capture; capture.frame_id = frame_id; @@ -612,18 +606,21 @@ void RemoteHeadset::on_transport_error() lava::log()->error("Remote Headset: Transport error detected!"); } -bool RemoteHeadset::convert_strategy(StereoStrategyType strategy_type, PacketStereoStrategyId& strategy_id) +bool RemoteHeadset::convert_strategy(StereoStrategyType strategy_type, RemoteStrategy& remote_strategy) { switch (strategy_type) { case STEREO_STRATEGY_TYPE_NATIVE_FORWARD: - strategy_id = STEREO_STRATEGY_ID_NATIVE_STEREO; + remote_strategy = REMOTE_STRATEGY_NATIVE; break; case STEREO_STRATEGY_TYPE_NATIVE_DEFERRED: - strategy_id = STEREO_STRATEGY_ID_NATIVE_STEREO; + remote_strategy = REMOTE_STRATEGY_NATIVE; + break; + case STEREO_STRATEGY_TYPE_MULTI_VIEW: + remote_strategy = REMOTE_STRATEGY_NATIVE; break; case STEREO_STRATEGY_TYPE_DEPTH_PEELING_REPROJECTION: - strategy_id = STEREO_STRATEGY_ID_NATIVE_STEREO; + remote_strategy = REMOTE_STRATEGY_NATIVE; break; default: lava::log()->error("Remote Headset: Unkown strategy type!"); diff --git a/src/headset/remote_headset.hpp b/src/headset/remote_headset.hpp index 0719de7f..fdc9dbc2 100644 --- a/src/headset/remote_headset.hpp +++ b/src/headset/remote_headset.hpp @@ -9,8 +9,7 @@ #include "headset.hpp" #include "strategy/stereo_strategy.hpp" - -#include "stream/transport.hpp" +#include "transport/transport.hpp" #include "utility/encoder.hpp" #include "utility/extern_fence.hpp" @@ -66,15 +65,15 @@ private: void on_setup(const glm::u32vec2& resolution); void on_projection_change(const glm::mat4& left_eye_projection, const glm::mat4& right_eye_projection, const glm::mat4& left_head_to_eye, const glm::mat4& right_head_to_eye); - void on_head_transform(PacketTransformId transform_id, const glm::mat4& head_transform); - void on_controller_transform(PacketControllerId controller_id, const glm::mat4& controller_transform); - void on_controller_event(PacketControllerId controller_id, PacketControllerState controller_state); - void on_controller_button(PacketControllerId controller_id, PacketButtonId button_id, PacketButtonState button_state); - void on_controller_thumbstick(PacketControllerId controller_id, const glm::vec2& thumbstick); - void on_latency(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, double timestamp_transform_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed); + void on_head_transform(TransformId transform_id, const glm::mat4& head_transform); + void on_controller_transform(Controller controller, const glm::mat4& controller_transform); + void on_controller_event(Controller controller, ControllerState controller_state); + void on_controller_button(Controller controller, Button button, ButtonState button_state); + void on_controller_thumbstick(Controller controller, const glm::vec2& thumbstick); + void on_latency(FrameNumber frame_number, FrameId frame_id, TransformId transform_id, double timestamp_transform_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed); void on_transport_error(); - static bool convert_strategy(StereoStrategyType strategy_type, PacketStereoStrategyId& strategy_id); + static bool convert_strategy(StereoStrategyType strategy_type, RemoteStrategy& remote_strategy); private: const uint32_t server_port = 4000; @@ -88,7 +87,7 @@ private: std::array<glm::mat4, 2> head_to_eye_matrices; std::array<glm::mat4, 2> projection_matrices; std::array<glm::mat4, 2> controller_matrices; - std::array<PacketControllerState, 2> controller_states; + std::array<ControllerState, 2> controller_states; Transport::Ptr transport; std::mutex transport_mutex; @@ -98,12 +97,12 @@ private: std::array<glm::mat4, 2> transport_head_to_eye_matrices; //NOTE: Protected by transport_mutex std::array<glm::mat4, 2> transport_projection_matrices; //NOTE: Protected by transport_mutex - std::array<PacketControllerState, 2> transport_controller_states; //NOTE: Protected by transport_mutex + std::array<ControllerState, 2> transport_controller_states; //NOTE: Protected by transport_mutex std::array<glm::mat4, 2> controller_transforms; //NOTE: Protected by transport_mutex std::array<glm::vec2, 2> controller_thumbsticks; //NOTE: Protected by transport_mutex - std::array<std::array<PacketButtonState, BUTTON_ID_MAX_COUNT>, 2> controller_buttons; //NOTE: Protected by transport_mutex + std::array<std::array<ButtonState, BUTTON_MAX_COUNT>, 2> controller_buttons; //NOTE: Protected by transport_mutex - PacketStereoStrategyId strategy_id = STEREO_STRATEGY_ID_NATIVE_STEREO; + RemoteStrategy remote_strategy = REMOTE_STRATEGY_NATIVE; uint32_t frame_number = 0; uint32_t frame_id_count = 0; uint32_t transform_id = 0; //NOTE: Protected by transport_mutex @@ -122,7 +121,6 @@ private: float encoder_last_submit = 0.0f; bool encoder_enable_submit = true; - //float send_bit_rate_limit = 15.0; //NOTE: Bitrate in Mbit per seconds uint32_t send_queue_size_limit = 100; //NOTE: Size in KBytes Statistic::Ptr statistic_send_bitrate; diff --git a/src/stream/packet.hpp b/src/stream/packet.hpp deleted file mode 100644 index 1fee1a93..00000000 --- a/src/stream/packet.hpp +++ /dev/null @@ -1,162 +0,0 @@ -#pragma once -#include <glm/glm.hpp> -#include <cstdint> - -#include "types.hpp" - -//All packages need to be smaller than 512, otherwise the package needs to be fragmented. -//In case of an fragmented package, all parts of it are gathered and sorted based on the information provide in the packet. -//Currently only PacketFrameNal, PacketFrameMetadata packages could get fragmented during transmission. -//Besides that the transport system has to wait for the PacketSetup, PacketSetupComplete to arrive before it can distribute other packages. -//This means that the application always sees the PacketSetup, PacketSetupComplete before any other package. - -//NOTE: Not an actual packet. -struct PacketHeader -{ - PacketId id = (PacketId)0; - uint32_t size = 0; -}; - -struct PacketSetup -{ - PacketId id = PACKET_ID_SETUP; - uint32_t size = 0; - - glm::u32vec2 resolution; -}; - -struct PacketSetupComplete -{ - PacketId id = PACKET_ID_SETUP_COMPLETE; - uint32_t size = 0; - - PacketStereoStrategyId strategy_id; - uint32_t max_frame_ids; - - float near_plane; - float far_plane; -}; - -struct PacketProjectionChange -{ - PacketId id = PACKET_ID_PROJECTION_CHANGE; - uint32_t size = 0; - - glm::mat4 left_eye_projection; - glm::mat4 right_eye_projection; - - glm::mat4 left_head_to_eye; - glm::mat4 right_head_to_eye; -}; - -struct PacketStageTransform -{ - PacketId id = PACKET_ID_STAGE_TRANSFORM; - uint32_t size = 0; - - glm::mat4 stage_transform; -}; - -struct PacketHeadTransform -{ - PacketId id = PACKET_ID_HEAD_TRANSFORM; - uint32_t size = 0; - - PacketTransformId transform_id; - glm::mat4 head_transform; -}; - -struct PacketControllerTransform -{ - PacketId id = PACKET_ID_CONTROLLER_TRANSFORM; - uint32_t size = 0; - - PacketControllerId controller_id; - glm::mat4 controller_transform; -}; - -struct PacketControllerEvent -{ - PacketId id = PACKET_ID_CONTROLLER_EVENT; - uint32_t size = 0; - - PacketControllerId controller_id; - PacketControllerState controller_state; -}; - -struct PacketControllerButton -{ - PacketId id = PACKET_ID_CONTROLLER_TRANSFORM; - uint32_t size = 0; - - PacketControllerId controller_id; - PacketButtonId button_id; - PacketButtonState button_state; -}; - -struct PacketControllerThumbstick -{ - PacketId id = PACKET_ID_CONTROLLER_THUMBSTICK; - uint32_t size = 0; - - PacketControllerId controller_id; - glm::vec2 thumbstick; -}; - -//NOTE: The entire size of the packet including the payload needs to be less than 512 bytes -struct PacketFrameConfigNal -{ - PacketId id = PACKET_ID_FRAME_NAL; - uint32_t size = 0; - - PacketFrameId frame_id; - - //Followed by: Config Nal -}; - -//NOTE: The packet needs to be fragmented if the payload is too big -struct PacketFrameNal -{ - PacketId id = PACKET_ID_FRAME_NAL; - uint32_t size = 0; - - PacketFrameNumber frame_number; //In case of fragmentation, frame number and frame id are used together as identifier - PacketFrameId frame_id; - PacketTransformId transform_id; - - uint32_t payload_offset; - uint32_t payload_size; - - //Followed by: Nal -}; - -//NOTE: Assume frame metadata of a particular frame number is always send before any frame nals of the same frame number -//NOTE: The packet needs to be fragmented if the payload is too big -struct PacketFrameMetadata -{ - PacketId id = PACKET_ID_FRAME_METADATA; - uint32_t size = 0; - - PacketFrameNumber frame_number; //In case of fragmentation, the frame number is used as identifier - - uint32_t payload_offset; - uint32_t payload_size; - - //Followed by: Metadata -}; - -struct PacketLatency -{ - PacketId id = PACKET_ID_LATENCY; - uint32_t size = 0; - - PacketFrameNumber frame_number; - PacketFrameId frame_id; - PacketTransformId transform_id; - - double timestamp_transform_query = 0.0; - double timestamp_server_response = 0.0; - double timestamp_frame_decoded = 0.0; - double timestamp_command_construct = 0.0; - double timestamp_command_executed = 0.0; -}; \ No newline at end of file diff --git a/src/stream/transport.hpp b/src/stream/transport.hpp deleted file mode 100644 index 1fd57a19..00000000 --- a/src/stream/transport.hpp +++ /dev/null @@ -1,136 +0,0 @@ -#pragma once -#include <glm/glm.hpp> -#include <asio.hpp> -#include <memory> -#include <span> -#include <optional> - -#include "datagram.hpp" -#include "packet.hpp" - -#define TRANSPORT_STATE_CHECK_INTERVAL 10 //NOTE: In milliseconds -#define TRANSPORT_CLIENT_CONNECT_TIMEOUT 60 //NOTE: In seconds -#define TRANSPORT_CLIENT_TIMEOUT 20 //NOTE: In seconds - -enum TransportState -{ - TRANSPORT_STATE_UNINITIALIZED, - TRANSPORT_STATE_WAITING, - TRANSPORT_STATE_CONNECTED, - TRANSPORT_STATE_ERROR -}; - -class Transport -{ -public: - typedef std::shared_ptr<Transport> Ptr; - - typedef std::function<void(const glm::u32vec2& resolution)> OnSetup; - typedef std::function<void(const glm::mat4& left_eye_projection, const glm::mat4& right_eye_projection, const glm::mat4& left_head_to_eye, const glm::mat4& right_head_to_eye)> OnProjectionChange; - typedef std::function<void(PacketTransformId transform_id, const glm::mat4& head_transform)> OnHeadTransform; - typedef std::function<void(PacketControllerId controller_id, const glm::mat4& controller_transform)> OnControllerTransform; - typedef std::function<void(PacketControllerId controller_id, PacketControllerState controller_state)> OnControllerEvent; - typedef std::function<void(PacketControllerId controller_id, PacketButtonId button_id, PacketButtonState button_state)> OnControllerButton; - typedef std::function<void(PacketControllerId controller_id, const glm::vec2& thumbstick)> OnControllerThumbstick; - typedef std::function<void(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, double timestamp_transform_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed)> OnLatency; - typedef std::function<void()> OnTransportError; - -public: - Transport() = default; - - bool create(uint32_t port_number); - void destroy(); - bool wait_connect(); - - //NOTE: The following functions should be thread safe - bool send_setup_complete(PacketStereoStrategyId strategy_id, uint32_t max_frame_ids, float near_plane, float far_plane); - bool send_stage_transform(const glm::mat4& stage_transform); - bool send_frame_config_nal(PacketFrameId frame_id, const std::span<uint8_t>& content); - bool send_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, const std::span<uint8_t>& content); - bool send_frame_metadata(PacketFrameNumber frame_number, const std::span<uint8_t>& content); - - //NOTE: The callbacks are only executed by the worker thread of the transport class. - //NOTE: Set the callbacks before calling create and don't do it again after calling create. - void set_on_setup(OnSetup function); - void set_on_projection_change(OnProjectionChange function); - void set_on_head_transform(OnHeadTransform function); - void set_on_controller_transform(OnControllerTransform function); - void set_on_controller_event(OnControllerEvent function); - void set_on_controller_button(OnControllerButton function); - void set_on_controller_thumbstick(OnControllerThumbstick function); - void set_on_latency(OnLatency function); - void set_on_transport_error(OnTransportError function); - - //NOTE: The following functions should be thread safe - //void set_max_bitrate_send(double bitrate); //NOTE: The bitrate is given in MBits/s - void set_max_send_queue_size(uint32_t size); //NOTE: The size is given in Bytes - - //NOTE: The following function should be thread safe - asio::io_context& get_context(); - TransportState get_state(); - double get_bitrate_send(); //NOTE: The bitrate is given in MBits/s - double get_bitrate_received(); //NOTE: The bitrate is given in MBits/s - //double get_max_bitrate_send(); //NOTE: The bitrate is given in MBits/s - uint32_t get_max_send_queue_size(); //NOTE: The size is given in Bytes - - uint32_t get_send_queue_size(); //NOTE: In Bytes - uint32_t get_receive_queue_size(); //NOTE: In Bytes - -private: - void set_state(TransportState state); - void check_state(); - - void send_datagram(const Datagram& datagram); - void receive_datagram(); - - void process_send_queue(); //NOTE: Protected by worker_mutex - void process_receive_queue(); - - void parse_setup(const Datagram& datagram); - void parse_projection_change(const Datagram& datagram); - void parse_head_transform(const Datagram& datagram); - void parse_controller_transform(const Datagram& datagram); - void parse_controller_event(const Datagram& datagram); - void parse_controller_button(const Datagram& datagram); - void parse_controller_thumbstick(const Datagram& datagram); - void parse_latency(const Datagram& datagram); - -private: - std::thread worker_thread; - std::mutex worker_mutex; - std::condition_variable worker_setup_condition; - - asio::io_context server_context; - std::optional<asio::high_resolution_timer> server_timer; - asio::ip::udp::endpoint server_endpoint; - std::optional<asio::ip::udp::socket> server_socket; - asio::ip::udp::endpoint receive_endpoint; - std::optional<asio::ip::udp::endpoint> client_endpoint; - - DatagramPool datagram_pool; //NOTE: Protected by worker_mutex - std::vector<Datagram> send_queue; //NOTE: Protected by worker_mutex - std::vector<Datagram> receive_queue; //NOTE: Owned by worker_thread and accessible after the thread has been stoped - std::vector<Datagram> error_queue; //NOTE: Owned by worker_thread and accessible after the thread has been stoped - - TransportState state = TRANSPORT_STATE_UNINITIALIZED; //NOTE: Protected by worker_mutex - uint32_t bits_send = 0; //NOTE: Owned by worker_thread - uint32_t bits_received = 0; //NOTE: Owned by worker_thread - double bitrate_send = 0.0; //NOTE: Protected by worker_mutex - double bitrate_received = 0.0; //NOTE: Protected by worker_mutex - //double max_bitrate_send = 1.0; //NOTE: Protected by worker_mutex - uint32_t max_send_queue_size = DATAGRAM_SIZE * 128; //NOTE: Protected by worker_mutex - bool send_active = false; //NOTE: Protected by worker_mutex - double inactive_time = 0.0; //NOTE: Owned by worker_thread - - OnSetup on_setup; - OnProjectionChange on_projection_change; - OnHeadTransform on_head_transform; - OnControllerTransform on_controller_transform; - OnControllerEvent on_controller_event; - OnControllerButton on_controller_button; - OnControllerThumbstick on_controller_thumbstick; - OnLatency on_latency; - OnTransportError on_transport_error; -}; - -Transport::Ptr make_transport(); \ No newline at end of file diff --git a/src/stream/types.hpp b/src/stream/types.hpp deleted file mode 100644 index 8428c166..00000000 --- a/src/stream/types.hpp +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once -#include <cstdint> - -enum PacketId : uint32_t -{ - PACKET_ID_SETUP = 0x00, - PACKET_ID_SETUP_COMPLETE = 0x01, - PACKET_ID_PROJECTION_CHANGE = 0x02, - PACKET_ID_STAGE_TRANSFORM = 0x03, - PACKET_ID_HEAD_TRANSFORM = 0x04, - PACKET_ID_CONTROLLER_TRANSFORM = 0x05, - PACKET_ID_CONTROLLER_EVENT = 0x06, - PACKET_ID_CONTROLLER_BUTTON = 0x07, - PACKET_ID_CONTROLLER_THUMBSTICK = 0x08, - PACKET_ID_FRAME_CONFIG_NAL = 0x09, - PACKET_ID_FRAME_NAL = 0x0A, - PACKET_ID_FRAME_METADATA = 0x0B, - PACKET_ID_LATENCY = 0x0C, - PACKET_ID_MAX_COUNT -}; - -enum PacketStereoStrategyId : uint32_t -{ - STEREO_STRATEGY_ID_DEBUG = 0x00, - STEREO_STRATEGY_ID_NATIVE_STEREO = 0x01, - STEREO_STRATEGY_ID_MAX_COUNT -}; - -enum PacketControllerId : uint32_t -{ - CONTROLLER_ID_LEFT = 0x00, - CONTROLLER_ID_RIGHT = 0x01, - CONTROLLER_ID_MAX_COUNT -}; - -enum PacketControllerState : uint32_t -{ - CONTROLLER_STATE_DETACHED = 0x00, - CONTROLLER_STATE_ATTACHED = 0x01 -}; - -enum PacketButtonId : uint32_t -{ - BUTTON_ID_A = 0x00, - BUTTON_ID_B = 0x01, - BUTTON_ID_X = 0x02, - BUTTON_ID_Y = 0x03, - BUTTON_ID_TRIGGER = 0x04, - BUTTON_ID_MAX_COUNT -}; - -enum PacketButtonState : uint32_t -{ - BUTTON_STATE_PRESSED = 0x00, - BUTTON_STATE_RELEASED = 0x01 -}; - -typedef uint32_t PacketFrameId; -typedef uint32_t PacketFrameNumber; -typedef uint32_t PacketTransformId; \ No newline at end of file diff --git a/src/transport/transport.cpp b/src/transport/transport.cpp new file mode 100644 index 00000000..2cdbf9e9 --- /dev/null +++ b/src/transport/transport.cpp @@ -0,0 +1,18 @@ +#include "transport.hpp" +#include "udp_transport.hpp" + +#include <liblava/lava.hpp> + +Transport::Ptr make_transport(TransportType transport_type) +{ + switch (transport_type) + { + case TRANSPORT_TYPE_UDP: + return std::make_shared<UDPTransport>(); + default: + lava::log()->error("Unkown transport type!"); + break; + } + + return nullptr; +} \ No newline at end of file diff --git a/src/transport/transport.hpp b/src/transport/transport.hpp new file mode 100644 index 00000000..2d3f6b3b --- /dev/null +++ b/src/transport/transport.hpp @@ -0,0 +1,83 @@ +/* + Hallo + Example: + + Hallo +*/ + +#pragma once +#include <glm/glm.hpp> +#include <functional> +#include <memory> +#include <span> + +#include "types.hpp" + +enum TransportType +{ + TRANSPORT_TYPE_UDP +}; + +enum TransportState +{ + TRANSPORT_STATE_UNINITIALIZED, + TRANSPORT_STATE_WAITING, + TRANSPORT_STATE_CONNECTED, + TRANSPORT_STATE_ERROR +}; + +class Transport +{ +public: + typedef std::shared_ptr<Transport> Ptr; + + typedef std::function<void(const glm::u32vec2& resolution)> OnSetup; + typedef std::function<void(const glm::mat4& left_eye_projection, const glm::mat4& right_eye_projection, const glm::mat4& left_head_to_eye, const glm::mat4& right_head_to_eye)> OnProjectionChange; + typedef std::function<void(TransformId transform, const glm::mat4& head_transform)> OnHeadTransform; + typedef std::function<void(Controller controller, const glm::mat4& controller_transform)> OnControllerTransform; + typedef std::function<void(Controller controller, ControllerState controller_state)> OnControllerEvent; + typedef std::function<void(Controller controller, Button button, ButtonState button_state)> OnControllerButton; + typedef std::function<void(Controller controller, const glm::vec2& thumbstick)> OnControllerThumbstick; + typedef std::function<void(FrameNumber frame_number, FrameId frame_id, TransformId transform_id, double timestamp_transform_query, double timestamp_server_response, double timestamp_frame_decoded, double timestamp_command_construct, double timestamp_command_executed)> OnLatency; + typedef std::function<void()> OnTransportError; + +public: + Transport() = default; + virtual ~Transport() = default; + + virtual bool create(uint32_t port_number) = 0; + virtual void destroy() = 0; + virtual bool wait_connect() = 0; + + //NOTE: The following functions should be thread safe + virtual bool send_setup_complete(RemoteStrategy remote_strategy, uint32_t max_frame_ids, float near_plane, float far_plane) = 0; + virtual bool send_stage_transform(const glm::mat4& stage_transform) = 0; + virtual bool send_frame_config_nal(FrameId frame_id, const std::span<uint8_t>& content) = 0; + virtual bool send_frame_nal(FrameNumber frame_number, FrameId frame_id, TransformId transform_id, const std::span<uint8_t>& content) = 0; + virtual bool send_frame_metadata(FrameNumber frame_number, const std::span<uint8_t>& content) = 0; + + //NOTE: The callbacks are only executed by the worker thread of the transport class. + //NOTE: Set the callbacks before calling create and don't do it again after calling create. + virtual void set_on_setup(OnSetup function) = 0; + virtual void set_on_projection_change(OnProjectionChange function) = 0; + virtual void set_on_head_transform(OnHeadTransform function) = 0; + virtual void set_on_controller_transform(OnControllerTransform function) = 0; + virtual void set_on_controller_event(OnControllerEvent function) = 0; + virtual void set_on_controller_button(OnControllerButton function) = 0; + virtual void set_on_controller_thumbstick(OnControllerThumbstick function) = 0; + virtual void set_on_latency(OnLatency function) = 0; + virtual void set_on_transport_error(OnTransportError function) = 0; + + //NOTE: The following functions should be thread safe + virtual void set_max_send_queue_size(uint32_t size) = 0; //NOTE: The size is given in Bytes + + //NOTE: The following function should be thread safe + virtual TransportState get_state() = 0; + virtual double get_bitrate_send() = 0; //NOTE: The bitrate is given in MBits/s + virtual double get_bitrate_received() = 0; //NOTE: The bitrate is given in MBits/s + virtual uint32_t get_max_send_queue_size() = 0; //NOTE: The size is given in Bytes + virtual uint32_t get_send_queue_size() = 0; //NOTE: In Bytes + virtual uint32_t get_receive_queue_size() = 0; //NOTE: In Bytes +}; + +Transport::Ptr make_transport(TransportType transport_type); \ No newline at end of file diff --git a/src/transport/udp_packet.cpp b/src/transport/udp_packet.cpp new file mode 100644 index 00000000..373ee6f9 --- /dev/null +++ b/src/transport/udp_packet.cpp @@ -0,0 +1,101 @@ +#include "udp_packet.hpp" +#include <liblava/lava.hpp> + +bool convert_udp_stereo_strategy(RemoteStrategy remote_strategey, UDPStereoStrategyId& udp_strategy) +{ + switch (remote_strategey) + { + case REMOTE_STRATEGY_DEBUG: + udp_strategy = UDP_STEREO_STRATEGY_ID_DEBUG; + break; + case REMOTE_STRATEGY_NATIVE: + udp_strategy = UDP_STEREO_STRATEGY_ID_NATIVE_STEREO; + break; + default: + lava::log()->error("UDP-Transport: Unkown remote strategy!"); + return false; + } + + return true; +} + +bool convert_udp_controller(UDPControllerId udp_controller, Controller& controller) +{ + switch (udp_controller) + { + case UDP_CONTROLLER_ID_LEFT: + controller = CONTROLLER_LEFT; + break; + case UDP_CONTROLLER_ID_RIGHT: + controller = CONTROLLER_RIGHT; + break; + default: + lava::log()->error("UDP-Transport: Unkown controller!"); + return false; + } + + return true; +} + +bool convert_udp_controller_state(UDPControllerState udp_controller_state, ControllerState& controller_state) +{ + switch (udp_controller_state) + { + case UDP_CONTROLLER_STATE_DETACHED: + controller_state = CONTROLLER_STATE_DETACHED; + break; + case UDP_CONTROLLER_STATE_ATTACHED: + controller_state = CONTROLLER_STATE_ATTACHED; + break; + default: + lava::log()->error("UDP-Transport: Unkown controller state!"); + return false; + } + + return true; +} + +bool convert_udp_button(UDPButtonId udp_button, Button& button) +{ + switch (udp_button) + { + case UDP_BUTTON_ID_A: + button = BUTTON_A; + break; + case UDP_BUTTON_ID_B: + button = BUTTON_B; + break; + case UDP_BUTTON_ID_X: + button = BUTTON_X; + break; + case UDP_BUTTON_ID_Y: + button = BUTTON_Y; + break; + case UDP_BUTTON_ID_TRIGGER: + button = BUTTON_TRIGGER; + break; + default: + lava::log()->error("UDP-Transport: Unkown button!"); + return false; + } + + return true; +} + +bool convert_udp_button_state(UDPButtonState udp_button_state, ButtonState& button_state) +{ + switch (udp_button_state) + { + case UDP_BUTTON_STATE_PRESSED: + button_state = BUTTON_STATE_PRESSED; + break; + case UDP_BUTTON_STATE_RELEASED: + button_state = BUTTON_STATE_RELEASED; + break; + default: + lava::log()->error("UDP-Transport: Unkown button state!"); + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/transport/udp_packet.hpp b/src/transport/udp_packet.hpp new file mode 100644 index 00000000..8af6a51d --- /dev/null +++ b/src/transport/udp_packet.hpp @@ -0,0 +1,226 @@ +#pragma once +#include <glm/glm.hpp> +#include <cstdint> + +#include "types.hpp" + +enum UDPPacketId : uint32_t +{ + UDP_PACKET_ID_SETUP = 0x00, + UDP_PACKET_ID_SETUP_COMPLETE = 0x01, + UDP_PACKET_ID_PROJECTION_CHANGE = 0x02, + UDP_PACKET_ID_STAGE_TRANSFORM = 0x03, + UDP_PACKET_ID_HEAD_TRANSFORM = 0x04, + UDP_PACKET_ID_CONTROLLER_TRANSFORM = 0x05, + UDP_PACKET_ID_CONTROLLER_EVENT = 0x06, + UDP_PACKET_ID_CONTROLLER_BUTTON = 0x07, + UDP_PACKET_ID_CONTROLLER_THUMBSTICK = 0x08, + UDP_PACKET_ID_FRAME_CONFIG_NAL = 0x09, + UDP_PACKET_ID_FRAME_NAL = 0x0A, + UDP_PACKET_ID_FRAME_METADATA = 0x0B, + UDP_PACKET_ID_LATENCY = 0x0C, + UDP_PACKET_ID_MAX_COUNT +}; + +enum UDPStereoStrategyId : uint32_t +{ + UDP_STEREO_STRATEGY_ID_DEBUG = 0x00, + UDP_STEREO_STRATEGY_ID_NATIVE_STEREO = 0x01, + UDP_STEREO_STRATEGY_ID_MAX_COUNT +}; + +enum UDPControllerId : uint32_t +{ + UDP_CONTROLLER_ID_LEFT = 0x00, + UDP_CONTROLLER_ID_RIGHT = 0x01, + UDP_CONTROLLER_ID_MAX_COUNT +}; + +enum UDPControllerState : uint32_t +{ + UDP_CONTROLLER_STATE_DETACHED = 0x00, + UDP_CONTROLLER_STATE_ATTACHED = 0x01 +}; + +enum UDPButtonId : uint32_t +{ + UDP_BUTTON_ID_A = 0x00, + UDP_BUTTON_ID_B = 0x01, + UDP_BUTTON_ID_X = 0x02, + UDP_BUTTON_ID_Y = 0x03, + UDP_BUTTON_ID_TRIGGER = 0x04, + UDP_BUTTON_ID_MAX_COUNT +}; + +enum UDPButtonState : uint32_t +{ + UDP_BUTTON_STATE_PRESSED = 0x00, + UDP_BUTTON_STATE_RELEASED = 0x01 +}; + +typedef uint32_t UDPFrameId; +typedef uint32_t UDPFrameNumber; +typedef uint32_t UDPTransformId; + +bool convert_udp_stereo_strategy(RemoteStrategy remote_strategey, UDPStereoStrategyId& udp_strategy); +bool convert_udp_controller(UDPControllerId udp_controller, Controller& controller); +bool convert_udp_controller_state(UDPControllerState udp_controller_state, ControllerState& controller_state); +bool convert_udp_button(UDPButtonId udp_button, Button& button); +bool convert_udp_button_state(UDPButtonState udp_button_state, ButtonState& button_state); + +//All packages need to be smaller than 512, otherwise the package needs to be fragmented. +//In case of an fragmented package, all parts of it are gathered and sorted based on the information provide in the packet. +//Currently only PacketFrameNal, PacketFrameMetadata packages could get fragmented during transmission. +//Besides that the transport system has to wait for the PacketSetup, PacketSetupComplete to arrive before it can distribute other packages. +//This means that the application always sees the PacketSetup, PacketSetupComplete before any other package. + +//NOTE: Not an actual packet. +struct UDPPacketHeader +{ + UDPPacketId id = (UDPPacketId)0; + uint32_t size = 0; +}; + +struct UDPPacketSetup +{ + UDPPacketId id = UDP_PACKET_ID_SETUP; + uint32_t size = 0; + + glm::u32vec2 resolution; +}; + +struct UDPPacketSetupComplete +{ + UDPPacketId id = UDP_PACKET_ID_SETUP_COMPLETE; + uint32_t size = 0; + + UDPStereoStrategyId strategy_id; + uint32_t max_frame_ids; + + float near_plane; + float far_plane; +}; + +struct UDPPacketProjectionChange +{ + UDPPacketId id = UDP_PACKET_ID_PROJECTION_CHANGE; + uint32_t size = 0; + + glm::mat4 left_eye_projection; + glm::mat4 right_eye_projection; + + glm::mat4 left_head_to_eye; + glm::mat4 right_head_to_eye; +}; + +struct UDPPacketStageTransform +{ + UDPPacketId id = UDP_PACKET_ID_STAGE_TRANSFORM; + uint32_t size = 0; + + glm::mat4 stage_transform; +}; + +struct UDPPacketHeadTransform +{ + UDPPacketId id = UDP_PACKET_ID_HEAD_TRANSFORM; + uint32_t size = 0; + + UDPTransformId transform_id; + glm::mat4 head_transform; +}; + +struct UDPPacketControllerTransform +{ + UDPPacketId id = UDP_PACKET_ID_CONTROLLER_TRANSFORM; + uint32_t size = 0; + + UDPControllerId controller_id; + glm::mat4 controller_transform; +}; + +struct UDPPacketControllerEvent +{ + UDPPacketId id = UDP_PACKET_ID_CONTROLLER_EVENT; + uint32_t size = 0; + + UDPControllerId controller_id; + UDPControllerState controller_state; +}; + +struct UDPPacketControllerButton +{ + UDPPacketId id = UDP_PACKET_ID_CONTROLLER_TRANSFORM; + uint32_t size = 0; + + UDPControllerId controller_id; + UDPButtonId button_id; + UDPButtonState button_state; +}; + +struct UDPPacketControllerThumbstick +{ + UDPPacketId id = UDP_PACKET_ID_CONTROLLER_THUMBSTICK; + uint32_t size = 0; + + UDPControllerId controller_id; + glm::vec2 thumbstick; +}; + +//NOTE: The entire size of the packet including the payload needs to be less than 512 bytes +struct UDPPacketFrameConfigNal +{ + UDPPacketId id = UDP_PACKET_ID_FRAME_NAL; + uint32_t size = 0; + + UDPFrameId frame_id; + + //Followed by: Config Nal +}; + +//NOTE: The packet needs to be fragmented if the payload is too big +struct UDPPacketFrameNal +{ + UDPPacketId id = UDP_PACKET_ID_FRAME_NAL; + uint32_t size = 0; + + UDPFrameNumber frame_number; //In case of fragmentation, frame number and frame id are used together as identifier + UDPFrameId frame_id; + UDPTransformId transform_id; + + uint32_t payload_offset; + uint32_t payload_size; + + //Followed by: Nal +}; + +//NOTE: Assume frame metadata of a particular frame number is always send before any frame nals of the same frame number +//NOTE: The packet needs to be fragmented if the payload is too big +struct UDPPacketFrameMetadata +{ + UDPPacketId id = UDP_PACKET_ID_FRAME_METADATA; + uint32_t size = 0; + + UDPFrameNumber frame_number; //In case of fragmentation, the frame number is used as identifier + + uint32_t payload_offset; + uint32_t payload_size; + + //Followed by: Metadata +}; + +struct UDPPacketLatency +{ + UDPPacketId id = UDP_PACKET_ID_LATENCY; + uint32_t size = 0; + + UDPFrameNumber frame_number; + UDPFrameId frame_id; + UDPTransformId transform_id; + + double timestamp_transform_query = 0.0; + double timestamp_server_response = 0.0; + double timestamp_frame_decoded = 0.0; + double timestamp_command_construct = 0.0; + double timestamp_command_executed = 0.0; +}; \ No newline at end of file diff --git a/src/stream/transport.cpp b/src/transport/udp_transport.cpp similarity index 57% rename from src/stream/transport.cpp rename to src/transport/udp_transport.cpp index 48b84ca2..0a276eb3 100644 --- a/src/stream/transport.cpp +++ b/src/transport/udp_transport.cpp @@ -1,8 +1,10 @@ -#include "transport.hpp" +#include "udp_transport.hpp" +#include "udp_packet.hpp" + #include <liblava/lava.hpp> #include <chrono> -bool Transport::create(uint32_t port_number) +bool UDPTransport::create(uint32_t port_number) { this->server_timer = asio::high_resolution_timer(this->server_context); this->server_endpoint = asio::ip::udp::endpoint(asio::ip::udp::v4(), port_number); @@ -22,7 +24,7 @@ bool Transport::create(uint32_t port_number) return true; } -void Transport::destroy() +void UDPTransport::destroy() { this->server_context.stop(); this->worker_thread.join(); @@ -68,18 +70,18 @@ void Transport::destroy() this->set_state(TRANSPORT_STATE_UNINITIALIZED); } -bool Transport::wait_connect() +bool UDPTransport::wait_connect() { if (this->get_state() != TRANSPORT_STATE_WAITING) { - lava::log()->error("Transport: Can't wait for connection!"); + lava::log()->error("UDP-Transport: Can't wait for connection!"); return false; } std::unique_lock<std::mutex> lock(this->worker_mutex); - std::chrono::seconds wait_time(TRANSPORT_CLIENT_CONNECT_TIMEOUT); + std::chrono::seconds wait_time(UDP_TRANSPORT_CLIENT_CONNECT_TIMEOUT); std::chrono::high_resolution_clock::time_point last_time = std::chrono::high_resolution_clock::now(); while (this->state == TRANSPORT_STATE_WAITING && wait_time.count() > 0) @@ -96,22 +98,30 @@ bool Transport::wait_connect() return this->get_state() == TRANSPORT_STATE_CONNECTED; } -bool Transport::send_setup_complete(PacketStereoStrategyId strategy_id, uint32_t max_frame_ids, float near_plane, float far_plane) +bool UDPTransport::send_setup_complete(RemoteStrategy remote_strategy, uint32_t max_frame_ids, float near_plane, float far_plane) { if (this->get_state() != TRANSPORT_STATE_CONNECTED) { - lava::log()->error("Transport: Can't send setup complete packet since not connection is established!"); + lava::log()->error("UDP-Transport: Can't send setup complete packet since not connection is established!"); + + return false; + } + + UDPStereoStrategyId strategy_id; + if (!convert_udp_stereo_strategy(remote_strategy, strategy_id)) + { return false; } + std::unique_lock<std::mutex> lock(this->worker_mutex); Datagram datagram = this->datagram_pool.acquire_datagram(); lock.unlock(); - PacketSetupComplete* packet = (PacketSetupComplete*) datagram.buffer; - packet->id = PACKET_ID_SETUP_COMPLETE; - packet->size = sizeof(PacketSetupComplete); + UDPPacketSetupComplete* packet = (UDPPacketSetupComplete*)datagram.buffer; + packet->id = UDP_PACKET_ID_SETUP_COMPLETE; + packet->size = sizeof(UDPPacketSetupComplete); packet->strategy_id = strategy_id; packet->max_frame_ids = max_frame_ids; packet->near_plane = near_plane; @@ -122,11 +132,11 @@ bool Transport::send_setup_complete(PacketStereoStrategyId strategy_id, uint32_t return true; } -bool Transport::send_stage_transform(const glm::mat4& stage_transform) +bool UDPTransport::send_stage_transform(const glm::mat4& stage_transform) { if (this->get_state() != TRANSPORT_STATE_CONNECTED) { - lava::log()->error("Transport: Can't send stage transform packet since not connection is established!"); + lava::log()->error("UDP-Transport: Can't send stage transform packet since not connection is established!"); return false; } @@ -135,9 +145,9 @@ bool Transport::send_stage_transform(const glm::mat4& stage_transform) Datagram datagram = this->datagram_pool.acquire_datagram(); lock.unlock(); - PacketStageTransform* packet = (PacketStageTransform*) datagram.buffer; - packet->id = PACKET_ID_STAGE_TRANSFORM; - packet->size = sizeof(PacketStageTransform); + UDPPacketStageTransform* packet = (UDPPacketStageTransform*)datagram.buffer; + packet->id = UDP_PACKET_ID_STAGE_TRANSFORM; + packet->size = sizeof(UDPPacketStageTransform); packet->stage_transform = stage_transform; this->send_datagram(datagram); @@ -145,11 +155,11 @@ bool Transport::send_stage_transform(const glm::mat4& stage_transform) return true; } -bool Transport::send_frame_config_nal(PacketFrameId frame_id, const std::span<uint8_t>& content) +bool UDPTransport::send_frame_config_nal(FrameId frame_id, const std::span<uint8_t>& content) { if (this->get_state() != TRANSPORT_STATE_CONNECTED) { - lava::log()->error("Transport: Can't send frame config nal packet since not connection is established!"); + lava::log()->error("UDP-Transport: Can't send frame config nal packet since not connection is established!"); return false; } @@ -158,32 +168,32 @@ bool Transport::send_frame_config_nal(PacketFrameId frame_id, const std::span<ui Datagram datagram = this->datagram_pool.acquire_datagram(); lock.unlock(); - uint32_t size = sizeof(PacketFrameConfigNal) + sizeof(uint8_t) * content.size(); + uint32_t size = sizeof(UDPPacketFrameConfigNal) + sizeof(uint8_t) * content.size(); if (size > datagram.size) { - lava::log()->error("Transport: Frame config nal packet exceeds datagram size!"); + lava::log()->error("UDP-Transport: Frame config nal packet exceeds datagram size!"); return false; } - PacketFrameConfigNal* packet = (PacketFrameConfigNal*)datagram.buffer; - packet->id = PACKET_ID_FRAME_CONFIG_NAL; + UDPPacketFrameConfigNal* packet = (UDPPacketFrameConfigNal*)datagram.buffer; + packet->id = UDP_PACKET_ID_FRAME_CONFIG_NAL; packet->size = size; packet->frame_id = frame_id; - memcpy(datagram.buffer + sizeof(PacketFrameConfigNal), content.data(), sizeof(uint8_t) * content.size()); + memcpy(datagram.buffer + sizeof(UDPPacketFrameConfigNal), content.data(), sizeof(uint8_t) * content.size()); this->send_datagram(datagram); return true; } -bool Transport::send_frame_nal(PacketFrameNumber frame_number, PacketFrameId frame_id, PacketTransformId transform_id, const std::span<uint8_t>& content) +bool UDPTransport::send_frame_nal(FrameNumber frame_number, FrameId frame_id, TransformId transform_id, const std::span<uint8_t>& content) { if (this->get_state() != TRANSPORT_STATE_CONNECTED) { - lava::log()->error("Transport: Can't send frame nal packet since not connection is established!"); + lava::log()->error("UDP-Transport: Can't send frame nal packet since not connection is established!"); return false; } @@ -194,18 +204,18 @@ bool Transport::send_frame_nal(PacketFrameNumber frame_number, PacketFrameId fra Datagram datagram = this->datagram_pool.acquire_datagram(); lock.unlock(); - uint32_t content_size = glm::min(datagram.size - (uint32_t) sizeof(PacketFrameNal), (uint32_t) content.size() - offset); + uint32_t content_size = glm::min(datagram.size - (uint32_t)sizeof(UDPPacketFrameNal), (uint32_t) content.size() - offset); - PacketFrameNal* packet = (PacketFrameNal*)datagram.buffer; - packet->id = PACKET_ID_FRAME_NAL; - packet->size = sizeof(PacketFrameNal) + content_size; + UDPPacketFrameNal* packet = (UDPPacketFrameNal*)datagram.buffer; + packet->id = UDP_PACKET_ID_FRAME_NAL; + packet->size = sizeof(UDPPacketFrameNal) + content_size; packet->frame_number = frame_number; packet->frame_id = frame_id; packet->transform_id = transform_id; packet->payload_offset = offset; packet->payload_size = content.size(); - memcpy(datagram.buffer + sizeof(PacketFrameNal), (uint8_t*) content.data() + offset, sizeof(uint8_t) * content_size); + memcpy(datagram.buffer + sizeof(UDPPacketFrameNal), (uint8_t*)content.data() + offset, sizeof(uint8_t) * content_size); this->send_datagram(datagram); @@ -215,11 +225,11 @@ bool Transport::send_frame_nal(PacketFrameNumber frame_number, PacketFrameId fra return true; } -bool Transport::send_frame_metadata(PacketFrameNumber frame_number, const std::span<uint8_t>& content) +bool UDPTransport::send_frame_metadata(FrameNumber frame_number, const std::span<uint8_t>& content) { if (this->get_state() != TRANSPORT_STATE_CONNECTED) { - lava::log()->error("Transport: Can't send frame metadata packet since not connection is established!"); + lava::log()->error("UDP-Transport: Can't send frame metadata packet since not connection is established!"); return false; } @@ -230,16 +240,16 @@ bool Transport::send_frame_metadata(PacketFrameNumber frame_number, const std::s Datagram datagram = this->datagram_pool.acquire_datagram(); lock.unlock(); - uint32_t content_size = glm::min(datagram.size - (uint32_t)sizeof(PacketFrameMetadata), (uint32_t)content.size() - offset); + uint32_t content_size = glm::min(datagram.size - (uint32_t)sizeof(UDPPacketFrameMetadata), (uint32_t) content.size() - offset); - PacketFrameMetadata* packet = (PacketFrameMetadata*)datagram.buffer; - packet->id = PACKET_ID_FRAME_METADATA; - packet->size = sizeof(PacketFrameMetadata) + content_size; + UDPPacketFrameMetadata* packet = (UDPPacketFrameMetadata*)datagram.buffer; + packet->id = UDP_PACKET_ID_FRAME_METADATA; + packet->size = sizeof(UDPPacketFrameMetadata) + content_size; packet->frame_number = frame_number; packet->payload_offset = offset; packet->payload_size = content.size(); - memcpy(datagram.buffer + sizeof(PacketFrameMetadata), (uint8_t*)content.data() + offset, sizeof(uint8_t) * content_size); + memcpy(datagram.buffer + sizeof(UDPPacketFrameMetadata), (uint8_t*)content.data() + offset, sizeof(uint8_t) * content_size); this->send_datagram(datagram); @@ -249,69 +259,58 @@ bool Transport::send_frame_metadata(PacketFrameNumber frame_number, const std::s return true; } -void Transport::set_on_setup(OnSetup function) +void UDPTransport::set_on_setup(OnSetup function) { this->on_setup = std::move(function); } -void Transport::set_on_projection_change(OnProjectionChange function) +void UDPTransport::set_on_projection_change(OnProjectionChange function) { this->on_projection_change = std::move(function); } -void Transport::set_on_head_transform(OnHeadTransform function) +void UDPTransport::set_on_head_transform(OnHeadTransform function) { this->on_head_transform = std::move(function); } -void Transport::set_on_controller_transform(OnControllerTransform function) +void UDPTransport::set_on_controller_transform(OnControllerTransform function) { this->on_controller_transform = std::move(function); } -void Transport::set_on_controller_event(OnControllerEvent function) +void UDPTransport::set_on_controller_event(OnControllerEvent function) { this->on_controller_event = std::move(function); } -void Transport::set_on_controller_button(OnControllerButton function) +void UDPTransport::set_on_controller_button(OnControllerButton function) { this->on_controller_button = std::move(function); } -void Transport::set_on_controller_thumbstick(OnControllerThumbstick function) +void UDPTransport::set_on_controller_thumbstick(OnControllerThumbstick function) { this->on_controller_thumbstick = std::move(function); } -void Transport::set_on_latency(OnLatency function) +void UDPTransport::set_on_latency(OnLatency function) { this->on_latency = std::move(function); } -void Transport::set_on_transport_error(OnTransportError function) +void UDPTransport::set_on_transport_error(OnTransportError function) { this->on_transport_error = std::move(function); } -/*void Transport::set_max_bitrate_send(double bitrate) -{ - std::unique_lock<std::mutex> lock(this->worker_mutex); - this->max_bitrate_send = bitrate; -}*/ - -void Transport::set_max_send_queue_size(uint32_t size) +void UDPTransport::set_max_send_queue_size(uint32_t size) { std::unique_lock<std::mutex> lock(this->worker_mutex); this->max_send_queue_size = size; } -asio::io_context& Transport::get_context() -{ - return this->server_context; -} - -TransportState Transport::get_state() +TransportState UDPTransport::get_state() { std::unique_lock<std::mutex> lock(this->worker_mutex); TransportState value = this->state; @@ -320,7 +319,7 @@ TransportState Transport::get_state() return value; } -double Transport::get_bitrate_send() +double UDPTransport::get_bitrate_send() { std::unique_lock<std::mutex> lock(this->worker_mutex); double value = this->bitrate_send; @@ -329,7 +328,7 @@ double Transport::get_bitrate_send() return value; } -double Transport::get_bitrate_received() +double UDPTransport::get_bitrate_received() { std::unique_lock<std::mutex> lock(this->worker_mutex); double value = this->bitrate_received; @@ -338,16 +337,7 @@ double Transport::get_bitrate_received() return value; } -/*double Transport::get_max_bitrate_send() -{ - std::unique_lock<std::mutex> lock(this->worker_mutex); - double value = this->max_bitrate_send; - lock.unlock(); - - return value; -}*/ - -uint32_t Transport::get_max_send_queue_size() +uint32_t UDPTransport::get_max_send_queue_size() { std::unique_lock<std::mutex> lock(this->worker_mutex); uint32_t value = this->max_send_queue_size; @@ -356,14 +346,14 @@ uint32_t Transport::get_max_send_queue_size() return value; } -uint32_t Transport::get_send_queue_size() +uint32_t UDPTransport::get_send_queue_size() { std::unique_lock<std::mutex> lock(this->worker_mutex); uint32_t bytes = 0; for (const Datagram& datagram : this->send_queue) { - const PacketHeader* header = (const PacketHeader*)datagram.buffer; + const UDPPacketHeader* header = (const UDPPacketHeader*)datagram.buffer; bytes += header->size; } lock.unlock(); @@ -371,14 +361,14 @@ uint32_t Transport::get_send_queue_size() return bytes; } -uint32_t Transport::get_receive_queue_size() +uint32_t UDPTransport::get_receive_queue_size() { std::unique_lock<std::mutex> lock(this->worker_mutex); uint32_t bytes = 0; for (const Datagram& datagram : this->receive_queue) { - const PacketHeader* header = (const PacketHeader*)datagram.buffer; + const UDPPacketHeader* header = (const UDPPacketHeader*)datagram.buffer; bytes += header->size; } lock.unlock(); @@ -386,23 +376,23 @@ uint32_t Transport::get_receive_queue_size() return bytes; } -void Transport::set_state(TransportState state) +void UDPTransport::set_state(TransportState state) { std::unique_lock<std::mutex> lock(this->worker_mutex); this->state = state; lock.unlock(); } -void Transport::check_state() +void UDPTransport::check_state() { std::chrono::high_resolution_clock::time_point start_time = std::chrono::high_resolution_clock::now(); - this->server_timer->expires_at(start_time + std::chrono::milliseconds(TRANSPORT_STATE_CHECK_INTERVAL)); + this->server_timer->expires_at(start_time + std::chrono::milliseconds(UDP_TRANSPORT_STATE_CHECK_INTERVAL)); this->server_timer->async_wait([this, start_time](const asio::error_code& error_code) { if (error_code) { - lava::log()->error("Transport: Error during state check! Message: {}", error_code.message()); + lava::log()->error("UDP-Transport: Error during state check! Message: {}", error_code.message()); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -432,9 +422,9 @@ void Transport::check_state() this->inactive_time += delta_time; } - if (this->inactive_time > TRANSPORT_CLIENT_TIMEOUT) + if (this->inactive_time > UDP_TRANSPORT_CLIENT_TIMEOUT) { - lava::log()->error("Transport: Client timeout"); + lava::log()->error("UDP-Transport: Client timeout"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -448,7 +438,7 @@ void Transport::check_state() }); } -void Transport::send_datagram(const Datagram& datagram) +void UDPTransport::send_datagram(const Datagram& datagram) { std::unique_lock<std::mutex> lock(this->worker_mutex); @@ -457,14 +447,14 @@ void Transport::send_datagram(const Datagram& datagram) for (uint32_t index = 1; index < this->send_queue.size(); index++) { const Datagram& datagram = this->send_queue[index]; - const PacketHeader* header = (const PacketHeader*)datagram.buffer; + const UDPPacketHeader* header = (const UDPPacketHeader*)datagram.buffer; bytes += header->size; } while (bytes > this->max_send_queue_size && this->send_queue.size() > 1) { const Datagram& datagram = this->send_queue[1]; - const PacketHeader* header = (const PacketHeader*) datagram.buffer; + const UDPPacketHeader* header = (const UDPPacketHeader*)datagram.buffer; bytes -= header->size; this->datagram_pool.release_datagram(datagram); @@ -479,7 +469,7 @@ void Transport::send_datagram(const Datagram& datagram) } } -void Transport::receive_datagram() +void UDPTransport::receive_datagram() { std::unique_lock<std::mutex> lock(this->worker_mutex); Datagram datagram = this->datagram_pool.acquire_datagram(); @@ -489,7 +479,7 @@ void Transport::receive_datagram() { if (error_code) { - lava::log()->error("Transport: Error during datagram receive! Message: {}", error_code.message()); + lava::log()->error("UDP-Transport: Error during datagram receive! Message: {}", error_code.message()); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -504,7 +494,7 @@ void Transport::receive_datagram() if (!this->client_endpoint.has_value()) { - lava::log()->debug("Transport: Client connected"); + lava::log()->debug("UDP-Transport: Client connected"); this->client_endpoint = this->receive_endpoint; this->check_state(); @@ -512,7 +502,7 @@ void Transport::receive_datagram() else if (this->receive_endpoint != this->client_endpoint) { - lava::log()->error("Transport: Client already connected!"); + lava::log()->error("UDP-Transport: Client already connected!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -521,46 +511,46 @@ void Transport::receive_datagram() return; } - const PacketHeader* packet = (const PacketHeader*)datagram.buffer; + const UDPPacketHeader* packet = (const UDPPacketHeader*)datagram.buffer; switch (packet->id) { - case PACKET_ID_SETUP: - case PACKET_ID_PROJECTION_CHANGE: - case PACKET_ID_HEAD_TRANSFORM: - case PACKET_ID_CONTROLLER_TRANSFORM: - case PACKET_ID_CONTROLLER_EVENT: - case PACKET_ID_CONTROLLER_BUTTON: - case PACKET_ID_CONTROLLER_THUMBSTICK: - case PACKET_ID_LATENCY: + case UDP_PACKET_ID_SETUP: + case UDP_PACKET_ID_PROJECTION_CHANGE: + case UDP_PACKET_ID_HEAD_TRANSFORM: + case UDP_PACKET_ID_CONTROLLER_TRANSFORM: + case UDP_PACKET_ID_CONTROLLER_EVENT: + case UDP_PACKET_ID_CONTROLLER_BUTTON: + case UDP_PACKET_ID_CONTROLLER_THUMBSTICK: + case UDP_PACKET_ID_LATENCY: this->receive_queue.push_back(datagram); break; - case PACKET_ID_SETUP_COMPLETE: - lava::log()->error("Transport: Received setup complete packet!"); + case UDP_PACKET_ID_SETUP_COMPLETE: + lava::log()->error("UDP-Transport: Received setup complete packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); this->error_queue.push_back(datagram); return; - case PACKET_ID_FRAME_CONFIG_NAL: - lava::log()->error("Transport: Received frame config nal packet!"); + case UDP_PACKET_ID_FRAME_CONFIG_NAL: + lava::log()->error("UDP-Transport: Received frame config nal packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); this->error_queue.push_back(datagram); return; - case PACKET_ID_FRAME_NAL: - lava::log()->error("Transport: Received frame nal packet!"); + case UDP_PACKET_ID_FRAME_NAL: + lava::log()->error("UDP-Transport: Received frame nal packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); this->error_queue.push_back(datagram); return; - case PACKET_ID_FRAME_METADATA: - lava::log()->error("Transport: Received frame metadata packet!"); + case UDP_PACKET_ID_FRAME_METADATA: + lava::log()->error("UDP-Transport: Received frame metadata packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); this->error_queue.push_back(datagram); return; default: - lava::log()->error("Transport: Received unknown packet!"); + lava::log()->error("UDP-Transport: Received unknown packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); this->error_queue.push_back(datagram); @@ -572,7 +562,7 @@ void Transport::receive_datagram() }); } -void Transport::process_send_queue() +void UDPTransport::process_send_queue() { if (this->send_queue.empty()) { @@ -585,13 +575,13 @@ void Transport::process_send_queue() } Datagram datagram = this->send_queue.front(); - const PacketHeader* header = (const PacketHeader*) datagram.buffer; + const UDPPacketHeader* header = (const UDPPacketHeader*)datagram.buffer; this->server_socket->async_send_to(asio::buffer(datagram.buffer, header->size), this->client_endpoint.value(), [this, datagram](const asio::error_code& error_code, std::size_t bytes) { if (error_code) { - lava::log()->error("Transport: Error during send of datagram! Message: {}", error_code.message()); + lava::log()->error("UDP-Transport: Error during send of datagram! Message: {}", error_code.message()); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -607,17 +597,14 @@ void Transport::process_send_queue() this->send_queue.erase(this->send_queue.begin()); this->datagram_pool.release_datagram(datagram); - //if (this->bits_send < (this->max_bitrate_send * (TRANSPORT_STATE_CHECK_INTERVAL / 1000.0)) * (1000.0 * 1000.0)) - //{ - this->process_send_queue(); - //} + this->process_send_queue(); lock.unlock(); }); this->send_active = true; } -void Transport::process_receive_queue() +void UDPTransport::process_receive_queue() { if (this->receive_queue.empty()) { @@ -627,9 +614,9 @@ void Transport::process_receive_queue() if (this->get_state() == TRANSPORT_STATE_WAITING) { const Datagram& datagram = this->receive_queue.back(); - const PacketHeader* packet = (const PacketHeader*)datagram.buffer; + const UDPPacketHeader* packet = (const UDPPacketHeader*)datagram.buffer; - if (packet->id == PACKET_ID_SETUP) + if (packet->id == UDP_PACKET_ID_SETUP) { this->set_state(TRANSPORT_STATE_CONNECTED); this->parse_setup(datagram); @@ -647,36 +634,36 @@ void Transport::process_receive_queue() { for (const Datagram& datagram : this->receive_queue) { - const PacketHeader* packet = (const PacketHeader*)datagram.buffer; + const UDPPacketHeader* packet = (const UDPPacketHeader*)datagram.buffer; switch (packet->id) { - case PACKET_ID_SETUP: + case UDP_PACKET_ID_SETUP: this->parse_setup(datagram); break; - case PACKET_ID_PROJECTION_CHANGE: + case UDP_PACKET_ID_PROJECTION_CHANGE: this->parse_projection_change(datagram); break; - case PACKET_ID_HEAD_TRANSFORM: + case UDP_PACKET_ID_HEAD_TRANSFORM: this->parse_head_transform(datagram); break; - case PACKET_ID_CONTROLLER_TRANSFORM: + case UDP_PACKET_ID_CONTROLLER_TRANSFORM: this->parse_controller_transform(datagram); break; - case PACKET_ID_CONTROLLER_EVENT: + case UDP_PACKET_ID_CONTROLLER_EVENT: this->parse_controller_event(datagram); break; - case PACKET_ID_CONTROLLER_BUTTON: + case UDP_PACKET_ID_CONTROLLER_BUTTON: this->parse_controller_button(datagram); break; - case PACKET_ID_CONTROLLER_THUMBSTICK: + case UDP_PACKET_ID_CONTROLLER_THUMBSTICK: this->parse_controller_thumbstick(datagram); break; - case PACKET_ID_LATENCY: + case UDP_PACKET_ID_LATENCY: this->parse_latency(datagram); break; default: - lava::log()->error("Transport: Unknown packet during receive queue process!"); + lava::log()->error("UDP-Transport: Unknown packet during receive queue process!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); return; @@ -691,13 +678,13 @@ void Transport::process_receive_queue() } } -void Transport::parse_setup(const Datagram& datagram) +void UDPTransport::parse_setup(const Datagram& datagram) { - PacketSetup* packet = (PacketSetup*)datagram.buffer; + UDPPacketSetup* packet = (UDPPacketSetup*)datagram.buffer; - if (packet->id != PACKET_ID_SETUP) + if (packet->id != UDP_PACKET_ID_SETUP) { - lava::log()->error("Transport: Wrong packet id for setup packet!"); + lava::log()->error("UDP-Transport: Wrong packet id for setup packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -707,13 +694,13 @@ void Transport::parse_setup(const Datagram& datagram) this->on_setup(packet->resolution); } -void Transport::parse_projection_change(const Datagram& datagram) +void UDPTransport::parse_projection_change(const Datagram& datagram) { - PacketProjectionChange* packet = (PacketProjectionChange*)datagram.buffer; + UDPPacketProjectionChange* packet = (UDPPacketProjectionChange*)datagram.buffer; - if (packet->id != PACKET_ID_PROJECTION_CHANGE) + if (packet->id != UDP_PACKET_ID_PROJECTION_CHANGE) { - lava::log()->error("Transport: Wrong packet id for projection change packet!"); + lava::log()->error("UDP-Transport: Wrong packet id for projection change packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -723,13 +710,13 @@ void Transport::parse_projection_change(const Datagram& datagram) this->on_projection_change(packet->left_eye_projection, packet->right_eye_projection, packet->left_head_to_eye, packet->right_head_to_eye); } -void Transport::parse_head_transform(const Datagram& datagram) +void UDPTransport::parse_head_transform(const Datagram& datagram) { - PacketHeadTransform* packet = (PacketHeadTransform*)datagram.buffer; + UDPPacketHeadTransform* packet = (UDPPacketHeadTransform*)datagram.buffer; - if (packet->id != PACKET_ID_HEAD_TRANSFORM) + if (packet->id != UDP_PACKET_ID_HEAD_TRANSFORM) { - lava::log()->error("Transport: Wrong packet id for head transform packet!"); + lava::log()->error("UDP-Transport: Wrong packet id for head transform packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); @@ -739,87 +726,149 @@ void Transport::parse_head_transform(const Datagram& datagram) this->on_head_transform(packet->transform_id, packet->head_transform); } -void Transport::parse_controller_transform(const Datagram& datagram) +void UDPTransport::parse_controller_transform(const Datagram& datagram) { - PacketControllerTransform* packet = (PacketControllerTransform*)datagram.buffer; + UDPPacketControllerTransform* packet = (UDPPacketControllerTransform*)datagram.buffer; + + if (packet->id != UDP_PACKET_ID_CONTROLLER_TRANSFORM) + { + lava::log()->error("UDP-Transport: Wrong packet id for controller transform packet!"); + this->on_transport_error(); + this->set_state(TRANSPORT_STATE_ERROR); - if (packet->id != PACKET_ID_CONTROLLER_TRANSFORM) + return; + } + + Controller controller; + + if (!convert_udp_controller(packet->controller_id, controller)) { - lava::log()->error("Transport: Wrong packet id for controller transform packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); return; } - this->on_controller_transform(packet->controller_id, packet->controller_transform); + this->on_controller_transform(controller, packet->controller_transform); } -void Transport::parse_controller_event(const Datagram& datagram) +void UDPTransport::parse_controller_event(const Datagram& datagram) { - PacketControllerEvent* packet = (PacketControllerEvent*)datagram.buffer; + UDPPacketControllerEvent* packet = (UDPPacketControllerEvent*)datagram.buffer; - if (packet->id != PACKET_ID_CONTROLLER_EVENT) + if (packet->id != UDP_PACKET_ID_CONTROLLER_EVENT) { - lava::log()->error("Transport: Wrong packet id for controller event packet!"); + lava::log()->error("UDP-Transport: Wrong packet id for controller event packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); return; } - this->on_controller_event(packet->controller_id, packet->controller_state); -} + Controller controller; + ControllerState controller_state; -void Transport::parse_controller_button(const Datagram& datagram) -{ - PacketControllerButton* packet = (PacketControllerButton*)datagram.buffer; + if (!convert_udp_controller(packet->controller_id, controller)) + { + this->on_transport_error(); + this->set_state(TRANSPORT_STATE_ERROR); - if (packet->id != PACKET_ID_CONTROLLER_BUTTON) + return; + } + + if (!convert_udp_controller_state(packet->controller_state, controller_state)) { - lava::log()->error("Transport: Wrong packet id for controller button packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); return; } - this->on_controller_button(packet->controller_id, packet->button_id, packet->button_state); + this->on_controller_event(controller, controller_state); } -void Transport::parse_controller_thumbstick(const Datagram& datagram) +void UDPTransport::parse_controller_button(const Datagram& datagram) { - PacketControllerThumbstick* packet = (PacketControllerThumbstick*)datagram.buffer; + UDPPacketControllerButton* packet = (UDPPacketControllerButton*)datagram.buffer; - if (packet->id != PACKET_ID_CONTROLLER_THUMBSTICK) + if (packet->id != UDP_PACKET_ID_CONTROLLER_BUTTON) { - lava::log()->error("Transport: Wrong packet id for controller thumbstick packet!"); + lava::log()->error("UDP-Transport: Wrong packet id for controller button packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); return; } - this->on_controller_thumbstick(packet->controller_id, packet->thumbstick); + Controller controller; + Button button; + ButtonState button_state; + + if (!convert_udp_controller(packet->controller_id, controller)) + { + this->on_transport_error(); + this->set_state(TRANSPORT_STATE_ERROR); + + return; + } + + if (!convert_udp_button(packet->button_id, button)) + { + this->on_transport_error(); + this->set_state(TRANSPORT_STATE_ERROR); + + return; + } + + if (!convert_udp_button_state(packet->button_state, button_state)) + { + this->on_transport_error(); + this->set_state(TRANSPORT_STATE_ERROR); + + return; + } + + this->on_controller_button(controller, button, button_state); } -void Transport::parse_latency(const Datagram& datagram) +void UDPTransport::parse_controller_thumbstick(const Datagram& datagram) { - PacketLatency* packet = (PacketLatency*)datagram.buffer; + UDPPacketControllerThumbstick* packet = (UDPPacketControllerThumbstick*)datagram.buffer; - if (packet->id != PACKET_ID_LATENCY) + if (packet->id != UDP_PACKET_ID_CONTROLLER_THUMBSTICK) { - lava::log()->error("Transport: Wrong packet id for latency packet!"); + lava::log()->error("UDP-Transport: Wrong packet id for controller thumbstick packet!"); this->on_transport_error(); this->set_state(TRANSPORT_STATE_ERROR); return; } - this->on_latency(packet->frame_number, packet->frame_id, packet->transform_id, packet->timestamp_transform_query, packet->timestamp_server_response, packet->timestamp_frame_decoded, packet->timestamp_command_construct, packet->timestamp_command_executed); + Controller controller; + + if (!convert_udp_controller(packet->controller_id, controller)) + { + this->on_transport_error(); + this->set_state(TRANSPORT_STATE_ERROR); + + return; + } + + this->on_controller_thumbstick(controller, packet->thumbstick); } -Transport::Ptr make_transport() +void UDPTransport::parse_latency(const Datagram& datagram) { - return std::make_shared<Transport>(); + UDPPacketLatency* packet = (UDPPacketLatency*)datagram.buffer; + + if (packet->id != UDP_PACKET_ID_LATENCY) + { + lava::log()->error("UDP-Transport: Wrong packet id for latency packet!"); + this->on_transport_error(); + this->set_state(TRANSPORT_STATE_ERROR); + + return; + } + + this->on_latency(packet->frame_number, packet->frame_id, packet->transform_id, packet->timestamp_transform_query, packet->timestamp_server_response, packet->timestamp_frame_decoded, packet->timestamp_command_construct, packet->timestamp_command_executed); } \ No newline at end of file diff --git a/src/transport/udp_transport.hpp b/src/transport/udp_transport.hpp new file mode 100644 index 00000000..116af2bf --- /dev/null +++ b/src/transport/udp_transport.hpp @@ -0,0 +1,105 @@ +#pragma once +#include <glm/glm.hpp> +#include <asio.hpp> +#include <thread> +#include <mutex> +#include <condition_variable> +#include <vector> +#include <optional> + +#include "transport.hpp" +#include "utility/datagram.hpp" + +#define UDP_TRANSPORT_STATE_CHECK_INTERVAL 10 //NOTE: In milliseconds +#define UDP_TRANSPORT_CLIENT_CONNECT_TIMEOUT 60 //NOTE: In seconds +#define UDP_TRANSPORT_CLIENT_TIMEOUT 20 //NOTE: In seconds + +class UDPTransport : public Transport +{ +public: + UDPTransport() = default; + + bool create(uint32_t port_number) override; + void destroy() override; + bool wait_connect() override; + + bool send_setup_complete(RemoteStrategy remote_strategy, uint32_t max_frame_ids, float near_plane, float far_plane) override; + bool send_stage_transform(const glm::mat4& stage_transform) override; + bool send_frame_config_nal(FrameId frame_id, const std::span<uint8_t>& content) override; + bool send_frame_nal(FrameNumber frame_number, FrameId frame_id, TransformId transform_id, const std::span<uint8_t>& content) override; + bool send_frame_metadata(FrameNumber frame_number, const std::span<uint8_t>& content) override; + + void set_on_setup(OnSetup function) override; + void set_on_projection_change(OnProjectionChange function) override; + void set_on_head_transform(OnHeadTransform function) override; + void set_on_controller_transform(OnControllerTransform function) override; + void set_on_controller_event(OnControllerEvent function) override; + void set_on_controller_button(OnControllerButton function) override; + void set_on_controller_thumbstick(OnControllerThumbstick function) override; + void set_on_latency(OnLatency function) override; + void set_on_transport_error(OnTransportError function) override; + + void set_max_send_queue_size(uint32_t size) override; + + TransportState get_state() override; + double get_bitrate_send() override; + double get_bitrate_received() override; + uint32_t get_max_send_queue_size() override; + uint32_t get_send_queue_size() override; + uint32_t get_receive_queue_size() override; + +private: + void set_state(TransportState state); + void check_state(); + + void send_datagram(const Datagram& datagram); + void receive_datagram(); + + void process_send_queue(); //NOTE: Protected by worker_mutex + void process_receive_queue(); + + void parse_setup(const Datagram& datagram); + void parse_projection_change(const Datagram& datagram); + void parse_head_transform(const Datagram& datagram); + void parse_controller_transform(const Datagram& datagram); + void parse_controller_event(const Datagram& datagram); + void parse_controller_button(const Datagram& datagram); + void parse_controller_thumbstick(const Datagram& datagram); + void parse_latency(const Datagram& datagram); + +private: + std::thread worker_thread; + std::mutex worker_mutex; + std::condition_variable worker_setup_condition; + + asio::io_context server_context; + std::optional<asio::high_resolution_timer> server_timer; + asio::ip::udp::endpoint server_endpoint; + std::optional<asio::ip::udp::socket> server_socket; + asio::ip::udp::endpoint receive_endpoint; + std::optional<asio::ip::udp::endpoint> client_endpoint; + + DatagramPool datagram_pool; //NOTE: Protected by worker_mutex + std::vector<Datagram> send_queue; //NOTE: Protected by worker_mutex + std::vector<Datagram> receive_queue; //NOTE: Owned by worker_thread and accessible after the thread has been stoped + std::vector<Datagram> error_queue; //NOTE: Owned by worker_thread and accessible after the thread has been stoped + + TransportState state = TRANSPORT_STATE_UNINITIALIZED; //NOTE: Protected by worker_mutex + uint32_t bits_send = 0; //NOTE: Owned by worker_thread + uint32_t bits_received = 0; //NOTE: Owned by worker_thread + double bitrate_send = 0.0; //NOTE: Protected by worker_mutex + double bitrate_received = 0.0; //NOTE: Protected by worker_mutex + uint32_t max_send_queue_size = DATAGRAM_SIZE * 128; //NOTE: Protected by worker_mutex + bool send_active = false; //NOTE: Protected by worker_mutex + double inactive_time = 0.0; //NOTE: Owned by worker_thread + + OnSetup on_setup; + OnProjectionChange on_projection_change; + OnHeadTransform on_head_transform; + OnControllerTransform on_controller_transform; + OnControllerEvent on_controller_event; + OnControllerButton on_controller_button; + OnControllerThumbstick on_controller_thumbstick; + OnLatency on_latency; + OnTransportError on_transport_error; +}; \ No newline at end of file diff --git a/src/types.hpp b/src/types.hpp new file mode 100644 index 00000000..f03364b6 --- /dev/null +++ b/src/types.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include <span> +#include <cstdint> + +typedef uint32_t FrameId; +typedef uint32_t FrameNumber; +typedef std::span<uint8_t> FrameMetadata; + +typedef uint32_t TransformId; + +// Defined the frame ids for the left and right eye in case of an non-remote headset. +enum Eye +{ + EYE_LEFT, + EYE_RIGHT +}; + +// Define left and right controller +enum Controller +{ + CONTROLLER_LEFT, + CONTROLLER_RIGHT, + CONTROLLER_MAX_COUNT +}; + +enum ControllerState +{ + CONTROLLER_STATE_ATTACHED, + CONTROLLER_STATE_DETACHED +}; + +// Define buttons +enum Button +{ + BUTTON_A, + BUTTON_B, + BUTTON_X, + BUTTON_Y, + BUTTON_TRIGGER, + BUTTON_MAX_COUNT +}; + +enum ButtonState +{ + BUTTON_STATE_PRESSED, + BUTTON_STATE_RELEASED +}; + +enum RemoteStrategy +{ + REMOTE_STRATEGY_DEBUG, + REMOTE_STRATEGY_NATIVE +}; \ No newline at end of file diff --git a/src/utility/command_parser.cpp b/src/utility/command_parser.cpp index 766e860d..da1fc427 100644 --- a/src/utility/command_parser.cpp +++ b/src/utility/command_parser.cpp @@ -74,6 +74,14 @@ bool CommandParser::parse_command(const argh::parser& cmd_line) } } + else if (parameter.first == "transport") + { + if (!this->set_transport(parameter.second)) + { + return false; + } + } + else if (parameter.first == "animation") { this->animation_name = parameter.second; @@ -213,6 +221,11 @@ StereoStrategyType CommandParser::get_stereo_strategy() const return this->stereo_strategy; } +TransportType CommandParser::get_transport() const +{ + return this->transport; +} + SceneUnit CommandParser::get_scene_unit() const { return this->scene_unit; @@ -284,6 +297,8 @@ void CommandParser::show_help() std::cout << " Options: emulated (default), openvr, openxr, remote" << std::endl; std::cout << " --stereo-strategy={strategy} The stereo strategy that should be used." << std::endl; std::cout << " Options: native-forward (default), native-deferred, multi-view, dpr" << std::endl; + std::cout << " --transport={transport} The transport method that should be used." << std::endl; + std::cout << " Options: udp (default)" << std::endl; std::cout << " --animation={animation_name} The name of the animation that should be played." << std::endl; std::cout << " This parameter is only allowed during a benchmark." << std::endl; std::cout << " --animation-index={animation_index} The index of the animation that should be played." << std::endl; @@ -365,6 +380,23 @@ bool CommandParser::set_stero_strategy(const std::string& name) return true; } +bool CommandParser::set_transport(const std::string& name) +{ + if (name == "udp") + { + this->transport = TRANSPORT_TYPE_UDP; + } + + else + { + std::cout << "Invalid option set for parameter 'transport'. Use option --help to get more information." << std::endl; + + return false; + } + + return true; +} + bool CommandParser::set_scene_unit(const std::string& name) { if (name == "meters") diff --git a/src/utility/command_parser.hpp b/src/utility/command_parser.hpp index 8cc7ea95..dbc07919 100644 --- a/src/utility/command_parser.hpp +++ b/src/utility/command_parser.hpp @@ -11,6 +11,7 @@ #include "../scene.hpp" #include "headset/headset.hpp" #include "strategy/stereo_strategy.hpp" +#include "transport/transport.hpp" class CommandParser { @@ -21,6 +22,7 @@ public: HeadsetType get_headset() const; StereoStrategyType get_stereo_strategy() const; + TransportType get_transport() const; SceneUnit get_scene_unit() const; const std::string& get_scene_path() const; @@ -43,11 +45,13 @@ private: bool set_headset(const std::string& name); bool set_stero_strategy(const std::string& name); + bool set_transport(const std::string& name); bool set_scene_unit(const std::string& name); private: HeadsetType headset = HEADSET_TYPE_EMULATED; StereoStrategyType stereo_strategy = STEREO_STRATEGY_TYPE_NATIVE_FORWARD; + TransportType transport = TRANSPORT_TYPE_UDP; SceneUnit scene_unit = SCENE_UNIT_METERS; std::string scene_path = ""; diff --git a/src/stream/datagram.cpp b/src/utility/datagram.cpp similarity index 100% rename from src/stream/datagram.cpp rename to src/utility/datagram.cpp diff --git a/src/stream/datagram.hpp b/src/utility/datagram.hpp similarity index 100% rename from src/stream/datagram.hpp rename to src/utility/datagram.hpp diff --git a/src/stream/ring_buffer.cpp b/src/utility/ring_buffer.cpp similarity index 100% rename from src/stream/ring_buffer.cpp rename to src/utility/ring_buffer.cpp diff --git a/src/stream/ring_buffer.hpp b/src/utility/ring_buffer.hpp similarity index 100% rename from src/stream/ring_buffer.hpp rename to src/utility/ring_buffer.hpp diff --git a/src/vr_application.cpp b/src/vr_application.cpp index d01d36c9..e4f7dec5 100644 --- a/src/vr_application.cpp +++ b/src/vr_application.cpp @@ -178,6 +178,11 @@ IndirectCache::Ptr VRApplication::get_indirect_cache() const return this->indirect_cache; } +const CommandParser& VRApplication::get_command_parser() const +{ + return this->command_parser; +} + lava::device_ptr VRApplication::get_device() const { return this->app->device; diff --git a/src/vr_application.hpp b/src/vr_application.hpp index 5ff7ab62..d1cd77da 100644 --- a/src/vr_application.hpp +++ b/src/vr_application.hpp @@ -46,6 +46,8 @@ public: ShadowCache::Ptr get_shadow_cache() const; IndirectCache::Ptr get_indirect_cache() const; + const CommandParser& get_command_parser() const; + lava::device_ptr get_device() const; lava::render_target::ptr get_target() const; -- GitLab