Skip to content
Snippets Groups Projects
Commit ba70bff4 authored by Azur Ponjavic's avatar Azur Ponjavic
Browse files

More stuff

parent deaa172b
Branches
No related tags found
No related merge requests found
......@@ -108,9 +108,6 @@ message(">>> spdlog")
add_subdirectory(${LIBLAVA_EXT_DIR}/spdlog spdlog EXCLUDE_FROM_ALL)
find_package(libdatachannel REQUIRED)
#find_package(jsoncpp REQUIRED)
#add_subdirectory(${LIBLAVA_EXT_DIR}/jsoncpp jsoncpp EXCLUDE_FROM_ALL)
......
......@@ -1151,6 +1151,8 @@ bool NvidiaEncoder::create_semaphore(NvidiaEncoderFrame::Ptr frame, lava::device
#if defined(_WIN32)
semaphore_description.type = CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32;
semaphore_description.handle.win32.handle = frame->semaphore_handle;
semaphore_description.handle.win32.name = nullptr;
#elif defined(__unix__)
semaphore_description.type = CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD;
semaphore_description.handle.fd = (int)frame->semaphore_handle;
......@@ -1158,6 +1160,7 @@ bool NvidiaEncoder::create_semaphore(NvidiaEncoderFrame::Ptr frame, lava::device
#error "Not implemented for this platform!"
#endif
semaphore_description.flags = 0;
memset(semaphore_description.reserved, 0, sizeof(semaphore_description.reserved));
if (cuImportExternalSemaphore(&frame->cuda_external_semaphore, &semaphore_description) != CUDA_SUCCESS)
{
......
......@@ -10,7 +10,6 @@ weak_ptr<T> make_weak_ptr(shared_ptr<T> ptr) {
return ptr;
}
std::string localId;
std::unordered_map<std::string, shared_ptr<rtc::PeerConnection>> peerConnectionMap;
std::unordered_map<std::string, shared_ptr<rtc::DataChannel>> dataChannelMap;
shared_ptr<rtc::PeerConnection> createPeerConnection(weak_ptr<rtc::WebSocket> wws, const std::string&, WebRTCTransport* tr);
......@@ -18,17 +17,6 @@ shared_ptr<rtc::WebSocket> ws;
std::future<void> wsFuture;
rtc::Configuration* config;
std::string randomId(size_t length) {
using std::chrono::high_resolution_clock;
static thread_local std::mt19937 rng(
static_cast<unsigned int>(high_resolution_clock::now().time_since_epoch().count()));
static const std::string characters(
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
std::string id(length, '0');
std::uniform_int_distribution<int> uniform(0, int(characters.size() - 1));
std::generate(id.begin(), id.end(), [&]() { return characters.at(uniform(rng)); });
return id;
}
bool WebRTCTransport::create(uint32_t port_number) {
try {
this->set_state(TRANSPORT_STATE_WAITING);
......@@ -37,95 +25,81 @@ bool WebRTCTransport::create(uint32_t port_number) {
config = new rtc::Configuration();
config->iceServers.emplace_back("stun:stun.l.google.com:19302"); // stun:stun.stunprotocol.org:3478
localId = randomId(4);
std::cout << "The local ID is " << localId << std::endl;
ws = std::make_shared<rtc::WebSocket>();
std::promise<void> wsPromise;
wsFuture = wsPromise.get_future();
ws->onOpen([&wsPromise]() {
std::cout << "WebSocket connected, signaling ready" << std::endl;
lava::log()->info("WebRTC-Transport: WebSocket connected, signaling ready");
wsPromise.set_value();
});
ws->onError([&wsPromise](std::string s) {
std::cout << "WebSocket error" << std::endl;
lava::log()->info("WebRTC-Transport: WebSocket error");
wsPromise.set_exception(std::make_exception_ptr(std::runtime_error(s)));
});
ws->onClosed([]() { std::cout << "WebSocket closed" << std::endl; });
ws->onClosed([]() { lava::log()->info("WebRTC-Transport: WebSocket closed");});
ws->onMessage([wws = make_weak_ptr(ws), tr = this](auto data) {
// data holds either std::string or rtc::binary
std::cout << "Got MSG" << std::endl;
lava::log()->info("WebRTC-Transport: Got MSG");
if (!std::holds_alternative<std::string>(data)) {
std::cout << "MSG is data!" << std::endl;
lava::log()->info("WebRTC-Transport: MSG is data!");
return;
}
Json::Value message;
Json::CharReaderBuilder builder;
Json::CharReader* reader = builder.newCharReader();
nlohmann::json message;
std::string msgStr = std::get<std::string>(data);
std::string errors;
bool parsingSuccessful = reader->parse(
msgStr.c_str(),
msgStr.c_str() + msgStr.size(),
&message,
&errors);
delete reader;
if (!parsingSuccessful) {
std::cout << "Error parsing the string: " + errors << std::endl;
return;
}
Json::Value uuid;
if (message.isMember("uuid")) {
message = nlohmann::json::parse(msgStr);
std::string uuid;
if (message.contains("uuid")) {
uuid = message["uuid"];
} else {
std::cout << "Error reading uuid" << std::endl;
lava::log()->info("WebRTC-Transport: Error reading uuid");
return;
}
std::string type;
if (message.isMember("sdp")) {
type = message["sdp"]["type"].asString();
} else if (message.isMember("ice")) {
if (message.contains("sdp")) {
type = message["sdp"]["type"];
} else if (message.contains("ice")) {
type = "candidate";
} else {
std::cout << "Error no sdp or ice member" << std::endl;
lava::log()->info("WebRTC-Transport: Error no sdp or ice member");
return;
}
std::cout << "MSG type " + type << std::endl;
lava::log()->info("WebRTC-Transport: MSG type " + type);
shared_ptr<rtc::PeerConnection> pc;
if (auto jt = peerConnectionMap.find(uuid.asString()); jt != peerConnectionMap.end()) {
if (auto jt = peerConnectionMap.find(uuid); jt != peerConnectionMap.end()) {
pc = jt->second;
} else if (type == "offer") {
std::cout << "Answering to " + uuid.asString() << std::endl;
pc = createPeerConnection(wws, uuid.asString(), tr);
lava::log()->info("WebRTC-Transport: Answering to " + uuid);
pc = createPeerConnection(wws, uuid, tr);
} else {
return;
}
if (type == "offer" || type == "answer") {
auto sdp = message["sdp"]["sdp"].asString();
auto sdp = message["sdp"]["sdp"];
pc->setRemoteDescription(rtc::Description(sdp, type));
} else if (type == "candidate") {
auto sdp = message["ice"]["candidate"].asString();
auto mid = message["ice"]["sdpMid"].asString(); //todo is this correct
auto sdp = message["ice"]["candidate"];
auto mid = message["ice"]["sdpMid"]; //todo is this correct
pc->addRemoteCandidate(rtc::Candidate(sdp, mid));
}
});
const std::string url = "wss://127.0.0.1:8443"; //todo maybe https?
std::cout << "WebSocket URL is " << url << std::endl;
lava::log()->info("WebRTC-Transport: WebSocket URL is " + url);
ws->open(url);
std::cout << "Waiting for signaling to be connected..." << std::endl;
lava::log()->info("WebRTC-Transport: Waiting for signaling to be connected...");
wsFuture.get();
} catch (const std::exception& e) {
std::cout << "Error: " << e.what() << std::endl;
lava::log()->info("WebRTC-Transport: Error: " + *e.what());
dataChannelMap.clear();
peerConnectionMap.clear();
return false;
......@@ -135,7 +109,7 @@ bool WebRTCTransport::create(uint32_t port_number) {
void WebRTCTransport::destroy() {
std::cout << "Cleaning up..." << std::endl;
lava::log()->info("WebRTC-Transport: Cleaning up...");
dataChannelMap.clear();
peerConnectionMap.clear();
......@@ -168,7 +142,7 @@ bool WebRTCTransport::wait_connect() {
std::unique_lock<std::mutex> lock(this->worker_mutex);
std::chrono::seconds wait_time(TRANSPORT_CLIENT_CONNECT_TIMEOUT);
std::chrono::seconds wait_time(600);
std::chrono::high_resolution_clock::time_point last_time = std::chrono::high_resolution_clock::now();
while (this->state == TRANSPORT_STATE_WAITING && wait_time.count() > 0)
......@@ -181,7 +155,6 @@ bool WebRTCTransport::wait_connect() {
}
lock.unlock();
return this->get_state() == TRANSPORT_STATE_CONNECTED;
}
......@@ -190,139 +163,135 @@ shared_ptr<rtc::PeerConnection> createPeerConnection(weak_ptr<rtc::WebSocket> ww
auto pc = std::make_shared<rtc::PeerConnection>(*config);
pc->onStateChange(
[](rtc::PeerConnection::State state) { std::cout << "State: " << state << std::endl; });
[](rtc::PeerConnection::State state) { lava::log()->info("WebRTC-Transport: State changed."); });
pc->onGatheringStateChange([](rtc::PeerConnection::GatheringState state) {
std::cout << "Gathering State: " << state << std::endl;
lava::log()->info("WebRTC-Transport: Gathering State changed.");
});
pc->onLocalDescription([wws, id, tr](rtc::Description description) {
std::unique_lock<std::mutex> lock(tr->worker_mutex);
Json::Value msgJSON_D;
Json::Value sdp_D;
sdp_D["type"] = description.typeString();
sdp_D["sdp"] = std::string(description);
msgJSON_D["sdp"] = sdp_D;
msgJSON_D["uuid"] = "remoteStreamerID";
Json::FastWriter fw;
std::string message = fw.write(msgJSON_D);
if (auto ws = wws.lock()) {
ws->send(message);
}
nlohmann::json msgJSON;
nlohmann::json sdp;
sdp["type"] = description.typeString();
sdp["sdp"] = std::string(description);
msgJSON["sdp"] = sdp;
msgJSON["uuid"] = "remoteStreamerID";
if (auto ws = wws.lock())
ws->send(msgJSON.dump());
lock.unlock();
});
pc->onLocalCandidate([wws, id, tr](rtc::Candidate candidate) {
std::unique_lock<std::mutex> lock(tr->worker_mutex);
Json::Value msgJSON_C;
Json::Value candidateJSON_C;
candidateJSON_C["candidate"] = std::string(candidate);
candidateJSON_C["sdpMid"] = candidate.mid();
nlohmann::json msgJSON;
nlohmann::json candidateJSON;
candidateJSON["candidate"] = std::string(candidate);
candidateJSON["sdpMid"] = candidate.mid();
//candidateJSON["sdpMLineIndex"] = candidate.mid(); //todo
msgJSON_C["ice"] = candidateJSON_C;
msgJSON_C["uuid"] = "remoteStreamerID";
msgJSON["ice"] = candidateJSON;
msgJSON["uuid"] = "remoteStreamerID";
Json::FastWriter fw;
std::string message = fw.write(msgJSON_C);
if (auto ws = wws.lock()) {
ws->send(message);
}
if (auto ws = wws.lock())
ws->send(msgJSON.dump());
lock.unlock();
});
pc->onDataChannel([id, tr](shared_ptr<rtc::DataChannel> dc) {
std::cout << "DataChannel from " << id << " received with label \"" << dc->label() << "\""
<< std::endl;
lava::log()->info("WebRTC-Transport: DataChannel from " + id + " received with label \"" + dc->label() + "\"");
dc->onOpen([wdc = make_weak_ptr(dc)]() {
});
dc->onClosed([id]() { std::cout << "DataChannel from " << id << " closed" << std::endl; });
dc->onClosed([id]() { lava::log()->info("WebRTC-Transport: DataChannel from " + id + " closed"); });
dc->onMessage([id, tr](auto data) {
// data holds either std::string or rtc::binary
if (std::holds_alternative<std::string>(data)) {
Json::Value msg;
Json::Reader reader;
bool parsingSuccessful = reader.parse(std::get<std::string>(data), msg);
if (parsingSuccessful) {
switch (msg["type"].asInt()) {
nlohmann::json msg;
msg = nlohmann::json::parse(std::get<std::string>(data));
if (!msg.empty()) {
int type = msg["type"];
switch (type) {
case 0: //SETUP
lava::log()->info("WebRTC-Transport: Setup Package received");
if (tr->get_state() == TRANSPORT_STATE_WAITING) {
tr->set_state(TRANSPORT_STATE_CONNECTED);
tr->parse_setup(msg["width"].asInt(), msg["height"].asInt());
tr->parse_setup(msg["width"], msg["height"]);
tr->worker_setup_condition.notify_all();
} else if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
tr->parse_setup(msg["width"].asInt(), msg["height"].asInt());
tr->parse_setup(msg["width"], msg["height"]);
}
break;
case 1: //PROJECTION_CHANGE
lava::log()->info("WebRTC-Transport: Projection Package received");
if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
Json::Value leftEye = msg["left_eye"];
Json::Value rightEye = msg["right_eye"];
Json::Value leftHead = msg["left_head"];
Json::Value rightHead = msg["right_head"];
auto leftEye = msg["left_eye"].get<std::array<float, 16>>();
auto rightEye = msg["right_eye"].get<std::array<float, 16>>();
auto leftHead = msg["left_head"].get<std::array<float, 16>>();
auto rightHead = msg["right_head"].get<std::array<float, 16>>();
float le[16], re[16], lh[16], rh[16];
for (int i = 0; i < leftEye.size(); ++i) {
le[i] = leftEye[i].asFloat();
le[i] = leftEye[i];
}
for (int i = 0; i < rightEye.size(); ++i) {
re[i] = rightEye[i].asFloat();
re[i] = rightEye[i];
}
for (int i = 0; i < leftHead.size(); ++i) {
lh[i] = leftHead[i].asFloat();
lh[i] = leftHead[i];
}
for (int i = 0; i < rightHead.size(); ++i) {
rh[i] = rightHead[i].asFloat();
rh[i] = rightHead[i];
}
tr->parse_projection_change(le, re, lh, rh);
}
break;
case 2: //HEAD_TRANSFORM
lava::log()->info("WebRTC-Transport: Head Transform Package received");
if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
Json::Value transform_id = msg["transform_id"];
Json::Value head_transform = msg["head_transform"];
int transform_id = msg["transform_id"];
auto head_transform = msg["head_transform"].get<std::array<float, 16>>();
float htr[16];
for (int i = 0; i < head_transform.size(); ++i) {
htr[i] = head_transform[i].asFloat();
htr[i] = head_transform[i];
}
tr->parse_head_transform(transform_id.asInt(), htr);
tr->parse_head_transform(transform_id, htr);
}
break;
case 3: //CONTROLLER_TRANSFORM
if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
Json::Value controller_left = msg["controller_left"];
Json::Value controller_transform = msg["controller_transform"];
bool controller_left = msg["controller_left"];
auto controller_transform = msg["controller_transform"].get<std::array<float, 16>>();
float ct[16];
for (int i = 0; i < controller_transform.size(); ++i) {
ct[i] = controller_transform[i].asFloat();
ct[i] = controller_transform[i];
}
tr->parse_controller_transform(controller_left.asBool(), ct);
tr->parse_controller_transform(controller_left, ct);
}
break;
case 4: //CONTROLLER_EVENT
if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
tr->parse_controller_event(msg["controller_left"].asBool(), msg["is_detached"].asBool());
tr->parse_controller_event(msg["controller_left"], msg["is_detached"]);
}
break;
case 5: //CONTROLLER_BUTTON
if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
tr->parse_controller_button(msg["controller_left"].asBool(), msg["button"].asInt(), msg["pressed"].asBool());
tr->parse_controller_button(msg["controller_left"], msg["button"], msg["pressed"]);
}
break;
case 6: //CONTROLLER_THUMBSTICK
if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
Json::Value thumbstick = msg["thumbstick"];
float vec[2];
vec[0] = thumbstick[0].asFloat();
vec[1] = thumbstick[1].asFloat();
tr->parse_controller_thumbstick(msg["controller_left"].asBool(), vec);
vec[0] = msg["thumbstick"][0];
vec[1] = msg["thumbstick"][1];
tr->parse_controller_thumbstick(msg["controller_left"], vec);
}
break;
case 7: //PERFORMANCE_SAMPLE
if (tr->get_state() == TRANSPORT_STATE_CONNECTED) {
tr->parse_performance_sample(msg["frame_number"].asUInt(), msg["frame_id"].asUInt(), msg["transform_id"].asUInt(), msg["unit"].asInt(), msg["name"].asString(), msg["log_only"].asBool(), msg["sample"].asDouble());
tr->parse_performance_sample(msg["frame_number"], msg["frame_id"], msg["transform_id"], msg["unit"], msg["name"], msg["log_only"], msg["sample"]);
}
break;
default:
......@@ -332,10 +301,10 @@ shared_ptr<rtc::PeerConnection> createPeerConnection(weak_ptr<rtc::WebSocket> ww
return;
}
} else {
std::cout << "Msg parsing failed" << std::endl;
lava::log()->info("WebRTC-Transport: Msg parsing failed");
}
} else
std::cout << "Binary message from " << id << std::endl;
lava::log()->info("WebRTC-Transport: Binary message from " + id);
});
dataChannelMap.emplace(id, dc);
......@@ -346,86 +315,83 @@ shared_ptr<rtc::PeerConnection> createPeerConnection(weak_ptr<rtc::WebSocket> ww
};
bool WebRTCTransport::send_setup_complete(RemoteStrategy remote_strategy, EncoderCodec codec, uint32_t max_frame_ids, float near_plane, float far_plane) {
lava::log()->info("WebRTC-Transport: Send Setup compleate.");
if (this->get_state() != TRANSPORT_STATE_CONNECTED){
lava::log()->error("WebRTC-Transport: Can't send setup complete packet since not connection is established!");
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "setup_complete";
msg["remote_strategy"] = remote_strategy;
msg["codec"] = codec;
msg["max_frame_ids"] = max_frame_ids;
msg["near_plane"] = near_plane;
msg["far_plane"] = near_plane;
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
this->send_data(msg.dump());
return true;
}
bool WebRTCTransport::send_stage_transform(const glm::mat4& stage_transform) {
lava::log()->info("WebRTC-Transport: Send stage transform.");
if (this->get_state() != TRANSPORT_STATE_CONNECTED) {
lava::log()->error("WebRTC-Transport: Can't send stage transform packet since not connection is established!");
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "stage_transform";
msg["stage_transform"] = glm::to_string(stage_transform);
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
this->send_data(msg.dump());
return true;
}
bool WebRTCTransport::send_frame_config_nal(FrameId frame_id, const std::span<const uint8_t>& content) {
lava::log()->info("WebRTC-Transport: Send frame config nal.");
if (this->get_state() != TRANSPORT_STATE_CONNECTED) {
lava::log()->error("WebRTC-Transport: Can't send frame config nal packet since not connection is established!");
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "frame_config_nal";
msg["frame_id"] = frame_id;
msg["content"] = std::string(content.begin(), content.end());
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
msg["content"] = content;
this->send_data(msg.dump());
return true;
}
bool WebRTCTransport::send_frame_nal(FrameNumber frame_number, FrameId frame_id, TransformId transform_id, const std::span<const uint8_t>& content) {
lava::log()->info("WebRTC-Transport: Send frame nal.");
if (this->get_state() != TRANSPORT_STATE_CONNECTED) {
lava::log()->error("WebRTC-Transport: Can't send frame nal packet since not connection is established!");
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "frame_nal";
msg["frame_number"] = frame_number;
msg["frame_id"] = frame_id;
msg["transform_id"] = transform_id;
msg["content"] = std::string(content.begin(), content.end());
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
msg["content"] = content;
this->send_data(msg.dump());
return true;
}
bool WebRTCTransport::send_frame_metadata(FrameNumber frame_number, const std::span<const uint8_t>& content) {
lava::log()->info("WebRTC-Transport: Send frame metadata.");
if (this->get_state() != TRANSPORT_STATE_CONNECTED) {
lava::log()->error("WebRTC-Transport: Can't send frame metadata packet since not connection is established!");
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "frame_metadata";
msg["frame_number"] = frame_number;
msg["content"] = std::string(content.begin(), content.end());
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
msg["content"] = content;
this->send_data(msg.dump());
return true;
}
......@@ -435,12 +401,10 @@ bool WebRTCTransport::send_overlay_config(bool overlay_enable) {
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "overlay_config";
msg["overlay_enable"] = overlay_enable;
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
this->send_data(msg.dump());
return true;
}
......@@ -450,15 +414,13 @@ bool WebRTCTransport::send_overlay_text(FrameNumber frame_number, uint32_t overl
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "overlay_text";
msg["frame_number"] = frame_number;
msg["overlay_index"] = overlay_index;
msg["label"] = label;
msg["text"] = text;
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
this->send_data(msg.dump());
return true;
}
......@@ -468,15 +430,13 @@ bool WebRTCTransport::send_overlay_graph(FrameNumber frame_number, uint32_t over
return false;
}
Json::FastWriter fwrite;
Json::Value msg;
nlohmann::json msg;
msg["type"] = "overlay_graph";
msg["frame_number"] = frame_number;
msg["overlay_index"] = overlay_index;
msg["label"] = label;
msg["samples"] = std::string(samples.begin(), samples.end());
std::string rawMsg = fwrite.write(msg);
this->send_data(rawMsg);
this->send_data(msg.dump());
return true;
}
......@@ -691,23 +651,23 @@ void WebRTCTransport::process_send_queue() {
return;
}
std::shared_ptr<rtc::DataChannel> dc;
dc = dataChannelMap.begin()->second;
//todo this is a bool, maybe use it for error detection
if (!dc->isOpen()) {
return;
}
dc->send(this->send_queue.front());
std::unique_lock<std::mutex> lock(this->worker_mutex);
this->send_active = false;
this->send_active = true;
this->bits_send += this->send_queue.front().size() * 8;
this->send_queue.erase(this->send_queue.begin());
this->process_send_queue();
lock.unlock();
//this->process_send_queue();
this->send_active = true;
this->send_active = false;
}
void WebRTCTransport::parse_setup(u_int width, u_int height) {
......
......@@ -11,8 +11,8 @@
#include "rtc/rtc.hpp"
#include <iostream>
#include "json/json.h"
#include <string>
#include <nlohmann/json.hpp>
#include <algorithm>
#include <chrono>
......@@ -23,6 +23,7 @@
#include <thread>
#include <unordered_map>
#define UDP_TRANSPORT_STATE_CHECK_INTERVAL 10 //NOTE: In milliseconds
#define TRANSPORT_CLIENT_CONNECT_TIMEOUT 60 //NOTE: In seconds
#define UDP_TRANSPORT_CLIENT_TIMEOUT 20 //NOTE: In seconds
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment