From 463a2e83f31a56a4eb9a915311cf5d7670da7895 Mon Sep 17 00:00:00 2001 From: phosit Date: Wed, 20 May 2026 21:12:39 +0200 Subject: [PATCH] Use QUIC as transport protocoll Fixes: #7001 --- source/network/NetClient.cpp | 9 +- source/network/NetClient.h | 2 +- source/network/NetClientSession.cpp | 549 +++++++++++++---- source/network/NetClientSession.h | 21 +- source/network/NetHost.cpp | 113 +++- source/network/NetHost.h | 71 ++- source/network/NetMessage.cpp | 19 +- source/network/NetMessage.h | 7 +- source/network/NetMessageSim.cpp | 1 + source/network/NetProtocol.cpp | 39 +- source/network/NetProtocol.h | 19 + source/network/NetServer.cpp | 558 ++++++++++++++---- source/network/NetServer.h | 15 +- source/network/NetServerSession.cpp | 336 ++++++++++- source/network/NetServerSession.h | 56 +- source/network/NetStats.cpp | 84 +-- source/network/NetStats.h | 13 +- .../network/scripting/JSInterface_Network.cpp | 2 +- source/network/tests/test_NetMessage.h | 2 +- 19 files changed, 1545 insertions(+), 371 deletions(-) diff --git a/source/network/NetClient.cpp b/source/network/NetClient.cpp index 0bb39d39d8..2903b3969c 100644 --- a/source/network/NetClient.cpp +++ b/source/network/NetClient.cpp @@ -72,7 +72,7 @@ CNetClient::CNetClient(CGame* game, std::string serverAddressOrHostname, std::ui CNetClient{PrivateTag{}, game, std::move(serverAddressOrHostname), serverPort, username, hostJID, std::move(hashedPassword), std::move(controllerSecret)} { - SetupConnection(nullptr); + SetupConnection(); } CNetClient::CNetClient(PrivateTag, CGame* game, std::string serverAddressOrHostname, @@ -169,10 +169,10 @@ CNetClient::~CNetClient() } -void CNetClient::SetupConnection(ENetHost* enetClient) +void CNetClient::SetupConnection() { CNetClientSession* session = new CNetClientSession(*this); - bool ok = session->Connect(m_ServerAddressOrHostname, m_ServerPort, enetClient); + bool ok = session->Connect(m_ServerAddressOrHostname, m_ServerPort); SetAndOwnSession(session); if (ok) m_PollingThread = std::thread(Threading::HandleExceptions::Wrapper, m_Session); @@ -284,7 +284,7 @@ bool CNetClient::TryToConnectWithSTUN(std::string serverAddressOrHostname, std:: try { - g_NetClient->SetupConnection(enetClient); + g_NetClient->SetupConnection(); } catch (...) { @@ -463,6 +463,7 @@ bool CNetClient::SendMessage(const CNetMessage* message) void CNetClient::HandleConnect() { + LOGMESSAGE("Net client: Connected", m_ServerAddressOrHostname, m_ServerPort); Update((uint)NMT_CONNECT_COMPLETE, NULL); } diff --git a/source/network/NetClient.h b/source/network/NetClient.h index c4cfb087eb..2716ca2b89 100644 --- a/source/network/NetClient.h +++ b/source/network/NetClient.h @@ -289,7 +289,7 @@ private: * Set up a connection to the remote networked server. * @return true on success, false on connection failure */ - void SetupConnection(ENetHost* enetClient); + void SetupConnection(); /** * Take ownership of a session object, and use it for all network communication. diff --git a/source/network/NetClientSession.cpp b/source/network/NetClientSession.cpp index 5cb995b4a0..4a87b74e44 100644 --- a/source/network/NetClientSession.cpp +++ b/source/network/NetClientSession.cpp @@ -22,8 +22,8 @@ #include "lib/code_generation.h" #include "lib/debug.h" #include "network/NetClient.h" -#include "network/NetEnet.h" #include "network/NetMessage.h" +#include "network/NetProtocol.h" #include "network/NetStats.h" #include "ps/CLogger.h" #include "ps/ProfileViewer.h" @@ -32,43 +32,405 @@ constexpr int NETCLIENT_POLL_TIMEOUT = 50; -constexpr int CHANNEL_COUNT = 1; +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +struct CNetClientSession::Quic +{ + AddressStorage localAddress; + std::unique_ptr credentials; + std::unique_ptr session; + std::unique_ptr quicConnection; + ngtcp2_crypto_conn_ref connectionReference; + int fd; + + std::optional streams; +}; + +namespace +{ + +struct CreateSocketResult +{ + int descriptor; + AddressStorage address; +}; +CreateSocketResult CreateSocket(const char* host, const std::uint16_t port) +{ + addrinfo hints{}; + hints.ai_flags = AF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; + + addrinfo* res; + const int rv{getaddrinfo(host, fmt::format("{}", port).c_str(), &hints, &res)}; + if (rv) + throw std::runtime_error{fmt::format("getaddrinfo: {}", gai_strerror(rv))}; + std::unique_ptr infoList{res, &freeaddrinfo}; + + addrinfo* rp; + int fd{-1}; + for (rp = res; rp; rp = rp->ai_next) + { + fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (fd != -1) + break; + } + + if (fd == -1) + throw std::runtime_error{"unable to create a socket"}; + + CreateSocketResult result{ + .descriptor{fd}, + .address{.length{rp->ai_addrlen}} + }; + if (rp->ai_family == AF_INET6) + result.address.address.in6 = reinterpret_cast(*rp->ai_addr); + else + result.address.address.in = reinterpret_cast(*rp->ai_addr); + + return result; +} + +AddressStorage ConnectSocket(const int fd, const AddressStorage& remoteAddress) +{ + if (connect(fd, &remoteAddress.address.sa, remoteAddress.length)) + throw std::runtime_error{fmt::format("connect: {}", strerror(errno))}; + + ngtcp2_sockaddr_union localAddress; + ngtcp2_socklen localAddressLength{sizeof(localAddress)}; + if (getsockname(fd, &localAddress.sa, &localAddressLength) == -1) + throw std::runtime_error{fmt::format("getsockname: {}", strerror(errno))}; + + return {localAddress, localAddressLength}; +} + +void ClientGnutlsInit(CNetClientSession::Quic* c) +{ + gnutls_certificate_credentials_t tempCred; + const int allocRet{gnutls_certificate_allocate_credentials(&tempCred)}; + if (allocRet) + { + throw std::runtime_error{fmt::format("cred init failed: {}: {}", allocRet, + gnutls_strerror(allocRet))}; + } + c->credentials.reset(tempCred); + + gnutls_session_t tempSession; + if (const int initRet{gnutls_init(&tempSession, GNUTLS_CLIENT | GNUTLS_ENABLE_EARLY_DATA | + GNUTLS_NO_END_OF_EARLY_DATA)}) + { + throw std::runtime_error{fmt::format("gnutls_init: {}", gnutls_strerror(initRet))}; + } + c->session.reset(tempSession); + + if (ngtcp2_crypto_gnutls_configure_client_session(c->session.get())) + throw std::runtime_error{"ngtcp2_crypto_gnutls_configure_client_session failed"}; + + if (const int priorityRet{gnutls_priority_set_direct(c->session.get(), TLS_PRIORITY, nullptr)}) + { + throw std::runtime_error{fmt::format("gnutls_priority_set_direct: {}", + gnutls_strerror(priorityRet))}; + } + + gnutls_session_set_ptr(c->session.get(), &c->connectionReference); + + if (const int setRet{gnutls_credentials_set(c->session.get(), GNUTLS_CRD_CERTIFICATE, + c->credentials.get())}) + { + throw std::runtime_error{fmt::format("gnutls_credentials_set: {}", gnutls_strerror(setRet))}; + } +} + +int OpenStream(ngtcp2_conn*, const std::int64_t streamId, void* userData) +{ + CNetClientSession& session{*static_cast(userData)}; + + session.m_Connected = true; + session.m_WasConnected = true; + session.m_Quic->streams.emplace(streamId); + session.m_IncomingMessages.push(CNetClientSession::ConnectionEstablished{}); + return 0; +} + +int OnStreamDataReceive(ngtcp2_conn* conn, const std::uint32_t /*flags*/, const std::int64_t streamId, + const std::size_t /*offset*/, const std::uint8_t* data, std::size_t dataSize, void* userData, void*) +{ + auto& session = *static_cast(userData); + auto message = session.m_Quic->streams.value().Receive({data, dataSize}); + if (message.has_value()) + session.m_IncomingMessages.push(new std::vector{std::move(message).value()}); + increaseWindow(conn, streamId, dataSize); + return 0; +} + +void ClientQuicInit(CNetClientSession& session, const AddressStorage& remote, const AddressStorage& local) +{ + const ngtcp2_path path{ + .local{ + .addr{const_cast(&local.address.sa)}, + .addrlen{local.length}, + }, + .remote{ + .addr{const_cast(&remote.address.sa)}, + .addrlen{remote.length}, + } + }; + constexpr ngtcp2_callbacks callbacks{ + .client_initial{&ngtcp2_crypto_client_initial_cb}, + .recv_crypto_data{&ngtcp2_crypto_recv_crypto_data_cb}, + .encrypt{&ngtcp2_crypto_encrypt_cb}, + .decrypt{&ngtcp2_crypto_decrypt_cb}, + .hp_mask{&ngtcp2_crypto_hp_mask_cb}, + .recv_stream_data{&OnStreamDataReceive}, + .stream_open{&OpenStream}, + .recv_retry{&ngtcp2_crypto_recv_retry_cb}, + .rand{&OnRandomRequest}, + .get_new_connection_id{&OnNewConnectionIdRequest}, + .update_key{&ngtcp2_crypto_update_key_cb}, + .delete_crypto_aead_ctx{&ngtcp2_crypto_delete_crypto_aead_ctx_cb}, + .delete_crypto_cipher_ctx{&ngtcp2_crypto_delete_crypto_cipher_ctx_cb}, + .get_path_challenge_data{&ngtcp2_crypto_get_path_challenge_data_cb}, + .version_negotiation{&ngtcp2_crypto_version_negotiation_cb} + }; + ngtcp2_cid dcid; + dcid.datalen = NGTCP2_MIN_INITIAL_DCIDLEN; + if (gnutls_rnd(GNUTLS_RND_RANDOM, dcid.data, dcid.datalen)) + throw std::runtime_error{"gnutls_rnd failed"}; + + ngtcp2_cid scid; + scid.datalen = 8; + if (gnutls_rnd(GNUTLS_RND_RANDOM, scid.data, scid.datalen)) + throw std::runtime_error{"gnutls_rnd failed"}; + + ngtcp2_settings settings; + ngtcp2_settings_default(&settings); + settings.initial_ts = timestamp(); + + ngtcp2_transport_params params; + ngtcp2_transport_params_default(¶ms); + params.initial_max_stream_data_bidi_remote = 128 * KiB; + params.initial_max_data = 1 * MiB; + params.initial_max_streams_bidi = 1; + params.max_udp_payload_size = MAX_UDP_PAYLOAD_SIZE; + params.grease_quic_bit = 1; + + ngtcp2_conn* tempConn; + if (const int rv{ngtcp2_conn_client_new(&tempConn, &dcid, &scid, &path, + NGTCP2_PROTO_VER_V1, &callbacks, &settings, ¶ms, nullptr, &session)}) + { + throw std::runtime_error{fmt::format("ngtcp2_conn_client_new: {}", ngtcp2_strerror(rv))}; + } + session.m_Quic->quicConnection.reset(tempConn); + + ngtcp2_conn_set_tls_native_handle(session.m_Quic->quicConnection.get(), + session.m_Quic->session.get()); +} + +void ClientRead(CNetClientSession::Quic* c) { + std::array buf; + struct sockaddr_storage addr; + iovec iov{ + .iov_base = buf.data(), + .iov_len = buf.size(), + }; + msghdr msg{}; + msg.msg_name = &addr; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + ngtcp2_pkt_info pi{}; + while (true) + { + msg.msg_namelen = sizeof(addr); + + const ssize_t nread{recvmsg(c->fd, &msg, MSG_DONTWAIT)}; + + if (nread == -1) + { + if (errno != EAGAIN && errno != EWOULDBLOCK) + LOGERROR("recvmsg: %s", strerror(errno)); + + break; + } + + ngtcp2_path path{ + .local{ + .addr{&c->localAddress.address.sa}, + .addrlen{c->localAddress.length} + }, + .remote{ + .addr{static_cast(msg.msg_name)}, + .addrlen{msg.msg_namelen} + } + }; + + const int rv{ngtcp2_conn_read_pkt(c->quicConnection.get(), &path, &pi, buf.data(), + static_cast(nread), timestamp())}; + if (rv != 0) + throw std::runtime_error{fmt::format("ngtcp2_conn_read_pkt: {}", ngtcp2_strerror(rv))}; + } +} + +void ClientSendDatagram(CNetClientSession::Quic* c, const std::span data) +{ + iovec iov{ + .iov_base = const_cast(data.data()), + .iov_len = data.size(), + }; + msghdr msg{}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + ssize_t nwrite; + do + { + nwrite = sendmsg(c->fd, &msg, 0); + } while (nwrite == -1 && errno == EINTR); + + if (nwrite == -1) + throw std::runtime_error{fmt::format("sendmsg: {}", strerror(errno))}; +} + +void ClientWriteStreams(CNetClientSession::Quic* c) +{ + const ngtcp2_tstamp ts{timestamp()}; + ngtcp2_pkt_info pi; + std::array buffer; + + ngtcp2_path_storage ps; + ngtcp2_path_storage_zero(&ps); + std::uint32_t flags{NGTCP2_WRITE_STREAM_FLAG_MORE}; + + while (true) + { + ngtcp2_vec datav; + std::int64_t streamId; + + const auto bytesToSend = c->streams.has_value() ? c->streams.value().PeekData() : std::nullopt; + if (c->streams.has_value() && bytesToSend.has_value()) + { + datav.base = const_cast(bytesToSend->data()); + datav.len = bytesToSend->size(); + streamId = c->streams.value().m_Id; + } + else + { + datav.base = nullptr; + datav.len = 0; + streamId = -1; + if (c->streams.has_value()) + flags &= ~NGTCP2_WRITE_STREAM_FLAG_MORE; + } + + ngtcp2_ssize wdatalen; + const ngtcp2_ssize nwrite = ngtcp2_conn_writev_stream(c->quicConnection.get(), &ps.path, &pi, + buffer.data(), buffer.size(), &wdatalen, flags, streamId, &datav, 1, ts); + if (nwrite < 0) + { + if (nwrite == NGTCP2_ERR_STREAM_DATA_BLOCKED) + { + LOGWARNING("blocked"); + break; + } + if (nwrite != NGTCP2_ERR_WRITE_MORE) + { + throw std::runtime_error{fmt::format("ngtcp2_conn_writev_stream: {}", + ngtcp2_strerror(static_cast(nwrite)))}; + } + if (c->streams.has_value() && wdatalen > 0) + c->streams.value().MarkSent(wdatalen); + continue; + } + + if (nwrite == 0) + break; + + if (c->streams.has_value() && wdatalen > 0) + c->streams.value().MarkSent(wdatalen); + + ClientSendDatagram(c, {buffer.data(), static_cast(nwrite)}); + } + ngtcp2_conn_update_pkt_tx_time(c->quicConnection.get(), timestamp()); +} + +void ClientHandleExpiry(CNetClientSession::Quic* c) +{ + if (const int rv = ngtcp2_conn_handle_expiry(c->quicConnection.get(), timestamp())) + throw std::runtime_error{fmt::format("ngtcp2_conn_handle_expiry: {}", ngtcp2_strerror(rv))}; +} + +ngtcp2_conn *GetConnection(ngtcp2_crypto_conn_ref* conn_ref) +{ + CNetClientSession::Quic* c{static_cast(conn_ref->user_data)}; + return c->quicConnection.get(); +} + +void ClientInit(CNetClientSession& session, const char* host, const std::uint16_t port) +{ + *session.m_Quic = CNetClientSession::Quic{}; + + const auto [descriptor, remoteAddress] = CreateSocket(host, port); + session.m_Quic->fd = descriptor; + + session.m_Quic->localAddress = ConnectSocket(session.m_Quic->fd, remoteAddress); + + ClientGnutlsInit(session.m_Quic.get()); + + ClientQuicInit(session, remoteAddress, session.m_Quic->localAddress); + + session.m_Quic->connectionReference.get_conn = &GetConnection; + session.m_Quic->connectionReference.user_data = session.m_Quic.get(); +} +} // anonymous namespace CNetClientSession::CNetClientSession(CNetClient& client) : - m_Client(client), m_FileTransferer(*this) + m_Client(client), m_FileTransferer(*this), + m_Quic(std::make_unique()) { } CNetClientSession::~CNetClientSession() { ENSURE(!m_LoopRunning); + + constexpr ngtcp2_ccerr reason{ + .type{NGTCP2_CCERR_TYPE_TRANSPORT}, + .error_code{0} + }; + + std::array buffer; + const ngtcp2_ssize amount{ngtcp2_conn_write_connection_close(m_Quic->quicConnection.get(), nullptr, nullptr, buffer.data(), + buffer.size(), &reason, timestamp())}; + if (amount <= 0) + { + LOGERROR("closing connection %s", ngtcp2_strerror(amount)); + } + + ClientSendDatagram(m_Quic.get(), {buffer.data(), static_cast(amount)}); } -bool CNetClientSession::Connect(const CStr& server, const u16 port, ENetHost* enetClient) +bool CNetClientSession::Connect(const CStr& server, const u16 port) { ENSURE(!m_LoopRunning); - ENSURE(!m_Host); - ENSURE(!m_Server); - // Create ENet host if necessary. - m_Host.reset(enetClient != nullptr ? enetClient : PS::Enet::CreateHost(nullptr, 1, CHANNEL_COUNT)); + ClientInit(*this, server.c_str(), port); + ClientWriteStreams(m_Quic.get()); - if (!m_Host) - return false; - - // Bind to specified host - ENetAddress addr; - addr.port = port; - if (enet_address_set_host(&addr, server.c_str()) < 0) - return false; - - // Initiate connection to server - m_Server.reset(enet_host_connect(m_Host.get(), &addr, CHANNEL_COUNT, 0)); - if (!m_Server) - return false; - - - m_Stats = std::make_unique(*m_Server); + m_Stats = std::make_unique(m_Quic->quicConnection.get()); if (CProfileViewer::IsInitialised()) g_ProfileViewer.AddRootTable(m_Stats.get()); @@ -84,14 +446,24 @@ void CNetClientSession::RunNetLoop(CNetClientSession* session) while (!session->m_ShouldShutdown) { - ENSURE(session->m_Host && session->m_Server); + // ENSURE(session->m_Host && session->m_Server); session->m_FileTransferer.Poll(); - session->Poll(); + try { + session->Poll(); + } + catch (std::runtime_error&) + { + // Report immediately. + LOGMESSAGE("Net client: Disconnected"); + session->m_Connected = false; + session->m_IncomingMessages.push(Disconnect{}); + return; + } session->Flush(); - session->m_LastReceivedTime = enet_time_get() - session->m_Server->lastReceiveTime; - session->m_MeanRTT = session->m_Server->roundTripTime; + // session->m_LastReceivedTime = timestamp() - session->m_Server->lastReceiveTime; + // session->m_MeanRTT = session->m_Server->roundTripTime; } session->m_LoopRunning = false; @@ -107,96 +479,77 @@ void CNetClientSession::Shutdown() void CNetClientSession::Poll() { - ENetEvent event; + pollfd pfd{ + .fd{m_Quic->fd}, + .events{POLLIN} + }; - // Use the timeout to make the thread wait and save CPU time. - if (enet_host_service(m_Host.get(), &event, NETCLIENT_POLL_TIMEOUT) <= 0) + const int ret{poll(&pfd, 1, NETCLIENT_POLL_TIMEOUT)}; + if (ret < 0) + { + LOGERROR("Error while waiting for poll: %s", std::strerror(errno)); return; - - if (event.type == ENET_EVENT_TYPE_CONNECT) - { - ENSURE(event.peer == m_Server.get()); - - // Report the server address immediately. - char hostname[256] = "(error)"; - enet_address_get_host_ip(&event.peer->address, hostname, ARRAY_SIZE(hostname)); - LOGMESSAGE("Net client: Connected to %s:%u", hostname, (unsigned int)event.peer->address.port); - m_Connected = true; - m_WasConnected = true; - - m_IncomingMessages.push(event); } - else if (event.type == ENET_EVENT_TYPE_DISCONNECT) + if (ret == 0) { - ENSURE(event.peer == m_Server.get()); - - // Report immediately. - LOGMESSAGE("Net client: Disconnected"); - m_Connected = false; - - m_IncomingMessages.push(event); + ClientHandleExpiry(m_Quic.get()); + ClientWriteStreams(m_Quic.get()); + return; } - else if (event.type == ENET_EVENT_TYPE_RECEIVE) - m_IncomingMessages.push(event); + + ClientRead(m_Quic.get()); + ClientWriteStreams(m_Quic.get()); } void CNetClientSession::Flush() { - ENetPacket* packet; - while (m_OutgoingMessages.pop(packet)) - if (enet_peer_send(m_Server.get(), CNetHost::DEFAULT_CHANNEL, packet) < 0) - { - // Report the error, but do so silently if we know we are disconnected. - if (m_Connected) - LOGERROR("NetClient: Failed to send packet to server"); - else - LOGMESSAGE("NetClient: Failed to send packet to server"); - } - - enet_host_flush(m_Host.get()); + std::vector* message; + while (m_OutgoingMessages.pop(message)) + { + std::unique_ptr> data{message}; + if (m_Quic->streams.has_value()) + m_Quic->streams.value().PushData(std::move(*data)); + else + LOGERROR("no stream to send message"); + } } void CNetClientSession::ProcessPolledMessages() { - ENetEvent event; - while(m_IncomingMessages.pop(event)) + IncommingMessage query{}; + while(m_IncomingMessages.pop(query)) { - if (event.type == ENET_EVENT_TYPE_CONNECT) - m_Client.HandleConnect(); - else if (event.type == ENET_EVENT_TYPE_DISCONNECT) + std::visit([&](Message message) { - // This deletes the session, so we must break; - if (event.data == 0 && !m_WasConnected) - m_Client.HandleDisconnect(NDR_CONNECTION_REQUEST_TIMED_OUT); - else - m_Client.HandleDisconnect(event.data); - break; - } - else if (event.type == ENET_EVENT_TYPE_RECEIVE) - { - CNetMessage* msg = CNetMessageFactory::CreateMessage(event.packet->data, event.packet->dataLength, m_Client.GetScriptInterface()); - if (msg) + if constexpr (std::same_as) { - LOGMESSAGE("Net client: Received message %s of size %lu from server", msg->ToString().c_str(), (unsigned long)msg->GetSerializedLength()); - - m_Client.HandleMessage(msg); + m_Client.HandleConnect(); } - // Thread-safe - enet_packet_destroy(event.packet); - } + else if constexpr (std::same_as) + { + m_Client.HandleDisconnect(NDR_UNKNOWN); + } + else + { + static_assert(std::same_as*>); + std::unique_ptr> data{message}; + CNetMessage* msg = CNetMessageFactory::CreateMessage(*data, m_Client.GetScriptInterface()); + if (msg) + { + LOGMESSAGE("Net client: Received message %s of size %lu from server", msg->ToString().c_str(), (unsigned long)msg->GetSerializedLength()); + + m_Client.HandleMessage(msg); + } + } + }, query); } } bool CNetClientSession::SendMessage(const CNetMessage* message) { - ENSURE(m_Host && m_Server); + // ENSURE(m_Host && m_Server); - // Thread-safe. - ENetPacket* packet = CNetHost::CreatePacket(message); - if (!packet) - return false; - - if (!m_OutgoingMessages.push(packet)) + if (!m_OutgoingMessages.push(new std::vector{CNetHost::CreatePacket(message)})) { LOGERROR("NetClient: Failed to push message on the outgoing queue."); return false; @@ -207,17 +560,11 @@ bool CNetClientSession::SendMessage(const CNetMessage* message) u32 CNetClientSession::GetLastReceivedTime() const { - if (!m_Server) - return 0; - return m_LastReceivedTime; } u32 CNetClientSession::GetMeanRTT() const { - if (!m_Server) - return 0; - return m_MeanRTT; } diff --git a/source/network/NetClientSession.h b/source/network/NetClientSession.h index 2446c85666..bd07e367ab 100644 --- a/source/network/NetClientSession.h +++ b/source/network/NetClientSession.h @@ -19,7 +19,6 @@ #define NETSESSION_H #include "lib/code_annotation.h" -#include "lib/external_libraries/enet.h" #include "lib/types.h" #include "network/NetFileTransfer.h" #include "network/NetHost.h" @@ -32,8 +31,6 @@ class CNetMessage; class CNetStatsTable; class CStr; -typedef struct _ENetHost ENetHost; - /** * @file * Network client/server sessions. @@ -53,10 +50,11 @@ class CNetClientSession NONCOPYABLE(CNetClientSession); public: + struct Quic; CNetClientSession(CNetClient& client); ~CNetClientSession(); - bool Connect(const CStr& server, const u16 port, ENetHost* enetClient); + bool Connect(const CStr& server, const u16 port); /** * The client NetSession is threaded to avoid getting timeouts if the main thread hangs. @@ -104,11 +102,14 @@ private: CNetClient& m_Client; CNetFileTransferer m_FileTransferer; - +public: // Net messages received and waiting for fetching. - boost::lockfree::queue m_IncomingMessages{16}; + struct ConnectionEstablished{}; + struct Disconnect{}; + using IncommingMessage = std::variant*, Disconnect>; + boost::lockfree::queue m_IncomingMessages{16}; // Net messages to send on the next flush() call. - boost::lockfree::queue m_OutgoingMessages{16}; + boost::lockfree::queue*> m_OutgoingMessages{16}; // Last known state. If false, flushing errors are silenced. bool m_Connected{false}; @@ -116,7 +117,7 @@ private: // Whether this session was ever connected to the server. bool m_WasConnected{false}; - // Wrapper around enet stats - those are atomic as the code is lock-free. + // Wrapper around stats - those are atomic as the code is lock-free. std::atomic m_LastReceivedTime{0}; std::atomic m_MeanRTT{0}; @@ -124,9 +125,9 @@ private: std::atomic m_LoopRunning{false}; std::atomic m_ShouldShutdown{false}; - std::unique_ptr m_Host; - std::unique_ptr m_Server; std::unique_ptr m_Stats; + + const std::unique_ptr m_Quic; }; #endif // NETSESSION_H diff --git a/source/network/NetHost.cpp b/source/network/NetHost.cpp index 5f07dea104..876167dcbd 100644 --- a/source/network/NetHost.cpp +++ b/source/network/NetHost.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Wildfire Games. +/* Copyright (C) 2026 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -25,32 +25,10 @@ #include "ps/CLogger.h" #include +#include #include -bool CNetHost::SendMessage(const CNetMessage* message, ENetPeer* peer, const char* peerName) -{ - ENetPacket* packet = CreatePacket(message); - if (!packet) - return false; - - LOGMESSAGE("Net: Sending message %s of size %lu to %s", message->ToString().c_str(), (unsigned long)packet->dataLength, peerName); - - // Let ENet send the message to peer - if (enet_peer_send(peer, DEFAULT_CHANNEL, packet) < 0) - { - LOGERROR("Net: Failed to send packet to peer"); - return false; - } - - // Don't call enet_host_flush now - let it queue up all the packets - // and send them during the next frame - // - // TODO: we should flush explicitly at some appropriate point before the next frame - - return true; -} - -ENetPacket* CNetHost::CreatePacket(const CNetMessage* message) +std::vector CNetHost::CreatePacket(const CNetMessage* message) { size_t size = message->GetSerializedLength(); @@ -63,12 +41,7 @@ ENetPacket* CNetHost::CreatePacket(const CNetMessage* message) // Save message to internal buffer message->Serialize(&buffer[0]); - // Create a reliable packet - ENetPacket* packet = enet_packet_create(&buffer[0], size, ENET_PACKET_FLAG_RELIABLE); - if (!packet) - LOGERROR("Net: Failed to construct packet"); - - return packet; + return buffer; } void CNetHost::Initialize() @@ -81,3 +54,81 @@ void CNetHost::Deinitialize() { enet_deinitialize(); } + + +Stream::Stream(const std::int64_t streamId): + m_Id{streamId} +{} + +void Stream::PushData(std::vector data) +{ + m_SendBuffer.push_back(std::move(data)); +} + +void Stream::PushMessage(const CNetMessage* message) +{ + m_SendBuffer.push_back(CNetHost::CreatePacket(message)); +} + +std::optional> Stream::PeekData() +{ + const std::size_t startOffset{m_SentOffset - m_AckedOffset}; + std::size_t offset{0}; + + for (std::vector& bytes : m_SendBuffer) + { + if (startOffset - offset < bytes.size()) + { + const std::size_t temp{startOffset - offset}; + return std::span{bytes.data() + temp, bytes.size() - temp}; + } + + offset += bytes.size(); + } + + return std::nullopt; +} + +void Stream::MarkSent(const std::size_t offset) +{ + m_SentOffset += offset; +} + +void Stream::MarkAcknowledged(const std::size_t offset) +{ + while (!m_SendBuffer.empty()) + { + std::vector& head{m_SendBuffer.front()}; + if (m_AckedOffset + head.size() > offset) + break; + + m_AckedOffset += head.size(); + m_SendBuffer.pop_front(); + } +} + +std::optional> Stream::Receive(const std::span data) +{ + m_ReceiveBuffer.emplace_back(data.begin(), data.end()); + const std::size_t bufferSize{std::transform_reduce(m_ReceiveBuffer.begin(), m_ReceiveBuffer.end(), + static_cast(0), std::plus<>{}, std::mem_fn(&std::vector::size))}; + if (bufferSize < 3) + return std::nullopt; + const auto& message = m_ReceiveBuffer.front(); + + auto bufferIter = message.begin(); + std::size_t messageSize; + Deserialize_int_1(bufferIter, std::ignore); + Deserialize_int_2(bufferIter, messageSize); + if (messageSize > bufferSize) + return std::nullopt; + + std::vector messageCopy; + while (messageCopy.size() < messageSize) + { + messageCopy.insert(messageCopy.end(), m_ReceiveBuffer.front().begin(), + m_ReceiveBuffer.front().end()); + m_ReceiveBuffer.pop_front(); + } + return messageCopy; +} diff --git a/source/network/NetHost.h b/source/network/NetHost.h index 7912cc9ee5..ade53bbf83 100644 --- a/source/network/NetHost.h +++ b/source/network/NetHost.h @@ -27,7 +27,14 @@ #include "lib/types.h" #include "ps/CStr.h" +#include #include +#include +#include +#include +#include + +#include class CNetMessage; @@ -35,6 +42,38 @@ typedef struct _ENetPeer ENetPeer; typedef struct _ENetPacket ENetPacket; typedef struct _ENetHost ENetHost; +constexpr std::size_t MAX_UDP_PAYLOAD_SIZE{64 * KiB}; + +struct CredentialsDeleter +{ + void operator()(const gnutls_certificate_credentials_t cred) const + { + gnutls_certificate_free_credentials(cred); + } +}; + +struct SessionDeleter +{ + void operator()(const gnutls_session_t p) const + { + gnutls_deinit(p); + } +}; + +struct ConnectionDeleter +{ + void operator()(ngtcp2_conn* p) const + { + ngtcp2_conn_del(p); + } +}; + +struct AddressStorage +{ + ngtcp2_sockaddr_union address; + ngtcp2_socklen length{sizeof(address)}; +}; + struct PlayerAssignment { /** @@ -106,20 +145,11 @@ class CNetHost public: static const int DEFAULT_CHANNEL = 0; - /** - * Transmit a message to the given peer. - * @param message message to send - * @param peer peer to send to - * @param peerName name of peer for debug logs - * @return true on success, false on failure - */ - static bool SendMessage(const CNetMessage* message, ENetPeer* peer, const char* peerName); - /** * Construct an ENet packet by serialising the given message. * @return NULL on failure */ - static ENetPacket* CreatePacket(const CNetMessage* message); + static std::vector CreatePacket(const CNetMessage* message); /** * Initialize ENet. @@ -133,4 +163,25 @@ public: static void Deinitialize(); }; +class Stream +{ +public: + Stream(const std::int64_t streamId); + + void PushData(std::vector data); + void PushMessage(const CNetMessage* message); + std::optional> PeekData(); + void MarkSent(const std::size_t offset); + void MarkAcknowledged(const std::size_t offset); + std::optional> Receive(const std::span data); + + std::int64_t m_Id; +private: + std::deque> m_SendBuffer; + std::deque> m_ReceiveBuffer; + /* invariant: m_SentOffset >= m_AckedOffset */ + std::size_t m_SentOffset{0}; + std::size_t m_AckedOffset{0}; +}; + #endif // NETHOST_H diff --git a/source/network/NetMessage.cpp b/source/network/NetMessage.cpp index f83047df64..bdb34a67b8 100644 --- a/source/network/NetMessage.cpp +++ b/source/network/NetMessage.cpp @@ -53,10 +53,7 @@ u8* CNetMessage::Serialize(u8* pBuffer) const const u8* CNetMessage::Deserialize(const u8* pStart, const u8* pEnd) { if (pStart + 3 > pEnd) - { - LOGERROR("CNetMessage: Corrupt packet (smaller than header)"); - return NULL; - } + throw std::invalid_argument{"CNetMessage: Corrupt packet (smaller than header)"}; const u8* pBuffer = pStart; @@ -67,10 +64,7 @@ const u8* CNetMessage::Deserialize(const u8* pStart, const u8* pEnd) m_Type = (NetMessageType)type; if (pStart + size != pEnd) - { - LOGERROR("CNetMessage: Corrupt packet (incorrect size)"); - return NULL; - } + throw std::invalid_argument{fmt::format("CNetMessage: Corrupt packet (incorrect size) %i %i", size, pEnd - pStart)}; return pBuffer; } @@ -91,15 +85,14 @@ CStr CNetMessage::ToString() const return fmt::format("Unknown Message {}", static_cast(GetType())); } -CNetMessage* CNetMessageFactory::CreateMessage(const void* pData, - size_t dataSize, - const Script::Interface& scriptInterface) +CNetMessage* CNetMessageFactory::CreateMessage(const std::span data, + const Script::Interface& scriptInterface) { CNetMessage* pNewMessage = NULL; CNetMessage header; // Figure out message type - header.Deserialize((const u8*)pData, (const u8*)pData + dataSize); + header.Deserialize(std::to_address(data.begin()), std::to_address(data.end())); switch (header.GetType()) { @@ -230,7 +223,7 @@ CNetMessage* CNetMessageFactory::CreateMessage(const void* pData, } if (pNewMessage) - pNewMessage->Deserialize((const u8*)pData, (const u8*)pData + dataSize); + pNewMessage->Deserialize(std::to_address(data.begin()), std::to_address(data.end())); return pNewMessage; } diff --git a/source/network/NetMessage.h b/source/network/NetMessage.h index 34bd81243e..0da2ba3990 100644 --- a/source/network/NetMessage.h +++ b/source/network/NetMessage.h @@ -27,6 +27,7 @@ #include #include #include +#include namespace Script { class Interface; } @@ -108,12 +109,12 @@ public: /** * Factory method which creates a message object based on the given data * - * @param pData Data buffer - * @param dataSize Size of data buffer + * @param data Data buffer * @param scriptInterface Script instance to use when constructing scripted messages * @return The new message created */ - static CNetMessage* CreateMessage(const void* pData, size_t dataSize, const Script::Interface& scriptInterface); + static CNetMessage* CreateMessage(const std::span data, + const Script::Interface& scriptInterface); }; /** diff --git a/source/network/NetMessageSim.cpp b/source/network/NetMessageSim.cpp index a57f24ee5e..9c9a7a17f3 100644 --- a/source/network/NetMessageSim.cpp +++ b/source/network/NetMessageSim.cpp @@ -35,6 +35,7 @@ #include #include #include +#include "ps/CLogger.h" namespace Script { class Interface; } diff --git a/source/network/NetProtocol.cpp b/source/network/NetProtocol.cpp index 75f0025a3f..a10b2a1608 100644 --- a/source/network/NetProtocol.cpp +++ b/source/network/NetProtocol.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Wildfire Games. +/* Copyright (C) 2026 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -19,10 +19,14 @@ #include "NetProtocol.h" +#include "ps/CLogger.h" #include "ps/CStr.h" #include +#include #include +#include +#include namespace { @@ -59,3 +63,36 @@ std::optional CheckHandshake(const CSrvHandshakeMessage& serverM return {}; } + +uint64_t timestamp() +{ + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +void increaseWindow(ngtcp2_conn* conn, const std::uint64_t streamId, const std::uint64_t size) +{ + ngtcp2_conn_extend_max_offset(conn, size); + ngtcp2_conn_extend_max_stream_offset(conn, streamId, size); +} + +void OnRandomRequest(std::uint8_t* destination, const std::size_t destinationLength, const ngtcp2_rand_ctx*) +{ + const int ret{gnutls_rnd(GNUTLS_RND_RANDOM, destination, destinationLength)}; + if (ret < 0) + LOGERROR("gnutls_rnd: %s", gnutls_strerror(ret)); +} + +int OnNewConnectionIdRequest(ngtcp2_conn*, ngtcp2_cid* cid, std::uint8_t* token, const std::size_t cidlen, + void* /*userData*/) +{ + if (gnutls_rnd(GNUTLS_RND_RANDOM, cid->data, cidlen)) + return NGTCP2_ERR_CALLBACK_FAILURE; + + cid->datalen = cidlen; + + if (gnutls_rnd (GNUTLS_RND_RANDOM, token, NGTCP2_STATELESS_RESET_TOKENLEN)) + return NGTCP2_ERR_CALLBACK_FAILURE; + + return 0; +} diff --git a/source/network/NetProtocol.h b/source/network/NetProtocol.h index 7faa82b721..8d6fe3a561 100644 --- a/source/network/NetProtocol.h +++ b/source/network/NetProtocol.h @@ -27,11 +27,17 @@ #include #include +#include + /** * Report the peer if we didn't receive a packet after this time (milliseconds). */ inline constexpr u32 NETWORK_WARNING_TIMEOUT{2000}; +inline constexpr const char* TLS_PRIORITY{ + "PERFORMANCE:-VERS-DTLS-ALL:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-TLS1.2:-SHA1:-AES-128-CBC:-AES-256-CBC:" + "-SIGN-RSA-SHA1:-SIGN-ECDSA-SHA1:%DISABLE_TLS13_COMPAT_MODE"}; + struct HandshakeError { std::string componentType; @@ -67,4 +73,17 @@ Message CreateHandshake() { std::optional CheckHandshake(const CSrvHandshakeMessage& serverMessage, const CCliHandshakeMessage& clientMessage); +uint64_t timestamp(); + +struct ngtcp2_conn; +void increaseWindow(ngtcp2_conn* conn, const std::uint64_t streamId, const std::uint64_t size); + +struct ngtcp2_rand_ctx; +void OnRandomRequest(std::uint8_t* destination, const std::size_t destinationLength, + const ngtcp2_rand_ctx*); + +struct ngtcp2_cid; +int OnNewConnectionIdRequest(ngtcp2_conn*, ngtcp2_cid* cid, std::uint8_t *token, const std::size_t cidlen, + void* /*userData*/); + #endif diff --git a/source/network/NetServer.cpp b/source/network/NetServer.cpp index c4198a797d..7afc9bc9fc 100644 --- a/source/network/NetServer.cpp +++ b/source/network/NetServer.cpp @@ -22,6 +22,7 @@ #include "lib/code_generation.h" #include "lib/debug.h" #include "lib/external_libraries/enet.h" +#include "lib/hash.h" #include "lib/secure_crt.h" #include "lib/status.h" #include "lib/types.h" @@ -39,6 +40,8 @@ #include "ps/CLogger.h" #include "ps/ConfigDB.h" #include "ps/GUID.h" +#include "ps/GameSetup/CmdLineArgs.h" +#include "ps/GameSetup/Paths.h" #include "ps/Hashing.h" #include "ps/ProfileViewer.h" #include "ps/Profiler2.h" @@ -56,8 +59,19 @@ #include #include #include +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -103,15 +117,366 @@ constexpr u32 NETWORK_BAD_PING = DEFAULT_TURN_LENGTH * COMMAND_DELAY_MP / 2; CNetServer* g_NetServer = NULL; -static CStr DebugName(CNetServerSession* session) +namespace { - if (session == NULL) - return "[unknown host]"; - if (session->GetGUID().empty()) - return "[unauthed host]"; - return "[" + session->GetGUID().substr(0, 8) + "...]"; +// static CStr DebugName(CNetServerSession* session) +// { +// if (session == NULL) +// return "[unknown host]"; +// if (session->GetGUID().empty()) +// return "[unauthed host]"; +// return "[" + session->GetGUID().substr(0, 8) + "...]"; +// } + +struct CidHash +{ + std::size_t operator()(const ngtcp2_cid& cid) const + { + return std::accumulate(cid.data, cid.data + cid.datalen, static_cast(0), + [](std::size_t carry, const std::uint8_t byte) + { + hash_combine(carry, byte); + return carry; + }); + } +}; +struct CidEqual +{ + bool operator()(const ngtcp2_cid& a, const ngtcp2_cid& b) const + { + return ngtcp2_cid_eq(&a, &b); + } +}; + +std::size_t ReceivePackage(const int fd, std::span data, AddressStorage& remoteAddress) +{ + iovec iov{ + .iov_base{data.data()}, + .iov_len{data.size()} + }; + + msghdr msg{}; + msg.msg_name = &remoteAddress.address.sa; + msg.msg_namelen = remoteAddress.length; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + ssize_t ret; + do + ret = recvmsg(fd, &msg, MSG_DONTWAIT); + while (ret < 0 && errno == EINTR); + if (ret < 0) + throw std::system_error{errno, std::generic_category(), "recvmsg"}; + + remoteAddress.length = msg.msg_namelen; + return ret; } +void GetRandomCid(ngtcp2_cid* cid) +{ + std::array buf; + const int ret{gnutls_rnd(GNUTLS_RND_RANDOM, buf.data(), buf.size())}; + if (ret < 0) + throw std::runtime_error{fmt::format("gnutls_rnd: {}", gnutls_strerror(ret))}; + ngtcp2_cid_init(cid, buf.data(), buf.size()); +} + +int ResolveAndBind(const char *port, AddressStorage& localAddress) +{ + addrinfo hints{}; + // IPv4 clients can connect to a IPv6 socket. + hints.ai_family = AF_INET6; + hints.ai_socktype = SOCK_DGRAM; + hints.ai_flags = AI_PASSIVE; + + addrinfo* res; + if (getaddrinfo(nullptr, port, &hints, &res)) + return -1; + std::unique_ptr infoList{res, &freeaddrinfo}; + + int fd; + addrinfo* rp; + for (rp = res; rp; rp = rp->ai_next) + { + fd = socket(AF_INET6, rp->ai_socktype, rp->ai_protocol); + if (fd == -1) + continue; + + // Allow IPv4 using the same socket. + const int onlyIPv6{0}; + if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &onlyIPv6, sizeof(onlyIPv6))) + continue; + + if (bind(fd, rp->ai_addr, rp->ai_addrlen)) + { + close(fd); + continue; + } + + break; + } + + if (!rp) + return -1; + + localAddress.address.in6 = reinterpret_cast(*rp->ai_addr); + localAddress.length = rp->ai_addrlen; + return fd; +} + +void GenerateKeyfiles(const std::filesystem::path& privatekey, const std::filesystem::path& certificate) +{ + constexpr auto binaryOverrideMode = std::ios::binary | std::ios::trunc; + gnutls_x509_privkey_t newKey; + if (const int ret{gnutls_x509_privkey_init(&newKey)}) + { + throw std::runtime_error{fmt::format("Error initializing privatkey: {}", + gnutls_strerror(ret))}; + } + std::unique_ptr keyDeleter( + newKey, gnutls_x509_privkey_deinit); + + if (const int ret{gnutls_x509_privkey_generate(newKey, GNUTLS_PK_RSA, 3072, 0)}) + { + throw std::runtime_error{fmt::format("Error generating privatkey: {}", + gnutls_strerror(ret))}; + } + + gnutls_datum_t exportedKey; + if (const int ret{gnutls_x509_privkey_export2_pkcs8(newKey, GNUTLS_X509_FMT_DER, nullptr, 0, + &exportedKey)}) + { + throw std::runtime_error{fmt::format("Error exporting privatkey: {}", + gnutls_strerror(ret))}; + } + const std::unique_ptr exportedKeyDeleter{exportedKey.data, + gnutls_free}; + + std::ofstream{privatekey, binaryOverrideMode}.write( + reinterpret_cast(exportedKey.data), exportedKey.size); + + + gnutls_x509_crt_t newCertificate; + if (const int ret{gnutls_x509_crt_init(&newCertificate)}) + { + throw std::runtime_error{fmt::format("Error initializing certificate: {}", + gnutls_strerror(ret))}; + } + std::unique_ptr certificateDeleter( + newCertificate, gnutls_x509_crt_deinit); + + std::array serialNumber; + auto generator = [&, randomDevice = std::random_device{}, + distribution = std::uniform_int_distribution{}]() mutable + { + return distribution(randomDevice); + }; + std::ranges::generate(serialNumber, std::ref(generator)); + std::get<0>(serialNumber) &= ~(1 << 8); + if (const int ret{gnutls_x509_crt_set_serial(newCertificate, serialNumber.data(), + serialNumber.size())}) + { + throw std::runtime_error{fmt::format("Error setting serial number: {}", + gnutls_strerror(ret))}; + } + + if (const int ret{gnutls_x509_crt_set_activation_time(newCertificate, 0)}) + { + throw std::runtime_error{fmt::format("Error setting activation time: {}", + gnutls_strerror(ret))}; + } + if (const int ret{gnutls_x509_crt_set_expiration_time(newCertificate, -1)}) + { + throw std::runtime_error{fmt::format("Error setting expiration time: {}", + gnutls_strerror(ret))}; + } + + if (const int ret{gnutls_x509_crt_set_key(newCertificate, newKey)}) + { + throw std::runtime_error{fmt::format("Error setting certificate key: {}", + gnutls_strerror(ret))}; + } + + if (const int ret{gnutls_x509_crt_sign(newCertificate, newCertificate, newKey)}) + { + throw std::runtime_error{fmt::format("Error signing certificate: {}", + gnutls_strerror(ret))}; + } + + gnutls_datum_t exportedCertificate; + if (const int ret{gnutls_x509_crt_export2(newCertificate, GNUTLS_X509_FMT_DER, + &exportedCertificate)}) + { + throw std::runtime_error{fmt::format("Error exporting certificate: {}", + gnutls_strerror(ret))}; + } + const std::unique_ptr exportedCertificateDeleter{ + exportedCertificate.data, gnutls_free}; + + std::ofstream{certificate, binaryOverrideMode}.write( + reinterpret_cast(exportedCertificate.data), exportedCertificate.size); +} + +std::unique_ptr CreateTlsServerCredentials() +{ + gnutls_certificate_credentials_t temp; + if (const int ret{gnutls_certificate_allocate_credentials(&temp)}) + { + throw std::runtime_error{fmt::format("gnutls_certificate_allocate_credentials: {}", + gnutls_strerror(ret))}; + } + std::unique_ptr cred{temp}; + + const auto folder = (Paths(g_CmdLineArgs).UserData() / "encryption").fileSystemPath(); + std::filesystem::create_directory(folder); + + const auto privatekeyPath{folder / "privatekey.der"}; + const auto certificatePath{folder / "certificate.der"}; + + if (!std::filesystem::is_regular_file(privatekeyPath) || + !std::filesystem::is_regular_file(certificatePath)) + { + GenerateKeyfiles(privatekeyPath, certificatePath); + } + + if (const int ret{gnutls_certificate_set_x509_key_file(cred.get(), certificatePath.c_str(), + privatekeyPath.c_str(), GNUTLS_X509_FMT_DER)}) + { + throw std::runtime_error{fmt::format("gnutls_certificate_set_x509_key_file: {}", + gnutls_strerror(ret))}; + } + + return cred; +} + +ngtcp2_settings InitSettings() +{ + ngtcp2_settings settings; + ngtcp2_settings_default(&settings); + settings.initial_ts = timestamp(); + return settings; +} +} // anonymous namespace + +class CNetServerWorker::Quic +{ +public: + explicit Quic(const std::uint16_t port): + m_SocketFd{ResolveAndBind(std::to_string(port).c_str(), m_LocalAddress)} + {} + ~Quic() + { + if (m_SocketFd >= 0) + close(m_SocketFd); + } + AddressStorage m_LocalAddress; + int m_SocketFd; + std::unique_ptr m_Credentials{ + CreateTlsServerCredentials()}; + ngtcp2_settings m_Settings{InitSettings()}; + + void HandleIncoming(CNetServerWorker& server) + { + std::array buf; + + while (true) + { + AddressStorage remoteAddress; + + std::size_t n_read; + try + { + n_read = ReceivePackage(m_SocketFd, buf, remoteAddress); + } + catch (std::system_error& e) + { + if (e.code().value() == EAGAIN || e.code().value() == EWOULDBLOCK) + return; + throw; + } + + ngtcp2_version_cid version; + if (const int ret{ngtcp2_pkt_decode_version_cid(&version, buf.data(), n_read, + NGTCP2_MAX_CIDLEN)}) + { + throw std::runtime_error{fmt::format("ngtcp2_pkt_decode_version_cid: {}", + ngtcp2_strerror(ret))}; + } + + const ngtcp2_addr remote{ + .addr{&remoteAddress.address.sa}, + .addrlen{remoteAddress.length} + }; + + /* Find any existing connection by DCID */ + ngtcp2_cid tempCid; + ngtcp2_cid_init(&tempCid, version.dcid, version.dcidlen); + + const auto connectionIter = std::ranges::find_if(server.m_Sessions, + [&tempCid](const std::vector association) + { + return std::ranges::any_of(association, [&tempCid](const ngtcp2_cid& elem) + { + return ngtcp2_cid_eq(&elem, &tempCid); + }); + }, + [](auto& connection) + { + const std::size_t amount{ngtcp2_conn_get_scid( + connection->m_Connection.m_QuicConnection.get(), nullptr)}; + std::vector cids(amount); + ngtcp2_conn_get_scid(connection->m_Connection.m_QuicConnection.get(), + cids.data()); + return cids; + }); + + const bool existing{connectionIter != server.m_Sessions.end()}; + if (!existing) + { + ngtcp2_pkt_hd header; + if (ngtcp2_accept(&header, buf.data(), n_read)) + throw std::invalid_argument{"Failed parsing package header"}; + + ngtcp2_cid newScid; + GetRandomCid(&newScid); + + const ngtcp2_path path{ + .local{ + .addr{&m_LocalAddress.address.sa}, + .addrlen{m_LocalAddress.length} + }, + .remote{remote} + }; + + server.m_Sessions.push_back(std::make_unique(server, m_SocketFd, + m_Settings, m_Credentials.get(), header, newScid, path)); + server.SetupSession(server.m_Sessions.back().get()); + } + auto& session{existing ? *connectionIter : server.m_Sessions.back()}; + + try + { + session->m_Connection.Read(remote, {buf.data(), n_read}); + } + catch (const std::system_error&) + { + throw; + } + catch (const std::runtime_error&) + { + const auto session = existing ? std::move(*connectionIter) : + std::move(server.m_Sessions.back()); + if (existing) + server.m_Sessions.erase(connectionIter); + else + server.m_Sessions.pop_back(); + + session->Update(static_cast(NMT_CONNECTION_LOST), nullptr); + } + } + } +}; + /* * XXX: We use some non-threadsafe functions from the worker thread. * See https://gitea.wildfiregames.com/0ad/0ad/issues/654 @@ -126,17 +491,17 @@ CNetServerWorker::CNetServerWorker(const bool continueSavedGame, std::uint16_t p m_Password{std::move(password)} { // Bind to default host - ENetAddress addr; - addr.host = ENET_HOST_ANY; - addr.port = port; + // ENetAddress addr; + // addr.host = ENET_HOST_ANY; + // addr.port = port; // Create ENet server - m_Host.reset(PS::Enet::CreateHost(&addr, MAX_CLIENTS, CHANNEL_COUNT)); - if (!m_Host) - { - LOGERROR("Net server: enet_host_create failed"); - throw std::runtime_error{"Failed to start server"}; - } + // m_Host.reset(PS::Enet::CreateHost(&addr, MAX_CLIENTS, CHANNEL_COUNT)); + // if (!m_Host) + // { + // LOGERROR("Net server: enet_host_create failed"); + // throw std::runtime_error{"Failed to start server"}; + // } m_Stats = std::make_unique(); if (CProfileViewer::IsInitialised()) @@ -146,7 +511,7 @@ CNetServerWorker::CNetServerWorker(const bool continueSavedGame, std::uint16_t p // Launch the worker thread m_WorkerThread = std::thread(Threading::HandleExceptions::Wrapper, this, - std::move(initAttributes)); + std::move(initAttributes), port); #if CONFIG2_MINIUPNPC // Launch the UPnP thread @@ -169,10 +534,6 @@ CNetServerWorker::~CNetServerWorker() if (m_UPnPThread.joinable()) m_UPnPThread.detach(); #endif - - // Clean up resources - for (const auto& session : m_Sessions) - session->DisconnectNow(NDR_SERVER_SHUTDOWN); } @@ -328,21 +689,17 @@ void CNetServerWorker::SetupUPnP(const u16 port) } #endif // CONFIG2_MINIUPNPC -bool CNetServerWorker::SendMessage(ENetPeer* peer, const CNetMessage* message) +bool CNetServerWorker::SendMessage(const CNetMessage* message) { - ENSURE(m_Host); + m_Sessions.front()->SendMessage(message); - CNetServerSession* session = static_cast(peer->data); - - return CNetHost::SendMessage(message, peer, DebugName(session).c_str()); + return true; } bool CNetServerWorker::Multicast(const CNetMessage* message, const std::vector& targetStates, const std::optional>& receivers /* = std::nullopt */) { - ENSURE(m_Host); - const auto isReceiver = [&](const CNetServerSession& session) { if (!PS::contains(targetStates, @@ -367,14 +724,14 @@ bool CNetServerWorker::Multicast(const CNetMessage* message, return ok; } -void CNetServerWorker::RunThread(CNetServerWorker* data, const std::string& initAttributes) +void CNetServerWorker::RunThread(CNetServerWorker* data, const std::string& initAttributes, u16 port) { debug_SetThreadName("NetServer"); - data->Run(initAttributes); + data->Run(initAttributes, port); } -void CNetServerWorker::Run(const std::string& initAttributes) +void CNetServerWorker::Run(const std::string& initAttributes, u16 port) { // The script context uses the profiler and therefore the thread must be registered before the context is created g_Profiler2.RegisterCurrentThread("Net server"); @@ -392,22 +749,27 @@ void CNetServerWorker::Run(const std::string& initAttributes) m_InitAttributes = gameAttributesVal; } + Quic quic{port}; + while (true) { - if (!RunStep()) + if (!RunStep(quic)) break; // Update profiler stats - m_Stats->LatchHostState(*m_Host); + m_Stats->LatchHostState(m_Sessions); } // Clear roots before deleting their context m_SavedCommands.clear(); SAFE_DELETE(m_ScriptInterface); + + for (const auto& session : m_Sessions) + session->Disconnect(NDR_SERVER_SHUTDOWN); } -bool CNetServerWorker::RunStep() +bool CNetServerWorker::RunStep(Quic& quic) { // Check for messages from the game thread. // (Do as little work as possible while the mutex is held open, @@ -448,103 +810,40 @@ bool CNetServerWorker::RunStep() CheckClientConnections(); - // Process network events: + pollfd pollFd{ + .fd{quic.m_SocketFd}, + .events{EPOLLIN | EPOLLOUT} + }; + const int ready{poll(&pollFd, 1, 25)}; - ENetEvent event; - int status = enet_host_service(m_Host.get(), &event, HOST_SERVICE_TIMEOUT); - if (status < 0) + if (ready < 0) + throw std::runtime_error{fmt::format("epoll_wait: {}", std::strerror(errno))}; + + if (ready == 0) { - LOGERROR("CNetServerWorker: enet_host_service failed (%d)", status); - // TODO: notify game that the server has shut down - return false; - } - - if (status == 0) - { - // Reached timeout with no events - try again - return true; - } - - // Process the event: - - switch (event.type) - { - case ENET_EVENT_TYPE_CONNECT: - { - // Report the client address - char hostname[256] = "(error)"; - enet_address_get_host_ip(&event.peer->address, hostname, ARRAY_SIZE(hostname)); - LOGMESSAGE("Net server: Received connection from %s:%u", hostname, (unsigned int)event.peer->address.port); - - // Set up a session object for this peer - - const std::unique_ptr& session{m_Sessions.emplace_back( - std::make_unique(*this, event.peer))}; - - SetupSession(session.get()); - - ENSURE(event.peer->data == NULL); - event.peer->data = session.get(); - - HandleConnect(session.get()); - - break; - } - - case ENET_EVENT_TYPE_DISCONNECT: - { - // If there is an active session with this peer, then reset and delete it - - CNetServerSession* session = static_cast(event.peer->data); - if (session) + for (auto& session : m_Sessions) { - LOGMESSAGE("Net server: Disconnected %s", DebugName(session).c_str()); - - // Remove the session first, so we won't send player-update messages to it - // when updating the FSM - const auto iter = std::ranges::find(m_Sessions, session, - &std::unique_ptr::get); - const std::unique_ptr _ = std::move(*iter); - m_Sessions.erase(iter); - - session->Update((uint)NMT_CONNECTION_LOST, NULL); - - event.peer->data = NULL; - } - - if (m_State == SERVER_STATE_LOADING) - CheckGameLoadStatus(NULL); - - break; - } - - case ENET_EVENT_TYPE_RECEIVE: - { - // If there is an active session with this peer, then process the message - - CNetServerSession* session = static_cast(event.peer->data); - if (session) - { - // Create message from raw data - CNetMessage* msg = CNetMessageFactory::CreateMessage(event.packet->data, event.packet->dataLength, GetScriptInterface()); - if (msg) + ngtcp2_conn *conn = session->m_Connection.m_QuicConnection.get(); + const int ret{ngtcp2_conn_handle_expiry(conn, timestamp())}; + if (ret < 0) { - LOGMESSAGE("Net server: Received message %s of size %lu from %s", msg->ToString().c_str(), (unsigned long)msg->GetSerializedLength(), DebugName(session).c_str()); - - HandleMessageReceive(msg, session); - - delete msg; + LOGERROR("ngtcp2_conn_handle_expiry: %s", ngtcp2_strerror(ret)); + continue; } + + session->m_Connection.Write(quic.m_SocketFd); } - - // Done using the packet - enet_packet_destroy(event.packet); - - break; } + else + { + if (pollFd.revents & EPOLLIN) + quic.HandleIncoming(*this); - case ENET_EVENT_TYPE_NONE: - break; + if (pollFd.revents & EPOLLOUT) + { + for (auto& session : m_Sessions) + session->m_Connection.Write(quic.m_SocketFd); + } } return true; @@ -1650,10 +1949,8 @@ CStrW CNetServerWorker::DeduplicatePlayerName(const CStrW& original) } } -void CNetServerWorker::SendHolePunchingMessage(const CStr& ipStr, u16 port) +void CNetServerWorker::SendHolePunchingMessage(const CStr& /*ipStr*/, u16 /*port*/) { - if (m_Host) - StunClient::SendHolePunchingMessages(*m_Host, ipStr, port); } @@ -1671,9 +1968,9 @@ CNetServer::CNetServer(const bool continueSavedGame, std::uint16_t port, const b // In lobby, we send our public ip and port on request to the players who want to connect. // Thus we need to know our public IP and use STUN to get it. - std::lock_guard lock(m_Worker.m_WorkerMutex); - if (!m_Worker.m_Host || !StunClient::FindPublicIP(*m_Worker.m_Host, m_PublicIp, m_PublicPort)) - throw std::runtime_error{"Failed to resolve public IP-address."}; + // std::lock_guard lock(m_Worker.m_WorkerMutex); + // if (!m_Worker.m_Host || !StunClient::FindPublicIP(*m_Worker.m_Host, m_PublicIp, m_PublicPort)) + // throw std::runtime_error{"Failed to resolve public IP-address."}; } bool CNetServer::UseLobbyAuth() const @@ -1694,9 +1991,7 @@ u16 CNetServer::GetPublicPort() const u16 CNetServer::GetLocalPort() const { std::lock_guard lock(m_Worker.m_WorkerMutex); - if (!m_Worker.m_Host) - return 0; - return m_Worker.m_Host->address.port; + return 0; // m_Worker.m_Host->address.port; } bool CNetServer::CheckPasswordAndIncrement(const std::string& username, const std::string& password, const std::string& salt) @@ -1743,3 +2038,4 @@ void CNetServer::SendHolePunchingMessage(const CStr& ip, u16 port) { m_Worker.SendHolePunchingMessage(ip, port); } + diff --git a/source/network/NetServer.h b/source/network/NetServer.h index 3f8577870f..6adad13ff6 100644 --- a/source/network/NetServer.h +++ b/source/network/NetServer.h @@ -119,7 +119,7 @@ public: /** * Send a message to the given network peer. */ - bool SendMessage(ENetPeer* peer, const CNetMessage* message); + bool SendMessage(const CNetMessage* message); /** * Disconnects a player from gamesetup or session. @@ -173,10 +173,12 @@ private: */ CStrW DeduplicatePlayerName(const CStrW& original); +public: /** * Get the script context used for init attributes. */ const Script::Interface& GetScriptInterface(); +private: /** * Set the turn length to a fixed value. @@ -227,7 +229,9 @@ private: void ConstructPlayerAssignmentMessage(CPlayerAssignmentMessage& message); +public: void HandleMessageReceive(CNetMessage* message, CNetServerSession* session); +public: /** * Send a network warning if the connection to a client is being lost or has bad latency. @@ -263,7 +267,8 @@ private: */ const bool m_LobbyAuth; - std::unique_ptr m_Host; + class Quic; + std::vector> m_Sessions; std::unique_ptr m_Stats; @@ -330,9 +335,9 @@ private: std::thread m_UPnPThread; #endif - static void RunThread(CNetServerWorker* data, const std::string& initAttributes); - void Run(const std::string& initAttributes); - bool RunStep(); + static void RunThread(CNetServerWorker* data, const std::string& initAttributes, u16 port); + void Run(const std::string& initAttributes, u16 port); + bool RunStep(Quic& quic); std::thread m_WorkerThread; mutable std::mutex m_WorkerMutex; diff --git a/source/network/NetServerSession.cpp b/source/network/NetServerSession.cpp index 4bd1793420..e079da0a39 100644 --- a/source/network/NetServerSession.cpp +++ b/source/network/NetServerSession.cpp @@ -25,8 +25,312 @@ #include "network/NetServer.h" #include "ps/CLogger.h" -CNetServerSession::CNetServerSession(CNetServerWorker& server, ENetPeer* peer) : - m_Server(server), m_FileTransferer(*this), m_Peer(peer) + +#include +#include + +namespace +{ +void SendPacket(const int socketFd, const std::span data, const ngtcp2_addr remote) +{ + iovec iov{ + .iov_base{const_cast(data.data())}, + .iov_len{data.size()} + }; + + msghdr msg{}; + msg.msg_name = remote.addr; + msg.msg_namelen = remote.addrlen; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + ssize_t ret; + do + ret = sendmsg(socketFd, &msg, MSG_DONTWAIT); + while (ret < 0 && errno == EINTR); + if (ret < 0) + throw std::system_error{errno, std::generic_category(), "Error sending message"}; +} + +ngtcp2_conn* GetConnection(ngtcp2_crypto_conn_ref* connRef) +{ + return reinterpret_cast*>(connRef->user_data)->get(); +} + +int OnReceiveStreamData(ngtcp2_conn* conn, const std::uint32_t /*flags*/, const std::int64_t streamId, + const std::uint64_t /*offset*/, const std::uint8_t* data, const std::size_t datalen, void* userData, + void* /*streamUserData*/) +{ + CNetServerSession& session{*static_cast(userData)}; + const auto messageData = session.m_Connection.m_Stream.value().Receive({data, datalen}); + if (messageData.has_value()) + { + std::unique_ptr message{CNetMessageFactory::CreateMessage(messageData.value(), + session.GetServer().GetScriptInterface())}; + session.GetServer().HandleMessageReceive(message.get(), &session); + } + increaseWindow(conn, streamId, datalen); + return 0; +} + +int OnAcknowledgedStreamData(ngtcp2_conn*, const std::int64_t, const std::uint64_t offset, + const std::uint64_t dataLength, void* userData, void*) +{ + Connection& connection{static_cast(userData)->m_Connection}; + Stream& stream{connection.GetStream()}; + stream.MarkAcknowledged(offset + dataLength); + return 0; +} + +int OnConnect(ngtcp2_conn*, const ngtcp2_encryption_level level, void* userData) +{ + if (level != NGTCP2_ENCRYPTION_LEVEL_1RTT) + return 0; + Connection& connection{static_cast(userData)->m_Connection}; + try + { + connection.OpenStream(); + } + catch (const std::runtime_error&) + { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + + const CSrvHandshakeMessage handshake(CreateHandshake()); + connection.GetStream().PushMessage(&handshake); + return 0; +} + +constexpr ngtcp2_callbacks callbacks{ + .recv_client_initial{&ngtcp2_crypto_recv_client_initial_cb}, + .recv_crypto_data{&ngtcp2_crypto_recv_crypto_data_cb}, + .encrypt{&ngtcp2_crypto_encrypt_cb}, + .decrypt{&ngtcp2_crypto_decrypt_cb}, + .hp_mask{&ngtcp2_crypto_hp_mask_cb}, + .recv_stream_data{&OnReceiveStreamData}, + .acked_stream_data_offset{&OnAcknowledgedStreamData}, + .recv_retry{&ngtcp2_crypto_recv_retry_cb}, + .rand{&OnRandomRequest}, + .get_new_connection_id{&OnNewConnectionIdRequest}, + .update_key{&ngtcp2_crypto_update_key_cb}, + .delete_crypto_aead_ctx{&ngtcp2_crypto_delete_crypto_aead_ctx_cb}, + .delete_crypto_cipher_ctx{&ngtcp2_crypto_delete_crypto_cipher_ctx_cb}, + .get_path_challenge_data{&ngtcp2_crypto_get_path_challenge_data_cb}, + .recv_tx_key{&OnConnect} +}; + +void WriteToStream(const int socketFd, ngtcp2_conn* conn, Stream* stream, const ngtcp2_addr remote) +{ + std::array buf; + + ngtcp2_path_storage ps; + ngtcp2_path_storage_zero(&ps); + + ngtcp2_pkt_info pi; + const std::uint64_t ts{timestamp()}; + + std::uint32_t flags{NGTCP2_WRITE_STREAM_FLAG_MORE}; + + while (true) + { + ngtcp2_vec datav; + std::int64_t stream_id; + + if (stream) + { + auto bytesToSend = stream->PeekData(); + if (bytesToSend.has_value()) + { + datav.base = const_cast(bytesToSend->data()); + datav.len = bytesToSend->size(); + stream_id = stream->m_Id; + } + else + { + /* No stream data to be sent */ + datav.base = nullptr; + datav.len = 0; + stream_id = -1; + flags &= ~NGTCP2_WRITE_STREAM_FLAG_MORE; + } + } + else + { + datav.base = NULL; + datav.len = 0; + stream_id = -1; + } + + ngtcp2_ssize n_read; + const ngtcp2_ssize n_written{ngtcp2_conn_writev_stream(conn, &ps.path, &pi, buf.data(), + buf.size(), &n_read, flags, stream_id, &datav, 1, ts)}; + if (n_written < 0) + { + if (n_written != NGTCP2_ERR_WRITE_MORE) + { + throw std::runtime_error{fmt::format("ngtcp2_conn_writev_stream: {}", + ngtcp2_strerror(static_cast(n_written)))}; + } + if (stream && n_read > 0) + stream->MarkSent(n_read); + continue; + } + + if (n_written == 0) + break; + + if (stream && n_read > 0) + stream->MarkSent(n_read); + + try + { + SendPacket(socketFd, {buf.data(), static_cast(n_written)}, remote); + } + catch (std::system_error& e) + { + if (e.code().value() == EAGAIN || e.code().value() == EWOULDBLOCK) + break; + throw; + } + + /* No stream data to be sent */ + if (stream && datav.len == 0) + break; + } + ngtcp2_conn_update_pkt_tx_time(conn, timestamp()); +} + +std::unique_ptr createTlsSession( + const gnutls_certificate_credentials_t cred) +{ + gnutls_session_t tempSession; + if (const int ret{gnutls_init(&tempSession, GNUTLS_SERVER | GNUTLS_ENABLE_EARLY_DATA | + GNUTLS_NO_END_OF_EARLY_DATA)}) + { + throw std::runtime_error{fmt::format("gnutls_init: {}", gnutls_strerror(ret))}; + } + std::unique_ptr session{tempSession}; + + if (const int ret{gnutls_priority_set_direct(session.get(), TLS_PRIORITY, NULL)}) + throw std::runtime_error{fmt::format("gnutls_priority_set_direct: {}", gnutls_strerror(ret))}; + + if (const int ret{gnutls_credentials_set(session.get(), GNUTLS_CRD_CERTIFICATE, cred)}) + throw std::runtime_error{fmt::format("gnutls_credentials_set: {}", gnutls_strerror(ret))}; + + return session; +} +} + +Connection::Connection(CNetServerSession& session, const ngtcp2_settings& settings, + gnutls_certificate_credentials_t credentials, const ngtcp2_pkt_hd& header, + const ngtcp2_cid& newScid, const ngtcp2_path& path): + m_TlsSession{createTlsSession(credentials)}, + m_ConnRef{ + .get_conn{&GetConnection}, + .user_data{&m_QuicConnection} + } +{ + ngtcp2_transport_params params; + ngtcp2_transport_params_default(¶ms); + params.initial_max_stream_data_bidi_local = 128 * KiB; + params.initial_max_data = 1 * MiB; + params.max_udp_payload_size = MAX_UDP_PAYLOAD_SIZE; + ngtcp2_cid_init(¶ms.original_dcid, header.dcid.data, header.dcid.datalen); + params.original_dcid_present = true; + params.grease_quic_bit = 1; + + ngtcp2_conn* tempConn; + if (const int ret = ngtcp2_conn_server_new(&tempConn, &header.scid, &newScid, &path, header.version, + &callbacks, &settings, ¶ms, nullptr, &session)) + { + throw std::runtime_error{fmt::format("ngtcp2_conn_server_new: {}", + ngtcp2_strerror (ret))}; + } + + m_QuicConnection.reset(tempConn); + + memcpy(&m_LocalAddress, path.local.addr, path.local.addrlen); + m_LocalAddressLength = path.local.addrlen; + memcpy(&m_RemoteAddress, path.remote.addr, path.remote.addrlen); + m_RemoteAddressLength = path.remote.addrlen; + + ngtcp2_crypto_gnutls_configure_server_session(m_TlsSession.get()); + ngtcp2_conn_set_tls_native_handle(m_QuicConnection.get(), m_TlsSession.get()); + gnutls_session_set_ptr(m_TlsSession.get(), &m_ConnRef); + m_TimerFd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); + if (m_TimerFd < 0) + throw std::system_error(errno, std::generic_category(), "timerfd_create"); +} + +void Connection::OpenStream() +{ + std::int64_t streamId; + if (ngtcp2_conn_open_bidi_stream(m_QuicConnection.get(), &streamId, nullptr)) + throw std::runtime_error{""}; + m_Stream.emplace(streamId); +} + +Stream& Connection::GetStream() +{ + return m_Stream.value(); +} + +void Connection::Read(const ngtcp2_addr remote, const std::span data) +{ + const ngtcp2_path path{ + .local{ngtcp2_conn_get_path(m_QuicConnection.get())->local}, + .remote{remote} + }; + + ngtcp2_pkt_info pi{}; + if (const int ret{ngtcp2_conn_read_pkt(m_QuicConnection.get(), &path, &pi, data.data(), data.size(), + timestamp())}) + { + throw std::runtime_error{"Destroy connection"}; + } +} + +void Connection::Write(const int socketFd) +{ + WriteToStream(socketFd, m_QuicConnection.get(), nullptr, + {&m_RemoteAddress.sa, m_RemoteAddressLength}); + + if (m_Stream.has_value()) + { + WriteToStream(socketFd, m_QuicConnection.get(), &m_Stream.value(), + {&m_RemoteAddress.sa, m_RemoteAddressLength}); + } + + const ngtcp2_tstamp expiry{ngtcp2_conn_get_expiry(m_QuicConnection.get())}; + const ngtcp2_tstamp now{timestamp()}; + itimerspec it{}; + + if (const int ret{timerfd_settime(m_TimerFd, 0, &it, nullptr)}) + throw std::system_error{errno, std::generic_category(), "timerfd_settime"}; + if (expiry < now) + { + it.it_value.tv_sec = 0; + it.it_value.tv_nsec = 1; + } + else + { + it.it_value.tv_sec = (expiry - now) / NGTCP2_SECONDS; + it.it_value.tv_nsec = ((expiry - now) % NGTCP2_SECONDS) / NGTCP2_NANOSECONDS; + } + + if (const int ret{timerfd_settime(m_TimerFd, 0, &it, nullptr)}) + throw std::system_error{errno, std::generic_category(), "timerfd_settime"}; + + return; +} + +CNetServerSession::CNetServerSession(CNetServerWorker& server, const int socketFd, + const ngtcp2_settings& settings, gnutls_certificate_credentials_t credentials, + const ngtcp2_pkt_hd& header, const ngtcp2_cid& newScid, const ngtcp2_path& path) : + m_Server(server), + m_Connection{*this, settings, credentials, header, newScid, path}, + m_FileTransferer(*this), + m_SocketFd{socketFd} { } @@ -58,18 +362,30 @@ void CNetServerSession::Disconnect(NetDisconnectReason reason) Update((uint)NMT_CONNECTION_LOST, NULL); - enet_peer_disconnect(m_Peer, static_cast(reason)); -} + const ngtcp2_ccerr quicReason{ + .type{NGTCP2_CCERR_TYPE_APPLICATION}, + .error_code{reason} + }; -void CNetServerSession::DisconnectNow(NetDisconnectReason reason) -{ - if (reason == NDR_UNKNOWN) - LOGWARNING("Disconnecting client without communicating the disconnect reason!"); + ngtcp2_sockaddr_union local; + ngtcp2_sockaddr_union remote; + ngtcp2_path path{ + .local{.addr{&local.sa}}, + .remote{.addr{&remote.sa}} + }; + std::array buffer; + const ngtcp2_ssize amount{ngtcp2_conn_write_connection_close(m_Connection.m_QuicConnection.get(), &path, nullptr, buffer.data(), + buffer.size(), &quicReason, timestamp())}; + if (amount <= 0) + LOGERROR("closing connection %s", ngtcp2_strerror(static_cast(amount))); - enet_peer_disconnect_now(m_Peer, static_cast(reason)); + SendPacket(m_SocketFd, {buffer.data(), static_cast(amount)}, path.remote); } bool CNetServerSession::SendMessage(const CNetMessage* message) { - return m_Server.SendMessage(m_Peer, message); + m_Connection.m_Stream.value().PushMessage(message); + return true; } + + diff --git a/source/network/NetServerSession.h b/source/network/NetServerSession.h index 635e33b252..e66e04c90b 100644 --- a/source/network/NetServerSession.h +++ b/source/network/NetServerSession.h @@ -23,10 +23,51 @@ #include "network/NetHost.h" #include "ps/CStr.h" +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "network/NetProtocol.h" +#include "ps/CLogger.h" + class CNetServerWorker; typedef struct _ENetPeer ENetPeer; +class CNetServerSession; +class Connection +{ +public: + Connection(CNetServerSession& session, const ngtcp2_settings& settings, + gnutls_certificate_credentials_t credentials, const ngtcp2_pkt_hd& header, + const ngtcp2_cid& scid, const ngtcp2_path& path); + + void OpenStream(); + Stream& GetStream(); + + void Read(const ngtcp2_addr remote, const std::span data); + void Write(const int socketFd); + + std::unique_ptr m_TlsSession; + std::unique_ptr m_QuicConnection; + int m_TimerFd{-1}; +public: + ngtcp2_sockaddr_union m_LocalAddress; + ngtcp2_socklen m_LocalAddressLength; + ngtcp2_sockaddr_union m_RemoteAddress; + ngtcp2_socklen m_RemoteAddressLength; + std::optional m_Stream; + ngtcp2_crypto_conn_ref m_ConnRef; +}; + /** * The server's end of a network session. * Represents an abstraction of the state of the client, storing all the per-client data @@ -40,7 +81,9 @@ class CNetServerSession : public CFsm NONCOPYABLE(CNetServerSession); public: - CNetServerSession(CNetServerWorker& server, ENetPeer* peer); + CNetServerSession(CNetServerWorker& server, const int socketFd, const ngtcp2_settings& settings, + gnutls_certificate_credentials_t credentials, const ngtcp2_pkt_hd& header, + const ngtcp2_cid& newScid, const ngtcp2_path& path); CNetServerWorker& GetServer() { return m_Server; } @@ -73,13 +116,6 @@ public: */ void Disconnect(NetDisconnectReason reason); - /** - * Sends an unreliable disconnection notification to the client. - * The server will not receive any disconnection notification. - * The server will not receive any further messages sent via this session. - */ - void DisconnectNow(NetDisconnectReason reason); - /** * Send a message to the client. */ @@ -89,7 +125,10 @@ public: private: CNetServerWorker& m_Server; +public: + Connection m_Connection; +private: CNetFileTransferer m_FileTransferer; ENetPeer* m_Peer; @@ -98,6 +137,7 @@ private: CStrW m_UserName; u32 m_HostID{0}; CStr m_Password; + int m_SocketFd; }; #endif // NET_SERVER_SESSION_H diff --git a/source/network/NetStats.cpp b/source/network/NetStats.cpp index cab95f2d0c..1f85c9fa59 100644 --- a/source/network/NetStats.cpp +++ b/source/network/NetStats.cpp @@ -19,7 +19,10 @@ #include "NetStats.h" +#include "network/NetServerSession.h" + #include +#include #include enum @@ -38,13 +41,8 @@ enum NumberRows }; -CNetStatsTable::CNetStatsTable(const ENetPeer& peer) - : m_Peer(&peer) -{ -} - -CNetStatsTable::CNetStatsTable() - : m_Peer(NULL) +CNetStatsTable::CNetStatsTable(ngtcp2_conn* conn): + m_Connection{conn} { } @@ -55,7 +53,7 @@ CStr CNetStatsTable::GetName() CStr CNetStatsTable::GetTitle() { - if (m_Peer) + if (m_Connection) return "Network client statistics"; else return "Network host statistics"; @@ -71,7 +69,7 @@ const std::vector& CNetStatsTable::GetColumns() m_ColumnDescriptions.clear(); m_ColumnDescriptions.push_back(ProfileColumn("Name", 200)); - if (m_Peer) + if (m_Connection) m_ColumnDescriptions.push_back(ProfileColumn("Value", 80)); else { @@ -96,22 +94,26 @@ CStr CNetStatsTable::GetCellText(size_t row, size_t col) #define ROW(id, title, member) \ case id: \ if (col == 0) return title; \ - if (m_Peer) return std::to_string(m_Peer->member); \ + if (m_Connection) return member; \ return "???" + ngtcp2_conn_info info; + if (col != 0) + ngtcp2_conn_get_conn_info(m_Connection, &info); + switch(row) { - ROW(Row_InData, "incoming bytes", incomingDataTotal); - ROW(Row_OutData, "outgoing bytes", outgoingDataTotal); - ROW(Row_LastSendTime, "last send time", lastSendTime); - ROW(Row_LastRecvTime, "last receive time", lastReceiveTime); - ROW(Row_NextTimeout, "next timeout", nextTimeout); - ROW(Row_PacketsSent, "packets sent", packetsSent); - ROW(Row_PacketsLost, "packets lost", packetsLost); - ROW(Row_LastRTT, "last RTT", lastRoundTripTime); - ROW(Row_RTT, "mean RTT", roundTripTime); - ROW(Row_MTU, "MTU", mtu); - ROW(Row_ReliableInTransit, "reliable data in transit", reliableDataInTransit); + ROW(Row_InData, "incoming bytes", {}); + ROW(Row_OutData, "outgoing bytes", {}); + ROW(Row_LastSendTime, "last send time", {}); + ROW(Row_LastRecvTime, "last receive time", {}); + ROW(Row_NextTimeout, "next timeout", std::to_string(ngtcp2_conn_get_expiry(m_Connection))); + ROW(Row_PacketsSent, "packets sent", {}); + ROW(Row_PacketsLost, "packets lost", {}); + ROW(Row_LastRTT, "last RTT", std::to_string(info.latest_rtt)); + ROW(Row_RTT, "mean RTT", std::to_string(info.smoothed_rtt)); + ROW(Row_MTU, "MTU", std::to_string(ngtcp2_conn_get_path_max_tx_udp_payload_size(m_Connection))); + ROW(Row_ReliableInTransit, "reliable data in transit", std::to_string(info.bytes_in_flight)); default: return "???"; @@ -125,29 +127,39 @@ AbstractProfileTable* CNetStatsTable::GetChild(size_t /*row*/) return 0; } -void CNetStatsTable::LatchHostState(const ENetHost& host) +void CNetStatsTable::LatchHostState(const std::span> sessions) { std::lock_guard lock(m_Mutex); #define ROW(id, title, member) \ - m_LatchedData[i].push_back(std::to_string(host.peers[i].member)); + m_LatchedData[i].push_back(std::to_string(info.member)); m_LatchedData.clear(); - m_LatchedData.resize(host.peerCount); + m_LatchedData.resize(sessions.size()); - for (size_t i = 0; i < host.peerCount; ++i) + for (size_t i = 0; i < sessions.size(); ++i) { - ROW(Row_InData, "incoming bytes", incomingDataTotal); - ROW(Row_OutData, "outgoing bytes", outgoingDataTotal); - ROW(Row_LastSendTime, "last send time", lastSendTime); - ROW(Row_LastRecvTime, "last receive time", lastReceiveTime); - ROW(Row_NextTimeout, "next timeout", nextTimeout); - ROW(Row_PacketsSent, "packets sent", packetsSent); - ROW(Row_PacketsLost, "packets lost", packetsLost); - ROW(Row_LastRTT, "last RTT", lastRoundTripTime); - ROW(Row_RTT, "mean RTT", roundTripTime); - ROW(Row_MTU, "MTU", mtu); - ROW(Row_ReliableInTransit, "reliable data in transit", reliableDataInTransit); + ngtcp2_conn_info info; + ngtcp2_conn_get_conn_info(sessions[i]->m_Connection.m_QuicConnection.get(), &info); + // ROW(Row_InData, "incoming bytes", bytes_recv); + m_LatchedData[i].push_back({}); + // ROW(Row_OutData, "outgoing bytes", bytes_sent); + m_LatchedData[i].push_back({}); + // ROW(Row_LastSendTime, "last send time", lastSendTime); + m_LatchedData[i].push_back({}); + // ROW(Row_LastRecvTime, "last receive time", lastReceiveTime); + m_LatchedData[i].push_back({}); + m_LatchedData[i].push_back(std::to_string(ngtcp2_conn_get_expiry( + sessions[i]->m_Connection.m_QuicConnection.get()))); + // ROW(Row_PacketsSent, "packets sent", pkt_sent); + m_LatchedData[i].push_back({}); + // ROW(Row_PacketsLost, "packets lost", pkt_lost); + m_LatchedData[i].push_back({}); + ROW(Row_LastRTT, "last RTT", latest_rtt); + ROW(Row_RTT, "mean RTT", smoothed_rtt); + m_LatchedData[i].push_back(std::to_string(ngtcp2_conn_get_path_max_tx_udp_payload_size( + sessions[i]->m_Connection.m_QuicConnection.get()))); + ROW(Row_ReliableInTransit, "reliable data in transit", bytes_in_flight); } #undef ROW } diff --git a/source/network/NetStats.h b/source/network/NetStats.h index 930360223a..c4f082ab01 100644 --- a/source/network/NetStats.h +++ b/source/network/NetStats.h @@ -25,10 +25,13 @@ #include #include +#include #include typedef struct _ENetPeer ENetPeer; -typedef struct _ENetHost ENetHost; + +class CNetServerSession; +struct ngtcp2_conn; /** * ENet connection statistics profiler table. @@ -43,8 +46,8 @@ class CNetStatsTable : public AbstractProfileTable { NONCOPYABLE(CNetStatsTable); public: - CNetStatsTable(); - CNetStatsTable(const ENetPeer& peer); + CNetStatsTable() = default; + CNetStatsTable(ngtcp2_conn* conn); CStr GetName() override; CStr GetTitle() override; @@ -53,10 +56,10 @@ public: CStr GetCellText(size_t row, size_t col) override; AbstractProfileTable* GetChild(size_t row) override; - void LatchHostState(const ENetHost& host); + void LatchHostState(const std::span> sessions); private: - const ENetPeer* m_Peer; + ngtcp2_conn* m_Connection{nullptr}; std::vector m_ColumnDescriptions; std::mutex m_Mutex; diff --git a/source/network/scripting/JSInterface_Network.cpp b/source/network/scripting/JSInterface_Network.cpp index 83cd27d970..925d574d94 100644 --- a/source/network/scripting/JSInterface_Network.cpp +++ b/source/network/scripting/JSInterface_Network.cpp @@ -111,7 +111,7 @@ void StartNetworkHost(const CStrW& playerName, const u16 serverPort, const CStr& // Generate a secret to identify the host client. g_Game = new CGame(storeReplay); - g_NetClient = new CNetClient(g_Game, "127.0.0.1", serverPort, playerName, hostJID, hashedPassword, + g_NetClient = new CNetClient(g_Game, "::1", serverPort, playerName, hostJID, hashedPassword, secret); } diff --git a/source/network/tests/test_NetMessage.h b/source/network/tests/test_NetMessage.h index b79c145632..d7040954d8 100644 --- a/source/network/tests/test_NetMessage.h +++ b/source/network/tests/test_NetMessage.h @@ -48,7 +48,7 @@ public: TS_ASSERT_EQUALS(msg.Serialize(buf) - (buf+len), 0); TS_ASSERT_EQUALS(buf[len], '!'); - CNetMessage* msg2 = CNetMessageFactory::CreateMessage(buf, len, script); + CNetMessage* msg2 = CNetMessageFactory::CreateMessage({buf, len}, script); TS_ASSERT_STR_EQUALS(((CSimulationMessage*)msg2)->ToString(), "CSimulationMessage { m_Client: 1, m_Player: 2, m_Turn: 3, m_Data: [4] }"); delete msg2;