From 0db049ca75866ccc6b3edfc21f7ec496d74ecba3 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 16:46:34 -0600 Subject: [PATCH 01/36] feat(online): validate handshake and trace position flow Co-Authored-By: Claude --- packages/net/Sailwind.Online.Net/NetClient.cs | 61 ++++- server/crates/sw-server/src/server.rs | 211 ++++++++++++++++-- .../NetClientTests.cs | 182 ++++++++++++++- tools/protocol-smoke/Checks.cs | 3 +- 4 files changed, 434 insertions(+), 23 deletions(-) diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index 386b6af..e80e16b 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.Globalization; using System.Net; using System.Net.Sockets; using LiteNetLib; @@ -66,6 +67,8 @@ public sealed class NetClient : IServerMessageHandler, IClientStateSender, IDisp private ulong _playerId; private uint _serverDay; private float _serverTimeOfDay; + private bool _loggedFirstOutboundPosition; + private bool _loggedFirstInboundPosition; /// Production constructor: drives the real LiteNetLib 1.3.1 transport. public NetClient(INetLog log) @@ -200,15 +203,27 @@ public void SendClientState(BoatPose pose) return; } + uint timestampMs = unchecked((uint)NowMs); byte[] bytes = _codec.EncodeClientState( NextSeq(), pose.Position.X, pose.Position.Y, pose.Position.Z, pose.Rotation.X, pose.Rotation.Y, pose.Rotation.Z, pose.Rotation.W, pose.Velocity.X, pose.Velocity.Y, pose.Velocity.Z, 0UL, - unchecked((uint)NowMs)); + timestampMs); - SendRaw(bytes); + if (SendRaw(bytes) && !_loggedFirstOutboundPosition) + { + _loggedFirstOutboundPosition = true; + _log.LogInfo(string.Format( + CultureInfo.InvariantCulture, + "[Sailwind.Online] First outbound position: player_id={0}, pos=({1}, {2}, {3}), t_ms={4}.", + _playerId, + pose.Position.X, + pose.Position.Y, + pose.Position.Z, + timestampMs)); + } } public void Dispose() @@ -226,6 +241,7 @@ private void OpenPeer() } _cache.Clear(); + ResetPositionObservability(); _transport.Connect(options.Host, options.Port, ConnectKey); _status = ConnectionStatus.Connecting; _log.LogInfo("[Sailwind.Online] Connecting to " + options.Host + ":" + options.Port + " ..."); @@ -264,20 +280,21 @@ private void SendHello() _lastHelloMs = NowMs; } - private void SendRaw(byte[] bytes) + private bool SendRaw(byte[] bytes) { if (!_transport.IsPeerConnected) { - return; + return false; } if (bytes.Length > Mtu) { _log.LogWarning("[Sailwind.Online] Dropping oversized packet (" + bytes.Length + " > " + Mtu + " bytes)."); - return; + return false; } _transport.Send(bytes, DeliveryMethod.Unreliable); + return true; } private void OnPeerConnected() @@ -292,6 +309,7 @@ private void OnPeerDisconnected(string reason) { _status = ConnectionStatus.Disconnected; _cache.Clear(); + ResetPositionObservability(); ScheduleReconnect(); _log.LogInfo("[Sailwind.Online] Disconnected (" + reason + "); will retry."); } @@ -314,16 +332,28 @@ void IServerMessageHandler.OnServerHello(ServerHello hello, uint seq) if (!hello.Accepted) { _status = ConnectionStatus.Disconnected; + ResetPositionObservability(); ScheduleReconnect(); _log.LogWarning("[Sailwind.Online] ServerHello rejected: " + (hello.Reason ?? "no reason")); return; } + CapabilityManifest? caps = hello.Capabilities; + if (caps.HasValue && caps.Value.ProtocolVersion != ProtocolVersion) + { + _status = ConnectionStatus.Disconnected; + ResetPositionObservability(); + ScheduleReconnect(); + _log.LogWarning( + "[Sailwind.Online] ServerHello protocol mismatch: client " + ProtocolVersion + + ", server " + caps.Value.ProtocolVersion + "; will retry."); + return; + } + _status = ConnectionStatus.Ready; _playerId = hello.PlayerId; _reconnectBackoffMs = DefaultReconnectMs; - CapabilityManifest? caps = hello.Capabilities; if (caps.HasValue && caps.Value.SnapshotHz > 0) { _snapshotHz = caps.Value.SnapshotHz; @@ -429,6 +459,19 @@ private void IngestPlayer(PlayerState ps, long now) Link = ps.AboardBoat }; _cache.UpsertPlayer(ps.PlayerId, sample); + + if (!_loggedFirstInboundPosition && HandshakeComplete && ps.PlayerId != _playerId) + { + _loggedFirstInboundPosition = true; + _log.LogInfo(string.Format( + CultureInfo.InvariantCulture, + "[Sailwind.Online] First inbound position: remote_player_id={0}, pos=({1}, {2}, {3}), t_ms={4}.", + ps.PlayerId, + sample.Pos.X, + sample.Pos.Y, + sample.Pos.Z, + sample.TMs)); + } } private void IngestBoat(BoatState bs, long now) @@ -467,6 +510,12 @@ private static NetQuat ToQuat(QuatC? q) return new NetQuat(value.X, value.Y, value.Z, value.W); } + private void ResetPositionObservability() + { + _loggedFirstOutboundPosition = false; + _loggedFirstInboundPosition = false; + } + public static string FormatTimeOfDay(float fractionOfDay) { float clamped = fractionOfDay - (float)Math.Floor(fractionOfDay); diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index d7863f5..9a8d093 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -220,24 +220,38 @@ impl Server { } fn on_hello(&mut self, peer: PeerId, hello: p::ClientHello<'_>) -> anyhow::Result<()> { + if hello.protocol_version() != sw_contracts::PROTOCOL_VERSION { + let reason = format!( + "protocol version mismatch: client {}, server {}", + hello.protocol_version(), + sw_contracts::PROTOCOL_VERSION + ); + self.reject_hello(peer, &reason); + return Ok(()); + } + + let max_string_len = self.cfg.max_wire_string_len_usize(); + let Some(api_surface_hash) = hello.api_surface_hash() else { + self.reject_hello(peer, "missing API surface hash"); + return Ok(()); + }; + if api_surface_hash.is_empty() { + self.reject_hello(peer, "missing API surface hash"); + return Ok(()); + } + if !validate::string_within_limit(api_surface_hash, max_string_len) { + let reason = + format!("API surface hash exceeds maximum length ({max_string_len} bytes)"); + self.reject_hello(peer, &reason); + return Ok(()); + } + let token = hello.token().unwrap_or(""); let name = hello.display_name().unwrap_or("sailor").to_string(); if token.is_empty() { // Auth is assertion-only, but a token must at least be present. - let bytes = codec::server_hello( - self.next_seq(), - false, - "missing token", - 0, - &self.cfg.server_name, - 0, - &self.caps(), - self.clock_now(), - self.weather_seed, - self.weather_epoch_day, - ); - self.send(peer, &bytes); + self.reject_hello(peer, "missing token"); return Ok(()); } @@ -303,6 +317,22 @@ impl Server { Ok(()) } + fn reject_hello(&mut self, peer: PeerId, reason: &str) { + let bytes = codec::server_hello( + self.next_seq(), + false, + reason, + 0, + &self.cfg.server_name, + 0, + &self.caps(), + self.clock_now(), + self.weather_seed, + self.weather_epoch_day, + ); + self.send(peer, &bytes); + } + fn on_client_state(&mut self, peer: PeerId, cs: p::ClientState<'_>, now_ms: i64) { let Some(player_id) = self.sessions.get(&peer).map(|s| s.player_id) else { return; @@ -843,6 +873,155 @@ mod tests { } } +#[cfg(test)] +mod handshake_tests { + use super::*; + use flatbuffers::FlatBufferBuilder; + use std::net::UdpSocket; + use std::time::Duration; + use sw_contracts::{decode_envelope, finish_envelope}; + use sw_net::{protocol, Event}; + use sw_world::Grid; + + fn make_server() -> Server { + let cfg = Config::default(); + let world = World::new(Grid::new(cfg.cell_size_m)); + Server { + host: Host::bind("127.0.0.1:0", CONNECT_KEY).unwrap(), + db: Db::open_in_memory().unwrap(), + world, + sessions: HashMap::new(), + seq: 0, + snapshot_tick: 0, + boot: Instant::now(), + epoch_ms: 0, + weather_seed: 0, + weather_epoch_day: 0, + trade_limiter: RateLimiter::new(cfg.trade_min_interval_ms_i64()), + client_state_limiter: RateLimiter::new(cfg.client_state_min_interval_ms_i64()), + chat_limiter: RateLimiter::new(cfg.chat_min_interval_ms_i64()), + econ_limiter: RateLimiter::new(cfg.econ_min_interval_ms_i64()), + moor_limiter: RateLimiter::new(cfg.moor_min_interval_ms_i64()), + running: Arc::new(AtomicBool::new(true)), + cfg, + } + } + + fn hello_envelope(protocol_version: u16, api_surface_hash: Option<&str>) -> Vec { + let mut fbb = FlatBufferBuilder::new(); + let token = fbb.create_string("handshake-token"); + let name = fbb.create_string("Sailor"); + let api_hash = api_surface_hash.map(|value| fbb.create_string(value)); + let hello = p::ClientHello::create( + &mut fbb, + &p::ClientHelloArgs { + protocol_version, + display_name: Some(name), + token: Some(token), + api_surface_hash: api_hash, + ..Default::default() + }, + ); + finish_envelope(&mut fbb, 1, p::Payload::ClientHello, hello.as_union_value()) + } + + fn deliver_hello(server: &mut Server, peer: PeerId, bytes: &[u8]) { + let env = decode_envelope(bytes).unwrap(); + server + .on_hello(peer, env.payload_as_client_hello().unwrap()) + .unwrap(); + } + + fn connect_peer(server: &mut Server) -> (UdpSocket, PeerId) { + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + client.connect(server.host.local_addr().unwrap()).unwrap(); + client + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let connect_data = protocol::write_litenet_string(CONNECT_KEY); + let request = protocol::build_connect_request(0, 1, 1, 16, &connect_data); + client.send(&request).unwrap(); + + let peer = match server.host.poll(Instant::now()).as_slice() { + [Event::Connected(peer)] => *peer, + events => panic!("expected one connected peer, got {events:?}"), + }; + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + client.recv(&mut accept).unwrap(); + (client, peer) + } + + fn receive_server_hello(client: &UdpSocket) -> (bool, String) { + let mut packet = [0u8; protocol::MTU]; + let received = client.recv(&mut packet).unwrap(); + assert_eq!( + protocol::Header::from_byte(packet[0]).property, + protocol::property::UNRELIABLE + ); + let env = decode_envelope(&packet[protocol::HEADER_SIZE..received]).unwrap(); + let hello = env.payload_as_server_hello().unwrap(); + (hello.accepted(), hello.reason().unwrap_or("").to_string()) + } + + #[test] + fn protocol_mismatch_is_rejected_before_session_creation() { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let bytes = hello_envelope(sw_contracts::PROTOCOL_VERSION + 1, Some("surface-hash")); + + deliver_hello(&mut server, peer, &bytes); + + assert!(!server.sessions.contains_key(&peer)); + assert_eq!( + receive_server_hello(&client), + ( + false, + format!( + "protocol version mismatch: client {}, server {}", + sw_contracts::PROTOCOL_VERSION + 1, + sw_contracts::PROTOCOL_VERSION + ) + ) + ); + } + + #[test] + fn api_surface_hash_must_be_present_nonempty_and_bounded() { + let max_len = Config::default().max_wire_string_len_usize(); + let too_long = "x".repeat(max_len + 1); + let cases = [ + (None, "missing API surface hash".to_string()), + (Some(""), "missing API surface hash".to_string()), + ( + Some(too_long.as_str()), + format!("API surface hash exceeds maximum length ({max_len} bytes)"), + ), + ]; + + for (hash, expected_reason) in cases { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let bytes = hello_envelope(sw_contracts::PROTOCOL_VERSION, hash); + deliver_hello(&mut server, peer, &bytes); + assert!( + !server.sessions.contains_key(&peer), + "invalid API surface hash created a session" + ); + assert_eq!(receive_server_hello(&client), (false, expected_reason)); + } + } + + #[test] + fn valid_protocol_and_api_surface_hash_create_session() { + let mut server = make_server(); + let bytes = hello_envelope(sw_contracts::PROTOCOL_VERSION, Some("surface-hash")); + + deliver_hello(&mut server, 1, &bytes); + + assert!(server.sessions.contains_key(&1)); + } +} + /// Interest-management hardening: these exercise the real message handlers and /// the per-recipient visibility decision against an in-memory server, pinning /// the quantitative AoI acceptance (initial interest set on join, added/removed @@ -887,12 +1066,14 @@ mod aoi_harden_tests { let mut fbb = FlatBufferBuilder::new(); let token_off = fbb.create_string(token); let name_off = fbb.create_string(name); + let api_hash_off = fbb.create_string("test-api-surface"); let hello = p::ClientHello::create( &mut fbb, &p::ClientHelloArgs { protocol_version: sw_contracts::PROTOCOL_VERSION, display_name: Some(name_off), token: Some(token_off), + api_surface_hash: Some(api_hash_off), ..Default::default() }, ); @@ -1081,12 +1262,14 @@ mod market_dispatch_tests { let mut fbb = FlatBufferBuilder::new(); let token_off = fbb.create_string(token); let name_off = fbb.create_string(name); + let api_hash_off = fbb.create_string("test-api-surface"); let hello = p::ClientHello::create( &mut fbb, &p::ClientHelloArgs { protocol_version: sw_contracts::PROTOCOL_VERSION, display_name: Some(name_off), token: Some(token_off), + api_surface_hash: Some(api_hash_off), ..Default::default() }, ); @@ -1303,12 +1486,14 @@ mod input_hardening_tests { let mut fbb = FlatBufferBuilder::new(); let token_off = fbb.create_string(token); let name_off = fbb.create_string("Sailor"); + let api_hash_off = fbb.create_string("test-api-surface"); let hello = p::ClientHello::create( &mut fbb, &p::ClientHelloArgs { protocol_version: sw_contracts::PROTOCOL_VERSION, display_name: Some(name_off), token: Some(token_off), + api_surface_hash: Some(api_hash_off), ..Default::default() }, ); diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index 2861c03..3ae11df 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Sockets; +using System.Numerics; using Google.FlatBuffers; using Sailwind.Api; using Sailwind.Online.Client.Net; @@ -99,6 +100,33 @@ public void Handshake_ServerHelloRejected_ReturnsToDisconnected() Assert.False(net.HandshakeComplete); } + [Fact] + public void Handshake_AcceptedServerHelloWithWrongProtocol_ReturnsToDisconnected() + { + var transport = new MockTransport(); + var log = new RecordingLog(); + long now = 0; + var net = new NetClient(log, transport, () => now); + net.Connect(Options); + transport.RaisePeerConnected(); + + transport.RaiseNetworkReceive(ServerHelloEnvelope( + accepted: true, + playerId: 77, + snapshotHz: 8, + protocolVersion: NetClient.ProtocolVersion + 1)); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.False(net.HandshakeComplete); + Assert.Contains(log.Warnings, message => message.Contains("protocol")); + Assert.DoesNotContain(log.Warnings, message => message.Contains(Options.Token)); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + } + [Fact] public void ClientHello_ResendsEveryIntervalUntilServerHello() { @@ -249,6 +277,120 @@ public void SendClientState_OnlySendsOnceReady() Assert.Equal(Payload.ClientState, Decode(transport.Sent[0]).PayloadType); } + [Fact] + public void AcceptedHello_BoatPoseOutbound_ThenRemoteSnapshotInbound_RoundTripsExactly() + { + long now = 4321; + var transport = new MockTransport(); + var net = new NetClient(new NullNetLog(), transport, () => now); + net.Connect(Options); + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope(accepted: true, playerId: 77, snapshotHz: 4)); + transport.Sent.Clear(); + + var pose = new BoatPose( + new Vector3(12.5f, -3.25f, 99.75f), + new Quaternion(0.1f, 0.2f, 0.3f, 0.9f), + new Vector3(-4.5f, 0.25f, 6.75f)); + net.SendClientState(pose); + + Assert.Single(transport.Sent); + byte[] expected = new Codec().EncodeClientState( + 2, + pose.Position.X, pose.Position.Y, pose.Position.Z, + pose.Rotation.X, pose.Rotation.Y, pose.Rotation.Z, pose.Rotation.W, + pose.Velocity.X, pose.Velocity.Y, pose.Velocity.Z, + 0, + (uint)now); + Assert.Equal(expected, transport.Sent[0]); + ClientState outbound = Decode(transport.Sent[0]).PayloadAsClientState(); + Assert.Equal(pose.Position.X, outbound.Pos.Value.X); + Assert.Equal(pose.Position.Y, outbound.Pos.Value.Y); + Assert.Equal(pose.Position.Z, outbound.Pos.Value.Z); + Assert.Equal(pose.Rotation.X, outbound.Rot.Value.X); + Assert.Equal(pose.Rotation.Y, outbound.Rot.Value.Y); + Assert.Equal(pose.Rotation.Z, outbound.Rot.Value.Z); + Assert.Equal(pose.Rotation.W, outbound.Rot.Value.W); + Assert.Equal(pose.Velocity.X, outbound.Vel.Value.X); + Assert.Equal(pose.Velocity.Y, outbound.Vel.Value.Y); + Assert.Equal(pose.Velocity.Z, outbound.Vel.Value.Z); + Assert.Equal((uint)now, outbound.TMs); + + transport.RaiseNetworkReceive(SnapshotDeltaEnvelope( + serverTick: 91, + playerId: 88, + x: -10.5f, + y: 2.25f, + z: 45.75f, + rx: 0.4f, + ry: 0.3f, + rz: 0.2f, + rw: 0.8f, + aboardBoat: 123, + tMs: 5678)); + + Assert.Equal(91u, net.Cache.LastServerTick); + Assert.True(net.Cache.TryGetPlayer(88, out var remote)); + Assert.NotNull(remote); + Assert.Equal(-10.5f, remote.Latest.Pos.X); + Assert.Equal(2.25f, remote.Latest.Pos.Y); + Assert.Equal(45.75f, remote.Latest.Pos.Z); + Assert.Equal(0.4f, remote.Latest.Rot.X); + Assert.Equal(0.3f, remote.Latest.Rot.Y); + Assert.Equal(0.2f, remote.Latest.Rot.Z); + Assert.Equal(0.8f, remote.Latest.Rot.W); + Assert.Equal(123ul, remote.Latest.Link); + Assert.Equal(5678u, remote.Latest.TMs); + Assert.Equal(now, remote.Latest.ReceivedMs); + } + + [Fact] + public void PositionObservability_LogsEachDirectionOncePerSession_AndResetsOnReconnect() + { + var transport = new MockTransport(); + var log = new RecordingLog(); + long now = 100; + var net = new NetClient(log, transport, () => now); + var pose = new BoatPose(new Vector3(1f, 2f, 3f), Quaternion.Identity, Vector3.Zero); + byte[] snapshot = SnapshotDeltaEnvelope(1, 8, 4f, 5f, 6f, 0f, 0f, 0f, 1f, 123, 10); + + net.Connect(Options); + net.SendClientState(pose); + Assert.DoesNotContain(log.Infos, message => message.Contains("First outbound position")); + + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope(true, 7, 4)); + net.SendClientState(pose); + net.SendClientState(pose); + transport.RaiseNetworkReceive(snapshot); + transport.RaiseNetworkReceive(snapshot); + + Assert.Equal(ConnectionStatus.Ready, net.Status); + Assert.True(Envelope.VerifyEnvelope(new ByteBuffer(snapshot))); + Assert.Equal(8ul, Decode(snapshot).PayloadAsSnapshotDelta().Players(0).Value.PlayerId); + string outboundLog = Assert.Single(log.Infos.FindAll(message => message.Contains("First outbound position"))); + Assert.Contains("player_id=7", outboundLog); + Assert.Contains("pos=(1, 2, 3)", outboundLog); + Assert.Contains("t_ms=100", outboundLog); + Assert.True(net.Cache.TryGetPlayer(8, out _)); + string inboundLog = Assert.Single(log.Infos.FindAll(message => message.Contains("First inbound position"))); + Assert.Contains("remote_player_id=8", inboundLog); + Assert.Contains("pos=(4, 5, 6)", inboundLog); + Assert.Contains("t_ms=10", inboundLog); + Assert.DoesNotContain(log.Infos, message => message.Contains(Options.Token)); + + transport.RaisePeerDisconnected(); + now += NetClient.DefaultReconnectMs; + net.Poll(); + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope(true, 7, 4)); + net.SendClientState(pose); + transport.RaiseNetworkReceive(snapshot); + + Assert.Equal(2, log.Infos.FindAll(message => message.Contains("First outbound position")).Count); + Assert.Equal(2, log.Infos.FindAll(message => message.Contains("First inbound position")).Count); + } + [Fact] public void Receive_WorldClock_DispatchesThroughCodecIntoState() { @@ -314,13 +456,17 @@ private static Envelope Decode(byte[] bytes) return Envelope.GetRootAsEnvelope(new ByteBuffer(bytes)); } - private static byte[] ServerHelloEnvelope(bool accepted, ulong playerId, byte snapshotHz) + private static byte[] ServerHelloEnvelope( + bool accepted, + ulong playerId, + byte snapshotHz, + ushort protocolVersion = NetClient.ProtocolVersion) { var b = new FlatBufferBuilder(128); StringOffset reason = b.CreateString(string.Empty); StringOffset serverName = b.CreateString("test-server"); Offset caps = CapabilityManifest.CreateCapabilityManifest( - b, protocol_version: NetClient.ProtocolVersion, snapshot_hz: snapshotHz); + b, protocol_version: protocolVersion, snapshot_hz: snapshotHz); Offset hello = ServerHello.CreateServerHello( b, accepted: accepted, @@ -332,6 +478,35 @@ private static byte[] ServerHelloEnvelope(bool accepted, ulong playerId, byte sn return Wrap(b, 1, Payload.ServerHello, hello.Value); } + private static byte[] SnapshotDeltaEnvelope( + uint serverTick, + ulong playerId, + float x, + float y, + float z, + float rx, + float ry, + float rz, + float rw, + ulong aboardBoat, + uint tMs) + { + var b = new FlatBufferBuilder(256); + PlayerState.StartPlayerState(b); + PlayerState.AddPlayerId(b, playerId); + PlayerState.AddPos(b, Vec3.CreateVec3(b, x, y, z)); + PlayerState.AddRot(b, QuatC.CreateQuatC(b, rx, ry, rz, rw)); + PlayerState.AddAboardBoat(b, aboardBoat); + PlayerState.AddTMs(b, tMs); + Offset player = PlayerState.EndPlayerState(b); + VectorOffset players = SnapshotDelta.CreatePlayersVector(b, new[] { player }); + Offset delta = SnapshotDelta.CreateSnapshotDelta( + b, + server_tick: serverTick, + playersOffset: players); + return Wrap(b, 3, Payload.SnapshotDelta, delta.Value); + } + private static byte[] WorldClockEnvelope(uint day, float timeOfDay) { var b = new FlatBufferBuilder(64); @@ -363,10 +538,11 @@ public void LogError(string message) { } private sealed class RecordingLog : INetLog { public readonly List Errors = new List(); + public readonly List Infos = new List(); public readonly List Warnings = new List(); public void LogDebug(string message) { } - public void LogInfo(string message) { } + public void LogInfo(string message) { Infos.Add(message); } public void LogWarning(string message) { Warnings.Add(message); } public void LogError(string message) { Errors.Add(message); } } diff --git a/tools/protocol-smoke/Checks.cs b/tools/protocol-smoke/Checks.cs index 54e903b..4b909ac 100644 --- a/tools/protocol-smoke/Checks.cs +++ b/tools/protocol-smoke/Checks.cs @@ -435,7 +435,8 @@ private void DrainPresence(SmokeClient b, ulong playerIdA, ref bool sawAddedCell private ServerHello? DoHello(SmokeClient c, string name, string token) { - var hello = Codec.EncodeClientHello(_seq++, name, token, "smoke", "0.0.0", string.Empty); + const string apiSurfaceHashSentinel = "protocol-smoke-surface-hash"; + var hello = Codec.EncodeClientHello(_seq++, name, token, "smoke", "0.0.0", apiSurfaceHashSentinel); var env = SendAndWait(c, hello, e => e.PayloadType == Payload.ServerHello, 6000, 250); return env.HasValue ? env.Value.PayloadAsServerHello() : (ServerHello?)null; } From 56319a7bbddcb03073d9d1948ffece4a7a2f4841 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 16:58:58 -0600 Subject: [PATCH 02/36] fix(net): require capabilities in accepted server hello Co-Authored-By: Claude --- packages/net/Sailwind.Online.Net/NetClient.cs | 13 +++++- .../NetClientTests.cs | 40 +++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index e80e16b..6aee8cf 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -339,7 +339,16 @@ void IServerMessageHandler.OnServerHello(ServerHello hello, uint seq) } CapabilityManifest? caps = hello.Capabilities; - if (caps.HasValue && caps.Value.ProtocolVersion != ProtocolVersion) + if (!caps.HasValue) + { + _status = ConnectionStatus.Disconnected; + ResetPositionObservability(); + ScheduleReconnect(); + _log.LogWarning("[Sailwind.Online] ServerHello missing capabilities; will retry."); + return; + } + + if (caps.Value.ProtocolVersion != ProtocolVersion) { _status = ConnectionStatus.Disconnected; ResetPositionObservability(); @@ -354,7 +363,7 @@ void IServerMessageHandler.OnServerHello(ServerHello hello, uint seq) _playerId = hello.PlayerId; _reconnectBackoffMs = DefaultReconnectMs; - if (caps.HasValue && caps.Value.SnapshotHz > 0) + if (caps.Value.SnapshotHz > 0) { _snapshotHz = caps.Value.SnapshotHz; } diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index 3ae11df..fceb7f8 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -127,6 +127,34 @@ public void Handshake_AcceptedServerHelloWithWrongProtocol_ReturnsToDisconnected Assert.Equal(ConnectionStatus.Connecting, net.Status); } + [Fact] + public void Handshake_AcceptedServerHelloWithoutCapabilities_ReturnsToDisconnected() + { + var transport = new MockTransport(); + var log = new RecordingLog(); + long now = 0; + var net = new NetClient(log, transport, () => now); + net.Connect(Options); + transport.RaisePeerConnected(); + + transport.RaiseNetworkReceive(ServerHelloEnvelope( + accepted: true, + playerId: 77, + snapshotHz: 8, + includeCapabilities: false)); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.False(net.HandshakeComplete); + Assert.Equal( + "[Sailwind.Online] ServerHello missing capabilities; will retry.", + Assert.Single(log.Warnings)); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + } + [Fact] public void ClientHello_ResendsEveryIntervalUntilServerHello() { @@ -460,13 +488,19 @@ private static byte[] ServerHelloEnvelope( bool accepted, ulong playerId, byte snapshotHz, - ushort protocolVersion = NetClient.ProtocolVersion) + ushort protocolVersion = NetClient.ProtocolVersion, + bool includeCapabilities = true) { var b = new FlatBufferBuilder(128); StringOffset reason = b.CreateString(string.Empty); StringOffset serverName = b.CreateString("test-server"); - Offset caps = CapabilityManifest.CreateCapabilityManifest( - b, protocol_version: protocolVersion, snapshot_hz: snapshotHz); + Offset caps = default(Offset); + if (includeCapabilities) + { + caps = CapabilityManifest.CreateCapabilityManifest( + b, protocol_version: protocolVersion, snapshot_hz: snapshotHz); + } + Offset hello = ServerHello.CreateServerHello( b, accepted: accepted, From 77a7e88f95c6e23339ceef1e2eda0ac50e50557f Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 17:25:18 -0600 Subject: [PATCH 03/36] fix(online): harden handshake retries Co-Authored-By: Claude --- .../net/Sailwind.Online.Net/ITransport.cs | 3 + .../LiteNetLibTransport.cs | 26 ++- packages/net/Sailwind.Online.Net/NetClient.cs | 24 +-- server/crates/sw-server/src/server.rs | 187 +++++++++++++++--- .../MockTransport.cs | 14 ++ .../NetClientTests.cs | 34 +++- 6 files changed, 249 insertions(+), 39 deletions(-) diff --git a/packages/net/Sailwind.Online.Net/ITransport.cs b/packages/net/Sailwind.Online.Net/ITransport.cs index 7dae4c5..1dc5815 100644 --- a/packages/net/Sailwind.Online.Net/ITransport.cs +++ b/packages/net/Sailwind.Online.Net/ITransport.cs @@ -29,6 +29,9 @@ public interface ITransport /// Open (or re-open) the single peer to : with the connect key. void Connect(string host, int port, string key); + /// Immediately drop the current peer without reporting a network-originated disconnect. + void DropPeer(); + /// Send one datagram to the current peer with the given delivery method. void Send(byte[] data, DeliveryMethod deliveryMethod); diff --git a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs index 938615d..3598100 100644 --- a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs +++ b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs @@ -17,6 +17,7 @@ public sealed class LiteNetLibTransport : ITransport private readonly EventBasedNetListener _listener = new EventBasedNetListener(); private readonly NetManager _manager; private NetPeer? _peer; + private NetPeer? _locallyDroppedPeer; public LiteNetLibTransport() { @@ -63,6 +64,19 @@ public void Connect(string host, int port, string key) _peer = _manager.Connect(host, port, key); } + public void DropPeer() + { + NetPeer? peer = _peer; + if (peer == null) + { + return; + } + + _locallyDroppedPeer = peer; + _peer = null; + _manager.DisconnectPeerForce(peer); + } + public void Send(byte[] data, DeliveryMethod deliveryMethod) { _peer?.Send(data, deliveryMethod); @@ -91,7 +105,17 @@ private void OnPeerConnected(NetPeer peer) private void OnPeerDisconnected(NetPeer peer, DisconnectInfo info) { - _peer = null; + if (ReferenceEquals(_locallyDroppedPeer, peer)) + { + _locallyDroppedPeer = null; + return; + } + + if (ReferenceEquals(_peer, peer)) + { + _peer = null; + } + PeerDisconnected?.Invoke(info.Reason.ToString()); } diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index 6aee8cf..94562d1 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -331,29 +331,20 @@ void IServerMessageHandler.OnServerHello(ServerHello hello, uint seq) { if (!hello.Accepted) { - _status = ConnectionStatus.Disconnected; - ResetPositionObservability(); - ScheduleReconnect(); - _log.LogWarning("[Sailwind.Online] ServerHello rejected: " + (hello.Reason ?? "no reason")); + RejectHandshake("[Sailwind.Online] ServerHello rejected: " + (hello.Reason ?? "no reason")); return; } CapabilityManifest? caps = hello.Capabilities; if (!caps.HasValue) { - _status = ConnectionStatus.Disconnected; - ResetPositionObservability(); - ScheduleReconnect(); - _log.LogWarning("[Sailwind.Online] ServerHello missing capabilities; will retry."); + RejectHandshake("[Sailwind.Online] ServerHello missing capabilities; will retry."); return; } if (caps.Value.ProtocolVersion != ProtocolVersion) { - _status = ConnectionStatus.Disconnected; - ResetPositionObservability(); - ScheduleReconnect(); - _log.LogWarning( + RejectHandshake( "[Sailwind.Online] ServerHello protocol mismatch: client " + ProtocolVersion + ", server " + caps.Value.ProtocolVersion + "; will retry."); return; @@ -525,6 +516,15 @@ private void ResetPositionObservability() _loggedFirstInboundPosition = false; } + private void RejectHandshake(string warning) + { + _transport.DropPeer(); + _status = ConnectionStatus.Disconnected; + ResetPositionObservability(); + ScheduleReconnect(); + _log.LogWarning(warning); + } + public static string FormatTimeOfDay(float fractionOfDay) { float clamped = fractionOfDay - (float)Math.Floor(fractionOfDay); diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 9a8d093..1ae8cf1 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -35,6 +35,7 @@ const FLUSH_INTERVAL: Duration = Duration::from_secs(5); struct Session { peer: PeerId, player_id: u64, + identity_hash: String, display_name: String, aboard_boat: u64, pos: [f32; 3], @@ -255,10 +256,24 @@ impl Server { return Ok(()); } + let identity_hash = token_hash(token); + if let Some((player_id, identity_matches)) = self + .sessions + .get(&peer) + .map(|session| (session.player_id, session.identity_hash == identity_hash)) + { + if !identity_matches { + self.reject_hello(peer, "identity change requires a new connection"); + return Ok(()); + } + + let balance = self.db.player_balance(player_id as i64)?; + self.send_accepted_hello(peer, player_id, balance); + return Ok(()); + } + let now = now_ms(); - let player = self - .db - .upsert_player_by_token(&token_hash(token), &name, now)?; + let player = self.db.upsert_player_by_token(&identity_hash, &name, now)?; let player_id = player.id as u64; // Drop any prior session for this identity (reconnect from a new peer). @@ -282,6 +297,7 @@ impl Server { Session { peer, player_id, + identity_hash, display_name: name, aboard_boat: 0, pos: [0.0, 0.0, 0.0], @@ -296,25 +312,29 @@ impl Server { tracing::info!(peer, player_id, name = %self.sessions[&peer].display_name, "hello accepted"); + self.send_accepted_hello(peer, player_id, player.gold); + + // Emit the join-time interest set so a freshly connected player learns + // its surrounding cells (and their contents, e.g. persisted moorings) + // without having to first cross a cell boundary. + self.emit_aoi(peer, &aoi); + Ok(()) + } + + fn send_accepted_hello(&mut self, peer: PeerId, player_id: u64, balance_gold: i64) { let bytes = codec::server_hello( self.next_seq(), true, "", player_id, &self.cfg.server_name, - player.gold, + balance_gold, &self.caps(), self.clock_now(), self.weather_seed, self.weather_epoch_day, ); self.send(peer, &bytes); - - // Emit the join-time interest set so a freshly connected player learns - // its surrounding cells (and their contents, e.g. persisted moorings) - // without having to first cross a cell boundary. - self.emit_aoi(peer, &aoi); - Ok(()) } fn reject_hello(&mut self, peer: PeerId, reason: &str) { @@ -907,9 +927,13 @@ mod handshake_tests { } } - fn hello_envelope(protocol_version: u16, api_surface_hash: Option<&str>) -> Vec { + fn hello_envelope( + token: &str, + protocol_version: u16, + api_surface_hash: Option<&str>, + ) -> Vec { let mut fbb = FlatBufferBuilder::new(); - let token = fbb.create_string("handshake-token"); + let token = fbb.create_string(token); let name = fbb.create_string("Sailor"); let api_hash = api_surface_hash.map(|value| fbb.create_string(value)); let hello = p::ClientHello::create( @@ -925,6 +949,24 @@ mod handshake_tests { finish_envelope(&mut fbb, 1, p::Payload::ClientHello, hello.as_union_value()) } + fn state_envelope() -> Vec { + let mut fbb = FlatBufferBuilder::new(); + let pos = p::Vec3::new(175.0, 4.0, -125.0); + let rot = p::QuatC::new(0.1, 0.2, 0.3, 0.9); + let vel = p::Vec3::new(2.0, 3.0, 4.0); + let state = p::ClientState::create( + &mut fbb, + &p::ClientStateArgs { + pos: Some(&pos), + rot: Some(&rot), + vel: Some(&vel), + aboard_boat: 99, + t_ms: 4321, + }, + ); + finish_envelope(&mut fbb, 2, p::Payload::ClientState, state.as_union_value()) + } + fn deliver_hello(server: &mut Server, peer: PeerId, bytes: &[u8]) { let env = decode_envelope(bytes).unwrap(); server @@ -952,22 +994,29 @@ mod handshake_tests { } fn receive_server_hello(client: &UdpSocket) -> (bool, String) { - let mut packet = [0u8; protocol::MTU]; - let received = client.recv(&mut packet).unwrap(); - assert_eq!( - protocol::Header::from_byte(packet[0]).property, - protocol::property::UNRELIABLE - ); - let env = decode_envelope(&packet[protocol::HEADER_SIZE..received]).unwrap(); - let hello = env.payload_as_server_hello().unwrap(); - (hello.accepted(), hello.reason().unwrap_or("").to_string()) + loop { + let mut packet = [0u8; protocol::MTU]; + let received = client.recv(&mut packet).unwrap(); + assert_eq!( + protocol::Header::from_byte(packet[0]).property, + protocol::property::UNRELIABLE + ); + let env = decode_envelope(&packet[protocol::HEADER_SIZE..received]).unwrap(); + if let Some(hello) = env.payload_as_server_hello() { + return (hello.accepted(), hello.reason().unwrap_or("").to_string()); + } + } } #[test] fn protocol_mismatch_is_rejected_before_session_creation() { let mut server = make_server(); let (client, peer) = connect_peer(&mut server); - let bytes = hello_envelope(sw_contracts::PROTOCOL_VERSION + 1, Some("surface-hash")); + let bytes = hello_envelope( + "handshake-token", + sw_contracts::PROTOCOL_VERSION + 1, + Some("surface-hash"), + ); deliver_hello(&mut server, peer, &bytes); @@ -1001,7 +1050,7 @@ mod handshake_tests { for (hash, expected_reason) in cases { let mut server = make_server(); let (client, peer) = connect_peer(&mut server); - let bytes = hello_envelope(sw_contracts::PROTOCOL_VERSION, hash); + let bytes = hello_envelope("handshake-token", sw_contracts::PROTOCOL_VERSION, hash); deliver_hello(&mut server, peer, &bytes); assert!( !server.sessions.contains_key(&peer), @@ -1014,12 +1063,102 @@ mod handshake_tests { #[test] fn valid_protocol_and_api_surface_hash_create_session() { let mut server = make_server(); - let bytes = hello_envelope(sw_contracts::PROTOCOL_VERSION, Some("surface-hash")); + let bytes = hello_envelope( + "handshake-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); deliver_hello(&mut server, 1, &bytes); assert!(server.sessions.contains_key(&1)); } + + #[test] + fn repeated_valid_hello_preserves_established_session_and_only_resends_server_hello() { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let hello = hello_envelope( + "handshake-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello(&mut server, peer, &hello); + assert_eq!(receive_server_hello(&client), (true, String::new())); + + let state = state_envelope(); + let env = decode_envelope(&state).unwrap(); + server.on_client_state(peer, env.payload_as_client_state().unwrap(), 1_000); + + let established = &server.sessions[&peer]; + let player_id = established.player_id; + let display_name = established.display_name.clone(); + let aboard_boat = established.aboard_boat; + let pos = established.pos; + let rot = established.rot; + let vel = established.vel; + let t_ms = established.t_ms; + let cell = established.cell; + let subscribed_cells = established.sub.cells().clone(); + let world_cell = server.world.cell_of_entity(player_id); + let seq_before_retry = server.seq; + + deliver_hello(&mut server, peer, &hello); + + let retried = &server.sessions[&peer]; + assert_eq!(retried.player_id, player_id); + assert_eq!(retried.display_name, display_name); + assert_eq!(retried.aboard_boat, aboard_boat); + assert_eq!(retried.pos, pos); + assert_eq!(retried.rot, rot); + assert_eq!(retried.vel, vel); + assert_eq!(retried.t_ms, t_ms); + assert_eq!(retried.cell, cell); + assert_eq!(retried.sub.cells(), &subscribed_cells); + assert_eq!(server.world.cell_of_entity(player_id), world_cell); + assert_eq!(server.seq, seq_before_retry.wrapping_add(1)); + assert_eq!(receive_server_hello(&client), (true, String::new())); + } + + #[test] + fn same_peer_cannot_replace_an_established_session_with_a_different_identity() { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let first = hello_envelope( + "first-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello(&mut server, peer, &first); + assert_eq!(receive_server_hello(&client), (true, String::new())); + + let player_id = server.sessions[&peer].player_id; + let subscribed_cells = server.sessions[&peer].sub.cells().clone(); + let world_len = server.world.len(); + let world_cell = server.world.cell_of_entity(player_id); + let seq_before_replacement = server.seq; + let replacement = hello_envelope( + "different-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + + deliver_hello(&mut server, peer, &replacement); + + assert_eq!(server.sessions.len(), 1); + assert_eq!(server.sessions[&peer].player_id, player_id); + assert_eq!(server.sessions[&peer].sub.cells(), &subscribed_cells); + assert_eq!(server.world.len(), world_len); + assert_eq!(server.world.cell_of_entity(player_id), world_cell); + assert_eq!(server.seq, seq_before_replacement.wrapping_add(1)); + assert_eq!( + receive_server_hello(&client), + ( + false, + "identity change requires a new connection".to_string() + ) + ); + } } /// Interest-management hardening: these exercise the real message handlers and diff --git a/tests/Sailwind.Online.Net.Tests/MockTransport.cs b/tests/Sailwind.Online.Net.Tests/MockTransport.cs index e827960..113445b 100644 --- a/tests/Sailwind.Online.Net.Tests/MockTransport.cs +++ b/tests/Sailwind.Online.Net.Tests/MockTransport.cs @@ -21,6 +21,8 @@ internal sealed class MockTransport : ITransport public int StartCalls; public int StopCalls; public int ConnectCalls; + public int FreshPeerConnectCalls; + public int DropPeerCalls; public int PollCalls; public string LastHost; @@ -58,11 +60,23 @@ public bool Start() public void Connect(string host, int port, string key) { ConnectCalls++; + if (!_peerConnected) + { + FreshPeerConnectCalls++; + } + LastHost = host; LastPort = port; LastKey = key; } + public void DropPeer() + { + DropPeerCalls++; + _peerConnected = false; + Ping = -1; + } + public void Send(byte[] data, DeliveryMethod deliveryMethod) { Sent.Add(data); diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index fceb7f8..60b51cd 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -90,7 +90,8 @@ public void Handshake_PeerConnectedThenServerHello_ReachesReady() public void Handshake_ServerHelloRejected_ReturnsToDisconnected() { var transport = new MockTransport(); - var net = new NetClient(new NullNetLog(), transport, () => 0); + long now = 0; + var net = new NetClient(new NullNetLog(), transport, () => now); net.Connect(Options); transport.RaisePeerConnected(); @@ -98,6 +99,22 @@ public void Handshake_ServerHelloRejected_ReturnsToDisconnected() Assert.Equal(ConnectionStatus.Disconnected, net.Status); Assert.False(net.HandshakeComplete); + Assert.Equal(1, transport.DropPeerCalls); + Assert.False(transport.IsPeerConnected); + + now = NetClient.DefaultReconnectMs - 1; + net.Poll(); + Assert.Equal(1, transport.ConnectCalls); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(2, transport.FreshPeerConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + + transport.RaisePeerConnected(); + Assert.Equal(ConnectionStatus.Handshaking, net.Status); + Assert.Equal(2, transport.Sent.Count); } [Fact] @@ -118,13 +135,19 @@ public void Handshake_AcceptedServerHelloWithWrongProtocol_ReturnsToDisconnected Assert.Equal(ConnectionStatus.Disconnected, net.Status); Assert.False(net.HandshakeComplete); + Assert.Equal(1, transport.DropPeerCalls); + Assert.False(transport.IsPeerConnected); Assert.Contains(log.Warnings, message => message.Contains("protocol")); Assert.DoesNotContain(log.Warnings, message => message.Contains(Options.Token)); now = NetClient.DefaultReconnectMs; net.Poll(); Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(2, transport.FreshPeerConnectCalls); Assert.Equal(ConnectionStatus.Connecting, net.Status); + + transport.RaisePeerConnected(); + Assert.Equal(ConnectionStatus.Handshaking, net.Status); } [Fact] @@ -145,6 +168,8 @@ public void Handshake_AcceptedServerHelloWithoutCapabilities_ReturnsToDisconnect Assert.Equal(ConnectionStatus.Disconnected, net.Status); Assert.False(net.HandshakeComplete); + Assert.Equal(1, transport.DropPeerCalls); + Assert.False(transport.IsPeerConnected); Assert.Equal( "[Sailwind.Online] ServerHello missing capabilities; will retry.", Assert.Single(log.Warnings)); @@ -152,7 +177,11 @@ public void Handshake_AcceptedServerHelloWithoutCapabilities_ReturnsToDisconnect now = NetClient.DefaultReconnectMs; net.Poll(); Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(2, transport.FreshPeerConnectCalls); Assert.Equal(ConnectionStatus.Connecting, net.Status); + + transport.RaisePeerConnected(); + Assert.Equal(ConnectionStatus.Handshaking, net.Status); } [Fact] @@ -200,6 +229,7 @@ public void Disconnect_ReconnectsWithExponentialBackoff() now = 0; transport.RaisePeerDisconnected(); Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.Equal(0, transport.DropPeerCalls); now = NetClient.DefaultReconnectMs - 1; net.Poll(); @@ -306,7 +336,7 @@ public void SendClientState_OnlySendsOnceReady() } [Fact] - public void AcceptedHello_BoatPoseOutbound_ThenRemoteSnapshotInbound_RoundTripsExactly() + public void ReadySession_EncodesOutboundPoseExactly_ThenIndependentlyIngestsRemoteSnapshot() { long now = 4321; var transport = new MockTransport(); From 23919192f9ddd6d2b36a84169b5055bb2bfb39c4 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 17:52:19 -0600 Subject: [PATCH 04/36] fix(net): ignore stale handshake events Co-Authored-By: Claude --- .../LiteNetLibTransport.cs | 84 +++++++++++++---- packages/net/Sailwind.Online.Net/NetClient.cs | 9 +- .../Sailwind.Online.Net.csproj | 4 + .../CurrentPeerSlotTests.cs | 28 ++++++ .../MockTransport.cs | 4 + .../NetClientTests.cs | 89 ++++++++++++++++++- 6 files changed, 198 insertions(+), 20 deletions(-) create mode 100644 tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs diff --git a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs index 3598100..8f5a800 100644 --- a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs +++ b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs @@ -16,8 +16,7 @@ public sealed class LiteNetLibTransport : ITransport { private readonly EventBasedNetListener _listener = new EventBasedNetListener(); private readonly NetManager _manager; - private NetPeer? _peer; - private NetPeer? _locallyDroppedPeer; + private readonly CurrentPeerSlot _peer = new CurrentPeerSlot(); public LiteNetLibTransport() { @@ -41,12 +40,20 @@ public bool IsRunning public bool IsPeerConnected { - get { return _peer != null && _peer.ConnectionState == ConnectionState.Connected; } + get + { + NetPeer? peer = _peer.Value; + return peer != null && peer.ConnectionState == ConnectionState.Connected; + } } public int Ping { - get { return _peer != null ? _peer.Ping : -1; } + get + { + NetPeer? peer = _peer.Value; + return peer != null ? peer.Ping : -1; + } } public event Action? PeerConnected; @@ -61,25 +68,23 @@ public bool Start() public void Connect(string host, int port, string key) { - _peer = _manager.Connect(host, port, key); + _peer.Set(_manager.Connect(host, port, key)); } public void DropPeer() { - NetPeer? peer = _peer; + NetPeer? peer = _peer.Clear(); if (peer == null) { return; } - _locallyDroppedPeer = peer; - _peer = null; _manager.DisconnectPeerForce(peer); } public void Send(byte[] data, DeliveryMethod deliveryMethod) { - _peer?.Send(data, deliveryMethod); + _peer.Value?.Send(data, deliveryMethod); } public void PollEvents() @@ -94,33 +99,37 @@ public void Stop() _manager.Stop(); } - _peer = null; + _peer.Clear(); } private void OnPeerConnected(NetPeer peer) { - _peer = peer; + if (!_peer.IsCurrent(peer)) + { + return; + } + PeerConnected?.Invoke(); } private void OnPeerDisconnected(NetPeer peer, DisconnectInfo info) { - if (ReferenceEquals(_locallyDroppedPeer, peer)) + if (!_peer.TryClear(peer)) { - _locallyDroppedPeer = null; return; } - if (ReferenceEquals(_peer, peer)) - { - _peer = null; - } - PeerDisconnected?.Invoke(info.Reason.ToString()); } private void OnNetworkReceive(NetPeer peer, NetPacketReader reader, byte channelNumber, DeliveryMethod deliveryMethod) { + if (!_peer.IsCurrent(peer)) + { + reader.Recycle(); + return; + } + byte[] data = reader.GetRemainingBytes(); reader.Recycle(); NetworkReceive?.Invoke(data); @@ -131,4 +140,43 @@ private void OnNetworkError(IPEndPoint endPoint, SocketError socketError) NetworkError?.Invoke(endPoint, socketError); } } + + internal sealed class CurrentPeerSlot + where TPeer : class + { + private TPeer? _value; + + public TPeer? Value + { + get { return _value; } + } + + public void Set(TPeer peer) + { + _value = peer; + } + + public TPeer? Clear() + { + TPeer? peer = _value; + _value = null; + return peer; + } + + public bool IsCurrent(TPeer peer) + { + return ReferenceEquals(_value, peer); + } + + public bool TryClear(TPeer peer) + { + if (!IsCurrent(peer)) + { + return false; + } + + _value = null; + return true; + } + } } diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index 94562d1..aa47fae 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -329,6 +329,13 @@ private void OnNetworkError(IPEndPoint endPoint, SocketError socketError) void IServerMessageHandler.OnServerHello(ServerHello hello, uint seq) { + if (_status != ConnectionStatus.Handshaking) + { + _log.LogDebug( + "[Sailwind.Online] Ignored ServerHello (seq " + seq + ") while " + StatusText + "."); + return; + } + if (!hello.Accepted) { RejectHandshake("[Sailwind.Online] ServerHello rejected: " + (hello.Reason ?? "no reason")); @@ -518,10 +525,10 @@ private void ResetPositionObservability() private void RejectHandshake(string warning) { - _transport.DropPeer(); _status = ConnectionStatus.Disconnected; ResetPositionObservability(); ScheduleReconnect(); + _transport.DropPeer(); _log.LogWarning(warning); } diff --git a/packages/net/Sailwind.Online.Net/Sailwind.Online.Net.csproj b/packages/net/Sailwind.Online.Net/Sailwind.Online.Net.csproj index 025b2a5..d7371f0 100644 --- a/packages/net/Sailwind.Online.Net/Sailwind.Online.Net.csproj +++ b/packages/net/Sailwind.Online.Net/Sailwind.Online.Net.csproj @@ -19,6 +19,10 @@ + + + + diff --git a/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs b/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs new file mode 100644 index 0000000..29d93b5 --- /dev/null +++ b/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs @@ -0,0 +1,28 @@ +using Sailwind.Online.Client.Net; +using Xunit; + +namespace Sailwind.Online.Net.Tests +{ + public sealed class CurrentPeerSlotTests + { + [Fact] + public void LocalDropThenReplacement_RejectsOldCallbacksButAllowsCurrentDisconnect() + { + var slot = new CurrentPeerSlot(); + var oldPeer = new object(); + var currentPeer = new object(); + + slot.Set(oldPeer); + Assert.Same(oldPeer, slot.Clear()); + + slot.Set(currentPeer); + Assert.False(slot.IsCurrent(oldPeer)); + Assert.False(slot.TryClear(oldPeer)); + Assert.True(slot.IsCurrent(currentPeer)); + Assert.Same(currentPeer, slot.Value); + + Assert.True(slot.TryClear(currentPeer)); + Assert.Null(slot.Value); + } + } +} diff --git a/tests/Sailwind.Online.Net.Tests/MockTransport.cs b/tests/Sailwind.Online.Net.Tests/MockTransport.cs index 113445b..f49e92d 100644 --- a/tests/Sailwind.Online.Net.Tests/MockTransport.cs +++ b/tests/Sailwind.Online.Net.Tests/MockTransport.cs @@ -25,6 +25,9 @@ internal sealed class MockTransport : ITransport public int DropPeerCalls; public int PollCalls; + /// Optional callback invoked during to model queued callbacks. + public Action DropPeerCallback; + public string LastHost; public int LastPort; public string LastKey; @@ -75,6 +78,7 @@ public void DropPeer() DropPeerCalls++; _peerConnected = false; Ping = -1; + DropPeerCallback?.Invoke(); } public void Send(byte[] data, DeliveryMethod deliveryMethod) diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index 60b51cd..82e5596 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -117,6 +117,92 @@ public void Handshake_ServerHelloRejected_ReturnsToDisconnected() Assert.Equal(2, transport.Sent.Count); } + [Fact] + public void Handshake_RejectedThenQueuedAcceptedHello_IgnoresStaleAcceptance() + { + var transport = new MockTransport(); + var log = new RecordingLog(); + long now = 0; + var net = new NetClient(log, transport, () => now); + net.Connect(Options); + transport.RaisePeerConnected(); + transport.DropPeerCallback = () => transport.RaiseNetworkReceive( + ServerHelloEnvelope(accepted: true, playerId: 99, snapshotHz: 12)); + + transport.RaiseNetworkReceive(ServerHelloEnvelope(accepted: false, playerId: 0, snapshotHz: 0)); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.False(net.HandshakeComplete); + Assert.Equal(0ul, net.PlayerId); + Assert.Equal((byte)4, net.SnapshotHz); + Assert.Equal(1, transport.DropPeerCalls); + Assert.Single(log.Warnings); + Assert.Single(log.Debugs); + + now = NetClient.DefaultReconnectMs - 1; + net.Poll(); + Assert.Equal(1, transport.ConnectCalls); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + + transport.RaiseNetworkReceive(ServerHelloEnvelope(accepted: true, playerId: 101, snapshotHz: 16)); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + Assert.Equal(0ul, net.PlayerId); + Assert.Equal((byte)4, net.SnapshotHz); + Assert.Equal(1, transport.DropPeerCalls); + Assert.Equal(2, log.Debugs.Count); + } + + [Fact] + public void Handshake_MultipleQueuedRejections_OnlyFirstChangesReconnectState() + { + var transport = new MockTransport(); + var log = new RecordingLog(); + long now = 0; + var net = new NetClient(log, transport, () => now); + net.Connect(Options); + transport.RaisePeerConnected(); + + byte[] rejection = ServerHelloEnvelope(accepted: false, playerId: 0, snapshotHz: 0); + transport.RaiseNetworkReceive(rejection); + transport.RaiseNetworkReceive(rejection); + transport.RaiseNetworkReceive(rejection); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.Equal(1, transport.DropPeerCalls); + Assert.Single(log.Warnings); + Assert.Equal(2, log.Debugs.Count); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + } + + [Fact] + public void ReadySession_DuplicateAcceptedHello_DoesNotReplaceIdentityOrCapabilities() + { + var transport = new MockTransport(); + var log = new RecordingLog(); + var net = new NetClient(log, transport, () => 0); + net.Connect(Options); + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope(accepted: true, playerId: 77, snapshotHz: 8)); + + transport.RaiseNetworkReceive(ServerHelloEnvelope(accepted: true, playerId: 99, snapshotHz: 12)); + + Assert.Equal(ConnectionStatus.Ready, net.Status); + Assert.True(net.HandshakeComplete); + Assert.Equal(77ul, net.PlayerId); + Assert.Equal((byte)8, net.SnapshotHz); + Assert.Equal(0, transport.DropPeerCalls); + Assert.Empty(log.Warnings); + Assert.Single(log.Debugs); + } + [Fact] public void Handshake_AcceptedServerHelloWithWrongProtocol_ReturnsToDisconnected() { @@ -601,11 +687,12 @@ public void LogError(string message) { } private sealed class RecordingLog : INetLog { + public readonly List Debugs = new List(); public readonly List Errors = new List(); public readonly List Infos = new List(); public readonly List Warnings = new List(); - public void LogDebug(string message) { } + public void LogDebug(string message) { Debugs.Add(message); } public void LogInfo(string message) { Infos.Add(message); } public void LogWarning(string message) { Warnings.Add(message); } public void LogError(string message) { Errors.Add(message); } From 43772906dd4613546804d00df93be5c1a80be10a Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 19:57:59 -0600 Subject: [PATCH 05/36] fix(server): throttle duplicate client hello Co-Authored-By: Claude --- server/config.example.toml | 6 + server/crates/sw-server/src/config.rs | 41 +++++- server/crates/sw-server/src/server.rs | 181 +++++++++++++++++++++++++- 3 files changed, 220 insertions(+), 8 deletions(-) diff --git a/server/config.example.toml b/server/config.example.toml index 9885116..f4ac691 100644 --- a/server/config.example.toml +++ b/server/config.example.toml @@ -36,6 +36,12 @@ cell_size_m = 1024.0 # throttled. Must be in 0..=3600000 (0 disables the throttle). trade_min_interval_ms = 250 +# Per-peer ClientHello throttle in milliseconds. This runs before protocol/hash +# validation, database access, and ServerHello generation. Keep it at or below +# the client's 250 ms retry cadence so a lost ServerHello can be retried on the +# next attempt. Must be in 0..=3600000 (0 disables the throttle). +hello_min_interval_ms = 250 + # Per-player throttles (milliseconds) for the other inbound message classes, # mirroring the rotation-proof, memory-bounded trade limiter above. A message # beyond the configured rate is dropped before it drives work; an idempotent diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index e5e286a..fe2f38d 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -23,8 +23,8 @@ const MAX_CELL_SIZE_M: f32 = 1_000_000.0; /// numerator, so the divisor needs its own floor here. const MIN_CELL_SIZE_M: f32 = 1.0; -/// Upper bound on any aggregate per-player message-class min-interval, in -/// milliseconds (market trade, client-state, chat, econ, moor). Bounds the +/// Upper bound on any aggregate peer/player message-class min-interval, in +/// milliseconds (hello, market trade, client-state, chat, econ, moor). Bounds the /// rate-limit knobs so a misconfiguration cannot wedge a message class behind an /// absurd cooldown, and so the saturating accessors have a finite ceiling. One /// hour is already far beyond any sane throttle. @@ -56,6 +56,12 @@ pub struct Config { pub aoi_radius_cells: u32, /// Grid cell size, in metres (advertised to clients). pub cell_size_m: f32, + /// Minimum interval, in milliseconds, between two processed `ClientHello` + /// messages from the same peer. A flood beyond this rate is dropped before + /// validation, persistence, or response generation. The 250 ms default + /// matches the client's handshake retry cadence. Bounded by + /// [`MAX_TRADE_MIN_INTERVAL_MS`]; 0 disables the throttle. + pub hello_min_interval_ms: u32, /// Minimum interval, in milliseconds, between two accepted market trades by /// the same player (an aggregate per-player throttle, independent of which /// port the request names). A new trade inside this window is rejected; an @@ -100,6 +106,7 @@ impl Default for Config { clock_broadcast_secs: 10, aoi_radius_cells: sw_world::AOI_RADIUS_CELLS as u32, cell_size_m: sw_world::Grid::DEFAULT_CELL_SIZE_M, + hello_min_interval_ms: 250, trade_min_interval_ms: 250, client_state_min_interval_ms: 20, chat_min_interval_ms: 500, @@ -192,6 +199,7 @@ impl Config { )); } for (name, value) in [ + ("hello_min_interval_ms", self.hello_min_interval_ms), ("trade_min_interval_ms", self.trade_min_interval_ms), ( "client_state_min_interval_ms", @@ -230,6 +238,13 @@ impl Config { self.trade_min_interval_ms.min(MAX_TRADE_MIN_INTERVAL_MS) as i64 } + /// Client-hello throttle min-interval as a bounded `i64` of milliseconds. + /// Saturates at [`MAX_TRADE_MIN_INTERVAL_MS`] so the limiter math stays + /// finite even if a caller bypasses [`Config::validate`]. + pub fn hello_min_interval_ms_i64(&self) -> i64 { + self.hello_min_interval_ms.min(MAX_TRADE_MIN_INTERVAL_MS) as i64 + } + /// Client-state throttle min-interval as a bounded `i64` of milliseconds. /// Saturates at [`MAX_TRADE_MIN_INTERVAL_MS`] so the limiter math stays /// finite even if a caller bypasses [`Config::validate`]. @@ -480,11 +495,23 @@ mod tests { cfg.validate().unwrap(); } + #[test] + fn parses_hello_interval_key() { + let toml_text = r#" + hello_min_interval_ms = 500 + "#; + let cfg: Config = toml::from_str(toml_text).unwrap(); + assert_eq!(cfg.hello_min_interval_ms, 500); + cfg.validate().unwrap(); + } + #[test] fn message_rate_limit_defaults_are_valid_and_bounded() { let cfg = Config::default(); cfg.validate().unwrap(); // A permissive-but-finite default for each per-class throttle. + assert_eq!(cfg.hello_min_interval_ms, 250); + assert!(cfg.hello_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); assert!(cfg.client_state_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); assert!(cfg.chat_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); assert!(cfg.econ_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); @@ -494,6 +521,10 @@ mod tests { let snapshot_period_ms = 1000 / cfg.snapshot_hz; assert!(cfg.client_state_min_interval_ms < snapshot_period_ms); // The bounded accessors mirror the config values for sane defaults. + assert_eq!( + cfg.hello_min_interval_ms_i64(), + cfg.hello_min_interval_ms as i64 + ); assert_eq!( cfg.client_state_min_interval_ms_i64(), cfg.client_state_min_interval_ms as i64 @@ -515,6 +546,7 @@ mod tests { #[test] fn rejects_out_of_range_message_intervals() { for mutate in [ + |c: &mut Config| c.hello_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, |c: &mut Config| c.client_state_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, |c: &mut Config| c.chat_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, |c: &mut Config| c.econ_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, @@ -532,12 +564,17 @@ mod tests { #[test] fn message_interval_accessors_saturate_at_the_bound() { let cfg = Config { + hello_min_interval_ms: u32::MAX, client_state_min_interval_ms: u32::MAX, chat_min_interval_ms: u32::MAX, econ_min_interval_ms: u32::MAX, moor_min_interval_ms: u32::MAX, ..Config::default() }; + assert_eq!( + cfg.hello_min_interval_ms_i64(), + MAX_TRADE_MIN_INTERVAL_MS as i64 + ); assert_eq!( cfg.client_state_min_interval_ms_i64(), MAX_TRADE_MIN_INTERVAL_MS as i64 diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 1ae8cf1..0e26014 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -60,6 +60,7 @@ pub struct Server { epoch_ms: i64, weather_seed: u64, weather_epoch_day: u32, + hello_limiter: RateLimiter, trade_limiter: RateLimiter, client_state_limiter: RateLimiter, chat_limiter: RateLimiter, @@ -89,6 +90,7 @@ impl Server { let host = Host::bind(&cfg.bind, CONNECT_KEY)?; let world = World::new(sw_world::Grid::new(cfg.cell_size_m)); + let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -107,6 +109,7 @@ impl Server { epoch_ms, weather_seed, weather_epoch_day, + hello_limiter, trade_limiter, client_state_limiter, chat_limiter, @@ -221,6 +224,19 @@ impl Server { } fn on_hello(&mut self, peer: PeerId, hello: p::ClientHello<'_>) -> anyhow::Result<()> { + self.on_hello_at(peer, hello, now_ms()) + } + + fn on_hello_at( + &mut self, + peer: PeerId, + hello: p::ClientHello<'_>, + now_ms: i64, + ) -> anyhow::Result<()> { + if !self.hello_limiter.allow(u64::from(peer), now_ms) { + return Ok(()); + } + if hello.protocol_version() != sw_contracts::PROTOCOL_VERSION { let reason = format!( "protocol version mismatch: client {}, server {}", @@ -272,8 +288,9 @@ impl Server { return Ok(()); } - let now = now_ms(); - let player = self.db.upsert_player_by_token(&identity_hash, &name, now)?; + let player = self + .db + .upsert_player_by_token(&identity_hash, &name, now_ms)?; let player_id = player.id as u64; // Drop any prior session for this identity (reconnect from a new peer). @@ -663,6 +680,7 @@ impl Server { } fn on_disconnect(&mut self, peer: PeerId, reason: DisconnectReason) -> anyhow::Result<()> { + self.hello_limiter.clear(u64::from(peer)); if let Some(s) = self.sessions.remove(&peer) { self.world.remove(s.player_id); // Drop the player's throttle state across every message class: a @@ -917,6 +935,7 @@ mod handshake_tests { epoch_ms: 0, weather_seed: 0, weather_epoch_day: 0, + hello_limiter: RateLimiter::new(cfg.hello_min_interval_ms_i64()), trade_limiter: RateLimiter::new(cfg.trade_min_interval_ms_i64()), client_state_limiter: RateLimiter::new(cfg.client_state_min_interval_ms_i64()), chat_limiter: RateLimiter::new(cfg.chat_min_interval_ms_i64()), @@ -974,6 +993,13 @@ mod handshake_tests { .unwrap(); } + fn deliver_hello_at(server: &mut Server, peer: PeerId, bytes: &[u8], now_ms: i64) { + let env = decode_envelope(bytes).unwrap(); + server + .on_hello_at(peer, env.payload_as_client_hello().unwrap(), now_ms) + .unwrap(); + } + fn connect_peer(server: &mut Server) -> (UdpSocket, PeerId) { let client = UdpSocket::bind("127.0.0.1:0").unwrap(); client.connect(server.host.local_addr().unwrap()).unwrap(); @@ -1008,6 +1034,21 @@ mod handshake_tests { } } + fn assert_no_outbound_datagram(client: &UdpSocket) { + client + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let mut packet = [0u8; protocol::MTU]; + let error = client.recv(&mut packet).unwrap_err(); + assert!( + matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ), + "expected no outbound datagram, got {error}" + ); + } + #[test] fn protocol_mismatch_is_rejected_before_session_creation() { let mut server = make_server(); @@ -1078,12 +1119,13 @@ mod handshake_tests { fn repeated_valid_hello_preserves_established_session_and_only_resends_server_hello() { let mut server = make_server(); let (client, peer) = connect_peer(&mut server); + let hello_interval_ms = server.cfg.hello_min_interval_ms_i64(); let hello = hello_envelope( "handshake-token", sw_contracts::PROTOCOL_VERSION, Some("surface-hash"), ); - deliver_hello(&mut server, peer, &hello); + deliver_hello_at(&mut server, peer, &hello, 1_000); assert_eq!(receive_server_hello(&client), (true, String::new())); let state = state_envelope(); @@ -1103,7 +1145,7 @@ mod handshake_tests { let world_cell = server.world.cell_of_entity(player_id); let seq_before_retry = server.seq; - deliver_hello(&mut server, peer, &hello); + deliver_hello_at(&mut server, peer, &hello, 1_000 + hello_interval_ms); let retried = &server.sessions[&peer]; assert_eq!(retried.player_id, player_id); @@ -1120,16 +1162,134 @@ mod handshake_tests { assert_eq!(receive_server_hello(&client), (true, String::new())); } + #[test] + fn duplicate_hello_burst_is_dropped_before_response_or_session_work() { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let hello_interval_ms = server.cfg.hello_min_interval_ms_i64(); + let hello = hello_envelope( + "handshake-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, peer, &hello, 1_000); + assert_eq!(receive_server_hello(&client), (true, String::new())); + + let state = state_envelope(); + let env = decode_envelope(&state).unwrap(); + server.on_client_state(peer, env.payload_as_client_state().unwrap(), 1_000); + + deliver_hello_at(&mut server, peer, &hello, 1_000 + hello_interval_ms); + assert_eq!(receive_server_hello(&client), (true, String::new())); + + let established = &server.sessions[&peer]; + let player_id = established.player_id; + let identity_hash = established.identity_hash.clone(); + let display_name = established.display_name.clone(); + let aboard_boat = established.aboard_boat; + let pos = established.pos; + let rot = established.rot; + let vel = established.vel; + let t_ms = established.t_ms; + let cell = established.cell; + let subscribed_cells = established.sub.cells().clone(); + let world_len = server.world.len(); + let world_cell = server.world.cell_of_entity(player_id); + let seq_after_allowed_retry = server.seq; + + for _ in 0..1_000 { + deliver_hello_at(&mut server, peer, &hello, 1_000 + hello_interval_ms + 1); + } + + let after_flood = &server.sessions[&peer]; + assert_eq!(after_flood.player_id, player_id); + assert_eq!(after_flood.identity_hash, identity_hash); + assert_eq!(after_flood.display_name, display_name); + assert_eq!(after_flood.aboard_boat, aboard_boat); + assert_eq!(after_flood.pos, pos); + assert_eq!(after_flood.rot, rot); + assert_eq!(after_flood.vel, vel); + assert_eq!(after_flood.t_ms, t_ms); + assert_eq!(after_flood.cell, cell); + assert_eq!(after_flood.sub.cells(), &subscribed_cells); + assert_eq!(server.sessions.len(), 1); + assert_eq!(server.world.len(), world_len); + assert_eq!(server.world.cell_of_entity(player_id), world_cell); + assert_eq!(server.seq, seq_after_allowed_retry); + assert_eq!(server.hello_limiter.tracked_count(), 1); + assert_no_outbound_datagram(&client); + } + + #[test] + fn hello_limit_is_independent_per_peer() { + let mut server = make_server(); + let first = hello_envelope( + "first-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let second = hello_envelope( + "second-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + + deliver_hello_at(&mut server, 1, &first, 1_000); + let seq_after_first = server.seq; + deliver_hello_at(&mut server, 1, &first, 1_001); + assert_eq!(server.seq, seq_after_first); + + deliver_hello_at(&mut server, 2, &second, 1_001); + assert_ne!(server.seq, seq_after_first); + assert!(server.sessions.contains_key(&1)); + assert!(server.sessions.contains_key(&2)); + assert_eq!(server.hello_limiter.tracked_count(), 2); + } + + #[test] + fn disconnect_clears_hello_limit_for_immediate_peer_id_reuse() { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let first = hello_envelope( + "first-token", + sw_contracts::PROTOCOL_VERSION + 1, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, peer, &first, 1_000); + assert!(!receive_server_hello(&client).0); + assert!(!server.sessions.contains_key(&peer)); + assert_eq!(server.hello_limiter.tracked_count(), 1); + + server + .on_disconnect(peer, DisconnectReason::Remote) + .unwrap(); + assert_eq!(server.hello_limiter.tracked_count(), 0); + let seq_before_reuse = server.seq; + + let reused = hello_envelope( + "reused-peer-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, peer, &reused, 1_001); + + assert!(server.sessions.contains_key(&peer)); + assert_ne!(server.seq, seq_before_reuse); + assert_eq!(server.hello_limiter.tracked_count(), 1); + assert_eq!(receive_server_hello(&client), (true, String::new())); + } + #[test] fn same_peer_cannot_replace_an_established_session_with_a_different_identity() { let mut server = make_server(); let (client, peer) = connect_peer(&mut server); + let hello_interval_ms = server.cfg.hello_min_interval_ms_i64(); let first = hello_envelope( "first-token", sw_contracts::PROTOCOL_VERSION, Some("surface-hash"), ); - deliver_hello(&mut server, peer, &first); + deliver_hello_at(&mut server, peer, &first, 1_000); assert_eq!(receive_server_hello(&client), (true, String::new())); let player_id = server.sessions[&peer].player_id; @@ -1143,7 +1303,7 @@ mod handshake_tests { Some("surface-hash"), ); - deliver_hello(&mut server, peer, &replacement); + deliver_hello_at(&mut server, peer, &replacement, 1_000 + hello_interval_ms); assert_eq!(server.sessions.len(), 1); assert_eq!(server.sessions[&peer].player_id, player_id); @@ -1175,6 +1335,7 @@ mod aoi_harden_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); + let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -1191,6 +1352,7 @@ mod aoi_harden_tests { epoch_ms: 0, weather_seed: 0, weather_epoch_day: 0, + hello_limiter, trade_limiter, client_state_limiter, chat_limiter, @@ -1371,6 +1533,7 @@ mod market_dispatch_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); + let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -1387,6 +1550,7 @@ mod market_dispatch_tests { epoch_ms: 0, weather_seed: 0, weather_epoch_day: 0, + hello_limiter, trade_limiter, client_state_limiter, chat_limiter, @@ -1595,6 +1759,7 @@ mod input_hardening_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); + let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -1611,6 +1776,7 @@ mod input_hardening_tests { epoch_ms: 0, weather_seed: 0, weather_epoch_day: 0, + hello_limiter, trade_limiter, client_state_limiter, chat_limiter, @@ -2004,6 +2170,7 @@ mod input_hardening_tests { #[test] fn disconnect_clears_every_message_class_limiter() { let mut server = make_server(Config { + hello_min_interval_ms: 250, client_state_min_interval_ms: 250, chat_min_interval_ms: 250, econ_min_interval_ms: 250, @@ -2027,6 +2194,7 @@ mod input_hardening_tests { // covered in the market dispatch suite); the point here is that // on_disconnect clears every class. server.trade_limiter.allow(pid, 1_000); + assert_eq!(server.hello_limiter.tracked_count(), 1); assert_eq!(server.client_state_limiter.tracked_count(), 1); assert_eq!(server.econ_limiter.tracked_count(), 1); assert_eq!(server.chat_limiter.tracked_count(), 1); @@ -2037,6 +2205,7 @@ mod input_hardening_tests { .on_disconnect(peer, DisconnectReason::Remote) .unwrap(); + assert_eq!(server.hello_limiter.tracked_count(), 0); assert_eq!(server.client_state_limiter.tracked_count(), 0); assert_eq!(server.econ_limiter.tracked_count(), 0); assert_eq!(server.chat_limiter.tracked_count(), 0); From f092a1dff128e1525d5da844c9d508a8c0eed1da Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 20:27:53 -0600 Subject: [PATCH 06/36] fix(net): preserve client reconnect liveness Co-Authored-By: Claude --- .../LiteNetLibTransport.cs | 9 +- packages/net/Sailwind.Online.Net/NetClient.cs | 6 +- .../CurrentPeerSlotTests.cs | 17 +++- .../NetClientTests.cs | 83 +++++++++++++++++++ 4 files changed, 110 insertions(+), 5 deletions(-) diff --git a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs index 8f5a800..6fb11a9 100644 --- a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs +++ b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs @@ -68,7 +68,7 @@ public bool Start() public void Connect(string host, int port, string key) { - _peer.Set(_manager.Connect(host, port, key)); + _peer.SetIfPresent(_manager.Connect(host, port, key)); } public void DropPeer() @@ -151,8 +151,13 @@ public TPeer? Value get { return _value; } } - public void Set(TPeer peer) + public void SetIfPresent(TPeer peer) { + if (peer == null) + { + return; + } + _value = peer; } diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index aa47fae..5ceacc3 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -162,6 +162,11 @@ public string StatusText /// Begin (or restart) a session with the given options. public void Connect(ConnectOptions options) { + if (_status == ConnectionStatus.Connecting && ReferenceEquals(_options, options)) + { + return; + } + _options = options; if (!_transport.IsRunning && !_transport.Start()) @@ -300,7 +305,6 @@ private bool SendRaw(byte[] bytes) private void OnPeerConnected() { _status = ConnectionStatus.Handshaking; - _reconnectBackoffMs = DefaultReconnectMs; SendHello(); _log.LogInfo("[Sailwind.Online] Transport up; sending ClientHello."); } diff --git a/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs b/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs index 29d93b5..854846c 100644 --- a/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs +++ b/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs @@ -5,6 +5,19 @@ namespace Sailwind.Online.Net.Tests { public sealed class CurrentPeerSlotTests { + [Fact] + public void PendingPeer_NullReplacementAttempt_PreservesCurrentPeer() + { + var slot = new CurrentPeerSlot(); + var pendingPeer = new object(); + + slot.SetIfPresent(pendingPeer); + slot.SetIfPresent(null); + + Assert.Same(pendingPeer, slot.Value); + Assert.True(slot.IsCurrent(pendingPeer)); + } + [Fact] public void LocalDropThenReplacement_RejectsOldCallbacksButAllowsCurrentDisconnect() { @@ -12,10 +25,10 @@ public void LocalDropThenReplacement_RejectsOldCallbacksButAllowsCurrentDisconne var oldPeer = new object(); var currentPeer = new object(); - slot.Set(oldPeer); + slot.SetIfPresent(oldPeer); Assert.Same(oldPeer, slot.Clear()); - slot.Set(currentPeer); + slot.SetIfPresent(currentPeer); Assert.False(slot.IsCurrent(oldPeer)); Assert.False(slot.TryClear(oldPeer)); Assert.True(slot.IsCurrent(currentPeer)); diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index 82e5596..2a7e9a7 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -47,6 +47,28 @@ public void Connect_StartsTransportAndOpensPeer_EntersConnecting() Assert.Equal(ConnectionStatus.Connecting, net.Status); } + [Fact] + public void Connect_SameOptionsWhilePending_DoesNotOpenDuplicatePeer() + { + var transport = new MockTransport(); + var net = new NetClient(new NullNetLog(), transport, () => 0); + + net.Connect(Options); + net.Connect(Options); + + Assert.Equal(1, transport.StartCalls); + Assert.Equal(1, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope( + accepted: true, + playerId: 77, + snapshotHz: 8)); + + Assert.Equal(ConnectionStatus.Ready, net.Status); + } + [Fact] public void Connect_WhenTransportStartFails_LogsAndDoesNotOpenPeer() { @@ -117,6 +139,67 @@ public void Handshake_ServerHelloRejected_ReturnsToDisconnected() Assert.Equal(2, transport.Sent.Count); } + [Fact] + public void Handshake_ConsecutiveRejections_BackOffUntilAcceptedHandshakeResets() + { + var transport = new MockTransport(); + long now = 0; + var net = new NetClient(new NullNetLog(), transport, () => now); + net.Connect(Options); + + long[] rejectionGaps = + { + NetClient.DefaultReconnectMs, + 2 * NetClient.DefaultReconnectMs, + 4 * NetClient.DefaultReconnectMs, + NetClient.MaxReconnectMs, + NetClient.MaxReconnectMs + }; + int expectedConnectCalls = 1; + + for (int i = 0; i < rejectionGaps.Length; i++) + { + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope( + accepted: false, + playerId: 0, + snapshotHz: 0)); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.Equal(i + 1, transport.DropPeerCalls); + + now += rejectionGaps[i] - 1; + net.Poll(); + Assert.Equal(expectedConnectCalls, transport.ConnectCalls); + + now++; + net.Poll(); + expectedConnectCalls++; + Assert.Equal(expectedConnectCalls, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + } + + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope( + accepted: true, + playerId: 77, + snapshotHz: 8)); + + Assert.Equal(ConnectionStatus.Ready, net.Status); + Assert.Equal(rejectionGaps.Length, transport.DropPeerCalls); + + transport.RaisePeerDisconnected(); + now += NetClient.DefaultReconnectMs - 1; + net.Poll(); + Assert.Equal(expectedConnectCalls, transport.ConnectCalls); + + now++; + net.Poll(); + expectedConnectCalls++; + Assert.Equal(expectedConnectCalls, transport.ConnectCalls); + Assert.Equal(rejectionGaps.Length, transport.DropPeerCalls); + } + [Fact] public void Handshake_RejectedThenQueuedAcceptedHello_IgnoresStaleAcceptance() { From c955030f1b06e3dee59323749fbce846e8129c23 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 20:40:21 -0600 Subject: [PATCH 07/36] fix(server): bound hello admission work Co-Authored-By: Claude --- server/config.example.toml | 3 +- server/crates/sw-server/src/config.rs | 75 ++++++++++--- server/crates/sw-server/src/ratelimit.rs | 131 +++++++++++------------ server/crates/sw-server/src/server.rs | 24 ++++- 4 files changed, 152 insertions(+), 81 deletions(-) diff --git a/server/config.example.toml b/server/config.example.toml index f4ac691..cf10fb8 100644 --- a/server/config.example.toml +++ b/server/config.example.toml @@ -39,7 +39,8 @@ trade_min_interval_ms = 250 # Per-peer ClientHello throttle in milliseconds. This runs before protocol/hash # validation, database access, and ServerHello generation. Keep it at or below # the client's 250 ms retry cadence so a lost ServerHello can be retried on the -# next attempt. Must be in 0..=3600000 (0 disables the throttle). +# next attempt. Must be in 1..=250; zero would disable pre-authentication flood +# protection. hello_min_interval_ms = 250 # Per-player throttles (milliseconds) for the other inbound message classes, diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index fe2f38d..1928eef 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -23,13 +23,23 @@ const MAX_CELL_SIZE_M: f32 = 1_000_000.0; /// numerator, so the divisor needs its own floor here. const MIN_CELL_SIZE_M: f32 = 1.0; -/// Upper bound on any aggregate peer/player message-class min-interval, in -/// milliseconds (hello, market trade, client-state, chat, econ, moor). Bounds the +/// Upper bound on any aggregate player message-class min-interval, in +/// milliseconds (market trade, client-state, chat, econ, moor). Bounds the /// rate-limit knobs so a misconfiguration cannot wedge a message class behind an /// absurd cooldown, and so the saturating accessors have a finite ceiling. One /// hour is already far beyond any sane throttle. pub const MAX_TRADE_MIN_INTERVAL_MS: u32 = 3_600_000; +/// Upper bound on the pre-authentication hello throttle. It matches the +/// client's fixed retry interval, so the first retry after a lost response is +/// eligible for admission. +pub const MAX_HELLO_MIN_INTERVAL_MS: u32 = 250; + +/// Lower bound on the pre-authentication hello throttle. Zero would disable the +/// limiter and expose validation, persistence, and response generation to an +/// unthrottled flood. +const MIN_HELLO_MIN_INTERVAL_MS: u32 = 1; + /// Upper bound on the per-message wire-string length cap, in bytes. Bounds the /// [`Config::max_wire_string_len`] knob so a misconfiguration cannot admit an /// unbounded string, and so the saturating accessor has a finite ceiling. The @@ -59,8 +69,8 @@ pub struct Config { /// Minimum interval, in milliseconds, between two processed `ClientHello` /// messages from the same peer. A flood beyond this rate is dropped before /// validation, persistence, or response generation. The 250 ms default - /// matches the client's handshake retry cadence. Bounded by - /// [`MAX_TRADE_MIN_INTERVAL_MS`]; 0 disables the throttle. + /// matches the client's handshake retry cadence. Bounded to + /// `1..=`[`MAX_HELLO_MIN_INTERVAL_MS`]. pub hello_min_interval_ms: u32, /// Minimum interval, in milliseconds, between two accepted market trades by /// the same player (an aggregate per-player throttle, independent of which @@ -198,8 +208,14 @@ impl Config { "cell_size_m must be a finite value in [{MIN_CELL_SIZE_M}, {MAX_CELL_SIZE_M}]" )); } + if !(MIN_HELLO_MIN_INTERVAL_MS..=MAX_HELLO_MIN_INTERVAL_MS) + .contains(&self.hello_min_interval_ms) + { + return Err(anyhow::anyhow!( + "hello_min_interval_ms must be in {MIN_HELLO_MIN_INTERVAL_MS}..={MAX_HELLO_MIN_INTERVAL_MS}" + )); + } for (name, value) in [ - ("hello_min_interval_ms", self.hello_min_interval_ms), ("trade_min_interval_ms", self.trade_min_interval_ms), ( "client_state_min_interval_ms", @@ -239,10 +255,11 @@ impl Config { } /// Client-hello throttle min-interval as a bounded `i64` of milliseconds. - /// Saturates at [`MAX_TRADE_MIN_INTERVAL_MS`] so the limiter math stays - /// finite even if a caller bypasses [`Config::validate`]. + /// Clamps to the security- and liveness-safe hello-specific range even if a + /// caller bypasses [`Config::validate`]. pub fn hello_min_interval_ms_i64(&self) -> i64 { - self.hello_min_interval_ms.min(MAX_TRADE_MIN_INTERVAL_MS) as i64 + self.hello_min_interval_ms + .clamp(MIN_HELLO_MIN_INTERVAL_MS, MAX_HELLO_MIN_INTERVAL_MS) as i64 } /// Client-state throttle min-interval as a bounded `i64` of milliseconds. @@ -498,20 +515,53 @@ mod tests { #[test] fn parses_hello_interval_key() { let toml_text = r#" - hello_min_interval_ms = 500 + hello_min_interval_ms = 249 "#; let cfg: Config = toml::from_str(toml_text).unwrap(); - assert_eq!(cfg.hello_min_interval_ms, 500); + assert_eq!(cfg.hello_min_interval_ms, 249); cfg.validate().unwrap(); } + #[test] + fn hello_interval_enforces_security_and_client_retry_liveness() { + for value in [0, MAX_HELLO_MIN_INTERVAL_MS + 1, 3_600_000] { + let cfg = Config { + hello_min_interval_ms: value, + ..Config::default() + }; + assert!( + cfg.validate().is_err(), + "hello interval {value} must be rejected" + ); + } + + let disabled = Config { + hello_min_interval_ms: 0, + ..Config::default() + }; + assert_eq!( + disabled.hello_min_interval_ms_i64(), + i64::from(MIN_HELLO_MIN_INTERVAL_MS) + ); + + let boundary = Config { + hello_min_interval_ms: MAX_HELLO_MIN_INTERVAL_MS, + ..Config::default() + }; + boundary.validate().unwrap(); + assert_eq!( + boundary.hello_min_interval_ms_i64(), + i64::from(MAX_HELLO_MIN_INTERVAL_MS) + ); + } + #[test] fn message_rate_limit_defaults_are_valid_and_bounded() { let cfg = Config::default(); cfg.validate().unwrap(); // A permissive-but-finite default for each per-class throttle. assert_eq!(cfg.hello_min_interval_ms, 250); - assert!(cfg.hello_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); + assert!(cfg.hello_min_interval_ms <= MAX_HELLO_MIN_INTERVAL_MS); assert!(cfg.client_state_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); assert!(cfg.chat_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); assert!(cfg.econ_min_interval_ms <= MAX_TRADE_MIN_INTERVAL_MS); @@ -546,7 +596,6 @@ mod tests { #[test] fn rejects_out_of_range_message_intervals() { for mutate in [ - |c: &mut Config| c.hello_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, |c: &mut Config| c.client_state_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, |c: &mut Config| c.chat_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, |c: &mut Config| c.econ_min_interval_ms = MAX_TRADE_MIN_INTERVAL_MS + 1, @@ -573,7 +622,7 @@ mod tests { }; assert_eq!( cfg.hello_min_interval_ms_i64(), - MAX_TRADE_MIN_INTERVAL_MS as i64 + MAX_HELLO_MIN_INTERVAL_MS as i64 ); assert_eq!( cfg.client_state_min_interval_ms_i64(), diff --git a/server/crates/sw-server/src/ratelimit.rs b/server/crates/sw-server/src/ratelimit.rs index cfc103d..321971a 100644 --- a/server/crates/sw-server/src/ratelimit.rs +++ b/server/crates/sw-server/src/ratelimit.rs @@ -1,30 +1,23 @@ -//! Aggregate per-player market-trade rate limiter. +//! Aggregate per-session-key rate limiter. //! -//! A minimum-interval throttle keyed by **player**: a given player may have at -//! most one *new* accepted trade within [`RateLimiter::min_interval_ms`], -//! regardless of which `port_id` the request names. Keying by player (never by -//! an attacker-chosen port) makes the throttle rotation-proof — a client cannot -//! escape the bound by sending a fresh `port_id` on every message — and bounds -//! the map to players active within the last window rather than to the 2^32 -//! possible port ids. +//! A minimum-interval throttle keyed by a connected peer or authenticated +//! player. Callers choose a lifecycle-bounded key, then call +//! [`RateLimiter::clear`] when that peer or player disconnects. This keeps the +//! map bounded to active/session keys without scanning all entries during +//! admission. //! -//! It is deliberately separate from the idempotency dedup in [`sw_econ::Market`] -//! — the caller checks dedup first, so an idempotent replay of an already-applied +//! It is deliberately separate from the idempotency dedup in [`sw_econ::Market`]: +//! the caller checks dedup first, so an idempotent replay of an already-applied //! trade is never throttled; only genuinely new trades are. //! -//! Memory stays bounded two ways: every [`RateLimiter::allow`] first evicts -//! entries whose window has fully elapsed (a stale entry can never throttle -//! anything, so it is pure waste), and [`RateLimiter::clear`] drops a player's -//! entry when they disconnect. -//! //! The interval is bounded/saturating at construction time (see //! [`crate::config::Config::trade_min_interval_ms_i64`]), so the window math //! never overflows. -use std::collections::HashMap; +use std::collections::{hash_map::Entry, HashMap}; -/// Tracks the last accepted trade time per player and admits a new trade only -/// once the configured interval has elapsed, pruning stale entries as it goes. +/// Tracks the last accepted message time per lifecycle-bounded key and admits a +/// new message only once the configured interval has elapsed. #[derive(Debug)] pub struct RateLimiter { min_interval_ms: i64, @@ -32,7 +25,7 @@ pub struct RateLimiter { } impl RateLimiter { - /// Build a limiter with the given aggregate per-player minimum interval in + /// Build a limiter with the given aggregate per-key minimum interval in /// milliseconds. A non-positive interval disables throttling. pub fn new(min_interval_ms: i64) -> RateLimiter { RateLimiter { @@ -41,37 +34,36 @@ impl RateLimiter { } } - /// Try to admit a new trade by `player` at time `now_ms`. + /// Try to admit a new message by `key` at time `now_ms`. /// - /// Returns `true` and records `now_ms` when the trade is allowed; returns + /// Returns `true` and records `now_ms` when the message is allowed; returns /// `false` without recording when it falls inside the throttle window, so a - /// rejected attempt never extends the cooldown. The decision is aggregate - /// per player: the `port_id` a request names cannot open a fresh bucket. - pub fn allow(&mut self, player: u64, now_ms: i64) -> bool { - // A non-positive interval disables throttling. Never store, so the map - // cannot grow at all in this mode. + /// rejected attempt never extends the cooldown. A timestamp rollback is + /// treated as no elapsed time and cannot bypass the window. + pub fn allow(&mut self, key: u64, now_ms: i64) -> bool { if self.min_interval_ms <= 0 { return true; } - // Evict every entry whose window has fully elapsed. A stale entry can no - // longer throttle anything, so keeping it is pure memory waste; pruning - // here bounds the live map to players who traded within the last window. - self.last - .retain(|_, &mut last| now_ms.saturating_sub(last) < self.min_interval_ms); - // A surviving entry means this player is still inside their window, so a - // new trade is throttled. - if self.last.contains_key(&player) { - return false; + + match self.last.entry(key) { + Entry::Vacant(entry) => { + entry.insert(now_ms); + true + } + Entry::Occupied(mut entry) => { + if now_ms.saturating_sub(*entry.get()) < self.min_interval_ms { + return false; + } + entry.insert(now_ms); + true + } } - self.last.insert(player, now_ms); - true } - /// Forget a player's throttle state, e.g. on disconnect. A departed - /// player's entry is useless and would otherwise linger until its window - /// elapsed; dropping it keeps the map bounded to connected players. - pub fn clear(&mut self, player: u64) { - self.last.remove(&player); + /// Forget a key's throttle state on disconnect, keeping storage bounded to + /// active peers or authenticated player sessions. + pub fn clear(&mut self, key: u64) { + self.last.remove(&key); } /// Number of live entries the limiter is tracking. Test-only: it lets the @@ -97,10 +89,7 @@ mod tests { fn a_second_trade_inside_the_window_is_rejected() { let mut rl = RateLimiter::new(250); assert!(rl.allow(1, 1000)); - // 100ms later, still inside the 250ms window -> throttled. assert!(!rl.allow(1, 1100)); - // A rejected attempt must not push the cooldown out: once the original - // window elapses the next trade is admitted. assert!(rl.allow(1, 1250)); } @@ -113,33 +102,46 @@ mod tests { #[test] fn the_throttle_is_aggregate_per_player_and_rotation_proof() { - // Regression for the security review's DoS finding. The pre-fix limiter - // keyed by (player, port) and admitted the first trade per key, so a - // client rotating port_ids got a fresh bucket every message and was - // never throttled. The throttle is now aggregate per PLAYER: once a - // player trades, every further trade inside the window is rejected no - // matter what port_id (or any other request field) it names. let mut rl = RateLimiter::new(250); assert!(rl.allow(1, 1000)); - // A different player keeps an independent bucket. assert!(rl.allow(2, 1000)); - // Player 1 is still throttled 50ms later — there is no per-port escape. assert!(!rl.allow(1, 1050)); } #[test] - fn stale_entries_are_evicted_so_the_map_stays_bounded() { - // An attacker sending distinct requests cannot grow the map without - // bound: the limiter is keyed by player and prunes entries whose window - // has elapsed, so the live size is bounded by the players active within - // one window, not by how many messages arrive over time. + fn clear_keeps_storage_bounded_to_active_keys() { let mut rl = RateLimiter::new(250); - for player in 0..10_000u64 { - // Each player trades once, a full window apart, so each new arrival - // makes the previous entry stale and it is pruned on the next check. - assert!(rl.allow(player, (player as i64) * 250)); + for key in 0..10_000u64 { + assert!(rl.allow(key, 1_000)); + rl.clear(key); } - assert_eq!(rl.tracked_count(), 1); + assert_eq!(rl.tracked_count(), 0); + } + + #[test] + fn admission_does_not_scan_all_tracked_keys() { + let source = include_str!("ratelimit.rs"); + let full_map_scan = [".ret", "ain("].concat(); + assert!( + !source.contains(&full_map_scan), + "allow must not scan every tracked key" + ); + + let mut rl = RateLimiter::new(250); + for key in 0..10_000 { + assert!(rl.allow(key, 1_000)); + } + assert_eq!(rl.tracked_count(), 10_000); + assert!(!rl.allow(9_999, 1_001)); + } + + #[test] + fn timestamp_rollback_does_not_bypass_the_window() { + let mut rl = RateLimiter::new(250); + assert!(rl.allow(1, 1_000)); + assert!(!rl.allow(1, 900)); + assert!(!rl.allow(1, 1_249)); + assert!(rl.allow(1, 1_250)); } #[test] @@ -149,7 +151,6 @@ mod tests { assert_eq!(rl.tracked_count(), 1); rl.clear(1); assert_eq!(rl.tracked_count(), 0); - // After a clear the player may trade again immediately. assert!(rl.allow(1, 1050)); } @@ -158,8 +159,6 @@ mod tests { let mut rl = RateLimiter::new(0); assert!(rl.allow(1, 1000)); assert!(rl.allow(1, 1000)); - // With throttling disabled the map never grows, so it cannot be a memory - // sink either. assert_eq!(rl.tracked_count(), 0); } } diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 0e26014..6351cf2 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -224,7 +224,7 @@ impl Server { } fn on_hello(&mut self, peer: PeerId, hello: p::ClientHello<'_>) -> anyhow::Result<()> { - self.on_hello_at(peer, hello, now_ms()) + self.on_hello_at(peer, hello, self.hello_admission_ms()) } fn on_hello_at( @@ -825,6 +825,10 @@ impl Server { self.boot.elapsed().as_millis() as u32 } + fn hello_admission_ms(&self) -> i64 { + self.boot.elapsed().as_millis().min(i64::MAX as u128) as i64 + } + fn send(&mut self, peer: PeerId, bytes: &[u8]) { if let Err(e) = self.host.send_unreliable(peer, bytes) { tracing::warn!(peer, error = %e, "send failed"); @@ -946,6 +950,24 @@ mod handshake_tests { } } + #[test] + fn production_hello_admission_uses_monotonic_uptime() { + let mut server = make_server(); + let hello = hello_envelope( + "handshake-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + assert!(server.hello_limiter.allow(1, 86_400_000)); + + deliver_hello(&mut server, 1, &hello); + + assert!( + !server.sessions.contains_key(&1), + "boot-elapsed time must not jump forward to the Unix epoch" + ); + } + fn hello_envelope( token: &str, protocol_version: u16, From 984b46f5183681b639f9a1d95adacfb780365857 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 21:10:19 -0600 Subject: [PATCH 08/36] fix(server): separate hello clock domains Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 79 +++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 6351cf2..66b70a2 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -224,16 +224,17 @@ impl Server { } fn on_hello(&mut self, peer: PeerId, hello: p::ClientHello<'_>) -> anyhow::Result<()> { - self.on_hello_at(peer, hello, self.hello_admission_ms()) + self.on_hello_at(peer, hello, self.hello_admission_ms(), now_ms()) } fn on_hello_at( &mut self, peer: PeerId, hello: p::ClientHello<'_>, - now_ms: i64, + admission_ms: i64, + persistence_ms: i64, ) -> anyhow::Result<()> { - if !self.hello_limiter.allow(u64::from(peer), now_ms) { + if !self.hello_limiter.allow(u64::from(peer), admission_ms) { return Ok(()); } @@ -290,7 +291,7 @@ impl Server { let player = self .db - .upsert_player_by_token(&identity_hash, &name, now_ms)?; + .upsert_player_by_token(&identity_hash, &name, persistence_ms)?; let player_id = player.id as u64; // Drop any prior session for this identity (reconnect from a new peer). @@ -968,6 +969,74 @@ mod handshake_tests { ); } + #[test] + fn hello_separates_admission_time_from_persistence_time() { + const ADMISSION_MS: i64 = 2_000; + const FIRST_EPOCH_MS: i64 = 1_700_000_000_000; + const RETURN_EPOCH_MS: i64 = 1_700_000_010_000; + + let mut server = make_server(); + let hello = hello_envelope( + "clock-domain-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let env = decode_envelope(&hello).unwrap(); + server + .on_hello_at( + 1, + env.payload_as_client_hello().unwrap(), + ADMISSION_MS, + FIRST_EPOCH_MS, + ) + .unwrap(); + + let player_id = server.sessions[&1].player_id as i64; + let created = server.db.player(player_id).unwrap().unwrap(); + assert_eq!(created.created_at, FIRST_EPOCH_MS); + assert_eq!(created.last_seen, FIRST_EPOCH_MS); + + let hello_interval_ms = server.cfg.hello_min_interval_ms_i64(); + let seq_after_first = server.seq; + let env = decode_envelope(&hello).unwrap(); + server + .on_hello_at( + 1, + env.payload_as_client_hello().unwrap(), + ADMISSION_MS + hello_interval_ms - 1, + RETURN_EPOCH_MS, + ) + .unwrap(); + assert_eq!(server.seq, seq_after_first); + assert_eq!(server.db.player(player_id).unwrap().unwrap(), created); + + let env = decode_envelope(&hello).unwrap(); + server + .on_hello_at( + 1, + env.payload_as_client_hello().unwrap(), + ADMISSION_MS + hello_interval_ms, + RETURN_EPOCH_MS, + ) + .unwrap(); + assert_ne!(server.seq, seq_after_first); + assert_eq!(server.db.player(player_id).unwrap().unwrap(), created); + + let env = decode_envelope(&hello).unwrap(); + server + .on_hello_at( + 2, + env.payload_as_client_hello().unwrap(), + ADMISSION_MS, + RETURN_EPOCH_MS, + ) + .unwrap(); + let returned = server.db.player(player_id).unwrap().unwrap(); + assert_eq!(server.sessions[&2].player_id as i64, player_id); + assert_eq!(returned.created_at, FIRST_EPOCH_MS); + assert_eq!(returned.last_seen, RETURN_EPOCH_MS); + } + fn hello_envelope( token: &str, protocol_version: u16, @@ -1018,7 +1087,7 @@ mod handshake_tests { fn deliver_hello_at(server: &mut Server, peer: PeerId, bytes: &[u8], now_ms: i64) { let env = decode_envelope(bytes).unwrap(); server - .on_hello_at(peer, env.payload_as_client_hello().unwrap(), now_ms) + .on_hello_at(peer, env.payload_as_client_hello().unwrap(), now_ms, now_ms) .unwrap(); } From 3f39e53e61ba5b944063c8b1defad06cb89786c6 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 21:42:40 -0600 Subject: [PATCH 09/36] fix(net): harden connection restart lifecycle Co-Authored-By: Claude --- .../net/Sailwind.Online.Net/ITransport.cs | 7 +- .../LiteNetLibTransport.cs | 12 +- packages/net/Sailwind.Online.Net/NetClient.cs | 72 +++++++- .../CurrentPeerSlotTests.cs | 13 +- .../MockTransport.cs | 26 ++- .../NetClientTests.cs | 163 ++++++++++++++++++ 6 files changed, 271 insertions(+), 22 deletions(-) diff --git a/packages/net/Sailwind.Online.Net/ITransport.cs b/packages/net/Sailwind.Online.Net/ITransport.cs index 1dc5815..dd053c8 100644 --- a/packages/net/Sailwind.Online.Net/ITransport.cs +++ b/packages/net/Sailwind.Online.Net/ITransport.cs @@ -26,8 +26,11 @@ public interface ITransport /// Bring the manager up. Returns false when the socket cannot bind. bool Start(); - /// Open (or re-open) the single peer to : with the connect key. - void Connect(string host, int port, string key); + /// + /// Open (or keep opening) the single peer to : + /// with the connect key. Returns true when a current peer exists after the attempt. + /// + bool Connect(string host, int port, string key); /// Immediately drop the current peer without reporting a network-originated disconnect. void DropPeer(); diff --git a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs index 6fb11a9..d45aeba 100644 --- a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs +++ b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs @@ -66,9 +66,9 @@ public bool Start() return _manager.Start(); } - public void Connect(string host, int port, string key) + public bool Connect(string host, int port, string key) { - _peer.SetIfPresent(_manager.Connect(host, port, key)); + return _peer.SetIfPresent(_manager.Connect(host, port, key)); } public void DropPeer() @@ -151,14 +151,14 @@ public TPeer? Value get { return _value; } } - public void SetIfPresent(TPeer peer) + public bool SetIfPresent(TPeer peer) { - if (peer == null) + if (peer != null) { - return; + _value = peer; } - _value = peer; + return _value != null; } public TPeer? Clear() diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index 5ceacc3..5a722ea 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -162,20 +162,27 @@ public string StatusText /// Begin (or restart) a session with the given options. public void Connect(ConnectOptions options) { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + if (_status == ConnectionStatus.Connecting && ReferenceEquals(_options, options)) { return; } - _options = options; - - if (!_transport.IsRunning && !_transport.Start()) + if (_status != ConnectionStatus.Disconnected) { - _log.LogError("[Sailwind.Online] Failed to start LiteNetLib NetManager."); - return; + _status = ConnectionStatus.Disconnected; + _transport.DropPeer(); } - OpenPeer(); + _options = options; + _reconnectBackoffMs = DefaultReconnectMs; + ResetSession(); + + StartTransportAndOpenPeer(); } /// Poll the transport and service the handshake/reconnect timers. Call every frame. @@ -194,7 +201,7 @@ public void Poll() } else if (_status == ConnectionStatus.Disconnected && _options != null && now >= _nextReconnectMs) { - OpenPeer(); + StartTransportAndOpenPeer(); } _cache.PruneStale(now, SnapshotCache.DefaultStaleMs); @@ -245,13 +252,42 @@ private void OpenPeer() return; } + if (!_transport.IsRunning) + { + _status = ConnectionStatus.Disconnected; + ScheduleReconnect(); + return; + } + _cache.Clear(); ResetPositionObservability(); - _transport.Connect(options.Host, options.Port, ConnectKey); + if (!_transport.Connect(options.Host, options.Port, ConnectKey)) + { + _status = ConnectionStatus.Disconnected; + ScheduleReconnect(); + _log.LogWarning( + "[Sailwind.Online] Transport did not create a peer for " + + options.Host + ":" + options.Port + "; will retry."); + return; + } + _status = ConnectionStatus.Connecting; _log.LogInfo("[Sailwind.Online] Connecting to " + options.Host + ":" + options.Port + " ..."); } + private void StartTransportAndOpenPeer() + { + if (!_transport.IsRunning && !_transport.Start()) + { + _status = ConnectionStatus.Disconnected; + ScheduleReconnect(); + _log.LogError("[Sailwind.Online] Failed to start LiteNetLib NetManager; will retry."); + return; + } + + OpenPeer(); + } + private void ScheduleReconnect() { _nextReconnectMs = NowMs + _reconnectBackoffMs; @@ -304,6 +340,13 @@ private bool SendRaw(byte[] bytes) private void OnPeerConnected() { + if (_status != ConnectionStatus.Connecting) + { + _log.LogDebug( + "[Sailwind.Online] Ignored transport connect while " + StatusText + "."); + return; + } + _status = ConnectionStatus.Handshaking; SendHello(); _log.LogInfo("[Sailwind.Online] Transport up; sending ClientHello."); @@ -527,6 +570,19 @@ private void ResetPositionObservability() _loggedFirstInboundPosition = false; } + private void ResetSession() + { + _cache.Clear(); + _seq = 0; + _lastHelloMs = 0; + _nextReconnectMs = 0; + _snapshotHz = 4; + _playerId = 0; + _serverDay = 0; + _serverTimeOfDay = 0; + ResetPositionObservability(); + } + private void RejectHandshake(string warning) { _status = ConnectionStatus.Disconnected; diff --git a/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs b/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs index 854846c..d540a1e 100644 --- a/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs +++ b/tests/Sailwind.Online.Net.Tests/CurrentPeerSlotTests.cs @@ -11,13 +11,22 @@ public void PendingPeer_NullReplacementAttempt_PreservesCurrentPeer() var slot = new CurrentPeerSlot(); var pendingPeer = new object(); - slot.SetIfPresent(pendingPeer); - slot.SetIfPresent(null); + Assert.True(slot.SetIfPresent(pendingPeer)); + Assert.True(slot.SetIfPresent(null)); Assert.Same(pendingPeer, slot.Value); Assert.True(slot.IsCurrent(pendingPeer)); } + [Fact] + public void NoCurrentPeer_NullCreation_ReportsFailure() + { + var slot = new CurrentPeerSlot(); + + Assert.False(slot.SetIfPresent(null)); + Assert.Null(slot.Value); + } + [Fact] public void LocalDropThenReplacement_RejectsOldCallbacksButAllowsCurrentDisconnect() { diff --git a/tests/Sailwind.Online.Net.Tests/MockTransport.cs b/tests/Sailwind.Online.Net.Tests/MockTransport.cs index f49e92d..c71cb5a 100644 --- a/tests/Sailwind.Online.Net.Tests/MockTransport.cs +++ b/tests/Sailwind.Online.Net.Tests/MockTransport.cs @@ -18,6 +18,12 @@ internal sealed class MockTransport : ITransport /// What returns; false simulates a socket that cannot bind. public bool StartResult = true; + /// Queued outcomes, consumed before . + public readonly Queue StartResults = new Queue(); + + /// What reports; false simulates no peer being created. + public bool ConnectResult = true; + public int StartCalls; public int StopCalls; public int ConnectCalls; @@ -36,6 +42,7 @@ internal sealed class MockTransport : ITransport public readonly List Sent = new List(); private bool _running; + private bool _hasPeer; private bool _peerConnected; public bool IsRunning => _running; @@ -52,30 +59,38 @@ internal sealed class MockTransport : ITransport public bool Start() { StartCalls++; - if (StartResult) + bool result = StartResults.Count > 0 ? StartResults.Dequeue() : StartResult; + if (result) { _running = true; } - return StartResult; + return result; } - public void Connect(string host, int port, string key) + public bool Connect(string host, int port, string key) { ConnectCalls++; - if (!_peerConnected) + if (ConnectResult && !_hasPeer) { FreshPeerConnectCalls++; } + if (ConnectResult) + { + _hasPeer = true; + } + LastHost = host; LastPort = port; LastKey = key; + return _hasPeer; } public void DropPeer() { DropPeerCalls++; + _hasPeer = false; _peerConnected = false; Ping = -1; DropPeerCallback?.Invoke(); @@ -95,6 +110,7 @@ public void Stop() { StopCalls++; _running = false; + _hasPeer = false; _peerConnected = false; } @@ -103,6 +119,7 @@ public void Stop() /// Simulate the transport completing its connect: the peer becomes send-ready first. public void RaisePeerConnected() { + _hasPeer = true; _peerConnected = true; PeerConnected?.Invoke(); } @@ -110,6 +127,7 @@ public void RaisePeerConnected() /// Simulate the peer dropping; the peer is no longer send-ready. public void RaisePeerDisconnected(string reason = "RemoteConnectionClose") { + _hasPeer = false; _peerConnected = false; PeerDisconnected?.Invoke(reason); } diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index 2a7e9a7..a68bea4 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -84,6 +84,169 @@ public void Connect_WhenTransportStartFails_LogsAndDoesNotOpenPeer() Assert.NotEmpty(log.Errors); } + [Fact] + public void Connect_WhenTransportStartFails_RetriesStartBeforeOpeningPeer() + { + var transport = new MockTransport(); + transport.StartResults.Enqueue(false); + transport.StartResults.Enqueue(true); + long now = 0; + var net = new NetClient(new NullNetLog(), transport, () => now); + + net.Connect(Options); + + now = NetClient.DefaultReconnectMs - 1; + net.Poll(); + Assert.Equal(1, transport.StartCalls); + Assert.Equal(0, transport.ConnectCalls); + Assert.False(transport.IsRunning); + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.StartCalls); + Assert.Equal(1, transport.ConnectCalls); + Assert.True(transport.IsRunning); + Assert.Equal(Options.Host, transport.LastHost); + Assert.Equal(Options.Port, transport.LastPort); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + } + + [Fact] + public void Connect_WhileHandshaking_RestartsSameOptionsWithFreshPeer() + { + var transport = new MockTransport(); + var net = new NetClient(new NullNetLog(), transport, () => 0); + net.Connect(Options); + transport.RaisePeerConnected(); + Assert.Equal(ConnectionStatus.Handshaking, net.Status); + Assert.Single(transport.Sent); + + net.Connect(Options); + + Assert.Equal(1, transport.DropPeerCalls); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(2, transport.FreshPeerConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + Assert.Single(transport.Sent); + + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Handshaking, net.Status); + Assert.Equal(2, transport.Sent.Count); + Assert.Equal(Options.DisplayName, Decode(transport.Sent[1]).PayloadAsClientHello().DisplayName); + } + + [Fact] + public void Connect_WhileReady_RestartsWithNewOptionsAndClearsSession() + { + var transport = new MockTransport(); + var log = new RecordingLog(); + var net = new NetClient(log, transport, () => 100); + net.Connect(Options); + transport.RaisePeerConnected(); + transport.RaiseNetworkReceive(ServerHelloEnvelope(accepted: true, playerId: 77, snapshotHz: 8)); + net.SendClientState(new BoatPose()); + byte[] snapshot = SnapshotDeltaEnvelope(1, 88, 1f, 2f, 3f, 0f, 0f, 0f, 1f, 123, 10); + transport.RaiseNetworkReceive(snapshot); + Assert.True(net.Cache.TryGetPlayer(88, out _)); + + var replacement = new ConnectOptions + { + Host = "replacement-host", + Port = 5252, + DisplayName = "Bea", + Token = "replacement-token", + GameBuild = "replacement-build", + ModVersion = "0.2.0", + ApiSurfaceHash = "replacement-hash" + }; + transport.Sent.Clear(); + + net.Connect(replacement); + + Assert.Equal(1, transport.DropPeerCalls); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(2, transport.FreshPeerConnectCalls); + Assert.Equal("replacement-host", transport.LastHost); + Assert.Equal(5252, transport.LastPort); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + Assert.Equal(0ul, net.PlayerId); + Assert.Equal((byte)4, net.SnapshotHz); + Assert.False(net.Cache.TryGetPlayer(88, out _)); + Assert.Empty(transport.Sent); + + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Handshaking, net.Status); + ClientHello hello = Decode(Assert.Single(transport.Sent)).PayloadAsClientHello(); + Assert.Equal("Bea", hello.DisplayName); + Assert.Equal("replacement-token", hello.Token); + + transport.RaiseNetworkReceive(ServerHelloEnvelope(accepted: true, playerId: 99, snapshotHz: 12)); + net.SendClientState(new BoatPose()); + transport.RaiseNetworkReceive(snapshot); + + Assert.Equal(ConnectionStatus.Ready, net.Status); + Assert.Equal(99ul, net.PlayerId); + Assert.Equal((byte)12, net.SnapshotHz); + Assert.Equal(2, log.Infos.FindAll(message => message.Contains("First outbound position")).Count); + Assert.Equal(2, log.Infos.FindAll(message => message.Contains("First inbound position")).Count); + Assert.DoesNotContain(log.Infos, message => message.Contains(replacement.Token)); + } + + [Fact] + public void Connect_Null_ThrowsWithoutChangingCurrentAttempt() + { + var transport = new MockTransport(); + var net = new NetClient(new NullNetLog(), transport, () => 0); + net.Connect(Options); + + Assert.Throws(() => net.Connect(null)); + + Assert.Equal(1, transport.StartCalls); + Assert.Equal(1, transport.ConnectCalls); + Assert.Equal(0, transport.DropPeerCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + Assert.Equal(Options.Host, transport.LastHost); + Assert.Equal(Options.Port, transport.LastPort); + + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Handshaking, net.Status); + ClientHello hello = Decode(Assert.Single(transport.Sent)).PayloadAsClientHello(); + Assert.Equal(Options.DisplayName, hello.DisplayName); + Assert.Equal(Options.Token, hello.Token); + } + + [Fact] + public void Connect_WhenTransportCreatesNoPeer_RemainsDisconnectedUntilRetrySucceeds() + { + var transport = new MockTransport { ConnectResult = false }; + long now = 0; + var net = new NetClient(new NullNetLog(), transport, () => now); + + net.Connect(Options); + + Assert.True(transport.IsRunning); + Assert.Equal(1, transport.StartCalls); + Assert.Equal(1, transport.ConnectCalls); + Assert.Equal(0, transport.FreshPeerConnectCalls); + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + + transport.ConnectResult = true; + now = NetClient.DefaultReconnectMs - 1; + net.Poll(); + Assert.Equal(1, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(1, transport.FreshPeerConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + } + [Fact] public void Handshake_PeerConnectedThenServerHello_ReachesReady() { From 9a5c0b71d90ad03c242465a2f4f8c6aec26e4292 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 22:15:47 -0600 Subject: [PATCH 10/36] fix(online): bound client hello lifecycle Co-Authored-By: Claude --- packages/net/Sailwind.Online.Net/NetClient.cs | 25 +- server/crates/sw-server/src/server.rs | 332 ++++++++++++++++-- .../NetClientTests.cs | 51 +++ 3 files changed, 382 insertions(+), 26 deletions(-) diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index 5a722ea..e9080b0 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -300,12 +300,12 @@ private uint NextSeq() return _seq; } - private void SendHello() + private bool SendHello() { ConnectOptions? options = _options; if (options == null) { - return; + return false; } byte[] bytes = _codec.EncodeClientHello( @@ -317,8 +317,14 @@ private void SendHello() options.ModVersion, options.ApiSurfaceHash); - SendRaw(bytes); + if (!SendRaw(bytes)) + { + EndHandshakeAttempt(); + return false; + } + _lastHelloMs = NowMs; + return true; } private bool SendRaw(byte[] bytes) @@ -348,8 +354,10 @@ private void OnPeerConnected() } _status = ConnectionStatus.Handshaking; - SendHello(); - _log.LogInfo("[Sailwind.Online] Transport up; sending ClientHello."); + if (SendHello()) + { + _log.LogInfo("[Sailwind.Online] Transport up; sending ClientHello."); + } } private void OnPeerDisconnected(string reason) @@ -584,12 +592,17 @@ private void ResetSession() } private void RejectHandshake(string warning) + { + EndHandshakeAttempt(); + _log.LogWarning(warning); + } + + private void EndHandshakeAttempt() { _status = ConnectionStatus.Disconnected; ResetPositionObservability(); ScheduleReconnect(); _transport.DropPeer(); - _log.LogWarning(warning); } public static string FormatTimeOfDay(float fractionOfDay) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 66b70a2..9c53a77 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -249,29 +249,49 @@ impl Server { } let max_string_len = self.cfg.max_wire_string_len_usize(); - let Some(api_surface_hash) = hello.api_surface_hash() else { - self.reject_hello(peer, "missing API surface hash"); + if let Err(reason) = validate_hello_string( + hello.api_surface_hash(), + "API surface hash", + true, + max_string_len, + ) { + self.reject_hello(peer, &reason); return Ok(()); + } + let token = match validate_hello_string(hello.token(), "token", true, max_string_len) { + Ok(Some(value)) => value, + Ok(None) => unreachable!("required hello string validated as absent"), + Err(reason) => { + self.reject_hello(peer, &reason); + return Ok(()); + } + }; + let display_name = match validate_hello_string( + hello.display_name(), + "display name", + false, + max_string_len, + ) { + Ok(value) => value, + Err(reason) => { + self.reject_hello(peer, &reason); + return Ok(()); + } }; - if api_surface_hash.is_empty() { - self.reject_hello(peer, "missing API surface hash"); + if let Err(reason) = + validate_hello_string(hello.game_build(), "game build", false, max_string_len) + { + self.reject_hello(peer, &reason); return Ok(()); } - if !validate::string_within_limit(api_surface_hash, max_string_len) { - let reason = - format!("API surface hash exceeds maximum length ({max_string_len} bytes)"); + if let Err(reason) = + validate_hello_string(hello.mod_version(), "mod version", false, max_string_len) + { self.reject_hello(peer, &reason); return Ok(()); } - let token = hello.token().unwrap_or(""); - let name = hello.display_name().unwrap_or("sailor").to_string(); - - if token.is_empty() { - // Auth is assertion-only, but a token must at least be present. - self.reject_hello(peer, "missing token"); - return Ok(()); - } + let name = display_name.unwrap_or("sailor").to_string(); let identity_hash = token_hash(token); if let Some((player_id, identity_matches)) = self @@ -898,6 +918,21 @@ fn token_hash(token: &str) -> String { format!("{h:016x}") } +fn validate_hello_string<'a>( + value: Option<&'a str>, + field: &str, + required: bool, + max_len: usize, +) -> Result, String> { + if required && value.is_none_or(str::is_empty) { + return Err(format!("missing {field}")); + } + if value.is_some_and(|string| !validate::string_within_limit(string, max_len)) { + return Err(format!("{field} exceeds maximum length ({max_len} bytes)")); + } + Ok(value) +} + #[cfg(test)] mod tests { use super::*; @@ -1041,19 +1076,40 @@ mod handshake_tests { token: &str, protocol_version: u16, api_surface_hash: Option<&str>, + ) -> Vec { + hello_envelope_with_strings( + Some(token), + Some("Sailor"), + Some("game-build"), + Some("mod-version"), + api_surface_hash, + protocol_version, + ) + } + + fn hello_envelope_with_strings( + token: Option<&str>, + display_name: Option<&str>, + game_build: Option<&str>, + mod_version: Option<&str>, + api_surface_hash: Option<&str>, + protocol_version: u16, ) -> Vec { let mut fbb = FlatBufferBuilder::new(); - let token = fbb.create_string(token); - let name = fbb.create_string("Sailor"); + let token = token.map(|value| fbb.create_string(value)); + let name = display_name.map(|value| fbb.create_string(value)); + let game_build = game_build.map(|value| fbb.create_string(value)); + let mod_version = mod_version.map(|value| fbb.create_string(value)); let api_hash = api_surface_hash.map(|value| fbb.create_string(value)); let hello = p::ClientHello::create( &mut fbb, &p::ClientHelloArgs { protocol_version, - display_name: Some(name), - token: Some(token), + display_name: name, + token, + game_build, + mod_version, api_surface_hash: api_hash, - ..Default::default() }, ); finish_envelope(&mut fbb, 1, p::Payload::ClientHello, hello.as_union_value()) @@ -1192,6 +1248,184 @@ mod handshake_tests { } } + #[test] + fn every_client_hello_string_is_bounded_before_persistence_or_session_creation() { + let max_len = Config::default().max_wire_string_len_usize(); + let too_long = "x".repeat(max_len + 1); + let cases = [ + ( + "token", + hello_envelope_with_strings( + Some(&too_long), + Some("Sailor"), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ), + format!("token exceeds maximum length ({max_len} bytes)"), + ), + ( + "display name", + hello_envelope_with_strings( + Some("handshake-token"), + Some(&too_long), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ), + format!("display name exceeds maximum length ({max_len} bytes)"), + ), + ( + "game build", + hello_envelope_with_strings( + Some("handshake-token"), + Some("Sailor"), + Some(&too_long), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ), + format!("game build exceeds maximum length ({max_len} bytes)"), + ), + ( + "mod version", + hello_envelope_with_strings( + Some("handshake-token"), + Some("Sailor"), + Some("game-build"), + Some(&too_long), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ), + format!("mod version exceeds maximum length ({max_len} bytes)"), + ), + ( + "API surface hash", + hello_envelope_with_strings( + Some("handshake-token"), + Some("Sailor"), + Some("game-build"), + Some("mod-version"), + Some(&too_long), + sw_contracts::PROTOCOL_VERSION, + ), + format!("API surface hash exceeds maximum length ({max_len} bytes)"), + ), + ]; + + for (field, bytes, expected_reason) in cases { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + + deliver_hello(&mut server, peer, &bytes); + + assert!( + !server.sessions.contains_key(&peer), + "over-long {field} created a session" + ); + assert!( + server.db.player(1).unwrap().is_none(), + "over-long {field} persisted a player" + ); + assert_eq!(receive_server_hello(&client), (false, expected_reason)); + } + } + + #[test] + fn token_and_api_hash_are_required_while_other_hello_strings_remain_optional() { + let required_cases = [ + ( + "missing token", + hello_envelope_with_strings( + None, + Some("Sailor"), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ), + "missing token", + ), + ( + "empty token", + hello_envelope_with_strings( + Some(""), + Some("Sailor"), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ), + "missing token", + ), + ( + "missing API surface hash", + hello_envelope_with_strings( + Some("handshake-token"), + Some("Sailor"), + Some("game-build"), + Some("mod-version"), + None, + sw_contracts::PROTOCOL_VERSION, + ), + "missing API surface hash", + ), + ( + "empty API surface hash", + hello_envelope_with_strings( + Some("handshake-token"), + Some("Sailor"), + Some("game-build"), + Some("mod-version"), + Some(""), + sw_contracts::PROTOCOL_VERSION, + ), + "missing API surface hash", + ), + ]; + + for (case, bytes, expected_reason) in required_cases { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + + deliver_hello(&mut server, peer, &bytes); + + assert!( + !server.sessions.contains_key(&peer), + "{case} created a session" + ); + assert!( + server.db.player(1).unwrap().is_none(), + "{case} persisted a player" + ); + assert_eq!( + receive_server_hello(&client), + (false, expected_reason.to_string()) + ); + } + + for (display_name, game_build, mod_version, expected_name) in [ + (None, None, None, "sailor"), + (Some(""), Some(""), Some(""), ""), + ] { + let mut server = make_server(); + let bytes = hello_envelope_with_strings( + Some("handshake-token"), + display_name, + game_build, + mod_version, + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ); + + deliver_hello(&mut server, 1, &bytes); + + assert_eq!(server.sessions[&1].display_name, expected_name); + } + } + #[test] fn valid_protocol_and_api_surface_hash_create_session() { let mut server = make_server(); @@ -1206,6 +1440,64 @@ mod handshake_tests { assert!(server.sessions.contains_key(&1)); } + #[test] + fn near_limit_display_name_and_chat_cannot_emit_an_oversized_datagram() { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let max_len = server.cfg.max_wire_string_len_usize(); + let display_name = "n".repeat(max_len); + let text = "t".repeat(max_len); + let hello = hello_envelope_with_strings( + Some("handshake-token"), + Some(&display_name), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ); + deliver_hello(&mut server, peer, &hello); + assert_eq!(receive_server_hello(&client), (true, String::new())); + client + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + loop { + let mut join_update = [0u8; sw_net::protocol::MTU]; + match client.recv(&mut join_update) { + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + break; + } + Err(error) => panic!("failed to drain join update: {error}"), + } + } + + let encoded = codec::chat_broadcast(2, 1, &display_name, &text, 0, 1_000); + assert!( + encoded.len() > sw_net::protocol::MTU - sw_net::protocol::HEADER_SIZE, + "the regression input must exceed the fixed unfragmented payload" + ); + + let mut fbb = FlatBufferBuilder::new(); + let text = fbb.create_string(&text); + let chat = p::ChatSend::create( + &mut fbb, + &p::ChatSendArgs { + text: Some(text), + channel: 0, + }, + ); + let bytes = finish_envelope(&mut fbb, 2, p::Payload::ChatSend, chat.as_union_value()); + let env = decode_envelope(&bytes).unwrap(); + server.on_chat(peer, env.payload_as_chat_send().unwrap(), 1_000); + + assert_no_outbound_datagram(&client); + } + #[test] fn repeated_valid_hello_preserves_established_session_and_only_resends_server_hello() { let mut server = make_server(); diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index a68bea4..3a2ead3 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -547,6 +547,57 @@ public void ClientHello_ResendsEveryIntervalUntilServerHello() Assert.Equal(3, transport.Sent.Count); } + [Fact] + public void ClientHello_WhenPacketCannotBeSent_DropsPeerAndReconnectsAfterBackoff() + { + var options = new ConnectOptions + { + Host = "test-host", + Port = 4242, + DisplayName = new string('x', NetClient.Mtu), + Token = "tok", + GameBuild = "build", + ModVersion = "0.1.0", + ApiSurfaceHash = "hash" + }; + var transport = new MockTransport(); + var log = new RecordingLog(); + long now = 0; + var net = new NetClient(log, transport, () => now); + net.Connect(options); + + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.Equal(1, transport.DropPeerCalls); + Assert.False(transport.IsPeerConnected); + Assert.Empty(transport.Sent); + int warningsAfterFailedSend = log.Warnings.Count; + Assert.True(warningsAfterFailedSend > 0); + Assert.DoesNotContain(log.Warnings, message => message.Contains(options.Token)); + + now = NetClient.HelloRetryMs; + net.Poll(); + Assert.Equal(warningsAfterFailedSend, log.Warnings.Count); + Assert.Equal(1, transport.ConnectCalls); + + options.DisplayName = "Ari"; + now = NetClient.DefaultReconnectMs - 1; + net.Poll(); + Assert.Equal(1, transport.ConnectCalls); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + Assert.Equal(ConnectionStatus.Connecting, net.Status); + + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Handshaking, net.Status); + Assert.Single(transport.Sent); + Assert.Equal("Ari", Decode(transport.Sent[0]).PayloadAsClientHello().DisplayName); + } + [Fact] public void Disconnect_ReconnectsWithExponentialBackoff() { From fc97a8b438bf82aad4df49d5bb75e4b390e2ad6f Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 22:46:35 -0600 Subject: [PATCH 11/36] fix(online): harden transport admission limits Co-Authored-By: Claude --- .../net/Sailwind.Online.Net/ITransport.cs | 6 + .../LiteNetLibTransport.cs | 11 + packages/net/Sailwind.Online.Net/NetClient.cs | 23 +- server/crates/sw-server/src/ratelimit.rs | 6 + server/crates/sw-server/src/server.rs | 245 ++++++++++++++---- .../MockTransport.cs | 10 + .../NetClientTests.cs | 65 +++-- 7 files changed, 302 insertions(+), 64 deletions(-) diff --git a/packages/net/Sailwind.Online.Net/ITransport.cs b/packages/net/Sailwind.Online.Net/ITransport.cs index dd053c8..c2e1778 100644 --- a/packages/net/Sailwind.Online.Net/ITransport.cs +++ b/packages/net/Sailwind.Online.Net/ITransport.cs @@ -20,6 +20,12 @@ public interface ITransport /// True while the current peer is fully connected, i.e. safe to send on. bool IsPeerConnected { get; } + /// + /// Maximum payload bytes the current peer can send as one unreliable packet, or zero when + /// there is no current peer. + /// + int MaxUnreliablePayloadSize { get; } + /// Round-trip estimate in milliseconds for the current peer, or -1 when there is none. int Ping { get; } diff --git a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs index d45aeba..c41c5f4 100644 --- a/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs +++ b/packages/net/Sailwind.Online.Net/LiteNetLibTransport.cs @@ -47,6 +47,17 @@ public bool IsPeerConnected } } + public int MaxUnreliablePayloadSize + { + get + { + NetPeer? peer = _peer.Value; + return peer != null + ? peer.GetMaxSinglePacketSize(DeliveryMethod.Unreliable) + : 0; + } + } + public int Ping { get diff --git a/packages/net/Sailwind.Online.Net/NetClient.cs b/packages/net/Sailwind.Online.Net/NetClient.cs index e9080b0..49ad92a 100644 --- a/packages/net/Sailwind.Online.Net/NetClient.cs +++ b/packages/net/Sailwind.Online.Net/NetClient.cs @@ -334,16 +334,33 @@ private bool SendRaw(byte[] bytes) return false; } - if (bytes.Length > Mtu) + int maxPayloadSize = _transport.MaxUnreliablePayloadSize; + if (maxPayloadSize <= 0 || bytes.Length > maxPayloadSize) { - _log.LogWarning("[Sailwind.Online] Dropping oversized packet (" + bytes.Length + " > " + Mtu + " bytes)."); + LogOversizedPacket(bytes.Length, maxPayloadSize); + return false; + } + + try + { + _transport.Send(bytes, DeliveryMethod.Unreliable); + } + catch (TooBigPacketException) + { + LogOversizedPacket(bytes.Length, maxPayloadSize); return false; } - _transport.Send(bytes, DeliveryMethod.Unreliable); return true; } + private void LogOversizedPacket(int packetSize, int maxPayloadSize) + { + _log.LogWarning( + "[Sailwind.Online] Dropping oversized packet (" + packetSize + + " bytes; current unreliable capacity " + maxPayloadSize + " payload bytes)."); + } + private void OnPeerConnected() { if (_status != ConnectionStatus.Connecting) diff --git a/server/crates/sw-server/src/ratelimit.rs b/server/crates/sw-server/src/ratelimit.rs index 321971a..3b8dcdf 100644 --- a/server/crates/sw-server/src/ratelimit.rs +++ b/server/crates/sw-server/src/ratelimit.rs @@ -73,6 +73,12 @@ impl RateLimiter { pub fn tracked_count(&self) -> usize { self.last.len() } + + /// Last admitted monotonic timestamp for a key. + #[cfg(test)] + pub fn last_accepted_ms(&self, key: u64) -> Option { + self.last.get(&key).copied() + } } #[cfg(test)] diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 9c53a77..67ce61f 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -14,7 +14,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use sw_contracts::decode_envelope; use sw_contracts::sw_proto as p; use sw_econ::{Ledger, Market, MarketAck, Trade, Txn}; -use sw_net::{DisconnectReason, Event, Host, PeerId}; +use sw_net::{protocol, DisconnectReason, Event, Host, PeerId}; use sw_persist::{Db, MooringRow}; use sw_world::{AoiUpdate, Cell, Subscription, World}; @@ -183,6 +183,16 @@ impl Server { } fn handle_data(&mut self, peer: PeerId, bytes: &[u8]) -> anyhow::Result<()> { + self.handle_data_at(peer, bytes, self.admission_ms(), now_ms()) + } + + fn handle_data_at( + &mut self, + peer: PeerId, + bytes: &[u8], + admission_ms: i64, + persistence_ms: i64, + ) -> anyhow::Result<()> { // Verified decode: hostile/garbage datagrams are simply dropped. let Ok(env) = decode_envelope(bytes) else { return Ok(()); @@ -190,32 +200,32 @@ impl Server { match env.payload_type() { p::Payload::ClientHello => { if let Some(h) = env.payload_as_client_hello() { - self.on_hello(peer, h)?; + self.on_hello_at(peer, h, admission_ms, persistence_ms)?; } } p::Payload::ClientState => { if let Some(cs) = env.payload_as_client_state() { - self.on_client_state(peer, cs, now_ms()); + self.on_client_state(peer, cs, admission_ms); } } p::Payload::EconTxn => { if let Some(t) = env.payload_as_econ_txn() { - self.on_econ(peer, t, now_ms())?; + self.on_econ(peer, t, admission_ms, persistence_ms)?; } } p::Payload::MarketTradeRequest => { if let Some(r) = env.payload_as_market_trade_request() { - self.on_trade(peer, r, now_ms())?; + self.on_trade(peer, r, admission_ms, persistence_ms)?; } } p::Payload::MoorRequest => { if let Some(m) = env.payload_as_moor_request() { - self.on_moor(peer, m, now_ms())?; + self.on_moor(peer, m, admission_ms, persistence_ms)?; } } p::Payload::ChatSend => { if let Some(c) = env.payload_as_chat_send() { - self.on_chat(peer, c, now_ms()); + self.on_chat(peer, c, admission_ms); } } _ => {} @@ -223,8 +233,9 @@ impl Server { Ok(()) } + #[cfg(test)] fn on_hello(&mut self, peer: PeerId, hello: p::ClientHello<'_>) -> anyhow::Result<()> { - self.on_hello_at(peer, hello, self.hello_admission_ms(), now_ms()) + self.on_hello_at(peer, hello, self.admission_ms(), now_ms()) } fn on_hello_at( @@ -463,7 +474,13 @@ impl Server { } } - fn on_econ(&mut self, peer: PeerId, txn: p::EconTxn<'_>, now_ms: i64) -> anyhow::Result<()> { + fn on_econ( + &mut self, + peer: PeerId, + txn: p::EconTxn<'_>, + admission_ms: i64, + persistence_ms: i64, + ) -> anyhow::Result<()> { let Some(player_id) = self.sessions.get(&peer).map(|s| s.player_id) else { return Ok(()); }; @@ -482,7 +499,7 @@ impl Server { // charged against the aggregate per-player econ throttle. let txn_id = txn.txn_id(); let already_applied = self.db.lookup_txn(txn_id as i64)?.is_some(); - if !already_applied && !self.econ_limiter.allow(player_id, now_ms) { + if !already_applied && !self.econ_limiter.allow(player_id, admission_ms) { tracing::debug!(player_id, txn_id, "econ txn rate limited"); return Ok(()); } @@ -495,7 +512,7 @@ impl Server { }; let ack = { - let mut ledger = Ledger::new(DbLedgerStore::new(&self.db, now_ms)); + let mut ledger = Ledger::new(DbLedgerStore::new(&self.db, persistence_ms)); ledger.apply(player_id, &txn)? }; tracing::debug!( @@ -518,7 +535,8 @@ impl Server { &mut self, peer: PeerId, req: p::MarketTradeRequest<'_>, - now_ms: i64, + admission_ms: i64, + persistence_ms: i64, ) -> anyhow::Result<()> { let Some(player_id) = self.sessions.get(&peer).map(|s| s.player_id) else { return Ok(()); @@ -538,7 +556,7 @@ impl Server { // attacker-supplied `port_id` never opens a fresh bucket, so rotating it // cannot raise a player's trade throughput. let already_applied = self.db.lookup_trade(trade.txn_id)?.is_some(); - if !already_applied && !self.trade_limiter.allow(player_id, now_ms) { + if !already_applied && !self.trade_limiter.allow(player_id, admission_ms) { let (stock, price) = self .db .market_state(trade.port_id, trade.item_id)? @@ -559,7 +577,7 @@ impl Server { } let ack = { - let mut market = Market::new(DbMarketStore::new(&self.db, now_ms)); + let mut market = Market::new(DbMarketStore::new(&self.db, persistence_ms)); market.apply(&trade)? }; tracing::debug!( @@ -582,7 +600,8 @@ impl Server { &mut self, peer: PeerId, req: p::MoorRequest<'_>, - now_ms: i64, + admission_ms: i64, + persistence_ms: i64, ) -> anyhow::Result<()> { let Some((owner, aboard)) = self .sessions @@ -615,13 +634,13 @@ impl Server { // class). Reject a new moor beyond the configured rate BEFORE the write, // keyed by player so it is rotation-proof and memory-bounded exactly like // the trade/econ/chat limiters. - if !self.moor_limiter.allow(owner, now_ms) { + if !self.moor_limiter.allow(owner, admission_ms) { tracing::debug!(owner, "moor request rate limited"); return Ok(()); } let cell = self.world.grid().cell_of(pos[0], pos[2]); - let created = now_ms; + let created = persistence_ms; let row = MooringRow { boat_id: boat_id as i64, @@ -684,6 +703,11 @@ impl Server { let text = text.to_string(); let channel = chat.channel(); let t_ms = self.uptime_ms(); + let bytes = + codec::chat_broadcast(self.next_seq(), sender_player, &name, &text, channel, t_ms); + if bytes.len() > protocol::MTU - protocol::HEADER_SIZE { + return; + } // Deliver to every session whose AoI currently includes the sender's cell. let recipients: Vec = self @@ -694,8 +718,6 @@ impl Server { .collect(); for target in recipients { - let bytes = - codec::chat_broadcast(self.next_seq(), sender_player, &name, &text, channel, t_ms); self.send(target, &bytes); } } @@ -846,7 +868,7 @@ impl Server { self.boot.elapsed().as_millis() as u32 } - fn hello_admission_ms(&self) -> i64 { + fn admission_ms(&self) -> i64 { self.boot.elapsed().as_millis().min(i64::MAX as u128) as i64 } @@ -1444,6 +1466,7 @@ mod handshake_tests { fn near_limit_display_name_and_chat_cannot_emit_an_oversized_datagram() { let mut server = make_server(); let (client, peer) = connect_peer(&mut server); + let (observer, observer_peer) = connect_peer(&mut server); let max_len = server.cfg.max_wire_string_len_usize(); let display_name = "n".repeat(max_len); let text = "t".repeat(max_len); @@ -1457,22 +1480,37 @@ mod handshake_tests { ); deliver_hello(&mut server, peer, &hello); assert_eq!(receive_server_hello(&client), (true, String::new())); + let observer_hello = hello_envelope_with_strings( + Some("observer-token"), + Some("Observer"), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ); + deliver_hello(&mut server, observer_peer, &observer_hello); + assert_eq!(receive_server_hello(&observer), (true, String::new())); client .set_read_timeout(Some(Duration::from_millis(20))) .unwrap(); - loop { - let mut join_update = [0u8; sw_net::protocol::MTU]; - match client.recv(&mut join_update) { - Ok(_) => {} - Err(error) - if matches!( - error.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) => - { - break; + for socket in [&client, &observer] { + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + loop { + let mut join_update = [0u8; sw_net::protocol::MTU]; + match socket.recv(&mut join_update) { + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + break; + } + Err(error) => panic!("failed to drain join update: {error}"), } - Err(error) => panic!("failed to drain join update: {error}"), } } @@ -1493,9 +1531,16 @@ mod handshake_tests { ); let bytes = finish_envelope(&mut fbb, 2, p::Payload::ChatSend, chat.as_union_value()); let env = decode_envelope(&bytes).unwrap(); + let seq_before_chat = server.seq; server.on_chat(peer, env.payload_as_chat_send().unwrap(), 1_000); + assert_eq!( + server.seq, + seq_before_chat.wrapping_add(1), + "one invalid broadcast must be rejected once, not encoded once per recipient" + ); assert_no_outbound_datagram(&client); + assert_no_outbound_datagram(&observer); } #[test] @@ -2003,7 +2048,12 @@ mod market_dispatch_tests { let bytes = trade_envelope(txn_id, port, item, qty, price); let env = decode_envelope(&bytes).unwrap(); server - .on_trade(peer, env.payload_as_market_trade_request().unwrap(), now_ms) + .on_trade( + peer, + env.payload_as_market_trade_request().unwrap(), + now_ms, + now_ms, + ) .unwrap(); } @@ -2235,7 +2285,7 @@ mod input_hardening_tests { fn send_econ(server: &mut Server, peer: PeerId, env_bytes: &[u8], now_ms: i64) { let env = decode_envelope(env_bytes).unwrap(); server - .on_econ(peer, env.payload_as_econ_txn().unwrap(), now_ms) + .on_econ(peer, env.payload_as_econ_txn().unwrap(), now_ms, now_ms) .unwrap(); } @@ -2258,10 +2308,124 @@ mod input_hardening_tests { fn send_moor(server: &mut Server, peer: PeerId, env_bytes: &[u8], now_ms: i64) { let env = decode_envelope(env_bytes).unwrap(); server - .on_moor(peer, env.payload_as_moor_request().unwrap(), now_ms) + .on_moor(peer, env.payload_as_moor_request().unwrap(), now_ms, now_ms) .unwrap(); } + fn trade_envelope(txn_id: u64, qty: i64) -> Vec { + let mut fbb = FlatBufferBuilder::new(); + let req = p::MarketTradeRequest::create( + &mut fbb, + &p::MarketTradeRequestArgs { + txn_id, + port_id: 10, + item_id: 5, + qty, + unit_price: 100, + }, + ); + finish_envelope( + &mut fbb, + 7, + p::Payload::MarketTradeRequest, + req.as_union_value(), + ) + } + + fn chat_envelope(text: &str) -> Vec { + let mut fbb = FlatBufferBuilder::new(); + let text_off = fbb.create_string(text); + let chat = p::ChatSend::create( + &mut fbb, + &p::ChatSendArgs { + text: Some(text_off), + channel: 0, + }, + ); + finish_envelope(&mut fbb, 5, p::Payload::ChatSend, chat.as_union_value()) + } + + #[test] + fn backward_wall_clock_does_not_wedge_any_message_rate_limiter() { + let mut server = make_server(Config { + client_state_min_interval_ms: 250, + chat_min_interval_ms: 250, + econ_min_interval_ms: 250, + trade_min_interval_ms: 250, + moor_min_interval_ms: 250, + ..Config::default() + }); + let peer: PeerId = 1; + let player_id = join(&mut server, peer, "tok-clock-liveness"); + + let first_admission_ms = 1_000; + let next_admission_ms = 1_250; + let first_epoch_ms = 2_000; + let corrected_epoch_ms = 500; + + for (bytes, admission_ms, epoch_ms) in [ + ( + motion_envelope(1.0, 0.0, 0.0, 0.0, 0.0, 0.0), + first_admission_ms, + first_epoch_ms, + ), + ( + motion_envelope(2.0, 0.0, 0.0, 0.0, 0.0, 0.0), + next_admission_ms, + corrected_epoch_ms, + ), + (chat_envelope("first"), first_admission_ms, first_epoch_ms), + ( + chat_envelope("second"), + next_admission_ms, + corrected_epoch_ms, + ), + ( + econ_envelope(1, 100, "first"), + first_admission_ms, + first_epoch_ms, + ), + ( + econ_envelope(2, 50, "second"), + next_admission_ms, + corrected_epoch_ms, + ), + (trade_envelope(11, 10), first_admission_ms, first_epoch_ms), + (trade_envelope(12, 5), next_admission_ms, corrected_epoch_ms), + ( + moor_envelope(0.0, 0.0, "first"), + first_admission_ms, + first_epoch_ms, + ), + ( + moor_envelope(0.0, 0.0, "second"), + next_admission_ms, + corrected_epoch_ms, + ), + ] { + server + .handle_data_at(peer, &bytes, admission_ms, epoch_ms) + .unwrap(); + } + + assert_eq!(server.sessions[&peer].pos[0], 2.0); + assert_eq!(server.db.player_balance(player_id as i64).unwrap(), 150); + assert_eq!(server.db.market_state(10, 5).unwrap(), Some((15, 100))); + for admitted_at in [ + server.client_state_limiter.last_accepted_ms(player_id), + server.chat_limiter.last_accepted_ms(player_id), + server.econ_limiter.last_accepted_ms(player_id), + server.trade_limiter.last_accepted_ms(player_id), + server.moor_limiter.last_accepted_ms(player_id), + ] { + assert_eq!(admitted_at, Some(next_admission_ms)); + } + let cell = server.world.grid().cell_of(0.0, 0.0); + let moorings = server.db.moorings_in_cell(cell.cx, cell.cz).unwrap(); + assert_eq!(moorings[0].name, "second"); + assert_eq!(moorings[0].created_at, first_epoch_ms); + } + // ---- PART 1a: per-message input validation ---- #[test] @@ -2508,16 +2672,7 @@ mod input_hardening_tests { } fn send_chat(server: &mut Server, peer: PeerId, text: &str, now_ms: i64) { - let mut fbb = FlatBufferBuilder::new(); - let text_off = fbb.create_string(text); - let chat = p::ChatSend::create( - &mut fbb, - &p::ChatSendArgs { - text: Some(text_off), - channel: 0, - }, - ); - let bytes = finish_envelope(&mut fbb, 5, p::Payload::ChatSend, chat.as_union_value()); + let bytes = chat_envelope(text); let env = decode_envelope(&bytes).unwrap(); server.on_chat(peer, env.payload_as_chat_send().unwrap(), now_ms); } diff --git a/tests/Sailwind.Online.Net.Tests/MockTransport.cs b/tests/Sailwind.Online.Net.Tests/MockTransport.cs index c71cb5a..df6f270 100644 --- a/tests/Sailwind.Online.Net.Tests/MockTransport.cs +++ b/tests/Sailwind.Online.Net.Tests/MockTransport.cs @@ -41,6 +41,9 @@ internal sealed class MockTransport : ITransport /// Every datagram NetClient handed to , in order. public readonly List Sent = new List(); + /// Queued send failures, consumed before a datagram is recorded. + public readonly Queue SendFailures = new Queue(); + private bool _running; private bool _hasPeer; private bool _peerConnected; @@ -49,6 +52,8 @@ internal sealed class MockTransport : ITransport public bool IsPeerConnected => _peerConnected; + public int MaxUnreliablePayloadSize { get; set; } = NetClient.Mtu - 1; + public int Ping { get; set; } = -1; public event Action PeerConnected; @@ -98,6 +103,11 @@ public void DropPeer() public void Send(byte[] data, DeliveryMethod deliveryMethod) { + if (SendFailures.Count > 0) + { + throw SendFailures.Dequeue(); + } + Sent.Add(data); } diff --git a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs index 3a2ead3..83b8654 100644 --- a/tests/Sailwind.Online.Net.Tests/NetClientTests.cs +++ b/tests/Sailwind.Online.Net.Tests/NetClientTests.cs @@ -4,6 +4,7 @@ using System.Net.Sockets; using System.Numerics; using Google.FlatBuffers; +using LiteNetLib; using Sailwind.Api; using Sailwind.Online.Client.Net; using SwProto; @@ -548,23 +549,13 @@ public void ClientHello_ResendsEveryIntervalUntilServerHello() } [Fact] - public void ClientHello_WhenPacketCannotBeSent_DropsPeerAndReconnectsAfterBackoff() + public void ClientHello_WhenPacketExceedsTransportCapacity_DropsPeerAndReconnectsAfterBackoff() { - var options = new ConnectOptions - { - Host = "test-host", - Port = 4242, - DisplayName = new string('x', NetClient.Mtu), - Token = "tok", - GameBuild = "build", - ModVersion = "0.1.0", - ApiSurfaceHash = "hash" - }; - var transport = new MockTransport(); + var transport = new MockTransport { MaxUnreliablePayloadSize = 1 }; var log = new RecordingLog(); long now = 0; var net = new NetClient(log, transport, () => now); - net.Connect(options); + net.Connect(Options); transport.RaisePeerConnected(); @@ -574,14 +565,14 @@ public void ClientHello_WhenPacketCannotBeSent_DropsPeerAndReconnectsAfterBackof Assert.Empty(transport.Sent); int warningsAfterFailedSend = log.Warnings.Count; Assert.True(warningsAfterFailedSend > 0); - Assert.DoesNotContain(log.Warnings, message => message.Contains(options.Token)); + Assert.DoesNotContain(log.Warnings, message => message.Contains(Options.Token)); now = NetClient.HelloRetryMs; net.Poll(); Assert.Equal(warningsAfterFailedSend, log.Warnings.Count); Assert.Equal(1, transport.ConnectCalls); - options.DisplayName = "Ari"; + transport.MaxUnreliablePayloadSize = NetClient.Mtu - 1; now = NetClient.DefaultReconnectMs - 1; net.Poll(); Assert.Equal(1, transport.ConnectCalls); @@ -595,7 +586,49 @@ public void ClientHello_WhenPacketCannotBeSent_DropsPeerAndReconnectsAfterBackof Assert.Equal(ConnectionStatus.Handshaking, net.Status); Assert.Single(transport.Sent); - Assert.Equal("Ari", Decode(transport.Sent[0]).PayloadAsClientHello().DisplayName); + Assert.Equal(Options.DisplayName, Decode(transport.Sent[0]).PayloadAsClientHello().DisplayName); + } + + [Fact] + public void ClientHello_WhenTransportRejectsPacketAsTooBig_DropsAndRecoversAfterBackoff() + { + var transport = new MockTransport(); + transport.SendFailures.Enqueue(new TooBigPacketException("first boundary failure")); + transport.SendFailures.Enqueue(new TooBigPacketException("second boundary failure")); + var log = new RecordingLog(); + long now = 0; + var net = new NetClient(log, transport, () => now); + net.Connect(Options); + + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.Equal(1, transport.DropPeerCalls); + Assert.Empty(transport.Sent); + + now = NetClient.DefaultReconnectMs; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Disconnected, net.Status); + Assert.Equal(2, transport.DropPeerCalls); + Assert.Empty(transport.Sent); + + now = NetClient.DefaultReconnectMs + (2 * NetClient.DefaultReconnectMs) - 1; + net.Poll(); + Assert.Equal(2, transport.ConnectCalls); + + now++; + net.Poll(); + Assert.Equal(3, transport.ConnectCalls); + transport.RaisePeerConnected(); + + Assert.Equal(ConnectionStatus.Handshaking, net.Status); + Assert.Single(transport.Sent); + Assert.Equal(2, log.Warnings.Count); + Assert.All(log.Warnings, message => Assert.Contains("oversized packet", message)); + Assert.DoesNotContain(log.Warnings, message => message.Contains(Options.Token)); } [Fact] From 5d99a58297d3a124252b5c35c3e805ea8a8325d7 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 23:01:30 -0600 Subject: [PATCH 12/36] test(online): cover production transport paths Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 76 +++++++++++++++++++ .../LiteNetLibTransportTests.cs | 55 ++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 tests/Sailwind.Online.Net.Tests/LiteNetLibTransportTests.cs diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 67ce61f..1da9bdc 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -1218,6 +1218,22 @@ mod handshake_tests { ); } + fn receive_payload(client: &UdpSocket, expected: p::Payload) -> Vec { + loop { + let mut packet = [0u8; protocol::MTU]; + let received = client.recv(&mut packet).unwrap(); + assert_eq!( + protocol::Header::from_byte(packet[0]).property, + protocol::property::UNRELIABLE + ); + let payload = packet[protocol::HEADER_SIZE..received].to_vec(); + let env = decode_envelope(&payload).unwrap(); + if env.payload_type() == expected { + return payload; + } + } + } + #[test] fn protocol_mismatch_is_rejected_before_session_creation() { let mut server = make_server(); @@ -1543,6 +1559,66 @@ mod handshake_tests { assert_no_outbound_datagram(&observer); } + #[test] + fn valid_chat_is_preencoded_once_and_fanned_out_through_connected_peers() { + let mut server = make_server(); + let (sender, sender_peer) = connect_peer(&mut server); + let (observer, observer_peer) = connect_peer(&mut server); + let sender_hello = hello_envelope_with_strings( + Some("sender-token"), + Some("Skipper"), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ); + deliver_hello(&mut server, sender_peer, &sender_hello); + assert_eq!(receive_server_hello(&sender), (true, String::new())); + let observer_hello = hello_envelope_with_strings( + Some("observer-token"), + Some("Observer"), + Some("game-build"), + Some("mod-version"), + Some("surface-hash"), + sw_contracts::PROTOCOL_VERSION, + ); + deliver_hello(&mut server, observer_peer, &observer_hello); + assert_eq!(receive_server_hello(&observer), (true, String::new())); + + let sender_player = server.sessions[&sender_peer].player_id; + let mut fbb = FlatBufferBuilder::new(); + let text = fbb.create_string("fair winds"); + let chat = p::ChatSend::create( + &mut fbb, + &p::ChatSendArgs { + text: Some(text), + channel: 2, + }, + ); + let bytes = finish_envelope(&mut fbb, 3, p::Payload::ChatSend, chat.as_union_value()); + let seq_before_chat = server.seq; + + server + .handle_data_at(sender_peer, &bytes, 1_000, 1_000) + .unwrap(); + + assert_eq!(server.seq, seq_before_chat.wrapping_add(1)); + let sender_payload = receive_payload(&sender, p::Payload::ChatBroadcast); + let observer_payload = receive_payload(&observer, p::Payload::ChatBroadcast); + assert_eq!( + sender_payload, observer_payload, + "every recipient must receive the one pre-encoded broadcast" + ); + + let env = decode_envelope(&sender_payload).unwrap(); + assert_eq!(env.seq(), seq_before_chat.wrapping_add(1)); + let broadcast = env.payload_as_chat_broadcast().unwrap(); + assert_eq!(broadcast.player_id(), sender_player); + assert_eq!(broadcast.display_name(), Some("Skipper")); + assert_eq!(broadcast.text(), Some("fair winds")); + assert_eq!(broadcast.channel(), 2); + } + #[test] fn repeated_valid_hello_preserves_established_session_and_only_resends_server_hello() { let mut server = make_server(); diff --git a/tests/Sailwind.Online.Net.Tests/LiteNetLibTransportTests.cs b/tests/Sailwind.Online.Net.Tests/LiteNetLibTransportTests.cs new file mode 100644 index 0000000..c7050b8 --- /dev/null +++ b/tests/Sailwind.Online.Net.Tests/LiteNetLibTransportTests.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading; +using LiteNetLib; +using Sailwind.Online.Client.Net; +using Xunit; + +namespace Sailwind.Online.Net.Tests +{ + public sealed class LiteNetLibTransportTests + { + private const string ConnectKey = "transport-capacity-test"; + + [Fact] + public void MaxUnreliablePayloadSize_MatchesConnectedLiteNetLibPeerCapacity() + { + var serverListener = new EventBasedNetListener(); + var server = new NetManager(serverListener); + var transport = new LiteNetLibTransport(); + NetPeer serverPeer = null; + + serverListener.ConnectionRequestEvent += request => request.AcceptIfKey(ConnectKey); + serverListener.PeerConnectedEvent += peer => serverPeer = peer; + + try + { + Assert.True(server.Start(0)); + Assert.True(transport.Start()); + Assert.True(transport.Connect("127.0.0.1", server.LocalPort, ConnectKey)); + Assert.True( + SpinWait.SpinUntil( + () => + { + server.PollEvents(); + transport.PollEvents(); + return transport.IsPeerConnected && serverPeer != null; + }, + TimeSpan.FromSeconds(5)), + "the real loopback LiteNetLib peers did not connect"); + + int peerCapacity = serverPeer!.GetMaxSinglePacketSize(DeliveryMethod.Unreliable); + + Assert.Equal(peerCapacity, transport.MaxUnreliablePayloadSize); + Assert.InRange(transport.MaxUnreliablePayloadSize, 1, NetClient.Mtu - 1); + } + finally + { + transport.Stop(); + if (server.IsRunning) + { + server.Stop(); + } + } + } + } +} From f701fdf0b3fa30e0a90687b9dd86619ac3239191 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 23:36:41 -0600 Subject: [PATCH 13/36] fix(server): bound new handshake admissions Co-Authored-By: Claude --- server/config.example.toml | 11 ++ server/crates/sw-persist/src/lib.rs | 112 +++++++++++++- server/crates/sw-server/src/config.rs | 96 ++++++++++++ server/crates/sw-server/src/ratelimit.rs | 54 +++++++ server/crates/sw-server/src/server.rs | 182 +++++++++++++++++++++-- 5 files changed, 443 insertions(+), 12 deletions(-) diff --git a/server/config.example.toml b/server/config.example.toml index cf10fb8..c3111a2 100644 --- a/server/config.example.toml +++ b/server/config.example.toml @@ -43,6 +43,17 @@ trade_min_interval_ms = 250 # protection. hello_min_interval_ms = 250 +# Process-wide minimum interval between database admissions for sessions that +# are not already established on their peer. Unlike the per-peer hello throttle, +# this constant-memory gate cannot be bypassed by reconnecting with new peer IDs +# or rotating tokens. Must be in 1..=250. +new_session_min_interval_ms = 30 + +# Hard ceiling on persistent player rows. Existing identities can reconnect +# after the table reaches this limit; new identities are refused without +# creating session or world state. Must be in 1..=1000000. +max_player_rows = 10000 + # Per-player throttles (milliseconds) for the other inbound message classes, # mirroring the rotation-proof, memory-bounded trade limiter above. A message # beyond the configured rate is dropped before it drives work; an idempotent diff --git a/server/crates/sw-persist/src/lib.rs b/server/crates/sw-persist/src/lib.rs index a77adeb..7b0a23a 100644 --- a/server/crates/sw-persist/src/lib.rs +++ b/server/crates/sw-persist/src/lib.rs @@ -8,7 +8,7 @@ //! earlier table. Synchronous by design — it is driven directly from the //! server's single-threaded tick loop. -use rusqlite::{params, Connection, OptionalExtension, Row}; +use rusqlite::{params, Connection, OptionalExtension, Row, TransactionBehavior}; pub use rusqlite::{Error, Result}; @@ -96,6 +96,14 @@ pub struct PlayerRow { pub last_seen: i64, } +/// Result of atomically admitting a player identity against the configured +/// persistent-row ceiling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlayerAdmission { + Admitted(PlayerRow), + CapacityReached, +} + /// A row of the `moorings` table (a persisted boat mooring). #[derive(Debug, Clone, PartialEq)] pub struct MooringRow { @@ -189,6 +197,63 @@ impl Db { ) } + /// Create or refresh a player while atomically enforcing `max_rows`. + /// + /// An immediate transaction serializes the existence check, count, and + /// insert across SQLite connections. Existing identities remain admissible + /// after the table reaches capacity; a refused new identity makes no change. + pub fn admit_player_by_token( + &mut self, + token_hash: &str, + name: &str, + now: i64, + max_rows: u32, + ) -> Result { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let existing_id = tx + .query_row( + "SELECT id FROM players WHERE token_hash = ?1", + params![token_hash], + |row| row.get::<_, i64>(0), + ) + .optional()?; + + if let Some(id) = existing_id { + tx.execute( + "UPDATE players SET name = ?2, last_seen = ?3 WHERE id = ?1", + params![id, name, now], + )?; + let player = tx.query_row( + "SELECT id, name, token_hash, gold, created_at, last_seen + FROM players WHERE id = ?1", + params![id], + player_from_row, + )?; + tx.commit()?; + return Ok(PlayerAdmission::Admitted(player)); + } + + let count = tx.query_row("SELECT COUNT(*) FROM players", [], |row| { + row.get::<_, u64>(0) + })?; + if count >= u64::from(max_rows) { + tx.commit()?; + return Ok(PlayerAdmission::CapacityReached); + } + + let player = tx.query_row( + "INSERT INTO players (name, token_hash, gold, created_at, last_seen) + VALUES (?1, ?2, 0, ?3, ?3) + RETURNING id, name, token_hash, gold, created_at, last_seen", + params![name, token_hash, now], + player_from_row, + )?; + tx.commit()?; + Ok(PlayerAdmission::Admitted(player)) + } + /// Fetch a player by id. pub fn player(&self, id: i64) -> Result> { self.conn @@ -535,6 +600,51 @@ mod tests { assert_ne!(c.id, a.id); } + #[test] + fn player_admission_enforces_capacity_without_blocking_existing_identity() { + let mut db = Db::open_in_memory().unwrap(); + + let first = match db + .admit_player_by_token("hash-one", "First", 100, 2) + .unwrap() + { + PlayerAdmission::Admitted(player) => player, + PlayerAdmission::CapacityReached => panic!("first player must fit"), + }; + let second = match db + .admit_player_by_token("hash-two", "Second", 100, 2) + .unwrap() + { + PlayerAdmission::Admitted(player) => player, + PlayerAdmission::CapacityReached => panic!("second player must fit"), + }; + assert_ne!(first.id, second.id); + + assert_eq!( + db.admit_player_by_token("hash-three", "Third", 200, 2) + .unwrap(), + PlayerAdmission::CapacityReached + ); + assert!( + db.player(second.id + 1).unwrap().is_none(), + "capacity refusal must not insert a player row" + ); + + let reconnected = match db + .admit_player_by_token("hash-one", "Renamed", 300, 2) + .unwrap() + { + PlayerAdmission::Admitted(player) => player, + PlayerAdmission::CapacityReached => { + panic!("an existing identity must reconnect at capacity") + } + }; + assert_eq!(reconnected.id, first.id); + assert_eq!(reconnected.name, "Renamed"); + assert_eq!(reconnected.created_at, first.created_at); + assert_eq!(reconnected.last_seen, 300); + } + #[test] fn ledger_commit_and_lookup_roundtrip() { let db = Db::open_in_memory().unwrap(); diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index 1928eef..a4cbd2e 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -40,6 +40,16 @@ pub const MAX_HELLO_MIN_INTERVAL_MS: u32 = 250; /// unthrottled flood. const MIN_HELLO_MIN_INTERVAL_MS: u32 = 1; +/// Upper bound on the process-wide admission interval for new sessions. It +/// matches the client's retry cadence so a busy rejection can be retried on +/// the next scheduled hello. +pub const MAX_NEW_SESSION_MIN_INTERVAL_MS: u32 = 250; + +/// Highest configurable persistent player-row ceiling. The server still uses +/// the operator's lower configured value; this only prevents an accidental +/// effectively-unbounded cap. +pub const MAX_PLAYER_ROWS: u32 = 1_000_000; + /// Upper bound on the per-message wire-string length cap, in bytes. Bounds the /// [`Config::max_wire_string_len`] knob so a misconfiguration cannot admit an /// unbounded string, and so the saturating accessor has a finite ceiling. The @@ -72,6 +82,15 @@ pub struct Config { /// matches the client's handshake retry cadence. Bounded to /// `1..=`[`MAX_HELLO_MIN_INTERVAL_MS`]. pub hello_min_interval_ms: u32, + /// Process-wide minimum interval, in milliseconds, between database + /// admissions for sessions not already established on their peer. This + /// constant-memory gate bounds connection/token rotation across peer IDs. + /// Bounded to `1..=`[`MAX_NEW_SESSION_MIN_INTERVAL_MS`]. + pub new_session_min_interval_ms: u32, + /// Hard ceiling on persistent rows in the `players` table. Existing + /// identities may reconnect at capacity; new identities are refused. + /// Bounded to `1..=`[`MAX_PLAYER_ROWS`]. + pub max_player_rows: u32, /// Minimum interval, in milliseconds, between two accepted market trades by /// the same player (an aggregate per-player throttle, independent of which /// port the request names). A new trade inside this window is rejected; an @@ -117,6 +136,8 @@ impl Default for Config { aoi_radius_cells: sw_world::AOI_RADIUS_CELLS as u32, cell_size_m: sw_world::Grid::DEFAULT_CELL_SIZE_M, hello_min_interval_ms: 250, + new_session_min_interval_ms: 30, + max_player_rows: 10_000, trade_min_interval_ms: 250, client_state_min_interval_ms: 20, chat_min_interval_ms: 500, @@ -215,6 +236,18 @@ impl Config { "hello_min_interval_ms must be in {MIN_HELLO_MIN_INTERVAL_MS}..={MAX_HELLO_MIN_INTERVAL_MS}" )); } + if !(MIN_HELLO_MIN_INTERVAL_MS..=MAX_NEW_SESSION_MIN_INTERVAL_MS) + .contains(&self.new_session_min_interval_ms) + { + return Err(anyhow::anyhow!( + "new_session_min_interval_ms must be in {MIN_HELLO_MIN_INTERVAL_MS}..={MAX_NEW_SESSION_MIN_INTERVAL_MS}" + )); + } + if self.max_player_rows == 0 || self.max_player_rows > MAX_PLAYER_ROWS { + return Err(anyhow::anyhow!( + "max_player_rows must be in 1..={MAX_PLAYER_ROWS}" + )); + } for (name, value) in [ ("trade_min_interval_ms", self.trade_min_interval_ms), ( @@ -262,6 +295,17 @@ impl Config { .clamp(MIN_HELLO_MIN_INTERVAL_MS, MAX_HELLO_MIN_INTERVAL_MS) as i64 } + /// Process-wide new-session admission interval as bounded milliseconds. + pub fn new_session_min_interval_ms_i64(&self) -> i64 { + self.new_session_min_interval_ms + .clamp(MIN_HELLO_MIN_INTERVAL_MS, MAX_NEW_SESSION_MIN_INTERVAL_MS) as i64 + } + + /// Persistent player-row ceiling with a defense-in-depth clamp. + pub fn max_player_rows_u32(&self) -> u32 { + self.max_player_rows.clamp(1, MAX_PLAYER_ROWS) + } + /// Client-state throttle min-interval as a bounded `i64` of milliseconds. /// Saturates at [`MAX_TRADE_MIN_INTERVAL_MS`] so the limiter math stays /// finite even if a caller bypasses [`Config::validate`]. @@ -679,6 +723,8 @@ mod tests { #[test] fn parses_new_hardening_keys() { let toml_text = r#" + new_session_min_interval_ms = 125 + max_player_rows = 5000 client_state_min_interval_ms = 33 chat_min_interval_ms = 750 econ_min_interval_ms = 200 @@ -686,6 +732,8 @@ mod tests { max_wire_string_len = 256 "#; let cfg: Config = toml::from_str(toml_text).unwrap(); + assert_eq!(cfg.new_session_min_interval_ms, 125); + assert_eq!(cfg.max_player_rows, 5000); assert_eq!(cfg.client_state_min_interval_ms, 33); assert_eq!(cfg.chat_min_interval_ms, 750); assert_eq!(cfg.econ_min_interval_ms, 200); @@ -694,6 +742,54 @@ mod tests { cfg.validate().unwrap(); } + #[test] + fn new_session_budget_defaults_and_bounds_are_safe() { + let cfg = Config::default(); + cfg.validate().unwrap(); + assert_eq!(cfg.new_session_min_interval_ms, 30); + assert_eq!(cfg.max_player_rows, 10_000); + assert_eq!( + cfg.new_session_min_interval_ms_i64(), + i64::from(cfg.new_session_min_interval_ms) + ); + assert_eq!(cfg.max_player_rows_u32(), cfg.max_player_rows); + + for value in [0, MAX_NEW_SESSION_MIN_INTERVAL_MS + 1, u32::MAX] { + let invalid = Config { + new_session_min_interval_ms: value, + ..Config::default() + }; + assert!( + invalid.validate().is_err(), + "new-session interval {value} must be rejected" + ); + } + for value in [0, MAX_PLAYER_ROWS + 1, u32::MAX] { + let invalid = Config { + max_player_rows: value, + ..Config::default() + }; + assert!( + invalid.validate().is_err(), + "player-row capacity {value} must be rejected" + ); + } + } + + #[test] + fn new_session_budget_accessors_clamp_bypassed_validation() { + let invalid = Config { + new_session_min_interval_ms: u32::MAX, + max_player_rows: u32::MAX, + ..Config::default() + }; + assert_eq!( + invalid.new_session_min_interval_ms_i64(), + i64::from(MAX_NEW_SESSION_MIN_INTERVAL_MS) + ); + assert_eq!(invalid.max_player_rows_u32(), MAX_PLAYER_ROWS); + } + #[test] fn parses_example_shape() { let toml_text = r#" diff --git a/server/crates/sw-server/src/ratelimit.rs b/server/crates/sw-server/src/ratelimit.rs index 3b8dcdf..289df85 100644 --- a/server/crates/sw-server/src/ratelimit.rs +++ b/server/crates/sw-server/src/ratelimit.rs @@ -16,6 +16,41 @@ use std::collections::{hash_map::Entry, HashMap}; +/// One process-wide minimum-interval gate with constant memory. +#[derive(Debug)] +pub struct GlobalRateLimiter { + min_interval_ms: i64, + last: Option, +} + +impl GlobalRateLimiter { + /// Build a global limiter with the given minimum interval in milliseconds. + pub fn new(min_interval_ms: i64) -> GlobalRateLimiter { + GlobalRateLimiter { + min_interval_ms, + last: None, + } + } + + /// Admit at most one operation in each process-wide interval. + /// + /// A rejected attempt does not extend the cooldown. Timestamp rollback is + /// treated as no elapsed time, matching [`RateLimiter`]. + pub fn allow(&mut self, now_ms: i64) -> bool { + if self.min_interval_ms <= 0 { + return true; + } + if self + .last + .is_some_and(|last| now_ms.saturating_sub(last) < self.min_interval_ms) + { + return false; + } + self.last = Some(now_ms); + true + } +} + /// Tracks the last accepted message time per lifecycle-bounded key and admits a /// new message only once the configured interval has elapsed. #[derive(Debug)] @@ -167,4 +202,23 @@ mod tests { assert!(rl.allow(1, 1000)); assert_eq!(rl.tracked_count(), 0); } + + #[test] + fn global_limiter_has_one_rotation_proof_admission_window() { + let mut limiter = GlobalRateLimiter::new(250); + assert!(limiter.allow(1_000)); + for _rotated_peer in 0..10_000 { + assert!(!limiter.allow(1_001)); + } + assert!(limiter.allow(1_250)); + } + + #[test] + fn global_limiter_rollback_does_not_bypass_window() { + let mut limiter = GlobalRateLimiter::new(250); + assert!(limiter.allow(1_000)); + assert!(!limiter.allow(900)); + assert!(!limiter.allow(1_249)); + assert!(limiter.allow(1_250)); + } } diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 1da9bdc..f243127 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -4,7 +4,7 @@ use crate::clock::{clock_from_epoch, WorldClock}; use crate::codec::{self, BoatSnap, Caps, MooringSnap, PlayerSnap}; use crate::config::Config; use crate::econ_store::{DbLedgerStore, DbMarketStore}; -use crate::ratelimit::RateLimiter; +use crate::ratelimit::{GlobalRateLimiter, RateLimiter}; use crate::validate; use std::collections::HashMap; use std::io::Write; @@ -15,7 +15,7 @@ use sw_contracts::decode_envelope; use sw_contracts::sw_proto as p; use sw_econ::{Ledger, Market, MarketAck, Trade, Txn}; use sw_net::{protocol, DisconnectReason, Event, Host, PeerId}; -use sw_persist::{Db, MooringRow}; +use sw_persist::{Db, MooringRow, PlayerAdmission}; use sw_world::{AoiUpdate, Cell, Subscription, World}; /// LiteNetLib connect key clients must present. @@ -61,6 +61,7 @@ pub struct Server { weather_seed: u64, weather_epoch_day: u32, hello_limiter: RateLimiter, + new_session_limiter: GlobalRateLimiter, trade_limiter: RateLimiter, client_state_limiter: RateLimiter, chat_limiter: RateLimiter, @@ -91,6 +92,7 @@ impl Server { let host = Host::bind(&cfg.bind, CONNECT_KEY)?; let world = World::new(sw_world::Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -110,6 +112,7 @@ impl Server { weather_seed, weather_epoch_day, hello_limiter, + new_session_limiter, trade_limiter, client_state_limiter, chat_limiter, @@ -320,9 +323,23 @@ impl Server { return Ok(()); } - let player = self - .db - .upsert_player_by_token(&identity_hash, &name, persistence_ms)?; + if !self.new_session_limiter.allow(admission_ms) { + self.reject_hello(peer, "server busy; retry"); + return Ok(()); + } + + let player = match self.db.admit_player_by_token( + &identity_hash, + &name, + persistence_ms, + self.cfg.max_player_rows_u32(), + )? { + PlayerAdmission::Admitted(player) => player, + PlayerAdmission::CapacityReached => { + self.reject_hello(peer, "server player capacity reached"); + return Ok(()); + } + }; let player_id = player.id as u64; // Drop any prior session for this identity (reconnect from a new peer). @@ -998,6 +1015,7 @@ mod handshake_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter: RateLimiter::new(cfg.hello_min_interval_ms_i64()), + new_session_limiter: GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()), trade_limiter: RateLimiter::new(cfg.trade_min_interval_ms_i64()), client_state_limiter: RateLimiter::new(cfg.client_state_min_interval_ms_i64()), chat_limiter: RateLimiter::new(cfg.chat_min_interval_ms_i64()), @@ -1084,7 +1102,7 @@ mod handshake_tests { .on_hello_at( 2, env.payload_as_client_hello().unwrap(), - ADMISSION_MS, + ADMISSION_MS + hello_interval_ms, RETURN_EPOCH_MS, ) .unwrap(); @@ -1494,7 +1512,7 @@ mod handshake_tests { Some("surface-hash"), sw_contracts::PROTOCOL_VERSION, ); - deliver_hello(&mut server, peer, &hello); + deliver_hello_at(&mut server, peer, &hello, 1_000); assert_eq!(receive_server_hello(&client), (true, String::new())); let observer_hello = hello_envelope_with_strings( Some("observer-token"), @@ -1504,7 +1522,13 @@ mod handshake_tests { Some("surface-hash"), sw_contracts::PROTOCOL_VERSION, ); - deliver_hello(&mut server, observer_peer, &observer_hello); + let observer_admission_ms = 1_000 + server.cfg.new_session_min_interval_ms_i64(); + deliver_hello_at( + &mut server, + observer_peer, + &observer_hello, + observer_admission_ms, + ); assert_eq!(receive_server_hello(&observer), (true, String::new())); client .set_read_timeout(Some(Duration::from_millis(20))) @@ -1572,7 +1596,7 @@ mod handshake_tests { Some("surface-hash"), sw_contracts::PROTOCOL_VERSION, ); - deliver_hello(&mut server, sender_peer, &sender_hello); + deliver_hello_at(&mut server, sender_peer, &sender_hello, 1_000); assert_eq!(receive_server_hello(&sender), (true, String::new())); let observer_hello = hello_envelope_with_strings( Some("observer-token"), @@ -1582,7 +1606,13 @@ mod handshake_tests { Some("surface-hash"), sw_contracts::PROTOCOL_VERSION, ); - deliver_hello(&mut server, observer_peer, &observer_hello); + let observer_admission_ms = 1_000 + server.cfg.new_session_min_interval_ms_i64(); + deliver_hello_at( + &mut server, + observer_peer, + &observer_hello, + observer_admission_ms, + ); assert_eq!(receive_server_hello(&observer), (true, String::new())); let sender_player = server.sessions[&sender_peer].player_id; @@ -1724,6 +1754,129 @@ mod handshake_tests { assert_no_outbound_datagram(&client); } + #[test] + fn rotating_fresh_peers_cannot_bypass_global_new_session_admission() { + let mut server = make_server(); + let admission_ms = 1_000; + + for peer in 1..=64 { + let hello = hello_envelope( + &format!("rotated-token-{peer}"), + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, peer, &hello, admission_ms); + server + .on_disconnect(peer, DisconnectReason::Remote) + .unwrap(); + } + + assert!( + server.db.player(1).unwrap().is_some(), + "the first valid new session should be admitted" + ); + assert!( + server.db.player(2).unwrap().is_none(), + "one admission window must create at most one player row across all peer ids" + ); + assert_eq!( + server.hello_limiter.tracked_count(), + 0, + "disconnect churn must still clear per-peer buckets" + ); + + let next = hello_envelope( + "next-window-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let next_admission_ms = admission_ms + server.cfg.new_session_min_interval_ms_i64(); + deliver_hello_at(&mut server, 65, &next, next_admission_ms); + assert!( + server.db.player(2).unwrap().is_some(), + "normal admission must resume after the monotonic global window" + ); + } + + #[test] + fn player_capacity_refuses_new_identity_without_state_and_allows_reconnect() { + let mut server = make_server(); + server.cfg.max_player_rows = 1; + let interval = server.cfg.new_session_min_interval_ms_i64(); + + let (first_client, first_peer) = connect_peer(&mut server); + let first = hello_envelope( + "capacity-existing-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, first_peer, &first, 1_000); + assert_eq!(receive_server_hello(&first_client), (true, String::new())); + let existing_player = server.sessions[&first_peer].player_id; + server + .on_disconnect(first_peer, DisconnectReason::Remote) + .unwrap(); + assert!(server.sessions.is_empty()); + assert_eq!(server.world.len(), 0); + + let (refused_client, refused_peer) = connect_peer(&mut server); + let new_identity = hello_envelope( + "capacity-new-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, refused_peer, &new_identity, 1_000 + interval); + assert_eq!( + receive_server_hello(&refused_client), + (false, "server player capacity reached".to_string()) + ); + assert!(!server.sessions.contains_key(&refused_peer)); + assert_eq!(server.world.len(), 0); + assert!( + server + .db + .player(existing_player as i64 + 1) + .unwrap() + .is_none(), + "capacity refusal must not create a player row" + ); + + let (returning_client, returning_peer) = connect_peer(&mut server); + deliver_hello_at(&mut server, returning_peer, &first, 1_000 + 2 * interval); + assert_eq!( + receive_server_hello(&returning_client), + (true, String::new()) + ); + assert_eq!(server.sessions[&returning_peer].player_id, existing_player); + assert_eq!(server.world.len(), 1); + } + + #[test] + fn established_duplicate_does_not_consume_global_new_session_admission() { + let mut server = make_server(); + let interval = server.cfg.new_session_min_interval_ms_i64(); + let established = hello_envelope( + "established-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, 1, &established, 1_000); + assert!(server.sessions.contains_key(&1)); + + deliver_hello_at(&mut server, 1, &established, 1_000 + interval); + let fresh = hello_envelope( + "fresh-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, 2, &fresh, 1_000 + interval); + + assert!( + server.sessions.contains_key(&2), + "idempotent established hello must not charge the global admission gate" + ); + } + #[test] fn hello_limit_is_independent_per_peer() { let mut server = make_server(); @@ -1743,7 +1896,8 @@ mod handshake_tests { deliver_hello_at(&mut server, 1, &first, 1_001); assert_eq!(server.seq, seq_after_first); - deliver_hello_at(&mut server, 2, &second, 1_001); + let second_admission_ms = 1_000 + server.cfg.new_session_min_interval_ms_i64(); + deliver_hello_at(&mut server, 2, &second, second_admission_ms); assert_ne!(server.seq, seq_after_first); assert!(server.sessions.contains_key(&1)); assert!(server.sessions.contains_key(&2)); @@ -1840,6 +1994,7 @@ mod aoi_harden_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -1857,6 +2012,7 @@ mod aoi_harden_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter, + new_session_limiter, trade_limiter, client_state_limiter, chat_limiter, @@ -2038,6 +2194,7 @@ mod market_dispatch_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -2055,6 +2212,7 @@ mod market_dispatch_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter, + new_session_limiter, trade_limiter, client_state_limiter, chat_limiter, @@ -2269,6 +2427,7 @@ mod input_hardening_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); let chat_limiter = RateLimiter::new(cfg.chat_min_interval_ms_i64()); @@ -2286,6 +2445,7 @@ mod input_hardening_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter, + new_session_limiter, trade_limiter, client_state_limiter, chat_limiter, From a44df120b5c9ea624b24af913566a396b27d4bfa Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Wed, 22 Jul 2026 23:52:31 -0600 Subject: [PATCH 14/36] test(persist): cover concurrent admission capacity Co-Authored-By: Claude --- server/crates/sw-persist/src/lib.rs | 113 ++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/server/crates/sw-persist/src/lib.rs b/server/crates/sw-persist/src/lib.rs index 7b0a23a..4f5e8bf 100644 --- a/server/crates/sw-persist/src/lib.rs +++ b/server/crates/sw-persist/src/lib.rs @@ -486,6 +486,45 @@ fn mooring_from_row(r: &Row<'_>) -> Result { #[cfg(test)] mod tests { use super::*; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::{mpsc, Arc, Barrier}; + use std::thread; + use std::time::Duration; + + struct TestDatabase { + path: PathBuf, + } + + impl TestDatabase { + fn new(stem: &str) -> Self { + static NEXT_ID: AtomicU64 = AtomicU64::new(0); + + let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir() + .join(format!("sw-persist-{stem}-{}-{id}.db", std::process::id())); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + let path = path.canonicalize().unwrap(); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } + } + + impl Drop for TestDatabase { + fn drop(&mut self) { + for suffix in ["-wal", "-shm", ""] { + let path = format!("{}{suffix}", self.path.display()); + let _ = std::fs::remove_file(path); + } + } + } fn sample_mooring(boat_id: i64, owner: i64) -> MooringRow { MooringRow { @@ -645,6 +684,80 @@ mod tests { assert_eq!(reconnected.last_seen, 300); } + #[test] + fn concurrent_player_admission_enforces_capacity_across_connections() { + const CONTENDERS: usize = 8; + const RESULT_TIMEOUT: Duration = Duration::from_secs(10); + + let database = TestDatabase::new("admission"); + let path = database.path().to_str().unwrap(); + + // Finish migrations before opening the independent connections that + // will contend on the same real SQLite file. + drop(Db::open(path).unwrap()); + let connections = (0..CONTENDERS) + .map(|_| Db::open(path).unwrap()) + .collect::>(); + + let start = Arc::new(Barrier::new(CONTENDERS)); + let (result_tx, result_rx) = mpsc::channel(); + let handles = connections + .into_iter() + .enumerate() + .map(|(index, mut db)| { + let start = Arc::clone(&start); + let result_tx = result_tx.clone(); + thread::spawn(move || { + start.wait(); + let result = db.admit_player_by_token( + &format!("token-{index}"), + &format!("Player {index}"), + index as i64, + 1, + ); + result_tx.send((index, result)).unwrap(); + }) + }) + .collect::>(); + drop(result_tx); + + let mut admissions = Vec::with_capacity(CONTENDERS); + for _ in 0..CONTENDERS { + admissions.push( + result_rx + .recv_timeout(RESULT_TIMEOUT) + .expect("all admission attempts must finish within the SQLite busy timeout"), + ); + } + for handle in handles { + handle.join().unwrap(); + } + + let mut admitted = 0; + let mut capacity_reached = 0; + for (index, result) in admissions { + match result.unwrap_or_else(|error| { + panic!( + "contender {index} returned a database error instead of an admission: {error}" + ) + }) { + PlayerAdmission::Admitted(_) => admitted += 1, + PlayerAdmission::CapacityReached => capacity_reached += 1, + } + } + assert_eq!(admitted, 1); + assert_eq!(capacity_reached, CONTENDERS - 1); + + let verifier = Db::open(path).unwrap(); + let player_count = verifier + .conn + .query_row("SELECT COUNT(*) FROM players", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + assert_eq!(player_count, 1); + } + #[test] fn ledger_commit_and_lookup_roundtrip() { let db = Db::open_in_memory().unwrap(); From 8c889019e5d76a8ed7f9cc3a376c73530637eb85 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 00:29:37 -0600 Subject: [PATCH 15/36] fix(server): respect admission window in load fixture Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 32 ++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index f243127..cedd117 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -2456,7 +2456,13 @@ mod input_hardening_tests { } } - fn join(server: &mut Server, peer: PeerId, token: &str) -> u64 { + fn join_at( + server: &mut Server, + peer: PeerId, + token: &str, + admission_ms: i64, + persistence_ms: i64, + ) -> u64 { let mut fbb = FlatBufferBuilder::new(); let token_off = fbb.create_string(token); let name_off = fbb.create_string("Sailor"); @@ -2474,11 +2480,21 @@ mod input_hardening_tests { let bytes = finish_envelope(&mut fbb, 1, p::Payload::ClientHello, hello.as_union_value()); let env = decode_envelope(&bytes).unwrap(); server - .on_hello(peer, env.payload_as_client_hello().unwrap()) + .on_hello_at( + peer, + env.payload_as_client_hello().unwrap(), + admission_ms, + persistence_ms, + ) .unwrap(); server.sessions[&peer].player_id } + fn join(server: &mut Server, peer: PeerId, token: &str) -> u64 { + let admission_ms = server.admission_ms(); + join_at(server, peer, token, admission_ms, now_ms()) + } + #[allow(clippy::too_many_arguments)] fn motion_envelope(px: f32, py: f32, pz: f32, vx: f32, vy: f32, vz: f32) -> Vec { let mut fbb = FlatBufferBuilder::new(); @@ -3000,18 +3016,27 @@ mod input_hardening_tests { // skips it) and run only by the non-blocking load job / `make load-test`. const N: u32 = 200; const TICKS: u32 = 60; + const LOAD_EPOCH_MS: i64 = 1_700_000_000_000; let cfg = Config::default(); let tick_dt = Duration::from_secs_f64(1.0 / cfg.tick_hz as f64); let cell = cfg.cell_size_m; let mut server = make_server(cfg); + let session_step_ms = server.cfg.new_session_min_interval_ms_i64(); // Join N clients, each seeded into a distinct cell on a roughly square // grid so AoI density is realistic and bounded, not all stacked together. let side = (N as f64).sqrt().ceil() as u32; for i in 0..N { let peer = (i + 1) as PeerId; - join(&mut server, peer, &format!("tok-load-{i}")); + let session_offset_ms = i64::from(i) * session_step_ms; + join_at( + &mut server, + peer, + &format!("tok-load-{i}"), + 1_000 + session_offset_ms, + LOAD_EPOCH_MS + session_offset_ms, + ); let cx = (i % side) as f32; let cz = (i / side) as f32; // The seed time advances per client so the client-state throttle never @@ -3023,6 +3048,7 @@ mod input_hardening_tests { 1_000 + i as i64, ); } + assert_eq!(server.sessions.len(), N as usize); assert_eq!(server.world.len(), N as usize); // Drive TICKS simulated ticks and measure the wall-clock server work. The From 7dcbf416ed810f874b5e4e4b0d582313637c4c38 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 12:39:52 -0600 Subject: [PATCH 16/36] fix(server): prevent superseded session reclaim Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 72 ++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index cedd117..9b3dc5f 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -6,7 +6,7 @@ use crate::config::Config; use crate::econ_store::{DbLedgerStore, DbMarketStore}; use crate::ratelimit::{GlobalRateLimiter, RateLimiter}; use crate::validate; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::Write; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -54,6 +54,9 @@ pub struct Server { db: Db, world: World, sessions: HashMap, + // Logical sessions replaced on another connection stay barred until their + // still-live transport peer disconnects. This set is bounded by live peers. + superseded_peers: HashSet, seq: u32, snapshot_tick: u32, boot: Instant, @@ -105,6 +108,7 @@ impl Server { db, world, sessions: HashMap::new(), + superseded_peers: HashSet::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -252,6 +256,11 @@ impl Server { return Ok(()); } + if self.superseded_peers.contains(&peer) { + self.reject_hello(peer, "session replaced by a newer connection"); + return Ok(()); + } + if hello.protocol_version() != sw_contracts::PROTOCOL_VERSION { let reason = format!( "protocol version mismatch: client {}, server {}", @@ -351,6 +360,7 @@ impl Server { .collect(); for pp in stale { self.sessions.remove(&pp); + self.superseded_peers.insert(pp); } let mut sub = Subscription::new(self.cfg.aoi_radius_i32()); @@ -741,6 +751,7 @@ impl Server { fn on_disconnect(&mut self, peer: PeerId, reason: DisconnectReason) -> anyhow::Result<()> { self.hello_limiter.clear(u64::from(peer)); + self.superseded_peers.remove(&peer); if let Some(s) = self.sessions.remove(&peer) { self.world.remove(s.player_id); // Drop the player's throttle state across every message class: a @@ -1008,6 +1019,7 @@ mod handshake_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), + superseded_peers: HashSet::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -1696,6 +1708,61 @@ mod handshake_tests { assert_eq!(receive_server_hello(&client), (true, String::new())); } + #[test] + fn superseded_peer_retry_cannot_reclaim_identity_from_accepted_replacement() { + let mut server = make_server(); + let (first_client, first_peer) = connect_peer(&mut server); + let hello = hello_envelope( + "shared-identity-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let first_admission_ms = 1_000; + + deliver_hello_at(&mut server, first_peer, &hello, first_admission_ms); + assert_eq!(receive_server_hello(&first_client), (true, String::new())); + + let (replacement_client, replacement_peer) = connect_peer(&mut server); + let replacement_admission_ms = + first_admission_ms + server.cfg.new_session_min_interval_ms_i64(); + deliver_hello_at( + &mut server, + replacement_peer, + &hello, + replacement_admission_ms, + ); + assert_eq!( + receive_server_hello(&replacement_client), + (true, String::new()) + ); + let player_id = server.sessions[&replacement_peer].player_id; + let world_cell = server.world.cell_of_entity(player_id); + assert!(!server.sessions.contains_key(&first_peer)); + + let stale_retry_ms = replacement_admission_ms + + server + .cfg + .hello_min_interval_ms_i64() + .max(server.cfg.new_session_min_interval_ms_i64()); + deliver_hello_at(&mut server, first_peer, &hello, stale_retry_ms); + + assert_eq!(server.sessions.len(), 1); + assert!(!server.sessions.contains_key(&first_peer)); + assert_eq!(server.sessions[&replacement_peer].player_id, player_id); + assert_eq!(server.world.len(), 1); + assert_eq!(server.world.cell_of_entity(player_id), world_cell); + assert_eq!( + receive_server_hello(&first_client), + (false, "session replaced by a newer connection".to_string()) + ); + + server + .on_disconnect(first_peer, DisconnectReason::Remote) + .unwrap(); + assert!(!server.superseded_peers.contains(&first_peer)); + assert_eq!(server.sessions[&replacement_peer].player_id, player_id); + } + #[test] fn duplicate_hello_burst_is_dropped_before_response_or_session_work() { let mut server = make_server(); @@ -2005,6 +2072,7 @@ mod aoi_harden_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), + superseded_peers: HashSet::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -2205,6 +2273,7 @@ mod market_dispatch_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), + superseded_peers: HashSet::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -2438,6 +2507,7 @@ mod input_hardening_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), + superseded_peers: HashSet::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), From 9d2b5dbae701a44b89f39cd633f2f50652ed583c Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 13:16:40 -0600 Subject: [PATCH 17/36] fix(server): bound rotating session admission Co-Authored-By: Claude --- server/config.example.toml | 26 ++- server/crates/sw-net/src/lib.rs | 100 ++++++++++ server/crates/sw-server/src/config.rs | 83 ++++++++- server/crates/sw-server/src/ratelimit.rs | 87 ++++++++- server/crates/sw-server/src/server.rs | 224 +++++++++++++++++++++-- 5 files changed, 493 insertions(+), 27 deletions(-) diff --git a/server/config.example.toml b/server/config.example.toml index c3111a2..b088093 100644 --- a/server/config.example.toml +++ b/server/config.example.toml @@ -36,19 +36,27 @@ cell_size_m = 1024.0 # throttled. Must be in 0..=3600000 (0 disables the throttle). trade_min_interval_ms = 250 -# Per-peer ClientHello throttle in milliseconds. This runs before protocol/hash -# validation, database access, and ServerHello generation. Keep it at or below -# the client's 250 ms retry cadence so a lost ServerHello can be retried on the -# next attempt. Must be in 1..=250; zero would disable pre-authentication flood -# protection. +# ClientHello throttle in milliseconds. Per-peer throttling runs before +# protocol/hash validation, database access, and ServerHello generation; the +# same interval also limits new-session/reconnect admission from one source IP. +# Keep it at or below the client's 250 ms retry cadence so a lost ServerHello +# can be retried on the next attempt. Must be in 1..=250; zero would disable +# pre-authentication flood protection. hello_min_interval_ms = 250 -# Process-wide minimum interval between database admissions for sessions that -# are not already established on their peer. Unlike the per-peer hello throttle, -# this constant-memory gate cannot be bypassed by reconnecting with new peer IDs -# or rotating tokens. Must be in 1..=250. +# Process-wide minimum interval between database admissions for identities +# without an active session. Active-identity reconnects have a separate +# per-player window, so reconnect churn cannot monopolize this new-identity +# budget. Must be in 1..=250. new_session_min_interval_ms = 30 +# Hard ceilings on live LiteNetLib transport peers, including peers that have +# not authenticated yet. The per-IP ceiling prevents source-port rotation from +# consuming the global peer budget. Both must be positive, the global value may +# not exceed 65535, and the per-IP value may not exceed the global value. +max_transport_peers = 1024 +max_transport_peers_per_ip = 16 + # Hard ceiling on persistent player rows. Existing identities can reconnect # after the table reaches this limit; new identities are refused without # creating session or world state. Must be in 1..=1000000. diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index 6125f5c..24dfca9 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -35,6 +35,12 @@ const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); /// How often the host sends its own Ping to each peer to measure RTT. const PING_INTERVAL: Duration = Duration::from_secs(1); +/// Default hard ceiling on live transport peers. +pub const DEFAULT_MAX_PEERS: usize = 1_024; + +/// Default hard ceiling on live transport peers sharing one source IP. +pub const DEFAULT_MAX_PEERS_PER_IP: usize = 16; + /// .NET `DateTime` ticks (100 ns units) at the Unix epoch (1970-01-01). const UNIX_EPOCH_TICKS: i64 = 621_355_968_000_000_000; @@ -83,12 +89,36 @@ pub struct Host { next_local_peer_id: i32, connect_key: String, timeout: Duration, + max_peers: usize, + max_peers_per_ip: usize, recv_buf: Box<[u8; RECV_BUFFER]>, } impl Host { /// Bind a non-blocking UDP host that accepts peers presenting `connect_key`. pub fn bind(addr: A, connect_key: &str) -> io::Result { + Self::bind_with_limits( + addr, + connect_key, + DEFAULT_MAX_PEERS, + DEFAULT_MAX_PEERS_PER_IP, + ) + } + + /// Bind with hard global and per-source-IP live-peer ceilings. + pub fn bind_with_limits( + addr: A, + connect_key: &str, + max_peers: usize, + max_peers_per_ip: usize, + ) -> io::Result { + if max_peers == 0 || max_peers_per_ip == 0 || max_peers_per_ip > max_peers { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "peer limits must be nonzero and per-IP must not exceed global", + )); + } + let socket = UdpSocket::bind(addr)?; socket.set_nonblocking(true)?; Ok(Host { @@ -99,6 +129,8 @@ impl Host { next_local_peer_id: 0, connect_key: connect_key.to_string(), timeout: DEFAULT_TIMEOUT, + max_peers, + max_peers_per_ip, recv_buf: Box::new([0u8; RECV_BUFFER]), }) } @@ -108,6 +140,11 @@ impl Host { self.socket.local_addr() } + /// Return the remote address for a live peer. + pub fn peer_addr(&self, peer: PeerId) -> Option { + self.by_id.get(&peer).copied() + } + /// Number of currently connected peers. pub fn peer_count(&self) -> usize { self.peers.len() @@ -224,6 +261,15 @@ impl Host { self.peers.remove(&addr); self.by_id.remove(&old_id); events.push(Event::Disconnected(old_id, DisconnectReason::Remote)); + } else if self.peers.len() >= self.max_peers + || self + .peers + .keys() + .filter(|connected| connected.ip() == addr.ip()) + .count() + >= self.max_peers_per_ip + { + return; } let id = self.next_id; @@ -387,6 +433,60 @@ mod tests { protocol::build_connect_request(0, connect_time, 7, 16, &data) } + fn client_from(source_ip: &str, server_addr: SocketAddr) -> UdpSocket { + let client = UdpSocket::bind((source_ip, 0)).unwrap(); + client.set_nonblocking(true).unwrap(); + client.connect(server_addr).unwrap(); + client + } + + #[test] + fn peer_limits_bound_global_and_per_source_transport_state() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 3, 2).unwrap(); + let server_addr = server.local_addr().unwrap(); + let first = client_from("127.0.0.1", server_addr); + let second = client_from("127.0.0.1", server_addr); + + first.send(&connect_datagram(1, "sailwind-online")).unwrap(); + assert_eq!(server.poll(Instant::now()), vec![Event::Connected(1)]); + second + .send(&connect_datagram(2, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(Instant::now()), vec![Event::Connected(2)]); + assert_eq!(server.peer_count(), 2); + + let same_source_excess = client_from("127.0.0.1", server_addr); + same_source_excess + .send(&connect_datagram(3, "sailwind-online")) + .unwrap(); + assert!(server.poll(Instant::now()).is_empty()); + assert_eq!(server.peer_count(), 2); + + let other_source = client_from("127.0.0.2", server_addr); + other_source + .send(&connect_datagram(4, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(Instant::now()), vec![Event::Connected(3)]); + assert_eq!(server.peer_count(), 3); + + let global_excess = client_from("127.0.0.2", server_addr); + global_excess + .send(&connect_datagram(5, "sailwind-online")) + .unwrap(); + assert!(server.poll(Instant::now()).is_empty()); + assert_eq!(server.peer_count(), 3); + + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + first.recv(&mut accept).unwrap(); + first.send(&connect_datagram(1, "sailwind-online")).unwrap(); + assert!(server.poll(Instant::now()).is_empty()); + assert_eq!( + first.recv(&mut accept).unwrap(), + protocol::CONNECT_ACCEPT_SIZE + ); + assert_eq!(server.peer_count(), 3); + } + #[test] fn handshake_then_data_then_disconnect() { let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index a4cbd2e..e2c2b3a 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -45,6 +45,9 @@ const MIN_HELLO_MIN_INTERVAL_MS: u32 = 1; /// the next scheduled hello. pub const MAX_NEW_SESSION_MIN_INTERVAL_MS: u32 = 250; +/// Highest configurable live transport-peer ceiling. +pub const MAX_TRANSPORT_PEERS: u32 = 65_535; + /// Highest configurable persistent player-row ceiling. The server still uses /// the operator's lower configured value; this only prevents an accidental /// effectively-unbounded cap. @@ -79,14 +82,22 @@ pub struct Config { /// Minimum interval, in milliseconds, between two processed `ClientHello` /// messages from the same peer. A flood beyond this rate is dropped before /// validation, persistence, or response generation. The 250 ms default - /// matches the client's handshake retry cadence. Bounded to + /// matches the client's handshake retry cadence. The same interval bounds + /// session admissions from one source IP. Bounded to /// `1..=`[`MAX_HELLO_MIN_INTERVAL_MS`]. pub hello_min_interval_ms: u32, /// Process-wide minimum interval, in milliseconds, between database - /// admissions for sessions not already established on their peer. This - /// constant-memory gate bounds connection/token rotation across peer IDs. - /// Bounded to `1..=`[`MAX_NEW_SESSION_MIN_INTERVAL_MS`]. + /// admissions for identities without an active session. Active-identity + /// reconnects use a separate per-player gate, so they cannot consume every + /// new-identity window. Bounded to + /// `1..=`[`MAX_NEW_SESSION_MIN_INTERVAL_MS`]. pub new_session_min_interval_ms: u32, + /// Hard ceiling on all live transport peers, including pre-authentication + /// peers. Bounded to `1..=`[`MAX_TRANSPORT_PEERS`]. + pub max_transport_peers: u32, + /// Hard ceiling on live transport peers sharing a source IP. Must not + /// exceed [`Config::max_transport_peers`]. + pub max_transport_peers_per_ip: u32, /// Hard ceiling on persistent rows in the `players` table. Existing /// identities may reconnect at capacity; new identities are refused. /// Bounded to `1..=`[`MAX_PLAYER_ROWS`]. @@ -137,6 +148,8 @@ impl Default for Config { cell_size_m: sw_world::Grid::DEFAULT_CELL_SIZE_M, hello_min_interval_ms: 250, new_session_min_interval_ms: 30, + max_transport_peers: sw_net::DEFAULT_MAX_PEERS as u32, + max_transport_peers_per_ip: sw_net::DEFAULT_MAX_PEERS_PER_IP as u32, max_player_rows: 10_000, trade_min_interval_ms: 250, client_state_min_interval_ms: 20, @@ -243,6 +256,18 @@ impl Config { "new_session_min_interval_ms must be in {MIN_HELLO_MIN_INTERVAL_MS}..={MAX_NEW_SESSION_MIN_INTERVAL_MS}" )); } + if self.max_transport_peers == 0 || self.max_transport_peers > MAX_TRANSPORT_PEERS { + return Err(anyhow::anyhow!( + "max_transport_peers must be in 1..={MAX_TRANSPORT_PEERS}" + )); + } + if self.max_transport_peers_per_ip == 0 + || self.max_transport_peers_per_ip > self.max_transport_peers + { + return Err(anyhow::anyhow!( + "max_transport_peers_per_ip must be in 1..=max_transport_peers" + )); + } if self.max_player_rows == 0 || self.max_player_rows > MAX_PLAYER_ROWS { return Err(anyhow::anyhow!( "max_player_rows must be in 1..={MAX_PLAYER_ROWS}" @@ -301,6 +326,17 @@ impl Config { .clamp(MIN_HELLO_MIN_INTERVAL_MS, MAX_NEW_SESSION_MIN_INTERVAL_MS) as i64 } + /// Global live-peer ceiling with a defense-in-depth clamp. + pub fn max_transport_peers_usize(&self) -> usize { + self.max_transport_peers.clamp(1, MAX_TRANSPORT_PEERS) as usize + } + + /// Per-source-IP live-peer ceiling with a defense-in-depth clamp. + pub fn max_transport_peers_per_ip_usize(&self) -> usize { + self.max_transport_peers_per_ip + .clamp(1, self.max_transport_peers.clamp(1, MAX_TRANSPORT_PEERS)) as usize + } + /// Persistent player-row ceiling with a defense-in-depth clamp. pub fn max_player_rows_u32(&self) -> u32 { self.max_player_rows.clamp(1, MAX_PLAYER_ROWS) @@ -724,6 +760,8 @@ mod tests { fn parses_new_hardening_keys() { let toml_text = r#" new_session_min_interval_ms = 125 + max_transport_peers = 2048 + max_transport_peers_per_ip = 24 max_player_rows = 5000 client_state_min_interval_ms = 33 chat_min_interval_ms = 750 @@ -733,6 +771,8 @@ mod tests { "#; let cfg: Config = toml::from_str(toml_text).unwrap(); assert_eq!(cfg.new_session_min_interval_ms, 125); + assert_eq!(cfg.max_transport_peers, 2048); + assert_eq!(cfg.max_transport_peers_per_ip, 24); assert_eq!(cfg.max_player_rows, 5000); assert_eq!(cfg.client_state_min_interval_ms, 33); assert_eq!(cfg.chat_min_interval_ms, 750); @@ -747,6 +787,11 @@ mod tests { let cfg = Config::default(); cfg.validate().unwrap(); assert_eq!(cfg.new_session_min_interval_ms, 30); + assert_eq!(cfg.max_transport_peers, sw_net::DEFAULT_MAX_PEERS as u32); + assert_eq!( + cfg.max_transport_peers_per_ip, + sw_net::DEFAULT_MAX_PEERS_PER_IP as u32 + ); assert_eq!(cfg.max_player_rows, 10_000); assert_eq!( cfg.new_session_min_interval_ms_i64(), @@ -774,12 +819,34 @@ mod tests { "player-row capacity {value} must be rejected" ); } + for value in [0, MAX_TRANSPORT_PEERS + 1, u32::MAX] { + let invalid = Config { + max_transport_peers: value, + ..Config::default() + }; + assert!( + invalid.validate().is_err(), + "transport-peer capacity {value} must be rejected" + ); + } + for value in [0, Config::default().max_transport_peers + 1, u32::MAX] { + let invalid = Config { + max_transport_peers_per_ip: value, + ..Config::default() + }; + assert!( + invalid.validate().is_err(), + "per-IP transport-peer capacity {value} must be rejected" + ); + } } #[test] fn new_session_budget_accessors_clamp_bypassed_validation() { let invalid = Config { new_session_min_interval_ms: u32::MAX, + max_transport_peers: u32::MAX, + max_transport_peers_per_ip: u32::MAX, max_player_rows: u32::MAX, ..Config::default() }; @@ -788,6 +855,14 @@ mod tests { i64::from(MAX_NEW_SESSION_MIN_INTERVAL_MS) ); assert_eq!(invalid.max_player_rows_u32(), MAX_PLAYER_ROWS); + assert_eq!( + invalid.max_transport_peers_usize(), + MAX_TRANSPORT_PEERS as usize + ); + assert_eq!( + invalid.max_transport_peers_per_ip_usize(), + MAX_TRANSPORT_PEERS as usize + ); } #[test] diff --git a/server/crates/sw-server/src/ratelimit.rs b/server/crates/sw-server/src/ratelimit.rs index 289df85..d30938b 100644 --- a/server/crates/sw-server/src/ratelimit.rs +++ b/server/crates/sw-server/src/ratelimit.rs @@ -14,7 +14,8 @@ //! [`crate::config::Config::trade_min_interval_ms_i64`]), so the window math //! never overflows. -use std::collections::{hash_map::Entry, HashMap}; +use std::collections::{hash_map::Entry, BTreeSet, HashMap}; +use std::hash::Hash; /// One process-wide minimum-interval gate with constant memory. #[derive(Debug)] @@ -116,6 +117,67 @@ impl RateLimiter { } } +/// A minimum-interval throttle with a hard ceiling on tracked keys. +/// +/// When the key budget is full, a new key may replace only the oldest entry +/// whose cooldown has elapsed. This retains disconnect churn protection +/// without allowing the limiter itself to grow beyond `max_keys`. +#[derive(Debug)] +pub struct BoundedRateLimiter { + min_interval_ms: i64, + max_keys: usize, + last: HashMap, + oldest: BTreeSet<(i64, K)>, +} + +impl BoundedRateLimiter +where + K: Clone + Eq + Hash + Ord, +{ + /// Build a bounded per-key limiter. + pub fn new(min_interval_ms: i64, max_keys: usize) -> BoundedRateLimiter { + BoundedRateLimiter { + min_interval_ms, + max_keys, + last: HashMap::new(), + oldest: BTreeSet::new(), + } + } + + /// Admit a key after its cooldown while retaining at most `max_keys`. + pub fn allow(&mut self, key: K, now_ms: i64) -> bool { + if self.min_interval_ms <= 0 { + return true; + } + + if let Some(&last_ms) = self.last.get(&key) { + if now_ms.saturating_sub(last_ms) < self.min_interval_ms { + return false; + } + self.oldest.remove(&(last_ms, key.clone())); + } else if self.last.len() >= self.max_keys { + let Some((oldest_ms, oldest_key)) = self.oldest.first().cloned() else { + return false; + }; + if now_ms.saturating_sub(oldest_ms) < self.min_interval_ms { + return false; + } + self.oldest.remove(&(oldest_ms, oldest_key.clone())); + self.last.remove(&oldest_key); + } + + self.last.insert(key.clone(), now_ms); + self.oldest.insert((now_ms, key)); + true + } + + /// Number of retained keys. + #[cfg(test)] + pub fn tracked_count(&self) -> usize { + self.last.len() + } +} + #[cfg(test)] mod tests { use super::*; @@ -221,4 +283,27 @@ mod tests { assert!(!limiter.allow(1_249)); assert!(limiter.allow(1_250)); } + + #[test] + fn bounded_limiter_rejects_new_keys_until_an_entry_expires() { + let mut limiter = BoundedRateLimiter::new(250, 2); + assert!(limiter.allow(1, 1_000)); + assert!(limiter.allow(2, 1_001)); + assert!(!limiter.allow(3, 1_249)); + assert_eq!(limiter.tracked_count(), 2); + + assert!(limiter.allow(3, 1_250)); + assert_eq!(limiter.tracked_count(), 2); + assert!(!limiter.allow(3, 1_251)); + } + + #[test] + fn bounded_limiter_never_exceeds_its_key_budget_under_churn() { + let mut limiter = BoundedRateLimiter::new(1, 4); + for key in 0..10_000 { + assert!(limiter.allow(key, key as i64)); + assert!(limiter.tracked_count() <= 4); + } + assert_eq!(limiter.tracked_count(), 4); + } } diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 9b3dc5f..77aa119 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -4,10 +4,11 @@ use crate::clock::{clock_from_epoch, WorldClock}; use crate::codec::{self, BoatSnap, Caps, MooringSnap, PlayerSnap}; use crate::config::Config; use crate::econ_store::{DbLedgerStore, DbMarketStore}; -use crate::ratelimit::{GlobalRateLimiter, RateLimiter}; +use crate::ratelimit::{BoundedRateLimiter, GlobalRateLimiter, RateLimiter}; use crate::validate; use std::collections::{HashMap, HashSet}; use std::io::Write; +use std::net::IpAddr; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -64,6 +65,8 @@ pub struct Server { weather_seed: u64, weather_epoch_day: u32, hello_limiter: RateLimiter, + source_session_limiter: BoundedRateLimiter, + reconnect_limiter: RateLimiter, new_session_limiter: GlobalRateLimiter, trade_limiter: RateLimiter, client_state_limiter: RateLimiter, @@ -92,9 +95,19 @@ impl Server { let weather_seed: u64 = seed_str.parse().unwrap_or(0); let weather_epoch_day = clock_from_epoch(epoch_ms, epoch_ms).day; - let host = Host::bind(&cfg.bind, CONNECT_KEY)?; + let host = Host::bind_with_limits( + &cfg.bind, + CONNECT_KEY, + cfg.max_transport_peers_usize(), + cfg.max_transport_peers_per_ip_usize(), + )?; let world = World::new(sw_world::Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let source_session_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_transport_peers_usize(), + ); + let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -116,6 +129,8 @@ impl Server { weather_seed, weather_epoch_day, hello_limiter, + source_session_limiter, + reconnect_limiter, new_session_limiter, trade_limiter, client_state_limiter, @@ -332,7 +347,30 @@ impl Server { return Ok(()); } - if !self.new_session_limiter.allow(admission_ms) { + let source_ip = match self.host.peer_addr(peer) { + Some(addr) => addr.ip(), + #[cfg(test)] + None => test_peer_ip(peer), + #[cfg(not(test))] + None => return Ok(()), + }; + if !self.source_session_limiter.allow(source_ip, admission_ms) { + self.reject_hello(peer, "server busy; retry"); + return Ok(()); + } + + let active_player_id = self + .sessions + .values() + .find(|session| session.identity_hash == identity_hash) + .map(|session| session.player_id); + if active_player_id + .is_some_and(|player_id| !self.reconnect_limiter.allow(player_id, admission_ms)) + { + self.reject_hello(peer, "server busy; retry"); + return Ok(()); + } + if active_player_id.is_none() && !self.new_session_limiter.allow(admission_ms) { self.reject_hello(peer, "server busy; retry"); return Ok(()); } @@ -350,6 +388,9 @@ impl Server { } }; let player_id = player.id as u64; + if active_player_id.is_none() { + let _ = self.reconnect_limiter.allow(player_id, admission_ms); + } // Drop any prior session for this identity (reconnect from a new peer). let stale: Vec = self @@ -758,6 +799,7 @@ impl Server { // departed player's entries are useless and leaving them behind would // let connection churn accrete stale entries in the limiter maps. self.trade_limiter.clear(s.player_id); + self.reconnect_limiter.clear(s.player_id); self.client_state_limiter.clear(s.player_id); self.chat_limiter.clear(s.player_id); self.econ_limiter.clear(s.player_id); @@ -968,6 +1010,11 @@ fn token_hash(token: &str) -> String { format!("{h:016x}") } +#[cfg(test)] +fn test_peer_ip(peer: PeerId) -> IpAddr { + IpAddr::V6(std::net::Ipv6Addr::from(u128::from(peer) + 1)) +} + fn validate_hello_string<'a>( value: Option<&'a str>, field: &str, @@ -1015,7 +1062,13 @@ mod handshake_tests { let cfg = Config::default(); let world = World::new(Grid::new(cfg.cell_size_m)); Server { - host: Host::bind("127.0.0.1:0", CONNECT_KEY).unwrap(), + host: Host::bind_with_limits( + "127.0.0.1:0", + CONNECT_KEY, + cfg.max_transport_peers_usize(), + cfg.max_transport_peers_per_ip_usize(), + ) + .unwrap(), db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), @@ -1027,6 +1080,11 @@ mod handshake_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter: RateLimiter::new(cfg.hello_min_interval_ms_i64()), + source_session_limiter: BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_transport_peers_usize(), + ), + reconnect_limiter: RateLimiter::new(cfg.hello_min_interval_ms_i64()), new_session_limiter: GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()), trade_limiter: RateLimiter::new(cfg.trade_min_interval_ms_i64()), client_state_limiter: RateLimiter::new(cfg.client_state_min_interval_ms_i64()), @@ -1200,7 +1258,11 @@ mod handshake_tests { } fn connect_peer(server: &mut Server) -> (UdpSocket, PeerId) { - let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + connect_peer_from(server, "127.0.0.1") + } + + fn connect_peer_from(server: &mut Server, source_ip: &str) -> (UdpSocket, PeerId) { + let client = UdpSocket::bind((source_ip, 0)).unwrap(); client.connect(server.host.local_addr().unwrap()).unwrap(); client .set_read_timeout(Some(Duration::from_secs(1))) @@ -1534,7 +1596,11 @@ mod handshake_tests { Some("surface-hash"), sw_contracts::PROTOCOL_VERSION, ); - let observer_admission_ms = 1_000 + server.cfg.new_session_min_interval_ms_i64(); + let observer_admission_ms = 1_000 + + server + .cfg + .hello_min_interval_ms_i64() + .max(server.cfg.new_session_min_interval_ms_i64()); deliver_hello_at( &mut server, observer_peer, @@ -1618,7 +1684,11 @@ mod handshake_tests { Some("surface-hash"), sw_contracts::PROTOCOL_VERSION, ); - let observer_admission_ms = 1_000 + server.cfg.new_session_min_interval_ms_i64(); + let observer_admission_ms = 1_000 + + server + .cfg + .hello_min_interval_ms_i64() + .max(server.cfg.new_session_min_interval_ms_i64()); deliver_hello_at( &mut server, observer_peer, @@ -1723,8 +1793,11 @@ mod handshake_tests { assert_eq!(receive_server_hello(&first_client), (true, String::new())); let (replacement_client, replacement_peer) = connect_peer(&mut server); - let replacement_admission_ms = - first_admission_ms + server.cfg.new_session_min_interval_ms_i64(); + let replacement_admission_ms = first_admission_ms + + server + .cfg + .hello_min_interval_ms_i64() + .max(server.cfg.new_session_min_interval_ms_i64()); deliver_hello_at( &mut server, replacement_peer, @@ -1763,6 +1836,89 @@ mod handshake_tests { assert_eq!(server.sessions[&replacement_peer].player_id, player_id); } + #[test] + fn same_identity_port_rotation_does_not_starve_another_source() { + let mut server = make_server(); + let (first_client, first_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let returning = hello_envelope( + "returning-identity-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let first_admission_ms = 1_000; + deliver_hello_at(&mut server, first_peer, &returning, first_admission_ms); + assert_eq!(receive_server_hello(&first_client), (true, String::new())); + + let (replacement_client, replacement_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let (other_client, other_peer) = connect_peer_from(&mut server, "127.0.0.2"); + let next_admission_ms = first_admission_ms + + server + .cfg + .hello_min_interval_ms_i64() + .max(server.cfg.new_session_min_interval_ms_i64()); + + deliver_hello_at(&mut server, replacement_peer, &returning, next_admission_ms); + assert_eq!( + receive_server_hello(&replacement_client), + (true, String::new()) + ); + + let other = hello_envelope( + "other-source-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, other_peer, &other, next_admission_ms); + + assert_eq!(receive_server_hello(&other_client), (true, String::new())); + assert_eq!(server.sessions.len(), 2); + assert!(!server.sessions.contains_key(&first_peer)); + assert!(server.sessions.contains_key(&replacement_peer)); + assert!(server.sessions.contains_key(&other_peer)); + } + + #[test] + fn one_source_cannot_monopolize_new_identity_admission() { + let mut server = make_server(); + let (first_client, first_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let first = hello_envelope( + "first-source-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let first_admission_ms = 1_000; + deliver_hello_at(&mut server, first_peer, &first, first_admission_ms); + assert_eq!(receive_server_hello(&first_client), (true, String::new())); + + let (rotated_client, rotated_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let rotated = hello_envelope( + "rotated-source-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let (other_client, other_peer) = connect_peer_from(&mut server, "127.0.0.2"); + let next_global_ms = first_admission_ms + server.cfg.new_session_min_interval_ms_i64(); + + deliver_hello_at(&mut server, rotated_peer, &rotated, next_global_ms); + assert_eq!( + receive_server_hello(&rotated_client), + (false, "server busy; retry".to_string()) + ); + + let other = hello_envelope( + "other-source-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, other_peer, &other, next_global_ms); + + assert_eq!(receive_server_hello(&other_client), (true, String::new())); + assert_eq!(server.sessions.len(), 2); + assert!(server.sessions.contains_key(&first_peer)); + assert!(!server.sessions.contains_key(&rotated_peer)); + assert!(server.sessions.contains_key(&other_peer)); + } + #[test] fn duplicate_hello_burst_is_dropped_before_response_or_session_work() { let mut server = make_server(); @@ -1869,7 +2025,10 @@ mod handshake_tests { fn player_capacity_refuses_new_identity_without_state_and_allows_reconnect() { let mut server = make_server(); server.cfg.max_player_rows = 1; - let interval = server.cfg.new_session_min_interval_ms_i64(); + let interval = server + .cfg + .hello_min_interval_ms_i64() + .max(server.cfg.new_session_min_interval_ms_i64()); let (first_client, first_peer) = connect_peer(&mut server); let first = hello_envelope( @@ -2061,6 +2220,11 @@ mod aoi_harden_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let source_session_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_transport_peers_usize(), + ); + let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -2068,7 +2232,13 @@ mod aoi_harden_tests { let econ_limiter = RateLimiter::new(cfg.econ_min_interval_ms_i64()); let moor_limiter = RateLimiter::new(cfg.moor_min_interval_ms_i64()); Server { - host: Host::bind("127.0.0.1:0", CONNECT_KEY).unwrap(), + host: Host::bind_with_limits( + "127.0.0.1:0", + CONNECT_KEY, + cfg.max_transport_peers_usize(), + cfg.max_transport_peers_per_ip_usize(), + ) + .unwrap(), db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), @@ -2080,6 +2250,8 @@ mod aoi_harden_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter, + source_session_limiter, + reconnect_limiter, new_session_limiter, trade_limiter, client_state_limiter, @@ -2262,6 +2434,11 @@ mod market_dispatch_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let source_session_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_transport_peers_usize(), + ); + let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -2269,7 +2446,13 @@ mod market_dispatch_tests { let econ_limiter = RateLimiter::new(cfg.econ_min_interval_ms_i64()); let moor_limiter = RateLimiter::new(cfg.moor_min_interval_ms_i64()); Server { - host: Host::bind("127.0.0.1:0", CONNECT_KEY).unwrap(), + host: Host::bind_with_limits( + "127.0.0.1:0", + CONNECT_KEY, + cfg.max_transport_peers_usize(), + cfg.max_transport_peers_per_ip_usize(), + ) + .unwrap(), db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), @@ -2281,6 +2464,8 @@ mod market_dispatch_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter, + source_session_limiter, + reconnect_limiter, new_session_limiter, trade_limiter, client_state_limiter, @@ -2496,6 +2681,11 @@ mod input_hardening_tests { fn make_server(cfg: Config) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let source_session_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_transport_peers_usize(), + ); + let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -2503,7 +2693,13 @@ mod input_hardening_tests { let econ_limiter = RateLimiter::new(cfg.econ_min_interval_ms_i64()); let moor_limiter = RateLimiter::new(cfg.moor_min_interval_ms_i64()); Server { - host: Host::bind("127.0.0.1:0", CONNECT_KEY).unwrap(), + host: Host::bind_with_limits( + "127.0.0.1:0", + CONNECT_KEY, + cfg.max_transport_peers_usize(), + cfg.max_transport_peers_per_ip_usize(), + ) + .unwrap(), db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), @@ -2515,6 +2711,8 @@ mod input_hardening_tests { weather_seed: 0, weather_epoch_day: 0, hello_limiter, + source_session_limiter, + reconnect_limiter, new_session_limiter, trade_limiter, client_state_limiter, From ed74c92d1c2e3da74f07e5ce7c7e2078c9d8d2c9 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 14:04:23 -0600 Subject: [PATCH 18/36] fix(server): harden reconnect admission state Co-Authored-By: Claude --- server/crates/sw-net/src/lib.rs | 132 +++++++++++++++---- server/crates/sw-persist/src/lib.rs | 28 ++++ server/crates/sw-server/src/server.rs | 180 ++++++++++++++++++++------ 3 files changed, 271 insertions(+), 69 deletions(-) diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index 24dfca9..404507d 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -17,7 +17,7 @@ pub mod protocol; use std::collections::HashMap; use std::io; -use std::net::{SocketAddr, ToSocketAddrs, UdpSocket}; +use std::net::{IpAddr, SocketAddr, ToSocketAddrs, UdpSocket}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; /// Opaque, stable identifier for a connected peer, assigned by the host. @@ -85,6 +85,7 @@ pub struct Host { socket: UdpSocket, peers: HashMap, by_id: HashMap, + peers_per_ip: HashMap, next_id: PeerId, next_local_peer_id: i32, connect_key: String, @@ -125,6 +126,7 @@ impl Host { socket, peers: HashMap::new(), by_id: HashMap::new(), + peers_per_ip: HashMap::new(), next_id: 1, next_local_peer_id: 0, connect_key: connect_key.to_string(), @@ -150,6 +152,11 @@ impl Host { self.peers.len() } + #[cfg(test)] + fn peer_count_for_ip(&self, ip: IpAddr) -> usize { + self.peers_per_ip.get(&ip).copied().unwrap_or(0) + } + /// Last measured round-trip time to `peer`, if a Pong has come back. pub fn rtt(&self, peer: PeerId) -> Option { let addr = self.by_id.get(&peer)?; @@ -258,16 +265,10 @@ impl Host { } // A genuinely new session from the same address replaces the old one. let old_id = existing.id; - self.peers.remove(&addr); - self.by_id.remove(&old_id); + self.remove_peer_at(addr); events.push(Event::Disconnected(old_id, DisconnectReason::Remote)); } else if self.peers.len() >= self.max_peers - || self - .peers - .keys() - .filter(|connected| connected.ip() == addr.ip()) - .count() - >= self.max_peers_per_ip + || self.peers_per_ip.get(&addr.ip()).copied().unwrap_or(0) >= self.max_peers_per_ip { return; } @@ -285,25 +286,47 @@ impl Host { ); let _ = self.socket.send_to(&accept, addr); - self.peers.insert( + self.insert_peer(Peer { + id, addr, - Peer { - id, - addr, - connect_time: req.connect_time, - connection_number: req.connection_number, - local_peer_id, - last_recv: now, - last_ping_sent: now, - ping_seq: 0, - ping_sent_at: None, - rtt: None, - }, - ); - self.by_id.insert(id, addr); + connect_time: req.connect_time, + connection_number: req.connection_number, + local_peer_id, + last_recv: now, + last_ping_sent: now, + ping_seq: 0, + ping_sent_at: None, + rtt: None, + }); events.push(Event::Connected(id)); } + fn insert_peer(&mut self, peer: Peer) { + let id = peer.id; + let addr = peer.addr; + *self.peers_per_ip.entry(addr.ip()).or_insert(0) += 1; + self.peers.insert(addr, peer); + self.by_id.insert(id, addr); + } + + fn remove_peer_at(&mut self, addr: SocketAddr) -> Option { + let peer_id = self.peers.get(&addr)?.id; + let ip = addr.ip(); + let remove_counter = match self.peers_per_ip.get_mut(&ip)? { + count if *count > 1 => { + *count -= 1; + false + } + _ => true, + }; + if remove_counter { + self.peers_per_ip.remove(&ip); + } + let peer = self.peers.remove(&addr)?; + self.by_id.remove(&peer_id); + Some(peer) + } + fn handle_ping(&mut self, data: &[u8], addr: SocketAddr, now: Instant) { let Some(seq) = protocol::read_sequence(data) else { return; @@ -344,8 +367,7 @@ impl Host { &protocol::build_control(protocol::property::SHUTDOWN_OK), addr, ); - self.peers.remove(&addr); - self.by_id.remove(&id); + self.remove_peer_at(addr); events.push(Event::Disconnected(id, DisconnectReason::Remote)); } @@ -358,8 +380,7 @@ impl Host { .map(|(addr, _)| *addr) .collect(); for addr in expired { - if let Some(peer) = self.peers.remove(&addr) { - self.by_id.remove(&peer.id); + if let Some(peer) = self.remove_peer_at(addr) { events.push(Event::Disconnected(peer.id, DisconnectReason::Timeout)); } } @@ -397,6 +418,23 @@ impl Host { Ok(()) } + /// Remove one peer immediately and send the LiteNetLib Disconnect packet. + /// + /// The caller already owns the corresponding application-session cleanup, + /// so this does not enqueue a second [`Event::Disconnected`]. + pub fn disconnect(&mut self, peer: PeerId) -> bool { + let Some(addr) = self.by_id.get(&peer).copied() else { + return false; + }; + let Some(peer) = self.remove_peer_at(addr) else { + return false; + }; + let _ = self + .socket + .send_to(&protocol::build_disconnect(peer.connect_time), peer.addr); + true + } + /// Gracefully disconnect every peer (Disconnect packets + `Shutdown` /// events). Called on ctrl-c so clients learn immediately instead of /// waiting for their own timeout. @@ -410,6 +448,7 @@ impl Host { } self.peers.clear(); self.by_id.clear(); + self.peers_per_ip.clear(); events } } @@ -440,6 +479,21 @@ mod tests { client } + #[test] + fn per_ip_admission_uses_bounded_counter_state_instead_of_peer_scans() { + let source = include_str!("lib.rs"); + let counter_field = ["peers_per_ip: HashMap"].concat(); + let full_peer_scan = [".fil", "ter(|connected| connected.ip() == addr.ip())"].concat(); + assert!( + source.contains(&counter_field), + "the transport must maintain one bounded counter per live source IP" + ); + assert!( + !source.contains(&full_peer_scan), + "connect admission must not scan every live peer" + ); + } + #[test] fn peer_limits_bound_global_and_per_source_transport_state() { let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 3, 2).unwrap(); @@ -454,6 +508,7 @@ mod tests { .unwrap(); assert_eq!(server.poll(Instant::now()), vec![Event::Connected(2)]); assert_eq!(server.peer_count(), 2); + assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 2); let same_source_excess = client_from("127.0.0.1", server_addr); same_source_excess @@ -468,6 +523,7 @@ mod tests { .unwrap(); assert_eq!(server.poll(Instant::now()), vec![Event::Connected(3)]); assert_eq!(server.peer_count(), 3); + assert_eq!(server.peer_count_for_ip("127.0.0.2".parse().unwrap()), 1); let global_excess = client_from("127.0.0.2", server_addr); global_excess @@ -485,6 +541,24 @@ mod tests { protocol::CONNECT_ACCEPT_SIZE ); assert_eq!(server.peer_count(), 3); + + first.send(&connect_datagram(6, "sailwind-online")).unwrap(); + assert_eq!( + server.poll(Instant::now()), + vec![ + Event::Disconnected(1, DisconnectReason::Remote), + Event::Connected(4) + ] + ); + assert_eq!(server.peer_count(), 3); + assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 2); + + assert!(server.disconnect(4)); + assert_eq!(server.peer_count(), 2); + assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 1); + server.shutdown(); + assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 0); + assert_eq!(server.peer_count_for_ip("127.0.0.2".parse().unwrap()), 0); } #[test] @@ -533,6 +607,7 @@ mod tests { vec![Event::Disconnected(1, DisconnectReason::Remote)] ); assert_eq!(server.peer_count(), 0); + assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 0); } #[test] @@ -620,6 +695,7 @@ mod tests { vec![Event::Disconnected(1, DisconnectReason::Timeout)] ); assert_eq!(server.peer_count(), 0); + assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 0); } #[test] diff --git a/server/crates/sw-persist/src/lib.rs b/server/crates/sw-persist/src/lib.rs index 4f5e8bf..aa25bd2 100644 --- a/server/crates/sw-persist/src/lib.rs +++ b/server/crates/sw-persist/src/lib.rs @@ -265,6 +265,20 @@ impl Db { .optional() } + /// Load at most `limit` persisted identity keys in stable player-id order. + /// + /// The caller requests one row past its own hard ceiling to detect an + /// oversized database without ever materializing an unbounded result. + pub fn player_identities(&self, limit: u32) -> Result> { + let mut statement = self + .conn + .prepare("SELECT token_hash, id FROM players ORDER BY id ASC LIMIT ?1")?; + let identities = statement + .query_map(params![limit], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::>>()?; + Ok(identities) + } + /// A player's current gold balance (0 if the player does not exist). pub fn player_balance(&self, id: i64) -> Result { Ok(self @@ -684,6 +698,20 @@ mod tests { assert_eq!(reconnected.last_seen, 300); } + #[test] + fn player_identity_load_is_stable_and_hard_limited() { + let db = Db::open_in_memory().unwrap(); + let first = db.upsert_player_by_token("hash-one", "First", 100).unwrap(); + db.upsert_player_by_token("hash-two", "Second", 100) + .unwrap(); + + assert_eq!( + db.player_identities(1).unwrap(), + vec![("hash-one".to_string(), first.id)] + ); + assert_eq!(db.player_identities(0).unwrap(), Vec::new()); + } + #[test] fn concurrent_player_admission_enforces_capacity_across_connections() { const CONTENDERS: usize = 8; diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 77aa119..a03d052 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -2,11 +2,11 @@ use crate::clock::{clock_from_epoch, WorldClock}; use crate::codec::{self, BoatSnap, Caps, MooringSnap, PlayerSnap}; -use crate::config::Config; +use crate::config::{Config, MAX_PLAYER_ROWS}; use crate::econ_store::{DbLedgerStore, DbMarketStore}; use crate::ratelimit::{BoundedRateLimiter, GlobalRateLimiter, RateLimiter}; use crate::validate; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::io::Write; use std::net::IpAddr; use std::sync::atomic::{AtomicBool, Ordering}; @@ -55,9 +55,7 @@ pub struct Server { db: Db, world: World, sessions: HashMap, - // Logical sessions replaced on another connection stay barred until their - // still-live transport peer disconnects. This set is bounded by live peers. - superseded_peers: HashSet, + identity_players: HashMap, seq: u32, snapshot_tick: u32, boot: Instant, @@ -81,6 +79,7 @@ impl Server { /// the socket. Fails before the readiness line if binding fails. pub fn new(cfg: Config, running: Arc) -> anyhow::Result { let db = Db::open(&cfg.db_path)?; + let identity_players = load_identity_players(&db)?; // World clock epoch: first boot stamps "now"; later boots reuse it. let now = now_ms(); @@ -121,7 +120,7 @@ impl Server { db, world, sessions: HashMap::new(), - superseded_peers: HashSet::new(), + identity_players, seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -271,11 +270,6 @@ impl Server { return Ok(()); } - if self.superseded_peers.contains(&peer) { - self.reject_hello(peer, "session replaced by a newer connection"); - return Ok(()); - } - if hello.protocol_version() != sw_contracts::PROTOCOL_VERSION { let reason = format!( "protocol version mismatch: client {}, server {}", @@ -359,18 +353,14 @@ impl Server { return Ok(()); } - let active_player_id = self - .sessions - .values() - .find(|session| session.identity_hash == identity_hash) - .map(|session| session.player_id); - if active_player_id + let known_player_id = self.identity_players.get(&identity_hash).copied(); + if known_player_id .is_some_and(|player_id| !self.reconnect_limiter.allow(player_id, admission_ms)) { self.reject_hello(peer, "server busy; retry"); return Ok(()); } - if active_player_id.is_none() && !self.new_session_limiter.allow(admission_ms) { + if known_player_id.is_none() && !self.new_session_limiter.allow(admission_ms) { self.reject_hello(peer, "server busy; retry"); return Ok(()); } @@ -387,8 +377,22 @@ impl Server { return Ok(()); } }; - let player_id = player.id as u64; - if active_player_id.is_none() { + let player_id = u64::try_from(player.id) + .map_err(|_| anyhow::anyhow!("admitted player id must be nonnegative"))?; + if let Some(expected_player_id) = known_player_id { + if player_id != expected_player_id { + return Err(anyhow::anyhow!( + "persisted identity index disagrees with admitted player" + )); + } + } else { + if self.identity_players.len() >= MAX_PLAYER_ROWS as usize { + return Err(anyhow::anyhow!( + "persisted identity index reached its hard ceiling" + )); + } + self.identity_players + .insert(identity_hash.clone(), player_id); let _ = self.reconnect_limiter.allow(player_id, admission_ms); } @@ -400,8 +404,11 @@ impl Server { .map(|(&pp, _)| pp) .collect(); for pp in stale { + if self.host.peer_addr(pp).is_some() && !self.host.disconnect(pp) { + return Err(anyhow::anyhow!("failed to evict superseded transport peer")); + } self.sessions.remove(&pp); - self.superseded_peers.insert(pp); + self.hello_limiter.clear(u64::from(pp)); } let mut sub = Subscription::new(self.cfg.aoi_radius_i32()); @@ -792,7 +799,6 @@ impl Server { fn on_disconnect(&mut self, peer: PeerId, reason: DisconnectReason) -> anyhow::Result<()> { self.hello_limiter.clear(u64::from(peer)); - self.superseded_peers.remove(&peer); if let Some(s) = self.sessions.remove(&peer) { self.world.remove(s.player_id); // Drop the player's throttle state across every message class: a @@ -1010,6 +1016,34 @@ fn token_hash(token: &str) -> String { format!("{h:016x}") } +fn load_identity_players(db: &Db) -> anyhow::Result> { + load_identity_players_up_to(db, MAX_PLAYER_ROWS) +} + +fn load_identity_players_up_to(db: &Db, max_players: u32) -> anyhow::Result> { + let rows = db.player_identities(max_players.saturating_add(1))?; + if rows.len() > max_players as usize { + return Err(anyhow::anyhow!( + "players table exceeds the hard identity ceiling of {max_players}" + )); + } + + rows.into_iter() + .map(|(identity_hash, player_id)| { + if identity_hash.len() != 16 + || !identity_hash.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(anyhow::anyhow!( + "persisted player identity hash has an invalid shape" + )); + } + let player_id = u64::try_from(player_id) + .map_err(|_| anyhow::anyhow!("persisted player id must be nonnegative"))?; + Ok((identity_hash, player_id)) + }) + .collect() +} + #[cfg(test)] fn test_peer_ip(peer: PeerId) -> IpAddr { IpAddr::V6(std::net::Ipv6Addr::from(u128::from(peer) + 1)) @@ -1041,6 +1075,17 @@ mod tests { assert_eq!(token_hash("abc").len(), 16); } + #[test] + fn persisted_identity_index_fails_closed_at_its_hard_bound() { + let db = Db::open_in_memory().unwrap(); + db.upsert_player_by_token(&token_hash("first"), "First", 1) + .unwrap(); + db.upsert_player_by_token(&token_hash("second"), "Second", 1) + .unwrap(); + + assert!(load_identity_players_up_to(&db, 1).is_err()); + } + #[test] fn vec_and_quat_defaults() { assert_eq!(vec3_of(None), [0.0, 0.0, 0.0]); @@ -1059,8 +1104,13 @@ mod handshake_tests { use sw_world::Grid; fn make_server() -> Server { + make_server_with_db(Db::open_in_memory().unwrap()) + } + + fn make_server_with_db(db: Db) -> Server { let cfg = Config::default(); let world = World::new(Grid::new(cfg.cell_size_m)); + let identity_players = load_identity_players(&db).unwrap(); Server { host: Host::bind_with_limits( "127.0.0.1:0", @@ -1069,10 +1119,10 @@ mod handshake_tests { cfg.max_transport_peers_per_ip_usize(), ) .unwrap(), - db: Db::open_in_memory().unwrap(), + db, world, sessions: HashMap::new(), - superseded_peers: HashSet::new(), + identity_players, seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -1811,29 +1861,77 @@ mod handshake_tests { let player_id = server.sessions[&replacement_peer].player_id; let world_cell = server.world.cell_of_entity(player_id); assert!(!server.sessions.contains_key(&first_peer)); + assert!( + server.host.peer_addr(first_peer).is_none(), + "a superseded logical session must be removed from the transport" + ); + assert_eq!( + server.host.peer_count(), + 1, + "superseded peers must not retain transport quota" + ); - let stale_retry_ms = replacement_admission_ms - + server - .cfg - .hello_min_interval_ms_i64() - .max(server.cfg.new_session_min_interval_ms_i64()); - deliver_hello_at(&mut server, first_peer, &hello, stale_retry_ms); + loop { + let mut packet = [0u8; protocol::MTU]; + let received = first_client.recv(&mut packet).unwrap(); + if protocol::Header::from_byte(packet[0]).property == protocol::property::DISCONNECT { + assert_eq!(received, protocol::DISCONNECT_SIZE); + break; + } + } + + first_client + .send(&protocol::build_unreliable(&hello)) + .unwrap(); + first_client.send(&protocol::build_ping(1)).unwrap(); + assert!( + server.host.poll(Instant::now()).is_empty(), + "data and keepalive traffic from the evicted address must be ignored" + ); assert_eq!(server.sessions.len(), 1); assert!(!server.sessions.contains_key(&first_peer)); assert_eq!(server.sessions[&replacement_peer].player_id, player_id); assert_eq!(server.world.len(), 1); assert_eq!(server.world.cell_of_entity(player_id), world_cell); - assert_eq!( - receive_server_hello(&first_client), - (false, "session replaced by a newer connection".to_string()) - ); + } - server - .on_disconnect(first_peer, DisconnectReason::Remote) + #[test] + fn persisted_offline_identity_does_not_compete_with_fresh_token_admission() { + let persisted_token = "persisted-offline-token"; + let db = Db::open_in_memory().unwrap(); + let persisted = db + .upsert_player_by_token(&token_hash(persisted_token), "Returning", 100) .unwrap(); - assert!(!server.superseded_peers.contains(&first_peer)); - assert_eq!(server.sessions[&replacement_peer].player_id, player_id); + let mut server = make_server_with_db(db); + let admission_ms = 1_000; + + let (fresh_client, fresh_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let fresh = hello_envelope( + "fresh-attacker-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, fresh_peer, &fresh, admission_ms); + assert_eq!(receive_server_hello(&fresh_client), (true, String::new())); + + let (returning_client, returning_peer) = connect_peer_from(&mut server, "127.0.0.2"); + let returning = hello_envelope( + persisted_token, + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, returning_peer, &returning, admission_ms); + + assert_eq!( + receive_server_hello(&returning_client), + (true, String::new()), + "a persisted identity must use reconnect admission even while the new-token gate is busy" + ); + assert_eq!( + server.sessions[&returning_peer].player_id, + persisted.id as u64 + ); } #[test] @@ -2242,7 +2340,7 @@ mod aoi_harden_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), - superseded_peers: HashSet::new(), + identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -2456,7 +2554,7 @@ mod market_dispatch_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), - superseded_peers: HashSet::new(), + identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -2703,7 +2801,7 @@ mod input_hardening_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), - superseded_peers: HashSet::new(), + identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, boot: Instant::now(), From 5b31d9df04c0c2b1ca7a0346d9875b8429533873 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 14:19:04 -0600 Subject: [PATCH 19/36] fix(smoke): retry transient hello admission Co-Authored-By: Claude --- tools/protocol-smoke/Checks.cs | 18 +++++++++++++++++- tools/protocol-smoke/Program.cs | 5 ++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tools/protocol-smoke/Checks.cs b/tools/protocol-smoke/Checks.cs index 4b909ac..6214e16 100644 --- a/tools/protocol-smoke/Checks.cs +++ b/tools/protocol-smoke/Checks.cs @@ -18,6 +18,7 @@ internal sealed class Checks { private const string TokenA = "smoke-token-A"; private const string TokenB = "smoke-token-B"; + private const string RetryableHelloReason = "server busy; retry"; private readonly string _serverPath; private readonly string _configPath; @@ -437,7 +438,22 @@ private void DrainPresence(SmokeClient b, ulong playerIdA, ref bool sawAddedCell { const string apiSurfaceHashSentinel = "protocol-smoke-surface-hash"; var hello = Codec.EncodeClientHello(_seq++, name, token, "smoke", "0.0.0", apiSurfaceHashSentinel); - var env = SendAndWait(c, hello, e => e.PayloadType == Payload.ServerHello, 6000, 250); + var env = SendAndWait( + c, + hello, + e => + { + if (e.PayloadType != Payload.ServerHello) + { + return false; + } + + var response = e.PayloadAsServerHello(); + return response.Accepted + || !string.Equals(response.Reason, RetryableHelloReason, StringComparison.Ordinal); + }, + 6000, + 250); return env.HasValue ? env.Value.PayloadAsServerHello() : (ServerHello?)null; } diff --git a/tools/protocol-smoke/Program.cs b/tools/protocol-smoke/Program.cs index 0d51564..1024485 100644 --- a/tools/protocol-smoke/Program.cs +++ b/tools/protocol-smoke/Program.cs @@ -112,7 +112,10 @@ private static string WriteConfig(string dir, string dbPath, int port) $"db = \"{db}\"\n" + "server_name = \"protocol-smoke\"\n" + "tick_hz = 30\n" + - "snapshot_hz = 4\n"; + "snapshot_hz = 4\n" + + // Force consecutive fresh identities through the retryable + // admission response so check 3 cannot pass by timing luck. + "new_session_min_interval_ms = 250\n"; File.WriteAllText(configPath, contents); return configPath; } From bca29f5995b623f1de4bd948ae59826321b60417 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 14:32:18 -0600 Subject: [PATCH 20/36] test(smoke): prove transient hello retry Co-Authored-By: Claude --- tools/protocol-smoke/Checks.cs | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tools/protocol-smoke/Checks.cs b/tools/protocol-smoke/Checks.cs index 6214e16..72fff8b 100644 --- a/tools/protocol-smoke/Checks.cs +++ b/tools/protocol-smoke/Checks.cs @@ -86,7 +86,7 @@ public bool Run() handshook ? "LiteNetLib Connect completed within 2s" : "no PeerConnected within 2s (sw-net LiteNetLib framing not interoperating yet)")); - ServerHello? helloA = handshook ? DoHello(a, "Smoke-A", TokenA) : null; + ServerHello? helloA = handshook ? DoHello(a, "Smoke-A", TokenA).Response : null; bool helloOk = IsValidHello(helloA); playerIdA = helloA.HasValue ? helloA.Value.PlayerId : 0; results.Add(new Result(2, "hello", helloOk, @@ -146,11 +146,17 @@ private Result RunPresence(ServerHarness server, IPEndPoint ep, SmokeClient a, u } var helloB = DoHello(b, "Smoke-B", TokenB); - if (!IsValidHello(helloB)) + if (!IsValidHello(helloB.Response)) { return new Result(3, "presence-aoi", false, "client B failed the hello exchange"); } + if (!helloB.SawRetryableResponse) + { + return new Result(3, "presence-aoi", false, + $"client B did not observe exact transient response '{RetryableHelloReason}' before acceptance"); + } + bool sawAddedCell = false; var xs = new List(); @@ -180,7 +186,7 @@ private Result RunPresence(ServerHarness server, IPEndPoint ep, SmokeClient a, u bool increasing = IsIncreasing(xs); bool ok = sawAddedCell && increasing; var detail = ok - ? $"B saw A's cell added and {xs.Count} monotonically increasing x samples" + ? $"B observed '{RetryableHelloReason}', then saw A's cell added and {xs.Count} monotonically increasing x samples" : $"sawAddedCell={sawAddedCell}, x-samples={xs.Count}, increasing={increasing}"; return new Result(3, "presence-aoi", ok, detail); } @@ -328,7 +334,7 @@ private Result RunRestartPersistence(ServerHarness server, SmokeClient a, ulong return new Result(6, "restart-persistence", false, "reconnect after restart failed"); } - var hello = DoHello(a2, "Smoke-A", TokenA); + var hello = DoHello(a2, "Smoke-A", TokenA).Response; if (!IsValidHello(hello)) { return new Result(6, "restart-persistence", false, "hello after restart failed"); @@ -434,9 +440,13 @@ private void DrainPresence(SmokeClient b, ulong playerIdA, ref bool sawAddedCell } } - private ServerHello? DoHello(SmokeClient c, string name, string token) + private (ServerHello? Response, bool SawRetryableResponse) DoHello( + SmokeClient c, + string name, + string token) { const string apiSurfaceHashSentinel = "protocol-smoke-surface-hash"; + var sawRetryableResponse = false; var hello = Codec.EncodeClientHello(_seq++, name, token, "smoke", "0.0.0", apiSurfaceHashSentinel); var env = SendAndWait( c, @@ -449,12 +459,17 @@ private void DrainPresence(SmokeClient b, ulong playerIdA, ref bool sawAddedCell } var response = e.PayloadAsServerHello(); - return response.Accepted - || !string.Equals(response.Reason, RetryableHelloReason, StringComparison.Ordinal); + if (string.Equals(response.Reason, RetryableHelloReason, StringComparison.Ordinal)) + { + sawRetryableResponse = true; + return false; + } + + return true; }, 6000, 250); - return env.HasValue ? env.Value.PayloadAsServerHello() : (ServerHello?)null; + return (env.HasValue ? env.Value.PayloadAsServerHello() : (ServerHello?)null, sawRetryableResponse); } // Resends `payload` every retryMs (safe because hello, econ and moorage From 72cb52cb3247e0f3100ee60759a0627055b11f5e Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 15:02:04 -0600 Subject: [PATCH 21/36] fix(server): retain reconnect admission cooldown Co-Authored-By: Claude --- server/config.example.toml | 2 +- server/crates/sw-server/src/config.rs | 8 +-- server/crates/sw-server/src/server.rs | 95 +++++++++++++++++++++++---- tools/protocol-smoke/Checks.cs | 53 +++++++++++++-- tools/protocol-smoke/Program.cs | 8 ++- 5 files changed, 138 insertions(+), 28 deletions(-) diff --git a/server/config.example.toml b/server/config.example.toml index b088093..ad979e0 100644 --- a/server/config.example.toml +++ b/server/config.example.toml @@ -47,7 +47,7 @@ hello_min_interval_ms = 250 # Process-wide minimum interval between database admissions for identities # without an active session. Active-identity reconnects have a separate # per-player window, so reconnect churn cannot monopolize this new-identity -# budget. Must be in 1..=250. +# budget. Must be in 1..=1000. new_session_min_interval_ms = 30 # Hard ceilings on live LiteNetLib transport peers, including peers that have diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index e2c2b3a..864970c 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -40,10 +40,10 @@ pub const MAX_HELLO_MIN_INTERVAL_MS: u32 = 250; /// unthrottled flood. const MIN_HELLO_MIN_INTERVAL_MS: u32 = 1; -/// Upper bound on the process-wide admission interval for new sessions. It -/// matches the client's retry cadence so a busy rejection can be retried on -/// the next scheduled hello. -pub const MAX_NEW_SESSION_MIN_INTERVAL_MS: u32 = 250; +/// Upper bound on the process-wide admission interval for new sessions. One +/// second is long enough to enforce a bounded retry exchange while keeping an +/// operator misconfiguration from wedging fresh identities indefinitely. +pub const MAX_NEW_SESSION_MIN_INTERVAL_MS: u32 = 1_000; /// Highest configurable live transport-peer ceiling. pub const MAX_TRANSPORT_PEERS: u32 = 65_535; diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index a03d052..b48d576 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -64,7 +64,7 @@ pub struct Server { weather_epoch_day: u32, hello_limiter: RateLimiter, source_session_limiter: BoundedRateLimiter, - reconnect_limiter: RateLimiter, + reconnect_limiter: BoundedRateLimiter, new_session_limiter: GlobalRateLimiter, trade_limiter: RateLimiter, client_state_limiter: RateLimiter, @@ -106,7 +106,12 @@ impl Server { cfg.hello_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); - let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let reconnect_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + identity_players + .len() + .max(cfg.max_player_rows_u32() as usize), + ); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -801,11 +806,11 @@ impl Server { self.hello_limiter.clear(u64::from(peer)); if let Some(s) = self.sessions.remove(&peer) { self.world.remove(s.player_id); - // Drop the player's throttle state across every message class: a - // departed player's entries are useless and leaving them behind would - // let connection churn accrete stale entries in the limiter maps. + // Drop session-scoped message throttles. The reconnect cooldown is + // intentionally retained in its bounded map, otherwise a known + // identity can disconnect and rotate source addresses to repeat + // persistence admission inside one per-player window. self.trade_limiter.clear(s.player_id); - self.reconnect_limiter.clear(s.player_id); self.client_state_limiter.clear(s.player_id); self.chat_limiter.clear(s.player_id); self.econ_limiter.clear(s.player_id); @@ -1111,6 +1116,9 @@ mod handshake_tests { let cfg = Config::default(); let world = World::new(Grid::new(cfg.cell_size_m)); let identity_players = load_identity_players(&db).unwrap(); + let reconnect_key_budget = identity_players + .len() + .max(cfg.max_player_rows_u32() as usize); Server { host: Host::bind_with_limits( "127.0.0.1:0", @@ -1134,7 +1142,10 @@ mod handshake_tests { cfg.hello_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ), - reconnect_limiter: RateLimiter::new(cfg.hello_min_interval_ms_i64()), + reconnect_limiter: BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + reconnect_key_budget, + ), new_session_limiter: GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()), trade_limiter: RateLimiter::new(cfg.trade_min_interval_ms_i64()), client_state_limiter: RateLimiter::new(cfg.client_state_min_interval_ms_i64()), @@ -1934,6 +1945,49 @@ mod handshake_tests { ); } + #[test] + fn reconnect_cooldown_survives_disconnect_and_source_rotation() { + let mut server = make_server(); + let hello = hello_envelope( + "disconnect-rotation-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let first_admission_ms = 1_000; + let (first_client, first_peer) = connect_peer_from(&mut server, "127.0.0.1"); + + deliver_hello_at(&mut server, first_peer, &hello, first_admission_ms); + assert_eq!(receive_server_hello(&first_client), (true, String::new())); + let player_id = server.sessions[&first_peer].player_id; + + server + .on_disconnect(first_peer, DisconnectReason::Remote) + .unwrap(); + assert_eq!( + server.reconnect_limiter.tracked_count(), + 1, + "a completed disconnect must retain the per-player reconnect window" + ); + + let (rotated_client, rotated_peer) = connect_peer_from(&mut server, "127.0.0.2"); + deliver_hello_at(&mut server, rotated_peer, &hello, first_admission_ms + 1); + assert_eq!( + receive_server_hello(&rotated_client), + (false, "server busy; retry".to_string()), + "source rotation inside the window must not repeat persistence admission" + ); + assert!(!server.sessions.contains_key(&rotated_peer)); + + let after_expiry_ms = first_admission_ms + server.cfg.hello_min_interval_ms_i64() + 1; + deliver_hello_at(&mut server, rotated_peer, &hello, after_expiry_ms); + assert_eq!( + receive_server_hello(&rotated_client), + (true, String::new()), + "the same reconnect must become eligible when the window expires" + ); + assert_eq!(server.sessions[&rotated_peer].player_id, player_id); + } + #[test] fn same_identity_port_rotation_does_not_starve_another_source() { let mut server = make_server(); @@ -2322,7 +2376,10 @@ mod aoi_harden_tests { cfg.hello_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); - let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let reconnect_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_player_rows_u32() as usize, + ); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -2536,7 +2593,10 @@ mod market_dispatch_tests { cfg.hello_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); - let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let reconnect_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_player_rows_u32() as usize, + ); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -2783,7 +2843,10 @@ mod input_hardening_tests { cfg.hello_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); - let reconnect_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); + let reconnect_limiter = BoundedRateLimiter::new( + cfg.hello_min_interval_ms_i64(), + cfg.max_player_rows_u32() as usize, + ); let new_session_limiter = GlobalRateLimiter::new(cfg.new_session_min_interval_ms_i64()); let trade_limiter = RateLimiter::new(cfg.trade_min_interval_ms_i64()); let client_state_limiter = RateLimiter::new(cfg.client_state_min_interval_ms_i64()); @@ -3321,10 +3384,10 @@ mod input_hardening_tests { ); } - // ---- disconnect clears every class ---- + // ---- disconnect clears session classes and retains reconnect cooldown ---- #[test] - fn disconnect_clears_every_message_class_limiter() { + fn disconnect_clears_session_limiters_and_retains_reconnect_cooldown() { let mut server = make_server(Config { hello_min_interval_ms: 250, client_state_min_interval_ms: 250, @@ -3348,9 +3411,10 @@ mod input_hardening_tests { send_moor(&mut server, peer, &moor_envelope(0.0, 0.0, "m"), 1_000); // Seed the trade limiter directly (its handler needs a market envelope, // covered in the market dispatch suite); the point here is that - // on_disconnect clears every class. + // on_disconnect clears each session-scoped class. server.trade_limiter.allow(pid, 1_000); assert_eq!(server.hello_limiter.tracked_count(), 1); + assert_eq!(server.reconnect_limiter.tracked_count(), 1); assert_eq!(server.client_state_limiter.tracked_count(), 1); assert_eq!(server.econ_limiter.tracked_count(), 1); assert_eq!(server.chat_limiter.tracked_count(), 1); @@ -3362,6 +3426,11 @@ mod input_hardening_tests { .unwrap(); assert_eq!(server.hello_limiter.tracked_count(), 0); + assert_eq!( + server.reconnect_limiter.tracked_count(), + 1, + "disconnect must retain the bounded per-player reconnect cooldown" + ); assert_eq!(server.client_state_limiter.tracked_count(), 0); assert_eq!(server.econ_limiter.tracked_count(), 0); assert_eq!(server.chat_limiter.tracked_count(), 0); diff --git a/tools/protocol-smoke/Checks.cs b/tools/protocol-smoke/Checks.cs index 72fff8b..7551227 100644 --- a/tools/protocol-smoke/Checks.cs +++ b/tools/protocol-smoke/Checks.cs @@ -76,6 +76,7 @@ public bool Run() SprayHostileInput(ep, 100); var a = new SmokeClient("A"); + var b = new SmokeClient("B"); ulong playerIdA = 0; bool serverSurvived; try @@ -86,6 +87,12 @@ public bool Run() handshook ? "LiteNetLib Connect completed within 2s" : "no PeerConnected within 2s (sw-net LiteNetLib framing not interoperating yet)")); + // Establish B's transport before A consumes the fresh-identity + // admission gate. Check 3 can then bound B's first hello + // exchange without including connection scheduling. + b.Connect(ep.Address.ToString(), ep.Port); + bool bConnected = PumpUntil(() => b.Connected, 2000, a, b); + ServerHello? helloA = handshook ? DoHello(a, "Smoke-A", TokenA).Response : null; bool helloOk = IsValidHello(helloA); playerIdA = helloA.HasValue ? helloA.Value.PlayerId : 0; @@ -93,7 +100,7 @@ public bool Run() helloOk ? $"ServerHello accepted, player_id={playerIdA}, protocol_version={helloA.Value.Capabilities.Value.ProtocolVersion}" : "no accepted ServerHello with player_id, capabilities, clock and weather")); - results.Add(RunPresence(server, ep, a, playerIdA)); + results.Add(RunPresence(server, a, b, bConnected, playerIdA)); results.Add(RunEcon(a)); results.Add(RunMarket(a)); results.Add(RunMoorage(a)); @@ -106,6 +113,7 @@ public bool Run() } finally { + b.Dispose(); a.Dispose(); } @@ -129,7 +137,12 @@ public bool Run() // Check 3: presence and server-driven AoI. A second client B joins the // same cell while A streams 20 ClientState updates moving +X; B must see // A's cell added and A's x increasing. - private Result RunPresence(ServerHarness server, IPEndPoint ep, SmokeClient a, ulong playerIdA) + private Result RunPresence( + ServerHarness server, + SmokeClient a, + SmokeClient b, + bool bConnected, + ulong playerIdA) { try { @@ -138,14 +151,12 @@ private Result RunPresence(ServerHarness server, IPEndPoint ep, SmokeClient a, u return new Result(3, "presence-aoi", false, "skipped: A has no established session"); } - using var b = new SmokeClient("B"); - b.Connect(ep.Address.ToString(), ep.Port); - if (!PumpUntil(() => b.Connected, 2000, a, b)) + if (!bConnected || !b.Connected) { return new Result(3, "presence-aoi", false, "client B failed to connect"); } - var helloB = DoHello(b, "Smoke-B", TokenB); + var helloB = DoHello(b, "Smoke-B", TokenB, requireRetryableFirst: true); if (!IsValidHello(helloB.Response)) { return new Result(3, "presence-aoi", false, "client B failed the hello exchange"); @@ -443,11 +454,39 @@ private void DrainPresence(SmokeClient b, ulong playerIdA, ref bool sawAddedCell private (ServerHello? Response, bool SawRetryableResponse) DoHello( SmokeClient c, string name, - string token) + string token, + bool requireRetryableFirst = false) { const string apiSurfaceHashSentinel = "protocol-smoke-surface-hash"; var sawRetryableResponse = false; var hello = Codec.EncodeClientHello(_seq++, name, token, "smoke", "0.0.0", apiSurfaceHashSentinel); + if (requireRetryableFirst) + { + // Program configures a one-second new-identity gate. Because B + // is transport-connected before A is admitted, this bounded + // half-second exchange must receive the exact retry response. + var first = SendAndWait( + c, + hello, + e => e.PayloadType == Payload.ServerHello, + 500, + 500); + if (!first.HasValue) + { + return (null, false); + } + + var firstHello = first.Value.PayloadAsServerHello(); + if (!string.Equals(firstHello.Reason, RetryableHelloReason, StringComparison.Ordinal)) + { + // Any permanent rejection or direct acceptance is terminal + // evidence that the required transient path did not occur. + return (firstHello, false); + } + + sawRetryableResponse = true; + } + var env = SendAndWait( c, hello, diff --git a/tools/protocol-smoke/Program.cs b/tools/protocol-smoke/Program.cs index 1024485..b45ccef 100644 --- a/tools/protocol-smoke/Program.cs +++ b/tools/protocol-smoke/Program.cs @@ -113,9 +113,11 @@ private static string WriteConfig(string dir, string dbPath, int port) "server_name = \"protocol-smoke\"\n" + "tick_hz = 30\n" + "snapshot_hz = 4\n" + - // Force consecutive fresh identities through the retryable - // admission response so check 3 cannot pass by timing luck. - "new_session_min_interval_ms = 250\n"; + // Check 3 bounds B's first hello exchange to half a second after + // B is already transport-connected. This one-second gate makes + // the exact transient response an enforced precondition rather + // than a scheduler-dependent observation. + "new_session_min_interval_ms = 1000\n"; File.WriteAllText(configPath, contents); return configPath; } From 88d215f2d06bd5b2e4651b8b190a1f39d6be839a Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 16:58:59 -0600 Subject: [PATCH 22/36] fix(server): bound transport poll admission Co-Authored-By: Claude --- server/config.example.toml | 11 +- server/crates/sw-net/src/lib.rs | 268 ++++++++++++++++++++++++-- server/crates/sw-server/src/config.rs | 36 +++- server/crates/sw-server/src/server.rs | 71 ++++++- 4 files changed, 358 insertions(+), 28 deletions(-) diff --git a/server/config.example.toml b/server/config.example.toml index ad979e0..cd020a9 100644 --- a/server/config.example.toml +++ b/server/config.example.toml @@ -37,11 +37,12 @@ cell_size_m = 1024.0 trade_min_interval_ms = 250 # ClientHello throttle in milliseconds. Per-peer throttling runs before -# protocol/hash validation, database access, and ServerHello generation; the -# same interval also limits new-session/reconnect admission from one source IP. -# Keep it at or below the client's 250 ms retry cadence so a lost ServerHello -# can be retried on the next attempt. Must be in 1..=250; zero would disable -# pre-authentication flood protection. +# protocol/hash validation, database access, and ServerHello generation. Source +# IP session admission uses max(hello_min_interval_ms, +# new_session_min_interval_ms + 1), which prevents one source from consuming +# consecutive global new-session slots. Keep the hello interval at or below the +# client's 250 ms retry cadence so a lost ServerHello can be retried on the next +# attempt. Must be in 1..=250; zero would disable pre-authentication protection. hello_min_interval_ms = 250 # Process-wide minimum interval between database admissions for identities diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index 404507d..f5e59a6 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -23,10 +23,27 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; /// Opaque, stable identifier for a connected peer, assigned by the host. pub type PeerId = u32; -/// Largest datagram the host will read. Above the fixed MTU so oversized or -/// hostile packets are received (and then rejected) rather than silently -/// truncated at the socket layer. -const RECV_BUFFER: usize = 2048; +/// Largest possible UDP payload. Receiving the complete datagram lets the host +/// reject every packet above the fixed LiteNetLib MTU without truncation. +const RECV_BUFFER: usize = 65_535; + +/// Hard ceiling on socket datagrams consumed by one fixed-tick poll. +const MAX_POLL_PACKETS: usize = 128; + +/// Hard ceiling on socket bytes consumed by one fixed-tick poll. Four maximum +/// UDP datagrams fit, while normal MTU-sized traffic reaches the packet ceiling +/// first. +const MAX_POLL_BYTES: usize = 256 * 1024; + +/// One datagram can emit at most a disconnect plus a replacement connect. +const MAX_POLL_SOCKET_EVENTS: usize = MAX_POLL_PACKETS * 2; + +/// Timeout cleanup gets a reserved slice of every poll's event budget, so a +/// socket flood cannot indefinitely retain expired peers. +const MAX_POLL_TIMEOUT_EVENTS: usize = 64; + +/// Hard ceiling on the event vector returned by one poll. +const MAX_POLL_EVENTS: usize = MAX_POLL_SOCKET_EVENTS + MAX_POLL_TIMEOUT_EVENTS; /// Idle timeout: a peer that sends nothing for this long is dropped /// (`Timeout`). Mirrors LiteNetLib's default 5 s disconnect timeout. @@ -67,6 +84,14 @@ pub enum Event { Disconnected(PeerId, DisconnectReason), } +#[derive(Debug, Clone, Copy, Default)] +struct PollWork { + packets: usize, + bytes: usize, + socket_events: usize, + timeout_events: usize, +} + struct Peer { id: PeerId, addr: SocketAddr, @@ -168,27 +193,59 @@ impl Host { /// `now` is the caller's tick timestamp; timeouts and ping scheduling are /// measured against it. pub fn poll(&mut self, now: Instant) -> Vec { - let mut events = Vec::new(); - self.drain_socket(now, &mut events); - self.process_timeouts(now, &mut events); - self.send_keepalive_pings(now); - events + self.poll_with_work(now).0 } - fn drain_socket(&mut self, now: Instant, events: &mut Vec) { - loop { + fn poll_with_work(&mut self, now: Instant) -> (Vec, PollWork) { + let mut events = Vec::with_capacity(MAX_POLL_EVENTS); + let mut work = PollWork::default(); + self.drain_socket(now, &mut events, &mut work); + work.socket_events = events.len(); + work.timeout_events = self.process_timeouts(now, &mut events, MAX_POLL_TIMEOUT_EVENTS); + self.send_keepalive_pings(now); + debug_assert!(work.packets <= MAX_POLL_PACKETS); + debug_assert!(work.bytes <= MAX_POLL_BYTES); + debug_assert!(work.socket_events <= MAX_POLL_SOCKET_EVENTS); + debug_assert!(work.timeout_events <= MAX_POLL_TIMEOUT_EVENTS); + debug_assert!(events.len() <= MAX_POLL_EVENTS); + (events, work) + } + + fn drain_socket(&mut self, now: Instant, events: &mut Vec, work: &mut PollWork) { + while work.packets < MAX_POLL_PACKETS && work.bytes < MAX_POLL_BYTES { + let remaining_bytes = MAX_POLL_BYTES - work.bytes; + match self.socket.peek_from(&mut self.recv_buf[..]) { + Ok((n, _)) if n > remaining_bytes => break, + Ok(_) => {} + Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(ref e) if e.kind() == io::ErrorKind::ConnectionReset => { + work.packets += 1; + continue; + } + Err(_) => break, + } + match self.socket.recv_from(&mut self.recv_buf[..]) { Ok((n, addr)) => { + work.packets += 1; + work.bytes += n; + if n > protocol::MTU { + continue; + } // Copy out of the shared buffer so packet handling can take // `&mut self` freely. let datagram = self.recv_buf[..n].to_vec(); + let events_before = events.len(); self.handle_datagram(&datagram, addr, now, events); + debug_assert!(events.len().saturating_sub(events_before) <= 2); } Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => break, // Windows raises WSAECONNRESET on the *next* recv after a send // to an unreachable port produced an ICMP message. It is not a // real error for a connectionless socket — keep draining. - Err(ref e) if e.kind() == io::ErrorKind::ConnectionReset => continue, + Err(ref e) if e.kind() == io::ErrorKind::ConnectionReset => { + work.packets += 1; + } Err(_) => break, } } @@ -371,19 +428,27 @@ impl Host { events.push(Event::Disconnected(id, DisconnectReason::Remote)); } - fn process_timeouts(&mut self, now: Instant, events: &mut Vec) { + fn process_timeouts( + &mut self, + now: Instant, + events: &mut Vec, + max_events: usize, + ) -> usize { let timeout = self.timeout; let expired: Vec = self .peers .iter() .filter(|(_, p)| now.saturating_duration_since(p.last_recv) > timeout) .map(|(addr, _)| *addr) + .take(max_events) .collect(); + let before = events.len(); for addr in expired { if let Some(peer) = self.remove_peer_at(addr) { events.push(Event::Disconnected(peer.id, DisconnectReason::Timeout)); } } + events.len() - before } fn send_keepalive_pings(&mut self, now: Instant) { @@ -479,6 +544,183 @@ mod tests { client } + fn assert_bounded_poll(work: PollWork, events: &[Event]) { + assert!(work.packets <= MAX_POLL_PACKETS); + assert!(work.bytes <= MAX_POLL_BYTES); + assert!(work.socket_events <= MAX_POLL_SOCKET_EVENTS); + assert!(work.timeout_events <= MAX_POLL_TIMEOUT_EVENTS); + assert!(events.len() <= MAX_POLL_EVENTS); + } + + #[test] + fn preauth_data_flood_is_bounded_and_a_queued_payload_progresses() { + let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + + client + .send(&connect_datagram(1, "sailwind-online")) + .unwrap(); + let (events, work) = server.poll_with_work(Instant::now()); + assert_bounded_poll(work, &events); + assert_eq!(events, vec![Event::Connected(1)]); + + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + client.recv(&mut accept).unwrap(); + + let flood = protocol::build_unreliable(b"flood"); + for _ in 0..MAX_POLL_PACKETS + 32 { + client.send(&flood).unwrap(); + } + let marker = protocol::build_unreliable(b"legitimate-marker"); + client.send(&marker).unwrap(); + + let mut saw_marker = false; + for _ in 0..8 { + for _ in 0..MAX_POLL_PACKETS / 4 { + client.send(&flood).unwrap(); + } + let (events, work) = server.poll_with_work(Instant::now()); + assert_bounded_poll(work, &events); + saw_marker |= events.iter().any( + |event| matches!(event, Event::Data(_, bytes) if bytes == b"legitimate-marker"), + ); + if saw_marker { + break; + } + } + assert!( + saw_marker, + "bounded polling must leave queued traffic for a later tick instead of starving it" + ); + } + + #[test] + fn preauth_garbage_and_replacement_churn_are_bounded_across_ticks() { + let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + + for _ in 0..MAX_POLL_PACKETS + 32 { + client.send(&[protocol::property::UNRELIABLE]).unwrap(); + } + client + .send(&connect_datagram(1, "sailwind-online")) + .unwrap(); + + let mut connected = false; + for _ in 0..8 { + for _ in 0..MAX_POLL_PACKETS / 4 { + client.send(&[protocol::property::UNRELIABLE]).unwrap(); + } + let (events, work) = server.poll_with_work(Instant::now()); + assert_bounded_poll(work, &events); + connected |= events + .iter() + .any(|event| matches!(event, Event::Connected(_))); + if connected { + break; + } + } + assert!( + connected, + "a valid request queued behind a pre-auth flood must progress" + ); + + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + client.recv(&mut accept).unwrap(); + let mut next_connect_time = 2; + for _ in 0..MAX_POLL_PACKETS + 32 { + client + .send(&connect_datagram(next_connect_time, "sailwind-online")) + .unwrap(); + next_connect_time += 1; + } + client + .send(&protocol::build_unreliable(b"replacement-marker")) + .unwrap(); + + let mut saw_marker = false; + for _ in 0..8 { + for _ in 0..MAX_POLL_PACKETS / 4 { + client + .send(&connect_datagram(next_connect_time, "sailwind-online")) + .unwrap(); + next_connect_time += 1; + } + let (events, work) = server.poll_with_work(Instant::now()); + assert_bounded_poll(work, &events); + saw_marker |= events.iter().any( + |event| matches!(event, Event::Data(_, bytes) if bytes == b"replacement-marker"), + ); + if saw_marker { + break; + } + } + assert!( + saw_marker, + "replacement churn must not prevent later queued data from progressing" + ); + assert_eq!(server.peer_count(), 1); + } + + #[test] + fn inbound_datagrams_over_the_fixed_mtu_are_dropped_before_payload_events() { + let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + + client + .send(&connect_datagram(1, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(Instant::now()), vec![Event::Connected(1)]); + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + client.recv(&mut accept).unwrap(); + + let oversized = vec![protocol::property::UNRELIABLE; protocol::MTU + 1]; + client.send(&oversized).unwrap(); + client + .send(&protocol::build_unreliable(b"after-oversized")) + .unwrap(); + + let (events, work) = server.poll_with_work(Instant::now()); + assert_bounded_poll(work, &events); + assert_eq!(events, vec![Event::Data(1, b"after-oversized".to_vec())]); + } + + #[test] + fn timeout_burst_uses_its_reserved_event_budget_and_finishes_on_later_ticks() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 80, 80).unwrap(); + let server_addr = server.local_addr().unwrap(); + let t0 = Instant::now(); + let mut clients = Vec::new(); + + for connect_time in 1..=70 { + let client = client_from("127.0.0.1", server_addr); + client + .send(&connect_datagram(connect_time, "sailwind-online")) + .unwrap(); + clients.push(client); + } + let (events, work) = server.poll_with_work(t0); + assert_bounded_poll(work, &events); + assert_eq!(events.len(), 70); + assert_eq!(server.peer_count(), 70); + + let expired_at = t0 + DEFAULT_TIMEOUT + Duration::from_secs(1); + let (events, work) = server.poll_with_work(expired_at); + assert_bounded_poll(work, &events); + assert_eq!(work.timeout_events, MAX_POLL_TIMEOUT_EVENTS); + assert_eq!(events.len(), MAX_POLL_TIMEOUT_EVENTS); + assert_eq!(server.peer_count(), 70 - MAX_POLL_TIMEOUT_EVENTS); + + let (events, work) = server.poll_with_work(expired_at); + assert_bounded_poll(work, &events); + assert_eq!(events.len(), 70 - MAX_POLL_TIMEOUT_EVENTS); + assert_eq!(server.peer_count(), 0); + drop(clients); + } + #[test] fn per_ip_admission_uses_bounded_counter_state_instead_of_peer_scans() { let source = include_str!("lib.rs"); diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index 864970c..c1f5b5f 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -82,9 +82,9 @@ pub struct Config { /// Minimum interval, in milliseconds, between two processed `ClientHello` /// messages from the same peer. A flood beyond this rate is dropped before /// validation, persistence, or response generation. The 250 ms default - /// matches the client's handshake retry cadence. The same interval bounds - /// session admissions from one source IP. Bounded to - /// `1..=`[`MAX_HELLO_MIN_INTERVAL_MS`]. + /// matches the client's handshake retry cadence. Source-IP session + /// admission uses a derived interval strictly longer than the global + /// new-session interval. Bounded to `1..=`[`MAX_HELLO_MIN_INTERVAL_MS`]. pub hello_min_interval_ms: u32, /// Process-wide minimum interval, in milliseconds, between database /// admissions for identities without an active session. Active-identity @@ -326,6 +326,17 @@ impl Config { .clamp(MIN_HELLO_MIN_INTERVAL_MS, MAX_NEW_SESSION_MIN_INTERVAL_MS) as i64 } + /// Per-source session-admission interval in bounded milliseconds. + /// + /// It is strictly greater than the global new-session interval, so the + /// source that consumed one global slot is ineligible at the next slot. + /// Another source therefore gets an uncontested admission opportunity for + /// every accepted configuration, including a 250/1000 hello/global pair. + pub fn source_session_min_interval_ms_i64(&self) -> i64 { + self.hello_min_interval_ms_i64() + .max(self.new_session_min_interval_ms_i64().saturating_add(1)) + } + /// Global live-peer ceiling with a defense-in-depth clamp. pub fn max_transport_peers_usize(&self) -> usize { self.max_transport_peers.clamp(1, MAX_TRANSPORT_PEERS) as usize @@ -841,6 +852,25 @@ mod tests { } } + #[test] + fn source_admission_window_is_strictly_longer_than_every_global_window() { + for hello_ms in MIN_HELLO_MIN_INTERVAL_MS..=MAX_HELLO_MIN_INTERVAL_MS { + for new_session_ms in MIN_HELLO_MIN_INTERVAL_MS..=MAX_NEW_SESSION_MIN_INTERVAL_MS { + let cfg = Config { + hello_min_interval_ms: hello_ms, + new_session_min_interval_ms: new_session_ms, + ..Config::default() + }; + cfg.validate().unwrap(); + assert!( + cfg.source_session_min_interval_ms_i64() + > cfg.new_session_min_interval_ms_i64(), + "one source must never be eligible for two consecutive global slots" + ); + } + } + } + #[test] fn new_session_budget_accessors_clamp_bypassed_validation() { let invalid = Config { diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index b48d576..b2a7f40 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -103,7 +103,7 @@ impl Server { let world = World::new(sw_world::Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let source_session_limiter = BoundedRateLimiter::new( - cfg.hello_min_interval_ms_i64(), + cfg.source_session_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); let reconnect_limiter = BoundedRateLimiter::new( @@ -1109,11 +1109,18 @@ mod handshake_tests { use sw_world::Grid; fn make_server() -> Server { - make_server_with_db(Db::open_in_memory().unwrap()) + make_server_with_config_and_db(Config::default(), Db::open_in_memory().unwrap()) } fn make_server_with_db(db: Db) -> Server { - let cfg = Config::default(); + make_server_with_config_and_db(Config::default(), db) + } + + fn make_server_with_config(cfg: Config) -> Server { + make_server_with_config_and_db(cfg, Db::open_in_memory().unwrap()) + } + + fn make_server_with_config_and_db(cfg: Config, db: Db) -> Server { let world = World::new(Grid::new(cfg.cell_size_m)); let identity_players = load_identity_players(&db).unwrap(); let reconnect_key_budget = identity_players @@ -1139,7 +1146,7 @@ mod handshake_tests { weather_epoch_day: 0, hello_limiter: RateLimiter::new(cfg.hello_min_interval_ms_i64()), source_session_limiter: BoundedRateLimiter::new( - cfg.hello_min_interval_ms_i64(), + cfg.source_session_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ), reconnect_limiter: BoundedRateLimiter::new( @@ -2071,6 +2078,56 @@ mod handshake_tests { assert!(server.sessions.contains_key(&other_peer)); } + #[test] + fn slow_global_gate_still_reserves_its_next_slot_for_another_source() { + let mut server = make_server_with_config(Config { + hello_min_interval_ms: 250, + new_session_min_interval_ms: 1_000, + ..Config::default() + }); + let first_admission_ms = 1_000; + + let (attacker_client, attacker_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let attacker = hello_envelope( + "first-attacker-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, attacker_peer, &attacker, first_admission_ms); + assert_eq!( + receive_server_hello(&attacker_client), + (true, String::new()) + ); + + let (rotated_client, rotated_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let rotated = hello_envelope( + "rotated-attacker-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let (legitimate_client, legitimate_peer) = connect_peer_from(&mut server, "127.0.0.2"); + let next_global_ms = first_admission_ms + 1_000; + + deliver_hello_at(&mut server, rotated_peer, &rotated, next_global_ms); + assert_eq!( + receive_server_hello(&rotated_client), + (false, "server busy; retry".to_string()), + "the source that consumed the prior global slot must not consume the next one" + ); + + let legitimate = hello_envelope( + "legitimate-other-source-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, legitimate_peer, &legitimate, next_global_ms); + assert_eq!( + receive_server_hello(&legitimate_client), + (true, String::new()), + "another source must be able to take the next global admission slot" + ); + } + #[test] fn duplicate_hello_burst_is_dropped_before_response_or_session_work() { let mut server = make_server(); @@ -2373,7 +2430,7 @@ mod aoi_harden_tests { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let source_session_limiter = BoundedRateLimiter::new( - cfg.hello_min_interval_ms_i64(), + cfg.source_session_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); let reconnect_limiter = BoundedRateLimiter::new( @@ -2590,7 +2647,7 @@ mod market_dispatch_tests { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let source_session_limiter = BoundedRateLimiter::new( - cfg.hello_min_interval_ms_i64(), + cfg.source_session_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); let reconnect_limiter = BoundedRateLimiter::new( @@ -2840,7 +2897,7 @@ mod input_hardening_tests { let world = World::new(Grid::new(cfg.cell_size_m)); let hello_limiter = RateLimiter::new(cfg.hello_min_interval_ms_i64()); let source_session_limiter = BoundedRateLimiter::new( - cfg.hello_min_interval_ms_i64(), + cfg.source_session_min_interval_ms_i64(), cfg.max_transport_peers_usize(), ); let reconnect_limiter = BoundedRateLimiter::new( From e13b4ce6964a94ef9a0c0d842f7d546fa1f058e9 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 17:32:21 -0600 Subject: [PATCH 23/36] fix(server): bound transport maintenance fairly Co-Authored-By: Claude --- server/config.example.toml | 9 +- server/crates/sw-net/src/lib.rs | 324 ++++++++++++++++++++++---- server/crates/sw-server/src/config.rs | 35 ++- server/crates/sw-server/src/server.rs | 56 +++++ 4 files changed, 363 insertions(+), 61 deletions(-) diff --git a/server/config.example.toml b/server/config.example.toml index cd020a9..149e7d8 100644 --- a/server/config.example.toml +++ b/server/config.example.toml @@ -39,10 +39,11 @@ trade_min_interval_ms = 250 # ClientHello throttle in milliseconds. Per-peer throttling runs before # protocol/hash validation, database access, and ServerHello generation. Source # IP session admission uses max(hello_min_interval_ms, -# new_session_min_interval_ms + 1), which prevents one source from consuming -# consecutive global new-session slots. Keep the hello interval at or below the -# client's 250 ms retry cadence so a lost ServerHello can be retried on the next -# attempt. Must be in 1..=250; zero would disable pre-authentication protection. +# 2 * new_session_min_interval_ms + 1), which reserves one complete global +# admission window for other source IPs after a source wins a slot. Keep the +# hello interval at or below the client's 250 ms retry cadence so a lost +# ServerHello can be retried on the next attempt. Must be in 1..=250; zero would +# disable pre-authentication protection. hello_min_interval_ms = 250 # Process-wide minimum interval between database admissions for identities diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index f5e59a6..c9db52a 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -42,6 +42,14 @@ const MAX_POLL_SOCKET_EVENTS: usize = MAX_POLL_PACKETS * 2; /// socket flood cannot indefinitely retain expired peers. const MAX_POLL_TIMEOUT_EVENTS: usize = 64; +/// Hard ceiling on live-peer slots inspected by one fixed-tick poll. At the +/// maximum supported 65,535 peers, the round-robin cursor covers every slot in +/// fewer than 128 ticks. +const MAX_POLL_MAINTENANCE_SCANS: usize = 512; + +/// Hard ceiling on host keepalive datagrams sent by one fixed-tick poll. +const MAX_POLL_KEEPALIVE_SENDS: usize = MAX_POLL_MAINTENANCE_SCANS; + /// Hard ceiling on the event vector returned by one poll. const MAX_POLL_EVENTS: usize = MAX_POLL_SOCKET_EVENTS + MAX_POLL_TIMEOUT_EVENTS; @@ -90,10 +98,13 @@ struct PollWork { bytes: usize, socket_events: usize, timeout_events: usize, + maintenance_scans: usize, + keepalive_sends: usize, } struct Peer { id: PeerId, + slot: usize, addr: SocketAddr, connect_time: i64, connection_number: u8, @@ -109,10 +120,10 @@ struct Peer { pub struct Host { socket: UdpSocket, peers: HashMap, - by_id: HashMap, + peer_slots: Vec>, + free_peer_slots: Vec, peers_per_ip: HashMap, - next_id: PeerId, - next_local_peer_id: i32, + maintenance_cursor: usize, connect_key: String, timeout: Duration, max_peers: usize, @@ -138,10 +149,14 @@ impl Host { max_peers: usize, max_peers_per_ip: usize, ) -> io::Result { - if max_peers == 0 || max_peers_per_ip == 0 || max_peers_per_ip > max_peers { + if max_peers == 0 + || max_peers > u16::MAX as usize + || max_peers_per_ip == 0 + || max_peers_per_ip > max_peers + { return Err(io::Error::new( io::ErrorKind::InvalidInput, - "peer limits must be nonzero and per-IP must not exceed global", + "peer limits must be nonzero, global must not exceed 65535, and per-IP must not exceed global", )); } @@ -150,10 +165,10 @@ impl Host { Ok(Host { socket, peers: HashMap::new(), - by_id: HashMap::new(), + peer_slots: Vec::new(), + free_peer_slots: Vec::new(), peers_per_ip: HashMap::new(), - next_id: 1, - next_local_peer_id: 0, + maintenance_cursor: 0, connect_key: connect_key.to_string(), timeout: DEFAULT_TIMEOUT, max_peers, @@ -169,7 +184,12 @@ impl Host { /// Return the remote address for a live peer. pub fn peer_addr(&self, peer: PeerId) -> Option { - self.by_id.get(&peer).copied() + let slot = usize::try_from(peer).ok()?.checked_sub(1)?; + let addr = self.peer_slots.get(slot)?.as_ref().copied()?; + self.peers + .get(&addr) + .filter(|connected| connected.id == peer) + .map(|_| addr) } /// Number of currently connected peers. @@ -184,8 +204,8 @@ impl Host { /// Last measured round-trip time to `peer`, if a Pong has come back. pub fn rtt(&self, peer: PeerId) -> Option { - let addr = self.by_id.get(&peer)?; - self.peers.get(addr)?.rtt + let addr = self.peer_addr(peer)?; + self.peers.get(&addr)?.rtt } /// Pump the socket and internal timers, returning everything that happened. @@ -201,12 +221,13 @@ impl Host { let mut work = PollWork::default(); self.drain_socket(now, &mut events, &mut work); work.socket_events = events.len(); - work.timeout_events = self.process_timeouts(now, &mut events, MAX_POLL_TIMEOUT_EVENTS); - self.send_keepalive_pings(now); + self.process_peer_maintenance(now, &mut events, &mut work); debug_assert!(work.packets <= MAX_POLL_PACKETS); debug_assert!(work.bytes <= MAX_POLL_BYTES); debug_assert!(work.socket_events <= MAX_POLL_SOCKET_EVENTS); debug_assert!(work.timeout_events <= MAX_POLL_TIMEOUT_EVENTS); + debug_assert!(work.maintenance_scans <= MAX_POLL_MAINTENANCE_SCANS); + debug_assert!(work.keepalive_sends <= MAX_POLL_KEEPALIVE_SENDS); debug_assert!(events.len() <= MAX_POLL_EVENTS); (events, work) } @@ -330,10 +351,11 @@ impl Host { return; } - let id = self.next_id; - self.next_id += 1; - let local_peer_id = self.next_local_peer_id; - self.next_local_peer_id += 1; + let Some(slot) = self.allocate_peer_slot() else { + return; + }; + let id = PeerId::try_from(slot + 1).expect("bounded peer slot fits PeerId"); + let local_peer_id = i32::try_from(slot).expect("bounded peer slot fits LiteNetLib peer id"); let accept = protocol::build_connect_accept( req.connect_time, @@ -345,6 +367,7 @@ impl Host { self.insert_peer(Peer { id, + slot, addr, connect_time: req.connect_time, connection_number: req.connection_number, @@ -360,14 +383,27 @@ impl Host { fn insert_peer(&mut self, peer: Peer) { let id = peer.id; + let slot = peer.slot; let addr = peer.addr; + debug_assert_eq!(id, PeerId::try_from(slot + 1).unwrap()); + debug_assert!(self.peer_slots[slot].is_none()); *self.peers_per_ip.entry(addr.ip()).or_insert(0) += 1; self.peers.insert(addr, peer); - self.by_id.insert(id, addr); + self.peer_slots[slot] = Some(addr); + } + + fn allocate_peer_slot(&mut self) -> Option { + if self.peer_slots.len() < self.max_peers { + let slot = self.peer_slots.len(); + self.peer_slots.push(None); + Some(slot) + } else { + self.free_peer_slots.pop() + } } fn remove_peer_at(&mut self, addr: SocketAddr) -> Option { - let peer_id = self.peers.get(&addr)?.id; + let slot = self.peers.get(&addr)?.slot; let ip = addr.ip(); let remove_counter = match self.peers_per_ip.get_mut(&ip)? { count if *count > 1 => { @@ -380,7 +416,9 @@ impl Host { self.peers_per_ip.remove(&ip); } let peer = self.peers.remove(&addr)?; - self.by_id.remove(&peer_id); + debug_assert_eq!(self.peer_slots[slot], Some(addr)); + self.peer_slots[slot] = None; + self.free_peer_slots.push(slot); Some(peer) } @@ -428,31 +466,50 @@ impl Host { events.push(Event::Disconnected(id, DisconnectReason::Remote)); } - fn process_timeouts( + fn process_peer_maintenance( &mut self, now: Instant, events: &mut Vec, - max_events: usize, - ) -> usize { - let timeout = self.timeout; - let expired: Vec = self - .peers - .iter() - .filter(|(_, p)| now.saturating_duration_since(p.last_recv) > timeout) - .map(|(addr, _)| *addr) - .take(max_events) - .collect(); - let before = events.len(); - for addr in expired { - if let Some(peer) = self.remove_peer_at(addr) { - events.push(Event::Disconnected(peer.id, DisconnectReason::Timeout)); - } + work: &mut PollWork, + ) { + let slot_count = self.peer_slots.len(); + if slot_count == 0 { + self.maintenance_cursor = 0; + return; } - events.len() - before - } - fn send_keepalive_pings(&mut self, now: Instant) { - for peer in self.peers.values_mut() { + let scan_count = slot_count.min(MAX_POLL_MAINTENANCE_SCANS); + for _ in 0..scan_count { + if self.maintenance_cursor >= slot_count { + self.maintenance_cursor = 0; + } + let slot = self.maintenance_cursor; + self.maintenance_cursor += 1; + work.maintenance_scans += 1; + + let Some(addr) = self.peer_slots[slot] else { + continue; + }; + let expired = self + .peers + .get(&addr) + .is_some_and(|peer| now.saturating_duration_since(peer.last_recv) > self.timeout); + if expired { + if work.timeout_events < MAX_POLL_TIMEOUT_EVENTS { + if let Some(peer) = self.remove_peer_at(addr) { + events.push(Event::Disconnected(peer.id, DisconnectReason::Timeout)); + work.timeout_events += 1; + } + } + continue; + } + + if work.keepalive_sends >= MAX_POLL_KEEPALIVE_SENDS { + continue; + } + let Some(peer) = self.peers.get_mut(&addr) else { + continue; + }; if now.saturating_duration_since(peer.last_ping_sent) < PING_INTERVAL { continue; } @@ -461,6 +518,7 @@ impl Host { peer.ping_sent_at = Some(now); let ping = protocol::build_ping(peer.ping_seq); let _ = self.socket.send_to(&ping, peer.addr); + work.keepalive_sends += 1; } } @@ -475,7 +533,7 @@ impl Host { "payload exceeds MTU; fragmentation is not supported at init-0", )); } - let Some(addr) = self.by_id.get(&peer).copied() else { + let Some(addr) = self.peer_addr(peer) else { return Ok(()); }; let packet = protocol::build_unreliable(bytes); @@ -488,7 +546,7 @@ impl Host { /// The caller already owns the corresponding application-session cleanup, /// so this does not enqueue a second [`Event::Disconnected`]. pub fn disconnect(&mut self, peer: PeerId) -> bool { - let Some(addr) = self.by_id.get(&peer).copied() else { + let Some(addr) = self.peer_addr(peer) else { return false; }; let Some(peer) = self.remove_peer_at(addr) else { @@ -512,8 +570,10 @@ impl Host { events.push(Event::Disconnected(peer.id, DisconnectReason::Shutdown)); } self.peers.clear(); - self.by_id.clear(); + self.peer_slots.clear(); + self.free_peer_slots.clear(); self.peers_per_ip.clear(); + self.maintenance_cursor = 0; events } } @@ -549,9 +609,71 @@ mod tests { assert!(work.bytes <= MAX_POLL_BYTES); assert!(work.socket_events <= MAX_POLL_SOCKET_EVENTS); assert!(work.timeout_events <= MAX_POLL_TIMEOUT_EVENTS); + assert!(work.maintenance_scans <= MAX_POLL_MAINTENANCE_SCANS); + assert!(work.keepalive_sends <= MAX_POLL_KEEPALIVE_SENDS); assert!(events.len() <= MAX_POLL_EVENTS); } + fn connect_many(server: &mut Host, count: usize, now: Instant) -> Vec { + let server_addr = server.local_addr().unwrap(); + let mut clients = Vec::with_capacity(count); + for connect_time in 1..=count { + let client = client_from("127.0.0.1", server_addr); + client + .send(&connect_datagram(connect_time as i64, "sailwind-online")) + .unwrap(); + clients.push(client); + if connect_time % MAX_POLL_PACKETS == 0 { + let (events, work) = server.poll_with_work(now); + assert_bounded_poll(work, &events); + } + } + while server.peer_count() < count { + let (events, work) = server.poll_with_work(now); + assert_bounded_poll(work, &events); + } + clients + } + + fn drain_packets(client: &UdpSocket) -> Vec> { + let mut packets = Vec::new(); + let mut buf = [0u8; 64]; + loop { + match client.recv(&mut buf) { + Ok(n) => packets.push(buf[..n].to_vec()), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => return packets, + Err(error) => panic!("reading test client: {error}"), + } + } + } + + fn assert_peer_indices_consistent(server: &Host) { + assert_eq!( + server.peer_slots.len(), + server.free_peer_slots.len() + server.peers.len() + ); + assert!(server.maintenance_cursor <= server.peer_slots.len()); + + let mut free = vec![false; server.peer_slots.len()]; + for &slot in &server.free_peer_slots { + assert!(slot < server.peer_slots.len()); + assert!(!free[slot], "free-list slot {slot} must appear once"); + free[slot] = true; + assert!(server.peer_slots[slot].is_none()); + } + for (slot, addr) in server.peer_slots.iter().enumerate() { + match addr { + Some(addr) => { + let peer = &server.peers[addr]; + assert_eq!(peer.slot, slot); + assert_eq!(peer.id, PeerId::try_from(slot + 1).unwrap()); + assert!(!free[slot]); + } + None => assert!(free[slot]), + } + } + } + #[test] fn preauth_data_flood_is_bounded_and_a_queued_payload_progresses() { let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); @@ -721,6 +843,115 @@ mod tests { drop(clients); } + #[test] + fn production_peers_have_bounded_fair_keepalive_maintenance() { + let peer_count = MAX_POLL_MAINTENANCE_SCANS + 1; + let mut server = + Host::bind_with_limits("127.0.0.1:0", "sailwind-online", peer_count, peer_count) + .unwrap(); + let t0 = Instant::now(); + let clients = connect_many(&mut server, peer_count, t0); + for client in &clients { + let packets = drain_packets(client); + assert!(packets.iter().any(|packet| { + protocol::Header::from_byte(packet[0]).property + == protocol::property::CONNECT_ACCEPT + })); + } + + let due_at = t0 + PING_INTERVAL; + let mut pinged = vec![false; peer_count]; + let mut keepalive_sends = 0; + for _ in 0..3 { + let (events, work) = server.poll_with_work(due_at); + assert!(events.is_empty()); + assert_bounded_poll(work, &events); + keepalive_sends += work.keepalive_sends; + for (index, client) in clients.iter().enumerate() { + pinged[index] |= drain_packets(client).iter().any(|packet| { + protocol::Header::from_byte(packet[0]).property == protocol::property::PING + }); + } + if pinged.iter().all(|ping| *ping) { + break; + } + } + assert!( + pinged.iter().all(|ping| *ping), + "round-robin maintenance must eventually send a keepalive to every live peer" + ); + assert_eq!( + keepalive_sends, peer_count, + "each due peer must receive exactly one keepalive during a complete pass" + ); + assert_peer_indices_consistent(&server); + } + + #[test] + fn production_peer_timeouts_are_scan_bounded_and_eventually_complete() { + let peer_count = MAX_POLL_MAINTENANCE_SCANS + 1; + let mut server = + Host::bind_with_limits("127.0.0.1:0", "sailwind-online", peer_count, peer_count) + .unwrap(); + let t0 = Instant::now(); + let clients = connect_many(&mut server, peer_count, t0); + let expired_at = t0 + DEFAULT_TIMEOUT + Duration::from_secs(1); + let mut timeout_events = 0; + for poll_index in 0..peer_count.div_ceil(MAX_POLL_TIMEOUT_EVENTS) + 3 { + let (events, work) = server.poll_with_work(expired_at); + assert_bounded_poll(work, &events); + if poll_index == 0 { + assert_eq!(work.maintenance_scans, MAX_POLL_MAINTENANCE_SCANS); + assert_eq!(work.timeout_events, MAX_POLL_TIMEOUT_EVENTS); + } + timeout_events += events + .iter() + .filter(|event| matches!(event, Event::Disconnected(_, DisconnectReason::Timeout))) + .count(); + assert_peer_indices_consistent(&server); + if server.peer_count() == 0 { + break; + } + } + assert_eq!(timeout_events, peer_count); + assert_eq!(server.peer_count(), 0); + assert_peer_indices_consistent(&server); + drop(clients); + } + + #[test] + fn bounded_slot_ids_cannot_wrap_or_collide_under_transport_churn() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 2, 2).unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + let now = Instant::now(); + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + + for connect_time in 1..=10_000i64 { + client + .send(&connect_datagram(connect_time, "sailwind-online")) + .unwrap(); + let events = server.poll(now); + let connected = events + .iter() + .find_map(|event| match event { + Event::Connected(peer) => Some(*peer), + _ => None, + }) + .expect("every replacement must connect"); + assert!( + connected <= 2, + "PeerId must be derived from a bounded live slot" + ); + + let n = client.recv(&mut accept).unwrap(); + assert_eq!(n, protocol::CONNECT_ACCEPT_SIZE); + let local_peer_id = i32::from_le_bytes(accept[11..15].try_into().unwrap()); + assert!((0..2).contains(&local_peer_id)); + assert_peer_indices_consistent(&server); + } + } + #[test] fn per_ip_admission_uses_bounded_counter_state_instead_of_peer_scans() { let source = include_str!("lib.rs"); @@ -789,18 +1020,21 @@ mod tests { server.poll(Instant::now()), vec![ Event::Disconnected(1, DisconnectReason::Remote), - Event::Connected(4) + Event::Connected(1) ] ); assert_eq!(server.peer_count(), 3); assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 2); + assert_peer_indices_consistent(&server); - assert!(server.disconnect(4)); + assert!(server.disconnect(1)); assert_eq!(server.peer_count(), 2); assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 1); + assert_peer_indices_consistent(&server); server.shutdown(); assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 0); assert_eq!(server.peer_count_for_ip("127.0.0.2".parse().unwrap()), 0); + assert_peer_indices_consistent(&server); } #[test] diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index c1f5b5f..66192b1 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -83,8 +83,9 @@ pub struct Config { /// messages from the same peer. A flood beyond this rate is dropped before /// validation, persistence, or response generation. The 250 ms default /// matches the client's handshake retry cadence. Source-IP session - /// admission uses a derived interval strictly longer than the global - /// new-session interval. Bounded to `1..=`[`MAX_HELLO_MIN_INTERVAL_MS`]. + /// admission uses a derived interval longer than two complete global + /// new-session intervals. Bounded to + /// `1..=`[`MAX_HELLO_MIN_INTERVAL_MS`]. pub hello_min_interval_ms: u32, /// Process-wide minimum interval, in milliseconds, between database /// admissions for identities without an active session. Active-identity @@ -328,13 +329,16 @@ impl Config { /// Per-source session-admission interval in bounded milliseconds. /// - /// It is strictly greater than the global new-session interval, so the - /// source that consumed one global slot is ineligible at the next slot. - /// Another source therefore gets an uncontested admission opportunity for - /// every accepted configuration, including a 250/1000 hello/global pair. + /// It is strictly greater than two global new-session intervals, so after + /// one source consumes a slot, every instant of the next complete global + /// window is uncontested by that source. This holds for every accepted + /// configuration, including a 250/1000 hello/global pair. pub fn source_session_min_interval_ms_i64(&self) -> i64 { - self.hello_min_interval_ms_i64() - .max(self.new_session_min_interval_ms_i64().saturating_add(1)) + self.hello_min_interval_ms_i64().max( + self.new_session_min_interval_ms_i64() + .saturating_mul(2) + .saturating_add(1), + ) } /// Global live-peer ceiling with a defense-in-depth clamp. @@ -853,7 +857,7 @@ mod tests { } #[test] - fn source_admission_window_is_strictly_longer_than_every_global_window() { + fn source_admission_reserves_a_complete_global_window_for_other_sources() { for hello_ms in MIN_HELLO_MIN_INTERVAL_MS..=MAX_HELLO_MIN_INTERVAL_MS { for new_session_ms in MIN_HELLO_MIN_INTERVAL_MS..=MAX_NEW_SESSION_MIN_INTERVAL_MS { let cfg = Config { @@ -862,11 +866,18 @@ mod tests { ..Config::default() }; cfg.validate().unwrap(); + let global_ms = cfg.new_session_min_interval_ms_i64(); + let source_ms = cfg.source_session_min_interval_ms_i64(); assert!( - cfg.source_session_min_interval_ms_i64() - > cfg.new_session_min_interval_ms_i64(), - "one source must never be eligible for two consecutive global slots" + source_ms > global_ms.saturating_mul(2), + "the winning source must remain ineligible throughout the next complete global window" ); + for other_source_offset_ms in 0..=global_ms { + assert!( + global_ms.saturating_add(other_source_offset_ms) < source_ms, + "another source must have every instant in the reserved global window" + ); + } } } } diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index b2a7f40..9f9445f 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -2128,6 +2128,62 @@ mod handshake_tests { ); } + #[test] + fn winning_source_cannot_retake_before_an_entire_other_source_window() { + let mut server = make_server_with_config(Config { + hello_min_interval_ms: 250, + new_session_min_interval_ms: 1_000, + ..Config::default() + }); + + let (first_client, first_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let first = hello_envelope( + "attacker-first-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, first_peer, &first, 1_000); + assert_eq!(receive_server_hello(&first_client), (true, String::new())); + + let (early_client, early_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let early = hello_envelope( + "attacker-at-2001", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, early_peer, &early, 2_001); + assert_eq!( + receive_server_hello(&early_client), + (false, "server busy; retry".to_string()) + ); + + let (legitimate_client, legitimate_peer) = connect_peer_from(&mut server, "127.0.0.2"); + let legitimate = hello_envelope( + "legitimate-at-window-end", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, legitimate_peer, &legitimate, 3_000); + assert_eq!( + receive_server_hello(&legitimate_client), + (true, String::new()), + "another source must be uncontested even at the last instant of its full window" + ); + + let (late_client, late_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let late = hello_envelope( + "attacker-at-3001", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, late_peer, &late, 3_001); + assert_eq!( + receive_server_hello(&late_client), + (false, "server busy; retry".to_string()), + "the attacker must not retake the global gate one millisecond after the reserved window" + ); + } + #[test] fn duplicate_hello_burst_is_dropped_before_response_or_session_work() { let mut server = make_server(); From b8f8fe1c833aeae711a2856df44ea532962d6feb Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 18:00:07 -0600 Subject: [PATCH 24/36] fix(net): quarantine retired peer slots Co-Authored-By: Claude --- server/crates/sw-net/src/lib.rs | 185 ++++++++++++++++++++++++-- server/crates/sw-server/src/server.rs | 66 +++++++++ 2 files changed, 241 insertions(+), 10 deletions(-) diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index c9db52a..7c06af9 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -15,7 +15,7 @@ pub mod protocol; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::io; use std::net::{IpAddr, SocketAddr, ToSocketAddrs, UdpSocket}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -121,7 +121,11 @@ pub struct Host { socket: UdpSocket, peers: HashMap, peer_slots: Vec>, - free_peer_slots: Vec, + // Retired slots join the back of this queue. Only the prefix counted by + // `available_free_peer_slots` existed when the current poll began, so slot + // reuse can never alias an event returned by that poll. + free_peer_slots: VecDeque, + available_free_peer_slots: usize, peers_per_ip: HashMap, maintenance_cursor: usize, connect_key: String, @@ -166,7 +170,8 @@ impl Host { socket, peers: HashMap::new(), peer_slots: Vec::new(), - free_peer_slots: Vec::new(), + free_peer_slots: VecDeque::new(), + available_free_peer_slots: 0, peers_per_ip: HashMap::new(), maintenance_cursor: 0, connect_key: connect_key.to_string(), @@ -211,12 +216,15 @@ impl Host { /// Pump the socket and internal timers, returning everything that happened. /// /// `now` is the caller's tick timestamp; timeouts and ping scheduling are - /// measured against it. + /// measured against it. The caller must finish consuming one returned event + /// batch before polling again: the next poll is the boundary at which peer + /// slots retired by the prior batch become eligible for reuse. pub fn poll(&mut self, now: Instant) -> Vec { self.poll_with_work(now).0 } fn poll_with_work(&mut self, now: Instant) -> (Vec, PollWork) { + self.available_free_peer_slots = self.free_peer_slots.len(); let mut events = Vec::with_capacity(MAX_POLL_EVENTS); let mut work = PollWork::default(); self.drain_socket(now, &mut events, &mut work); @@ -397,8 +405,11 @@ impl Host { let slot = self.peer_slots.len(); self.peer_slots.push(None); Some(slot) + } else if self.available_free_peer_slots > 0 { + self.available_free_peer_slots -= 1; + self.free_peer_slots.pop_front() } else { - self.free_peer_slots.pop() + None } } @@ -418,7 +429,7 @@ impl Host { let peer = self.peers.remove(&addr)?; debug_assert_eq!(self.peer_slots[slot], Some(addr)); self.peer_slots[slot] = None; - self.free_peer_slots.push(slot); + self.free_peer_slots.push_back(slot); Some(peer) } @@ -572,6 +583,7 @@ impl Host { self.peers.clear(); self.peer_slots.clear(); self.free_peer_slots.clear(); + self.available_free_peer_slots = 0; self.peers_per_ip.clear(); self.maintenance_cursor = 0; events @@ -652,6 +664,7 @@ mod tests { server.peer_slots.len(), server.free_peer_slots.len() + server.peers.len() ); + assert!(server.available_free_peer_slots <= server.free_peer_slots.len()); assert!(server.maintenance_cursor <= server.peer_slots.len()); let mut free = vec![false; server.peer_slots.len()]; @@ -952,6 +965,154 @@ mod tests { } } + #[test] + fn removed_peer_id_cannot_alias_a_cross_address_connect_in_the_same_poll() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 1, 1).unwrap(); + let server_addr = server.local_addr().unwrap(); + let old_client = client_from("127.0.0.1", server_addr); + let new_client = client_from("127.0.0.2", server_addr); + let now = Instant::now(); + + old_client + .send(&connect_datagram(1, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(now), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&old_client).len(), 1); + + old_client + .send(&protocol::build_unreliable(b"old-before-disconnect")) + .unwrap(); + old_client.send(&protocol::build_disconnect(1)).unwrap(); + new_client + .send(&connect_datagram(2, "sailwind-online")) + .unwrap(); + + let events = server.poll(now); + assert_eq!( + events, + vec![ + Event::Data(1, b"old-before-disconnect".to_vec()), + Event::Disconnected(1, DisconnectReason::Remote), + ] + ); + assert_eq!( + server.peer_addr(1), + None, + "an id in an earlier event must not resolve to a later peer from the same batch" + ); + server + .send_unreliable(1, b"must-not-reach-the-new-peer") + .unwrap(); + assert!( + !server.disconnect(1), + "acting on the old event id must not disconnect the new peer" + ); + assert!( + drain_packets(&new_client).is_empty(), + "the new endpoint must receive neither data nor disconnect for the old id" + ); + + new_client + .send(&connect_datagram(2, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(now), vec![Event::Connected(1)]); + assert_eq!(server.peer_addr(1), Some(new_client.local_addr().unwrap())); + let packets = drain_packets(&new_client); + assert_eq!(packets.len(), 1); + assert_eq!( + protocol::Header::from_byte(packets[0][0]).property, + protocol::property::CONNECT_ACCEPT + ); + assert_peer_indices_consistent(&server); + } + + #[test] + fn full_capacity_same_address_replacement_connects_on_the_next_poll_retry() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 1, 1).unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + let now = Instant::now(); + + client + .send(&connect_datagram(1, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(now), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + + client + .send(&connect_datagram(2, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(now), + vec![Event::Disconnected(1, DisconnectReason::Remote)], + "a full-capacity replacement must retire the old id for the complete event batch" + ); + assert_eq!(server.peer_addr(1), None); + assert!(drain_packets(&client).is_empty()); + + client + .send(&connect_datagram(2, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(now), vec![Event::Connected(1)]); + assert_eq!(server.peer_addr(1), Some(client.local_addr().unwrap())); + let packets = drain_packets(&client); + assert_eq!(packets.len(), 1); + let local_peer_id = i32::from_le_bytes(packets[0][11..15].try_into().unwrap()); + assert_eq!(local_peer_id, 0); + assert_peer_indices_consistent(&server); + } + + #[test] + fn local_disconnect_during_batch_processing_keeps_remaining_events_unresolvable() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 1, 1).unwrap(); + let server_addr = server.local_addr().unwrap(); + let old_client = client_from("127.0.0.1", server_addr); + let retry_client = client_from("127.0.0.2", server_addr); + let now = Instant::now(); + + old_client + .send(&connect_datagram(1, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(now), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&old_client).len(), 1); + + old_client + .send(&protocol::build_unreliable(b"first")) + .unwrap(); + old_client + .send(&protocol::build_unreliable(b"second")) + .unwrap(); + let events = server.poll(now); + assert_eq!( + events, + vec![ + Event::Data(1, b"first".to_vec()), + Event::Data(1, b"second".to_vec()), + ] + ); + + assert!(server.disconnect(1)); + for event in events.iter().skip(1) { + let Event::Data(peer, _) = event else { + panic!("expected the remaining data event"); + }; + assert_eq!(server.peer_addr(*peer), None); + server.send_unreliable(*peer, b"stale-reply").unwrap(); + assert!(!server.disconnect(*peer)); + } + + retry_client + .send(&connect_datagram(2, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(now), vec![Event::Connected(1)]); + assert_eq!( + server.peer_addr(1), + Some(retry_client.local_addr().unwrap()) + ); + assert_eq!(drain_packets(&retry_client).len(), 1); + assert_peer_indices_consistent(&server); + } + #[test] fn per_ip_admission_uses_bounded_counter_state_instead_of_peer_scans() { let source = include_str!("lib.rs"); @@ -1018,11 +1179,15 @@ mod tests { first.send(&connect_datagram(6, "sailwind-online")).unwrap(); assert_eq!( server.poll(Instant::now()), - vec![ - Event::Disconnected(1, DisconnectReason::Remote), - Event::Connected(1) - ] + vec![Event::Disconnected(1, DisconnectReason::Remote)] ); + assert_eq!(server.peer_count(), 2); + assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 1); + assert_eq!(server.peer_addr(1), None); + assert_peer_indices_consistent(&server); + + first.send(&connect_datagram(6, "sailwind-online")).unwrap(); + assert_eq!(server.poll(Instant::now()), vec![Event::Connected(1)]); assert_eq!(server.peer_count(), 3); assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 2); assert_peer_indices_consistent(&server); diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 9f9445f..31932cc 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -1914,6 +1914,72 @@ mod handshake_tests { assert_eq!(server.world.cell_of_entity(player_id), world_cell); } + #[test] + fn retired_transport_event_cannot_resolve_to_a_later_endpoint_in_its_batch() { + let cfg = Config { + max_transport_peers: 1, + max_transport_peers_per_ip: 1, + ..Config::default() + }; + let mut server = make_server_with_config(cfg); + let (old_client, old_peer) = connect_peer_from(&mut server, "127.0.0.1"); + let server_addr = server.host.local_addr().unwrap(); + let new_client = UdpSocket::bind(("127.0.0.2", 0)).unwrap(); + new_client.connect(server_addr).unwrap(); + new_client + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let old_hello = hello_envelope( + "retired-peer-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + let connect_data = protocol::write_litenet_string(CONNECT_KEY); + let new_request = protocol::build_connect_request(0, 2, 1, 16, &connect_data); + + old_client + .send(&protocol::build_unreliable(&old_hello)) + .unwrap(); + old_client.send(&protocol::build_disconnect(1)).unwrap(); + new_client.send(&new_request).unwrap(); + std::thread::sleep(Duration::from_millis(10)); + + let events = server.host.poll(Instant::now()); + assert_eq!( + events, + vec![ + Event::Data(old_peer, old_hello), + Event::Disconnected(old_peer, DisconnectReason::Remote), + ] + ); + let Event::Data(data_peer, _) = &events[0] else { + panic!("the old data event must remain first"); + }; + assert_eq!( + server.host.peer_addr(*data_peer), + None, + "production event handling must not resolve old data against the queued endpoint" + ); + let Event::Disconnected(disconnected_peer, _) = &events[1] else { + panic!("the old disconnect event must follow its data"); + }; + assert_eq!(data_peer, disconnected_peer); + assert_no_outbound_datagram(&new_client); + + new_client.send(&new_request).unwrap(); + let events = server.host.poll(Instant::now()); + assert_eq!(events, vec![Event::Connected(old_peer)]); + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + assert_eq!( + new_client.recv(&mut accept).unwrap(), + protocol::CONNECT_ACCEPT_SIZE + ); + assert_eq!( + server.host.peer_addr(old_peer), + Some(new_client.local_addr().unwrap()) + ); + } + #[test] fn persisted_offline_identity_does_not_compete_with_fresh_token_admission() { let persisted_token = "persisted-offline-token"; From 20ee901d60582c36301215a215abd9208a1ce199 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 18:42:51 -0600 Subject: [PATCH 25/36] fix(net): enforce peer connection numbers Reject stale connected packets before they can refresh liveness, emit events, or trigger replies. Stamp every connected outbound packet with the negotiated LiteNetLib discriminator and cover nonzero reconnects over real UDP. Co-Authored-By: Claude --- server/crates/sw-net/src/lib.rs | 298 +++++++++++++++++++++----- server/crates/sw-net/src/protocol.rs | 70 ++++-- server/crates/sw-server/src/server.rs | 115 +++++++++- 3 files changed, 410 insertions(+), 73 deletions(-) diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index 7c06af9..ea93fd0 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -295,6 +295,14 @@ impl Host { if header.fragmented { return; } + if header.property != protocol::property::CONNECT_REQUEST { + let Some(peer) = self.peers.get(&addr) else { + return; + }; + if header.connection_number != peer.connection_number { + return; + } + } match header.property { protocol::property::CONNECT_REQUEST => { self.handle_connect_request(data, addr, now, events) @@ -439,7 +447,7 @@ impl Host { }; if let Some(peer) = self.peers.get_mut(&addr) { peer.last_recv = now; - let pong = protocol::build_pong(seq, dotnet_ticks_now()); + let pong = protocol::build_pong(peer.connection_number, seq, dotnet_ticks_now()); let _ = self.socket.send_to(&pong, peer.addr); } } @@ -527,7 +535,7 @@ impl Host { peer.ping_seq = peer.ping_seq.wrapping_add(1); peer.last_ping_sent = now; peer.ping_sent_at = Some(now); - let ping = protocol::build_ping(peer.ping_seq); + let ping = protocol::build_ping(peer.connection_number, peer.ping_seq); let _ = self.socket.send_to(&ping, peer.addr); work.keepalive_sends += 1; } @@ -547,7 +555,8 @@ impl Host { let Some(addr) = self.peer_addr(peer) else { return Ok(()); }; - let packet = protocol::build_unreliable(bytes); + let connection_number = self.peers[&addr].connection_number; + let packet = protocol::build_unreliable(connection_number, bytes); self.socket.send_to(&packet, addr)?; Ok(()) } @@ -563,9 +572,10 @@ impl Host { let Some(peer) = self.remove_peer_at(addr) else { return false; }; - let _ = self - .socket - .send_to(&protocol::build_disconnect(peer.connect_time), peer.addr); + let _ = self.socket.send_to( + &protocol::build_disconnect(peer.connection_number, peer.connect_time), + peer.addr, + ); true } @@ -575,9 +585,10 @@ impl Host { pub fn shutdown(&mut self) -> Vec { let mut events = Vec::new(); for peer in self.peers.values() { - let _ = self - .socket - .send_to(&protocol::build_disconnect(peer.connect_time), peer.addr); + let _ = self.socket.send_to( + &protocol::build_disconnect(peer.connection_number, peer.connect_time), + peer.addr, + ); events.push(Event::Disconnected(peer.id, DisconnectReason::Shutdown)); } self.peers.clear(); @@ -604,9 +615,9 @@ fn dotnet_ticks_now() -> i64 { mod tests { use super::*; - fn connect_datagram(connect_time: i64, key: &str) -> Vec { + fn connect_datagram(connection_number: u8, connect_time: i64, key: &str) -> Vec { let data = protocol::write_litenet_string(key); - protocol::build_connect_request(0, connect_time, 7, 16, &data) + protocol::build_connect_request(connection_number, connect_time, 7, 16, &data) } fn client_from(source_ip: &str, server_addr: SocketAddr) -> UdpSocket { @@ -632,7 +643,7 @@ mod tests { for connect_time in 1..=count { let client = client_from("127.0.0.1", server_addr); client - .send(&connect_datagram(connect_time as i64, "sailwind-online")) + .send(&connect_datagram(0, connect_time as i64, "sailwind-online")) .unwrap(); clients.push(client); if connect_time % MAX_POLL_PACKETS == 0 { @@ -687,6 +698,180 @@ mod tests { } } + #[test] + fn nonzero_connection_numbers_isolate_connected_udp_sessions() { + for connection_number in 1..protocol::MAX_CONNECTION_NUMBER { + let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + let t0 = Instant::now(); + let connect_time = i64::from(connection_number) + 100; + + client + .send(&connect_datagram( + connection_number, + connect_time, + "sailwind-online", + )) + .unwrap(); + assert_eq!(server.poll(t0), vec![Event::Connected(1)]); + + let accept = drain_packets(&client); + assert_eq!(accept.len(), 1); + assert_eq!( + protocol::Header::from_byte(accept[0][0]).property, + protocol::property::CONNECT_ACCEPT + ); + assert_eq!(accept[0][9], connection_number); + + let current_data = protocol::build_unreliable(connection_number, b"current-session"); + client.send(¤t_data).unwrap(); + assert_eq!( + server.poll(t0), + vec![Event::Data(1, b"current-session".to_vec())] + ); + + server.send_unreliable(1, b"server-current").unwrap(); + let outbound = drain_packets(&client); + assert_eq!(outbound.len(), 1); + assert_eq!( + protocol::Header::from_byte(outbound[0][0]), + protocol::Header { + property: protocol::property::UNRELIABLE, + connection_number, + fragmented: false, + } + ); + + let stale_number = (connection_number + protocol::MAX_CONNECTION_NUMBER - 1) + % protocol::MAX_CONNECTION_NUMBER; + let stale_data = protocol::build_unreliable(stale_number, b"stale-session"); + client.send(&stale_data).unwrap(); + assert!( + server.poll(t0).is_empty(), + "stale-number data must not emit an application event" + ); + + let stale_ping = protocol::build_ping(stale_number, 0x1234); + client.send(&stale_ping).unwrap(); + assert!(server.poll(t0).is_empty()); + assert!( + drain_packets(&client).is_empty(), + "stale-number ping must not amplify into a pong" + ); + + let keepalive_at = t0 + PING_INTERVAL; + assert!(server.poll(keepalive_at).is_empty()); + let keepalive = drain_packets(&client); + assert_eq!(keepalive.len(), 1); + let keepalive_header = protocol::Header::from_byte(keepalive[0][0]); + assert_eq!(keepalive_header.property, protocol::property::PING); + assert_eq!(keepalive_header.connection_number, connection_number); + let sequence = protocol::read_sequence(&keepalive[0]).unwrap(); + + let stale_pong = protocol::build_pong(stale_number, sequence, 0); + client.send(&stale_pong).unwrap(); + assert!(server.poll(keepalive_at).is_empty()); + assert_eq!( + server.rtt(1), + None, + "stale-number pong must not mutate RTT state" + ); + + let stale_disconnect = protocol::build_disconnect(stale_number, connect_time); + client.send(&stale_disconnect).unwrap(); + assert!(server.poll(keepalive_at).is_empty()); + assert_eq!(server.peer_count(), 1); + assert!( + drain_packets(&client).is_empty(), + "stale-number disconnect must not receive ShutdownOk" + ); + + let current_disconnect = protocol::build_disconnect(connection_number, connect_time); + client.send(¤t_disconnect).unwrap(); + assert_eq!( + server.poll(keepalive_at), + vec![Event::Disconnected(1, DisconnectReason::Remote)] + ); + assert_eq!(server.peer_count(), 0); + } + } + + #[test] + fn stale_connection_number_cannot_refresh_idle_timeout() { + let stale_packets = [ + protocol::build_unreliable(1, b"stale-data"), + protocol::build_ping(1, 7).to_vec(), + protocol::build_pong(1, 7, 0).to_vec(), + protocol::build_disconnect(1, 20).to_vec(), + ]; + + for stale in stale_packets { + let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + let t0 = Instant::now(); + + client + .send(&connect_datagram(2, 20, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(t0), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + + client.send(&stale).unwrap(); + let expired_at = t0 + DEFAULT_TIMEOUT + Duration::from_millis(1); + assert_eq!( + server.poll(expired_at), + vec![Event::Disconnected(1, DisconnectReason::Timeout)] + ); + assert_eq!(server.peer_count(), 0); + assert!( + drain_packets(&client).is_empty(), + "stale connected traffic must not receive any reply" + ); + } + } + + #[test] + fn replaced_endpoint_rejects_packets_from_the_retired_connection_number() { + let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + let now = Instant::now(); + + client + .send(&connect_datagram(1, 10, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(now), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + + client + .send(&connect_datagram(2, 20, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(now), + vec![ + Event::Disconnected(1, DisconnectReason::Remote), + Event::Connected(2), + ] + ); + assert_eq!(drain_packets(&client).len(), 1); + + let retired = protocol::build_unreliable(1, b"retired-hello"); + client.send(&retired).unwrap(); + assert!( + server.poll(now).is_empty(), + "a packet from the replaced connection must not execute in the new session" + ); + + let current = protocol::build_unreliable(2, b"current-hello"); + client.send(¤t).unwrap(); + assert_eq!( + server.poll(now), + vec![Event::Data(2, b"current-hello".to_vec())] + ); + } + #[test] fn preauth_data_flood_is_bounded_and_a_queued_payload_progresses() { let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); @@ -694,7 +879,7 @@ mod tests { let client = client_from("127.0.0.1", server_addr); client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); let (events, work) = server.poll_with_work(Instant::now()); assert_bounded_poll(work, &events); @@ -703,11 +888,11 @@ mod tests { let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; client.recv(&mut accept).unwrap(); - let flood = protocol::build_unreliable(b"flood"); + let flood = protocol::build_unreliable(0, b"flood"); for _ in 0..MAX_POLL_PACKETS + 32 { client.send(&flood).unwrap(); } - let marker = protocol::build_unreliable(b"legitimate-marker"); + let marker = protocol::build_unreliable(0, b"legitimate-marker"); client.send(&marker).unwrap(); let mut saw_marker = false; @@ -740,7 +925,7 @@ mod tests { client.send(&[protocol::property::UNRELIABLE]).unwrap(); } client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); let mut connected = false; @@ -767,19 +952,19 @@ mod tests { let mut next_connect_time = 2; for _ in 0..MAX_POLL_PACKETS + 32 { client - .send(&connect_datagram(next_connect_time, "sailwind-online")) + .send(&connect_datagram(0, next_connect_time, "sailwind-online")) .unwrap(); next_connect_time += 1; } client - .send(&protocol::build_unreliable(b"replacement-marker")) + .send(&protocol::build_unreliable(0, b"replacement-marker")) .unwrap(); let mut saw_marker = false; for _ in 0..8 { for _ in 0..MAX_POLL_PACKETS / 4 { client - .send(&connect_datagram(next_connect_time, "sailwind-online")) + .send(&connect_datagram(0, next_connect_time, "sailwind-online")) .unwrap(); next_connect_time += 1; } @@ -806,7 +991,7 @@ mod tests { let client = client_from("127.0.0.1", server_addr); client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); assert_eq!(server.poll(Instant::now()), vec![Event::Connected(1)]); let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; @@ -815,7 +1000,7 @@ mod tests { let oversized = vec![protocol::property::UNRELIABLE; protocol::MTU + 1]; client.send(&oversized).unwrap(); client - .send(&protocol::build_unreliable(b"after-oversized")) + .send(&protocol::build_unreliable(0, b"after-oversized")) .unwrap(); let (events, work) = server.poll_with_work(Instant::now()); @@ -833,7 +1018,7 @@ mod tests { for connect_time in 1..=70 { let client = client_from("127.0.0.1", server_addr); client - .send(&connect_datagram(connect_time, "sailwind-online")) + .send(&connect_datagram(0, connect_time, "sailwind-online")) .unwrap(); clients.push(client); } @@ -942,7 +1127,7 @@ mod tests { for connect_time in 1..=10_000i64 { client - .send(&connect_datagram(connect_time, "sailwind-online")) + .send(&connect_datagram(0, connect_time, "sailwind-online")) .unwrap(); let events = server.poll(now); let connected = events @@ -974,17 +1159,17 @@ mod tests { let now = Instant::now(); old_client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); assert_eq!(server.poll(now), vec![Event::Connected(1)]); assert_eq!(drain_packets(&old_client).len(), 1); old_client - .send(&protocol::build_unreliable(b"old-before-disconnect")) + .send(&protocol::build_unreliable(0, b"old-before-disconnect")) .unwrap(); - old_client.send(&protocol::build_disconnect(1)).unwrap(); + old_client.send(&protocol::build_disconnect(0, 1)).unwrap(); new_client - .send(&connect_datagram(2, "sailwind-online")) + .send(&connect_datagram(0, 2, "sailwind-online")) .unwrap(); let events = server.poll(now); @@ -1013,7 +1198,7 @@ mod tests { ); new_client - .send(&connect_datagram(2, "sailwind-online")) + .send(&connect_datagram(0, 2, "sailwind-online")) .unwrap(); assert_eq!(server.poll(now), vec![Event::Connected(1)]); assert_eq!(server.peer_addr(1), Some(new_client.local_addr().unwrap())); @@ -1034,13 +1219,13 @@ mod tests { let now = Instant::now(); client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); assert_eq!(server.poll(now), vec![Event::Connected(1)]); assert_eq!(drain_packets(&client).len(), 1); client - .send(&connect_datagram(2, "sailwind-online")) + .send(&connect_datagram(0, 2, "sailwind-online")) .unwrap(); assert_eq!( server.poll(now), @@ -1051,7 +1236,7 @@ mod tests { assert!(drain_packets(&client).is_empty()); client - .send(&connect_datagram(2, "sailwind-online")) + .send(&connect_datagram(0, 2, "sailwind-online")) .unwrap(); assert_eq!(server.poll(now), vec![Event::Connected(1)]); assert_eq!(server.peer_addr(1), Some(client.local_addr().unwrap())); @@ -1071,16 +1256,16 @@ mod tests { let now = Instant::now(); old_client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); assert_eq!(server.poll(now), vec![Event::Connected(1)]); assert_eq!(drain_packets(&old_client).len(), 1); old_client - .send(&protocol::build_unreliable(b"first")) + .send(&protocol::build_unreliable(0, b"first")) .unwrap(); old_client - .send(&protocol::build_unreliable(b"second")) + .send(&protocol::build_unreliable(0, b"second")) .unwrap(); let events = server.poll(now); assert_eq!( @@ -1102,7 +1287,7 @@ mod tests { } retry_client - .send(&connect_datagram(2, "sailwind-online")) + .send(&connect_datagram(0, 2, "sailwind-online")) .unwrap(); assert_eq!(server.poll(now), vec![Event::Connected(1)]); assert_eq!( @@ -1135,10 +1320,12 @@ mod tests { let first = client_from("127.0.0.1", server_addr); let second = client_from("127.0.0.1", server_addr); - first.send(&connect_datagram(1, "sailwind-online")).unwrap(); + first + .send(&connect_datagram(0, 1, "sailwind-online")) + .unwrap(); assert_eq!(server.poll(Instant::now()), vec![Event::Connected(1)]); second - .send(&connect_datagram(2, "sailwind-online")) + .send(&connect_datagram(0, 2, "sailwind-online")) .unwrap(); assert_eq!(server.poll(Instant::now()), vec![Event::Connected(2)]); assert_eq!(server.peer_count(), 2); @@ -1146,14 +1333,14 @@ mod tests { let same_source_excess = client_from("127.0.0.1", server_addr); same_source_excess - .send(&connect_datagram(3, "sailwind-online")) + .send(&connect_datagram(0, 3, "sailwind-online")) .unwrap(); assert!(server.poll(Instant::now()).is_empty()); assert_eq!(server.peer_count(), 2); let other_source = client_from("127.0.0.2", server_addr); other_source - .send(&connect_datagram(4, "sailwind-online")) + .send(&connect_datagram(0, 4, "sailwind-online")) .unwrap(); assert_eq!(server.poll(Instant::now()), vec![Event::Connected(3)]); assert_eq!(server.peer_count(), 3); @@ -1161,14 +1348,16 @@ mod tests { let global_excess = client_from("127.0.0.2", server_addr); global_excess - .send(&connect_datagram(5, "sailwind-online")) + .send(&connect_datagram(0, 5, "sailwind-online")) .unwrap(); assert!(server.poll(Instant::now()).is_empty()); assert_eq!(server.peer_count(), 3); let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; first.recv(&mut accept).unwrap(); - first.send(&connect_datagram(1, "sailwind-online")).unwrap(); + first + .send(&connect_datagram(0, 1, "sailwind-online")) + .unwrap(); assert!(server.poll(Instant::now()).is_empty()); assert_eq!( first.recv(&mut accept).unwrap(), @@ -1176,7 +1365,9 @@ mod tests { ); assert_eq!(server.peer_count(), 3); - first.send(&connect_datagram(6, "sailwind-online")).unwrap(); + first + .send(&connect_datagram(0, 6, "sailwind-online")) + .unwrap(); assert_eq!( server.poll(Instant::now()), vec![Event::Disconnected(1, DisconnectReason::Remote)] @@ -1186,7 +1377,9 @@ mod tests { assert_eq!(server.peer_addr(1), None); assert_peer_indices_consistent(&server); - first.send(&connect_datagram(6, "sailwind-online")).unwrap(); + first + .send(&connect_datagram(0, 6, "sailwind-online")) + .unwrap(); assert_eq!(server.poll(Instant::now()), vec![Event::Connected(1)]); assert_eq!(server.peer_count(), 3); assert_eq!(server.peer_count_for_ip("127.0.0.1".parse().unwrap()), 2); @@ -1214,7 +1407,7 @@ mod tests { // 1) Connect. client - .send(&connect_datagram(0x1234, "sailwind-online")) + .send(&connect_datagram(0, 0x1234, "sailwind-online")) .unwrap(); let events = server.poll(Instant::now()); assert_eq!(events, vec![Event::Connected(1)]); @@ -1230,7 +1423,7 @@ mod tests { ); // 2) Application data over the Unreliable channel. - let payload = protocol::build_unreliable(b"hello-server"); + let payload = protocol::build_unreliable(0, b"hello-server"); client.send(&payload).unwrap(); let events = server.poll(Instant::now()); assert_eq!(events, vec![Event::Data(1, b"hello-server".to_vec())]); @@ -1238,10 +1431,13 @@ mod tests { // 3) Server -> client unreliable send. server.send_unreliable(1, b"hello-client").unwrap(); let n = client.recv(&mut buf).unwrap(); - assert_eq!(&buf[..n], &protocol::build_unreliable(b"hello-client")[..]); + assert_eq!( + &buf[..n], + &protocol::build_unreliable(0, b"hello-client")[..] + ); // 4) Disconnect. - client.send(&protocol::build_disconnect(0x1234)).unwrap(); + client.send(&protocol::build_disconnect(0, 0x1234)).unwrap(); let events = server.poll(Instant::now()); assert_eq!( events, @@ -1258,7 +1454,7 @@ mod tests { let client = UdpSocket::bind("127.0.0.1:0").unwrap(); client.connect(server_addr).unwrap(); - client.send(&connect_datagram(1, "wrong-key")).unwrap(); + client.send(&connect_datagram(0, 1, "wrong-key")).unwrap(); let events = server.poll(Instant::now()); assert!(events.is_empty()); assert_eq!(server.peer_count(), 0); @@ -1298,13 +1494,13 @@ mod tests { client.connect(server_addr).unwrap(); client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); server.poll(Instant::now()); let mut buf = [0u8; 64]; let _ = client.recv(&mut buf).unwrap(); // drain accept - client.send(&protocol::build_ping(0x00AB)).unwrap(); + client.send(&protocol::build_ping(0, 0x00AB)).unwrap(); server.poll(Instant::now()); let n = client.recv(&mut buf).unwrap(); assert_eq!(n, protocol::PONG_SIZE); @@ -1324,7 +1520,7 @@ mod tests { let t0 = Instant::now(); client - .send(&connect_datagram(1, "sailwind-online")) + .send(&connect_datagram(0, 1, "sailwind-online")) .unwrap(); assert_eq!(server.poll(t0), vec![Event::Connected(1)]); diff --git a/server/crates/sw-net/src/protocol.rs b/server/crates/sw-net/src/protocol.rs index 37362bf..a1d0d57 100644 --- a/server/crates/sw-net/src/protocol.rs +++ b/server/crates/sw-net/src/protocol.rs @@ -104,6 +104,17 @@ impl Header { fragmented: false, } } + + /// A non-fragmented header belonging to one connected session. + #[inline] + pub fn connected(property: u8, connection_number: u8) -> Header { + debug_assert!(connection_number < MAX_CONNECTION_NUMBER); + Header { + property, + connection_number, + fragmented: false, + } + } } /// A parsed ConnectRequest packet (fields after the LiteNetLib target address). @@ -199,26 +210,27 @@ pub fn build_connect_accept( b } -/// Wrap `data` in an Unreliable packet (property byte + payload). -pub fn build_unreliable(data: &[u8]) -> Vec { +/// Wrap `data` in an Unreliable packet for `connection_number`. +pub fn build_unreliable(connection_number: u8, data: &[u8]) -> Vec { let mut out = Vec::with_capacity(HEADER_SIZE + data.len()); - out.push(Header::plain(property::UNRELIABLE).to_byte()); + out.push(Header::connected(property::UNRELIABLE, connection_number).to_byte()); out.extend_from_slice(data); out } -/// Build a Ping packet with the given sequence number. -pub fn build_ping(sequence: u16) -> [u8; PING_SIZE] { +/// Build a Ping packet for `connection_number` with the given sequence number. +pub fn build_ping(connection_number: u8, sequence: u16) -> [u8; PING_SIZE] { let mut b = [0u8; PING_SIZE]; - b[0] = Header::plain(property::PING).to_byte(); + b[0] = Header::connected(property::PING, connection_number).to_byte(); b[1..3].copy_from_slice(&sequence.to_le_bytes()); b } -/// Build a Pong packet echoing `sequence`, carrying our local time in .NET ticks. -pub fn build_pong(sequence: u16, time_ticks: i64) -> [u8; PONG_SIZE] { +/// Build a Pong for `connection_number`, echoing `sequence` and carrying our +/// local time in .NET ticks. +pub fn build_pong(connection_number: u8, sequence: u16, time_ticks: i64) -> [u8; PONG_SIZE] { let mut b = [0u8; PONG_SIZE]; - b[0] = Header::plain(property::PONG).to_byte(); + b[0] = Header::connected(property::PONG, connection_number).to_byte(); b[1..3].copy_from_slice(&sequence.to_le_bytes()); b[3..11].copy_from_slice(&time_ticks.to_le_bytes()); b @@ -232,10 +244,10 @@ pub fn read_sequence(buf: &[u8]) -> Option { Some(u16::from_le_bytes([buf[1], buf[2]])) } -/// Build a Disconnect packet carrying `connect_time` (validated by the peer). -pub fn build_disconnect(connect_time: i64) -> [u8; DISCONNECT_SIZE] { +/// Build a Disconnect for `connection_number`, carrying `connect_time`. +pub fn build_disconnect(connection_number: u8, connect_time: i64) -> [u8; DISCONNECT_SIZE] { let mut b = [0u8; DISCONNECT_SIZE]; - b[0] = Header::plain(property::DISCONNECT).to_byte(); + b[0] = Header::connected(property::DISCONNECT, connection_number).to_byte(); b[1..9].copy_from_slice(&connect_time.to_le_bytes()); b } @@ -356,8 +368,8 @@ mod tests { #[test] fn ping_pong_golden_bytes() { - assert_eq!(build_ping(0x0201), [property::PING, 0x01, 0x02]); - let pong = build_pong(0x0201, 0x0A09_0807_0605_0403); + assert_eq!(build_ping(0, 0x0201), [property::PING, 0x01, 0x02]); + let pong = build_pong(0, 0x0201, 0x0A09_0807_0605_0403); assert_eq!( pong, [ @@ -379,17 +391,43 @@ mod tests { #[test] fn disconnect_golden_roundtrip() { - let d = build_disconnect(0x0102_0304_0506_0708); + let d = build_disconnect(0, 0x0102_0304_0506_0708); assert_eq!(d[0], property::DISCONNECT); assert_eq!(read_disconnect_time(&d), Some(0x0102_0304_0506_0708)); } #[test] fn unreliable_wraps_payload() { - let p = build_unreliable(&[0xDE, 0xAD, 0xBE, 0xEF]); + let p = build_unreliable(0, &[0xDE, 0xAD, 0xBE, 0xEF]); assert_eq!(p, [property::UNRELIABLE, 0xDE, 0xAD, 0xBE, 0xEF]); } + #[test] + fn connected_packet_builders_stamp_every_connection_number() { + for connection_number in 0..MAX_CONNECTION_NUMBER { + let unreliable = build_unreliable(connection_number, &[0xAA]); + let ping = build_ping(connection_number, 0x0201); + let pong = build_pong(connection_number, 0x0201, 7); + let disconnect = build_disconnect(connection_number, 9); + + for (packet, property) in [ + (unreliable.as_slice(), property::UNRELIABLE), + (ping.as_slice(), property::PING), + (pong.as_slice(), property::PONG), + (disconnect.as_slice(), property::DISCONNECT), + ] { + assert_eq!( + Header::from_byte(packet[0]), + Header { + property, + connection_number, + fragmented: false, + } + ); + } + } + } + #[test] fn litenet_string_roundtrip() { let key = "sailwind-online"; diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 31932cc..1caf9ee 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -1326,17 +1326,25 @@ mod handshake_tests { } fn connect_peer(server: &mut Server) -> (UdpSocket, PeerId) { - connect_peer_from(server, "127.0.0.1") + connect_peer_from_with_number(server, "127.0.0.1", 0) } fn connect_peer_from(server: &mut Server, source_ip: &str) -> (UdpSocket, PeerId) { + connect_peer_from_with_number(server, source_ip, 0) + } + + fn connect_peer_from_with_number( + server: &mut Server, + source_ip: &str, + connection_number: u8, + ) -> (UdpSocket, PeerId) { let client = UdpSocket::bind((source_ip, 0)).unwrap(); client.connect(server.host.local_addr().unwrap()).unwrap(); client .set_read_timeout(Some(Duration::from_secs(1))) .unwrap(); let connect_data = protocol::write_litenet_string(CONNECT_KEY); - let request = protocol::build_connect_request(0, 1, 1, 16, &connect_data); + let request = protocol::build_connect_request(connection_number, 1, 1, 16, &connect_data); client.send(&request).unwrap(); let peer = match server.host.poll(Instant::now()).as_slice() { @@ -1345,6 +1353,7 @@ mod handshake_tests { }; let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; client.recv(&mut accept).unwrap(); + assert_eq!(accept[9], connection_number); (client, peer) } @@ -1394,6 +1403,100 @@ mod handshake_tests { } } + #[test] + fn nonzero_transport_session_round_trips_hello_with_its_connection_number() { + for connection_number in 1..protocol::MAX_CONNECTION_NUMBER { + let mut server = make_server(); + let (client, peer) = + connect_peer_from_with_number(&mut server, "127.0.0.1", connection_number); + let hello = hello_envelope( + &format!("nonzero-session-{connection_number}"), + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + + client + .send(&protocol::build_unreliable(connection_number, &hello)) + .unwrap(); + let events = server.host.poll(Instant::now()); + assert_eq!(events, vec![Event::Data(peer, hello)]); + for event in events { + server.handle_event(event).unwrap(); + } + + let mut packet = [0u8; protocol::MTU]; + let received = client.recv(&mut packet).unwrap(); + assert_eq!( + protocol::Header::from_byte(packet[0]), + protocol::Header { + property: protocol::property::UNRELIABLE, + connection_number, + fragmented: false, + } + ); + let envelope = decode_envelope(&packet[protocol::HEADER_SIZE..received]).unwrap(); + let server_hello = envelope.payload_as_server_hello().unwrap(); + assert!(server_hello.accepted()); + assert_eq!(server_hello.reason(), Some("")); + } + } + + #[test] + fn retired_connection_number_cannot_authenticate_replacement_endpoint() { + let mut server = make_server(); + let (client, first_peer) = connect_peer_from_with_number(&mut server, "127.0.0.1", 1); + let connect_data = protocol::write_litenet_string(CONNECT_KEY); + let replacement_request = protocol::build_connect_request(2, 2, 1, 16, &connect_data); + client.send(&replacement_request).unwrap(); + + let events = server.host.poll(Instant::now()); + assert_eq!( + events, + vec![ + Event::Disconnected(first_peer, DisconnectReason::Remote), + Event::Connected(2), + ] + ); + for event in events { + server.handle_event(event).unwrap(); + } + let mut accept = [0u8; protocol::CONNECT_ACCEPT_SIZE]; + assert_eq!( + client.recv(&mut accept).unwrap(), + protocol::CONNECT_ACCEPT_SIZE + ); + assert_eq!(accept[9], 2); + + let hello = hello_envelope( + "replacement-number-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + client.send(&protocol::build_unreliable(1, &hello)).unwrap(); + assert!( + server.host.poll(Instant::now()).is_empty(), + "the retired connection number must not reach authentication" + ); + assert!(server.sessions.is_empty()); + + client.send(&protocol::build_unreliable(2, &hello)).unwrap(); + let current_events = server.host.poll(Instant::now()); + assert_eq!(current_events, vec![Event::Data(2, hello)]); + for event in current_events { + server.handle_event(event).unwrap(); + } + assert_eq!(server.sessions.len(), 1); + assert!(server.sessions.contains_key(&2)); + + let mut response = [0u8; protocol::MTU]; + let received = client.recv(&mut response).unwrap(); + let header = protocol::Header::from_byte(response[0]); + assert_eq!(header.property, protocol::property::UNRELIABLE); + assert_eq!(header.connection_number, 2); + let envelope = decode_envelope(&response[protocol::HEADER_SIZE..received]).unwrap(); + assert!(envelope.payload_as_server_hello().unwrap().accepted()); + } + #[test] fn protocol_mismatch_is_rejected_before_session_creation() { let mut server = make_server(); @@ -1899,9 +2002,9 @@ mod handshake_tests { } first_client - .send(&protocol::build_unreliable(&hello)) + .send(&protocol::build_unreliable(0, &hello)) .unwrap(); - first_client.send(&protocol::build_ping(1)).unwrap(); + first_client.send(&protocol::build_ping(0, 1)).unwrap(); assert!( server.host.poll(Instant::now()).is_empty(), "data and keepalive traffic from the evicted address must be ignored" @@ -1938,9 +2041,9 @@ mod handshake_tests { let new_request = protocol::build_connect_request(0, 2, 1, 16, &connect_data); old_client - .send(&protocol::build_unreliable(&old_hello)) + .send(&protocol::build_unreliable(0, &old_hello)) .unwrap(); - old_client.send(&protocol::build_disconnect(1)).unwrap(); + old_client.send(&protocol::build_disconnect(0, 1)).unwrap(); new_client.send(&new_request).unwrap(); std::thread::sleep(Duration::from_millis(10)); From 9d0c9ff42c73c166569143a2e23ad1719fe95797 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 19:57:11 -0600 Subject: [PATCH 26/36] docs(agents): record connection number semantics Co-Authored-By: Claude --- .agents/lessons-learned.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.agents/lessons-learned.md b/.agents/lessons-learned.md index 529088e..ebbdf03 100644 --- a/.agents/lessons-learned.md +++ b/.agents/lessons-learned.md @@ -25,3 +25,4 @@ fact. 2026-07-21: A Harmony POSTFIX on the game load method cannot stall the loading screen: it runs only after the load method returns, so a stuck load means the hook never fired (a fired-but-slow postfix would show as a hang after the screen clears, not a frozen loading screen). Still, wrap the event invoke in try/catch so a subscriber exception can never propagate back into the game's load and abort it. 2026-07-22: Bash fixture runners must remove CR from .env input before sourcing because Windows checkout conversion otherwise leaves carriage returns in exported values under POSIX shells. 2026-07-22: SaveLoadManager.readyToSave is the common static world-ready marker set by both StartMenu new-game and continue coroutines; LoadGame runs only for continue, so API readiness must poll readyToSave rather than depend on the LoadGame postfix. +2026-07-24: LiteNetLib 1.3.1 connected packets use a two-bit ConnectionNumber: stale mismatches are rejected before liveness or state handling, while user data, Ping, Pong, and Disconnect carry the active number. From 3bf0e4a661dde376d78bdf1e4ca7cb4dadc172e3 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 20:25:57 -0600 Subject: [PATCH 27/36] fix(net): ignore stale reconnect requests Co-Authored-By: Claude --- server/crates/sw-net/src/lib.rs | 186 ++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index ea93fd0..437441d 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -346,6 +346,12 @@ impl Host { } if let Some(existing) = self.peers.get(&addr) { + // LiteNetLib orders this signed Int64 directly. Do not subtract: + // a wrapped comparison could let a delayed request evict its live + // successor. + if req.connect_time < existing.connect_time { + return; + } if existing.connect_time == req.connect_time { // Retransmitted request: the client missed our accept. Resend it. let accept = protocol::build_connect_accept( @@ -670,6 +676,36 @@ mod tests { } } + #[derive(Debug, PartialEq, Eq)] + struct PeerState { + id: PeerId, + slot: usize, + connect_time: i64, + connection_number: u8, + local_peer_id: i32, + last_recv: Instant, + last_ping_sent: Instant, + ping_seq: u16, + ping_sent_at: Option, + rtt: Option, + } + + fn peer_state(server: &Host, addr: SocketAddr) -> PeerState { + let peer = &server.peers[&addr]; + PeerState { + id: peer.id, + slot: peer.slot, + connect_time: peer.connect_time, + connection_number: peer.connection_number, + local_peer_id: peer.local_peer_id, + last_recv: peer.last_recv, + last_ping_sent: peer.last_ping_sent, + ping_seq: peer.ping_seq, + ping_sent_at: peer.ping_sent_at, + rtt: peer.rtt, + } + } + fn assert_peer_indices_consistent(server: &Host) { assert_eq!( server.peer_slots.len(), @@ -872,6 +908,156 @@ mod tests { ); } + #[test] + fn same_endpoint_connect_requests_only_replace_with_a_strictly_newer_time() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 3, 3).unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + let addr = client.local_addr().unwrap(); + let t0 = Instant::now(); + + client + .send(&connect_datagram(1, 30, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(t0), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + + client + .send(&connect_datagram(2, 40, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(t0 + Duration::from_millis(1)), + vec![ + Event::Disconnected(1, DisconnectReason::Remote), + Event::Connected(2), + ] + ); + let time_40_accept = drain_packets(&client); + assert_eq!(time_40_accept.len(), 1); + assert_eq!( + i64::from_le_bytes(time_40_accept[0][1..9].try_into().unwrap()), + 40 + ); + assert_eq!(time_40_accept[0][9], 2); + + let state_after_time_40 = peer_state(&server, addr); + client + .send(&connect_datagram(1, 30, "sailwind-online")) + .unwrap(); + assert!(server.poll(t0 + Duration::from_millis(2)).is_empty()); + assert!(drain_packets(&client).is_empty()); + assert_eq!( + peer_state(&server, addr), + state_after_time_40, + "an older request must not mutate live-session or liveness state" + ); + assert_eq!(server.peer_count(), 1); + assert_eq!(server.peer_addr(2), Some(addr)); + assert_peer_indices_consistent(&server); + + client + .send(&protocol::build_unreliable(1, b"retired-session")) + .unwrap(); + client + .send(&protocol::build_unreliable(2, b"current-session")) + .unwrap(); + assert_eq!( + server.poll(t0 + Duration::from_millis(3)), + vec![Event::Data(2, b"current-session".to_vec())] + ); + + let state_before_duplicate = peer_state(&server, addr); + client + .send(&connect_datagram(2, 40, "sailwind-online")) + .unwrap(); + assert!(server.poll(t0 + Duration::from_millis(4)).is_empty()); + assert_eq!(drain_packets(&client), time_40_accept); + assert_eq!( + peer_state(&server, addr), + state_before_duplicate, + "an exact retry must only resend the original accept" + ); + + client + .send(&connect_datagram(3, 50, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(t0 + Duration::from_millis(5)), + vec![ + Event::Disconnected(2, DisconnectReason::Remote), + Event::Connected(3), + ] + ); + let time_50_accept = drain_packets(&client); + assert_eq!(time_50_accept.len(), 1); + assert_eq!( + i64::from_le_bytes(time_50_accept[0][1..9].try_into().unwrap()), + 50 + ); + assert_eq!(time_50_accept[0][9], 3); + assert_eq!(server.peer_count(), 1); + assert_eq!(server.peer_addr(3), Some(addr)); + assert_peer_indices_consistent(&server); + } + + #[test] + fn delayed_older_request_cannot_black_hole_a_full_capacity_peer() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 1, 1).unwrap(); + let server_addr = server.local_addr().unwrap(); + let client = client_from("127.0.0.1", server_addr); + let addr = client.local_addr().unwrap(); + let t0 = Instant::now(); + + client + .send(&connect_datagram(1, 30, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(t0), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + + client + .send(&connect_datagram(2, 40, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(t0 + Duration::from_millis(1)), + vec![Event::Disconnected(1, DisconnectReason::Remote)] + ); + assert_eq!(server.peer_count(), 0); + assert_eq!(server.free_peer_slots, VecDeque::from([0])); + + client + .send(&connect_datagram(2, 40, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(t0 + Duration::from_millis(2)), + vec![Event::Connected(1)] + ); + assert_eq!(drain_packets(&client).len(), 1); + assert!(server.free_peer_slots.is_empty()); + + client + .send(&connect_datagram(1, 30, "sailwind-online")) + .unwrap(); + client + .send(&protocol::build_unreliable(1, b"retired-session")) + .unwrap(); + client + .send(&protocol::build_unreliable(2, b"current-session")) + .unwrap(); + assert_eq!( + server.poll(t0 + Duration::from_millis(3)), + vec![Event::Data(1, b"current-session".to_vec())], + "the delayed request must not retire the only slot or drop current-session traffic" + ); + assert!(drain_packets(&client).is_empty()); + assert_eq!(server.peer_count(), 1); + assert_eq!(server.peer_addr(1), Some(addr)); + assert_eq!(server.peers[&addr].connect_time, 40); + assert_eq!(server.peers[&addr].connection_number, 2); + assert_eq!(server.peers[&addr].last_recv, t0 + Duration::from_millis(3)); + assert!(server.free_peer_slots.is_empty()); + assert_peer_indices_consistent(&server); + } + #[test] fn preauth_data_flood_is_bounded_and_a_queued_payload_progresses() { let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); From b21d535133c98ed472ffc9eccb38ddb806c5f435 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 20:55:04 -0600 Subject: [PATCH 28/36] fix(net): retain reconnect ordering through slot retry Co-Authored-By: Claude --- server/crates/sw-net/src/lib.rs | 207 +++++++++++++++++++++++++++++++- 1 file changed, 201 insertions(+), 6 deletions(-) diff --git a/server/crates/sw-net/src/lib.rs b/server/crates/sw-net/src/lib.rs index 437441d..e09f241 100644 --- a/server/crates/sw-net/src/lib.rs +++ b/server/crates/sw-net/src/lib.rs @@ -50,6 +50,9 @@ const MAX_POLL_MAINTENANCE_SCANS: usize = 512; /// Hard ceiling on host keepalive datagrams sent by one fixed-tick poll. const MAX_POLL_KEEPALIVE_SENDS: usize = MAX_POLL_MAINTENANCE_SCANS; +/// Hard ceiling on expired reconnect watermarks removed by one fixed-tick poll. +const MAX_POLL_WATERMARK_CLEANUPS: usize = MAX_POLL_MAINTENANCE_SCANS; + /// Hard ceiling on the event vector returned by one poll. const MAX_POLL_EVENTS: usize = MAX_POLL_SOCKET_EVENTS + MAX_POLL_TIMEOUT_EVENTS; @@ -100,6 +103,7 @@ struct PollWork { timeout_events: usize, maintenance_scans: usize, keepalive_sends: usize, + watermark_cleanups: usize, } struct Peer { @@ -116,6 +120,12 @@ struct Peer { rtt: Option, } +struct ReconnectWatermark { + connect_time: i64, + expires_at: Instant, + generation: u64, +} + /// A UDP host: binds a socket, tracks peers, and exposes an event/send API. pub struct Host { socket: UdpSocket, @@ -127,6 +137,9 @@ pub struct Host { free_peer_slots: VecDeque, available_free_peer_slots: usize, peers_per_ip: HashMap, + reconnect_watermarks: HashMap, + reconnect_watermark_order: VecDeque<(SocketAddr, u64)>, + next_reconnect_watermark_generation: u64, maintenance_cursor: usize, connect_key: String, timeout: Duration, @@ -173,6 +186,9 @@ impl Host { free_peer_slots: VecDeque::new(), available_free_peer_slots: 0, peers_per_ip: HashMap::new(), + reconnect_watermarks: HashMap::new(), + reconnect_watermark_order: VecDeque::new(), + next_reconnect_watermark_generation: 0, maintenance_cursor: 0, connect_key: connect_key.to_string(), timeout: DEFAULT_TIMEOUT, @@ -227,6 +243,7 @@ impl Host { self.available_free_peer_slots = self.free_peer_slots.len(); let mut events = Vec::with_capacity(MAX_POLL_EVENTS); let mut work = PollWork::default(); + self.expire_reconnect_watermarks(now, &mut work); self.drain_socket(now, &mut events, &mut work); work.socket_events = events.len(); self.process_peer_maintenance(now, &mut events, &mut work); @@ -236,6 +253,7 @@ impl Host { debug_assert!(work.timeout_events <= MAX_POLL_TIMEOUT_EVENTS); debug_assert!(work.maintenance_scans <= MAX_POLL_MAINTENANCE_SCANS); debug_assert!(work.keepalive_sends <= MAX_POLL_KEEPALIVE_SENDS); + debug_assert!(work.watermark_cleanups <= MAX_POLL_WATERMARK_CLEANUPS); debug_assert!(events.len() <= MAX_POLL_EVENTS); (events, work) } @@ -345,6 +363,7 @@ impl Host { _ => return, } + let mut replaced_existing = false; if let Some(existing) = self.peers.get(&addr) { // LiteNetLib orders this signed Int64 directly. Do not subtract: // a wrapped comparison could let a delayed request evict its live @@ -367,13 +386,29 @@ impl Host { let old_id = existing.id; self.remove_peer_at(addr); events.push(Event::Disconnected(old_id, DisconnectReason::Remote)); - } else if self.peers.len() >= self.max_peers - || self.peers_per_ip.get(&addr.ip()).copied().unwrap_or(0) >= self.max_peers_per_ip - { - return; + replaced_existing = true; + } else { + if let Some(watermark) = self.reconnect_watermarks.get_mut(&addr) { + // Keep the same direct signed ordering used for a live peer. + // Subtraction could overflow across i64::MIN/i64::MAX. + if req.connect_time < watermark.connect_time { + return; + } + if req.connect_time > watermark.connect_time { + watermark.connect_time = req.connect_time; + } + } + if self.peers.len() >= self.max_peers + || self.peers_per_ip.get(&addr.ip()).copied().unwrap_or(0) >= self.max_peers_per_ip + { + return; + } } let Some(slot) = self.allocate_peer_slot() else { + if replaced_existing { + self.record_reconnect_watermark(addr, req.connect_time, now); + } return; }; let id = PeerId::try_from(slot + 1).expect("bounded peer slot fits PeerId"); @@ -400,9 +435,77 @@ impl Host { ping_sent_at: None, rtt: None, }); + self.reconnect_watermarks.remove(&addr); events.push(Event::Connected(id)); } + fn record_reconnect_watermark(&mut self, addr: SocketAddr, connect_time: i64, now: Instant) { + if self.next_reconnect_watermark_generation == u64::MAX { + self.reconnect_watermarks.clear(); + self.reconnect_watermark_order.clear(); + self.next_reconnect_watermark_generation = 0; + } + self.next_reconnect_watermark_generation += 1; + let generation = self.next_reconnect_watermark_generation; + + if self.reconnect_watermark_order.len() == self.max_peers { + let (oldest_addr, oldest_generation) = self + .reconnect_watermark_order + .pop_front() + .expect("a full reconnect watermark order has an entry"); + if self + .reconnect_watermarks + .get(&oldest_addr) + .is_some_and(|watermark| watermark.generation == oldest_generation) + { + self.reconnect_watermarks.remove(&oldest_addr); + } + } + + let expires_at = now.checked_add(self.timeout).unwrap_or(now); + match self.reconnect_watermarks.get_mut(&addr) { + Some(watermark) => { + if connect_time > watermark.connect_time { + watermark.connect_time = connect_time; + } + watermark.expires_at = expires_at; + watermark.generation = generation; + } + None => { + self.reconnect_watermarks.insert( + addr, + ReconnectWatermark { + connect_time, + expires_at, + generation, + }, + ); + } + } + self.reconnect_watermark_order.push_back((addr, generation)); + debug_assert!(self.reconnect_watermarks.len() <= self.max_peers); + debug_assert!(self.reconnect_watermark_order.len() <= self.max_peers); + } + + fn expire_reconnect_watermarks(&mut self, now: Instant, work: &mut PollWork) { + while work.watermark_cleanups < MAX_POLL_WATERMARK_CLEANUPS { + let Some((addr, generation)) = self.reconnect_watermark_order.front().copied() else { + break; + }; + match self.reconnect_watermarks.get(&addr) { + Some(watermark) if watermark.generation == generation => { + if watermark.expires_at > now { + break; + } + self.reconnect_watermarks.remove(&addr); + } + _ => {} + } + self.reconnect_watermark_order.pop_front(); + work.watermark_cleanups += 1; + } + } + fn insert_peer(&mut self, peer: Peer) { let id = peer.id; let slot = peer.slot; @@ -602,6 +705,9 @@ impl Host { self.free_peer_slots.clear(); self.available_free_peer_slots = 0; self.peers_per_ip.clear(); + self.reconnect_watermarks.clear(); + self.reconnect_watermark_order.clear(); + self.next_reconnect_watermark_generation = 0; self.maintenance_cursor = 0; events } @@ -640,6 +746,7 @@ mod tests { assert!(work.timeout_events <= MAX_POLL_TIMEOUT_EVENTS); assert!(work.maintenance_scans <= MAX_POLL_MAINTENANCE_SCANS); assert!(work.keepalive_sends <= MAX_POLL_KEEPALIVE_SENDS); + assert!(work.watermark_cleanups <= MAX_POLL_WATERMARK_CLEANUPS); assert!(events.len() <= MAX_POLL_EVENTS); } @@ -1024,14 +1131,28 @@ mod tests { assert_eq!(server.peer_count(), 0); assert_eq!(server.free_peer_slots, VecDeque::from([0])); + client + .send(&connect_datagram(1, 30, "sailwind-online")) + .unwrap(); client .send(&connect_datagram(2, 40, "sailwind-online")) .unwrap(); assert_eq!( server.poll(t0 + Duration::from_millis(2)), - vec![Event::Connected(1)] + vec![Event::Connected(1)], + "the stale request queued first must not reclaim the quarantined slot" ); - assert_eq!(drain_packets(&client).len(), 1); + let retry_packets = drain_packets(&client); + assert_eq!( + retry_packets.len(), + 1, + "the stale request must receive no reply" + ); + assert_eq!( + i64::from_le_bytes(retry_packets[0][1..9].try_into().unwrap()), + 40 + ); + assert_eq!(retry_packets[0][9], 2); assert!(server.free_peer_slots.is_empty()); client @@ -1058,6 +1179,80 @@ mod tests { assert_peer_indices_consistent(&server); } + #[test] + fn reconnect_watermarks_are_capacity_bounded_expire_and_allow_future_sessions() { + let mut server = Host::bind_with_limits("127.0.0.1:0", "sailwind-online", 1, 1).unwrap(); + server.timeout = Duration::from_millis(3); + let server_addr = server.local_addr().unwrap(); + let t0 = Instant::now(); + let mut last_client = None; + + for source in 0..8 { + let client = client_from("127.0.0.1", server_addr); + client + .send(&connect_datagram(1, source * 10 + 1, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(t0), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + + client + .send(&connect_datagram(2, source * 10 + 2, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(t0), + vec![Event::Disconnected(1, DisconnectReason::Remote)] + ); + assert!(server.reconnect_watermarks.len() <= server.max_peers); + assert!(server.reconnect_watermark_order.len() <= server.max_peers); + client + .send(&connect_datagram(2, source * 10 + 2, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(t0), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + assert!(server.reconnect_watermarks.is_empty()); + assert!(server.reconnect_watermark_order.len() <= server.max_peers); + + assert!(server.disconnect(1)); + assert_eq!(server.peer_count(), 0); + last_client = Some(client); + } + + let client = last_client.unwrap(); + let _ = drain_packets(&client); + client + .send(&connect_datagram(1, i64::MIN, "sailwind-online")) + .unwrap(); + assert_eq!(server.poll(t0), vec![Event::Connected(1)]); + assert_eq!(drain_packets(&client).len(), 1); + client + .send(&connect_datagram(2, i64::MAX, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(t0), + vec![Event::Disconnected(1, DisconnectReason::Remote)] + ); + assert_eq!(server.reconnect_watermarks.len(), 1); + assert_eq!(server.reconnect_watermark_order.len(), 1); + assert!(server.poll(t0 + Duration::from_millis(4)).is_empty()); + assert!(server.reconnect_watermarks.is_empty()); + assert!(server.reconnect_watermark_order.is_empty()); + + client + .send(&connect_datagram(3, i64::MIN, "sailwind-online")) + .unwrap(); + assert_eq!( + server.poll(t0 + Duration::from_millis(5)), + vec![Event::Connected(1)], + "expiry must not permanently reject a legitimate lower signed timestamp" + ); + let accept = drain_packets(&client); + assert_eq!(accept.len(), 1); + assert_eq!( + i64::from_le_bytes(accept[0][1..9].try_into().unwrap()), + i64::MIN + ); + } + #[test] fn preauth_data_flood_is_bounded_and_a_queued_payload_progresses() { let mut server = Host::bind("127.0.0.1:0", "sailwind-online").unwrap(); From 178650fd943cb0dd1b66ff109561d8e951ec09bd Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 21:37:07 -0600 Subject: [PATCH 29/36] fix(server): bound snapshot and AoI work Co-Authored-By: Claude --- server/crates/sw-persist/src/lib.rs | 100 +++ server/crates/sw-server/src/server.rs | 852 +++++++++++++++++++++++--- 2 files changed, 861 insertions(+), 91 deletions(-) diff --git a/server/crates/sw-persist/src/lib.rs b/server/crates/sw-persist/src/lib.rs index aa25bd2..244f519 100644 --- a/server/crates/sw-persist/src/lib.rs +++ b/server/crates/sw-persist/src/lib.rs @@ -15,6 +15,9 @@ pub use rusqlite::{Error, Result}; /// The latest schema version this build knows how to produce. pub const SCHEMA_VERSION: i64 = 2; +/// Hard ceiling for one keyset-paginated mooring cell read. +pub const MAX_MOORINGS_PAGE_ROWS: usize = 8; + /// Embedded schema for `user_version = 1`. Applied once, in a transaction. const MIGRATION_V1: &str = r#" CREATE TABLE players ( @@ -425,6 +428,52 @@ impl Db { rows.collect() } + /// One stable, bounded page of moorings in a cell ordered by `boat_id`. + pub fn moorings_in_cell_after( + &self, + cell_x: i32, + cell_z: i32, + after_boat_id: Option, + limit: usize, + ) -> Result> { + if !(1..=MAX_MOORINGS_PAGE_ROWS).contains(&limit) { + return Err(Error::InvalidParameterName(format!( + "mooring page limit must be in 1..={MAX_MOORINGS_PAGE_ROWS}" + ))); + } + let limit = i64::try_from(limit).expect("validated mooring page limit fits i64"); + let mut rows = Vec::with_capacity(limit as usize); + if let Some(after_boat_id) = after_boat_id { + let mut stmt = self.conn.prepare( + "SELECT boat_id, owner, cell_x, cell_z, pos_x, pos_y, pos_z, rot_x, rot_y, rot_z, rot_w, name, created_at + FROM moorings + WHERE cell_x = ?1 AND cell_z = ?2 AND boat_id > ?3 + ORDER BY boat_id + LIMIT ?4", + )?; + let mapped = stmt.query_map( + params![cell_x, cell_z, after_boat_id, limit], + mooring_from_row, + )?; + for row in mapped { + rows.push(row?); + } + } else { + let mut stmt = self.conn.prepare( + "SELECT boat_id, owner, cell_x, cell_z, pos_x, pos_y, pos_z, rot_x, rot_y, rot_z, rot_w, name, created_at + FROM moorings + WHERE cell_x = ?1 AND cell_z = ?2 + ORDER BY boat_id + LIMIT ?3", + )?; + let mapped = stmt.query_map(params![cell_x, cell_z, limit], mooring_from_row)?; + for row in mapped { + rows.push(row?); + } + } + Ok(rows) + } + /// Count of all moorings (used in tests / diagnostics). pub fn mooring_count(&self) -> Result { self.conn @@ -813,6 +862,57 @@ mod tests { assert_eq!(db.moorings_in_cell(3, -2).unwrap()[0].name, "Renamed"); } + #[test] + fn mooring_cell_pages_are_bounded_stable_and_keyset_paginated() { + let db = Db::open_in_memory().unwrap(); + for boat_id in [9, 2, 7, 4, 1] { + db.upsert_mooring(&sample_mooring(boat_id, 7)).unwrap(); + } + let mut other_cell = sample_mooring(3, 7); + other_cell.cell_x = 99; + db.upsert_mooring(&other_cell).unwrap(); + + let first = db.moorings_in_cell_after(3, -2, None, 2).unwrap(); + let second = db + .moorings_in_cell_after(3, -2, first.last().map(|row| row.boat_id), 2) + .unwrap(); + let third = db + .moorings_in_cell_after(3, -2, second.last().map(|row| row.boat_id), 2) + .unwrap(); + + assert_eq!( + first + .iter() + .chain(&second) + .chain(&third) + .map(|row| row.boat_id) + .collect::>(), + vec![1, 2, 4, 7, 9] + ); + assert_eq!(first.len(), 2); + assert_eq!(second.len(), 2); + assert_eq!(third.len(), 1); + assert!(db.moorings_in_cell_after(3, -2, None, 0).is_err()); + assert!(db + .moorings_in_cell_after(3, -2, None, MAX_MOORINGS_PAGE_ROWS + 1) + .is_err()); + let query_plan: String = db + .conn + .query_row( + "EXPLAIN QUERY PLAN + SELECT boat_id FROM moorings + WHERE cell_x = ?1 AND cell_z = ?2 AND boat_id > ?3 + ORDER BY boat_id LIMIT ?4", + params![3, -2, 2, 2], + |row| row.get(3), + ) + .unwrap(); + assert!( + query_plan.contains("idx_moorings_cell"), + "keyset page must use the cell index, got: {query_plan}" + ); + } + #[test] fn mooring_survives_reopen_and_is_found_by_cell() { // Mirrors the restart-persistence conformance check: a mooring written diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 1caf9ee..7fbc4a5 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -6,9 +6,10 @@ use crate::config::{Config, MAX_PLAYER_ROWS}; use crate::econ_store::{DbLedgerStore, DbMarketStore}; use crate::ratelimit::{BoundedRateLimiter, GlobalRateLimiter, RateLimiter}; use crate::validate; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::io::Write; use std::net::IpAddr; +use std::ops::Bound::{Excluded, Unbounded}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -16,7 +17,7 @@ use sw_contracts::decode_envelope; use sw_contracts::sw_proto as p; use sw_econ::{Ledger, Market, MarketAck, Trade, Txn}; use sw_net::{protocol, DisconnectReason, Event, Host, PeerId}; -use sw_persist::{Db, MooringRow, PlayerAdmission}; +use sw_persist::{Db, MooringRow, PlayerAdmission, MAX_MOORINGS_PAGE_ROWS}; use sw_world::{AoiUpdate, Cell, Subscription, World}; /// LiteNetLib connect key clients must present. @@ -32,6 +33,28 @@ const KEY_WEATHER_SEED: &str = "weather_seed"; /// How often dirty player state is flushed to the database. const FLUSH_INTERVAL: Duration = Duration::from_secs(5); +/// Global snapshot transport and CPU budgets charged on each snapshot tick. +const SNAPSHOT_PACKETS_PER_TICK: usize = 8; +const SNAPSHOT_ENTITY_SCAN_PER_PACKET: usize = 8; +const SNAPSHOT_ENTITIES_PER_PACKET: usize = 5; + +/// Global AoI delivery budget charged on every fixed server tick. +const AOI_WORK_ITEMS_PER_TICK: usize = 8; +const AOI_CELLS_PER_UPDATE: usize = 32; +const CELL_ENTITY_SCAN_PER_WORK: usize = 8; +const CELL_ENTITIES_PER_PACKET: usize = 5; +const CELL_MOORINGS_PER_PACKET: usize = 1; +const MAX_SNAPSHOT_MOORING_NAME_BYTES: usize = 512; + +struct CellHydration { + cell: Cell, + player_cursor: Option, + players_remaining: usize, + mooring_cursor: Option, + players_complete: bool, + sent_any: bool, +} + /// Per-connection state, created on ClientHello. struct Session { peer: PeerId, @@ -46,6 +69,32 @@ struct Session { sub: Subscription, cell: Option, dirty: bool, + snapshot_cursor: Option, + snapshot_remaining: usize, + published_cells: HashSet, + hydration_cells: VecDeque, + hydration_pending: HashSet, + active_hydration: Option, + aoi_queued: bool, +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct SnapshotTickWork { + recipient_visits: usize, + candidates_examined: usize, + player_states_encoded: usize, + packets: usize, + encoded_bytes: usize, +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct AoiTickWork { + recipient_visits: usize, + persisted_queries: usize, + packets: usize, + cells_completed: usize, + moorings_encoded: usize, + oversized_moorings_skipped: usize, } /// The server. @@ -55,6 +104,10 @@ pub struct Server { db: Db, world: World, sessions: HashMap, + player_peers: HashMap, + player_order: BTreeSet, + snapshot_recipients: VecDeque, + aoi_recipients: VecDeque, identity_players: HashMap, seq: u32, snapshot_tick: u32, @@ -125,6 +178,10 @@ impl Server { db, world, sessions: HashMap::new(), + player_peers: HashMap::new(), + player_order: BTreeSet::new(), + snapshot_recipients: VecDeque::new(), + aoi_recipients: VecDeque::new(), identity_players, seq: 0, snapshot_tick: 0, @@ -169,6 +226,8 @@ impl Server { } } + self.process_aoi_work(); + if tick % ticks_per_snapshot == 0 { self.broadcast_snapshots(); } @@ -402,17 +461,16 @@ impl Server { } // Drop any prior session for this identity (reconnect from a new peer). - let stale: Vec = self - .sessions - .iter() - .filter(|(&pp, s)| pp != peer && s.player_id == player_id) - .map(|(&pp, _)| pp) - .collect(); - for pp in stale { + if let Some(pp) = self + .player_peers + .get(&player_id) + .copied() + .filter(|&pp| pp != peer) + { if self.host.peer_addr(pp).is_some() && !self.host.disconnect(pp) { return Err(anyhow::anyhow!("failed to evict superseded transport peer")); } - self.sessions.remove(&pp); + self.unregister_session(pp); self.hello_limiter.clear(u64::from(pp)); } @@ -421,7 +479,7 @@ impl Server { let aoi = sub.recenter(origin); self.world.place_in_cell(player_id, origin); - self.sessions.insert( + self.register_session( peer, Session { peer, @@ -436,8 +494,15 @@ impl Server { sub, cell: Some(origin), dirty: true, + snapshot_cursor: None, + snapshot_remaining: 0, + published_cells: HashSet::new(), + hydration_cells: VecDeque::new(), + hydration_pending: HashSet::new(), + active_hydration: None, + aoi_queued: false, }, - ); + )?; tracing::info!(peer, player_id, name = %self.sessions[&peer].display_name, "hello accepted"); @@ -530,27 +595,10 @@ impl Server { self.emit_aoi(peer, &aoi); } - /// Send an AoI delta to `peer`: the added/removed cell list followed by a - /// full [`codec::cell_snapshot`] for each newly entered cell. A no-op when - /// the delta is empty (the player stayed in the same cell). + /// Schedule a changed AoI for bounded delivery from the fixed-tick loop. fn emit_aoi(&mut self, peer: PeerId, aoi: &AoiUpdate) { - if aoi.is_empty() { - return; - } - - let bytes = codec::aoi_update(self.next_seq(), &aoi.added, &aoi.removed); - self.send(peer, &bytes); - - for &cell in &aoi.added { - let (players, boats, moorings) = match self.gather_cell(cell) { - Ok(v) => v, - Err(e) => { - tracing::warn!(error = %e, "gather cell failed"); - continue; - } - }; - let bytes = codec::cell_snapshot(self.next_seq(), cell, &players, &boats, &moorings); - self.send(peer, &bytes); + if !aoi.is_empty() { + self.schedule_aoi(peer); } } @@ -804,7 +852,7 @@ impl Server { fn on_disconnect(&mut self, peer: PeerId, reason: DisconnectReason) -> anyhow::Result<()> { self.hello_limiter.clear(u64::from(peer)); - if let Some(s) = self.sessions.remove(&peer) { + if let Some(s) = self.unregister_session(peer) { self.world.remove(s.player_id); // Drop session-scoped message throttles. The reconnect cooldown is // intentionally retained in its bounded map, otherwise a known @@ -821,38 +869,382 @@ impl Server { Ok(()) } - fn broadcast_snapshots(&mut self) { - self.snapshot_tick = self.snapshot_tick.wrapping_add(1); - let server_tick = self.snapshot_tick; + fn register_session(&mut self, peer: PeerId, session: Session) -> anyhow::Result<()> { + if self.sessions.contains_key(&peer) { + return Err(anyhow::anyhow!("peer already owns a session")); + } + if let Some(existing_peer) = self.player_peers.get(&session.player_id) { + return Err(anyhow::anyhow!( + "player already owns session peer {existing_peer}" + )); + } + let player_id = session.player_id; + self.sessions.insert(peer, session); + self.player_peers.insert(player_id, peer); + self.player_order.insert(player_id); + self.snapshot_recipients.push_back(peer); + self.schedule_aoi(peer); + Ok(()) + } - let recipients: Vec<(PeerId, u64, Cell)> = self - .sessions - .values() - .filter_map(|s| s.cell.map(|c| (s.peer, s.player_id, c))) - .collect(); + fn unregister_session(&mut self, peer: PeerId) -> Option { + let session = self.sessions.remove(&peer)?; + if self.player_peers.get(&session.player_id) == Some(&peer) { + self.player_peers.remove(&session.player_id); + self.player_order.remove(&session.player_id); + } + self.snapshot_recipients.retain(|&queued| queued != peer); + self.aoi_recipients.retain(|&queued| queued != peer); + Some(session) + } + + fn schedule_aoi(&mut self, peer: PeerId) { + let Some(session) = self.sessions.get_mut(&peer) else { + return; + }; + if !session.aoi_queued { + session.aoi_queued = true; + self.aoi_recipients.push_back(peer); + } + } - for (peer, self_pid, cell) in recipients { + fn broadcast_snapshots(&mut self) -> SnapshotTickWork { + self.snapshot_tick = self.snapshot_tick.wrapping_add(1); + let server_tick = self.snapshot_tick; + let mut work = SnapshotTickWork::default(); + let recipients = self + .snapshot_recipients + .len() + .min(SNAPSHOT_PACKETS_PER_TICK); + for _ in 0..recipients { + let Some(peer) = self.snapshot_recipients.pop_front() else { + break; + }; + let Some((self_pid, cell, cursor, remaining)) = + self.sessions.get(&peer).and_then(|s| { + s.cell + .map(|cell| (s.player_id, cell, s.snapshot_cursor, s.snapshot_remaining)) + }) + else { + continue; + }; + work.recipient_visits += 1; + let remaining = if remaining == 0 { + self.player_order.len() + } else { + remaining + }; + let candidates = self + .player_candidates_after(cursor, remaining.min(SNAPSHOT_ENTITY_SCAN_PER_PACKET)); let mut players = Vec::new(); let mut boats = Vec::new(); - for eid in self.players_in_view(cell, self_pid) { - if let Some(s) = self.session_by_player(eid) { - players.push(player_snap(s)); - if s.aboard_boat != 0 { - boats.push(boat_snap(s)); - } + let mut last_examined = None; + let mut examined = 0usize; + for player_id in candidates { + last_examined = Some(player_id); + examined += 1; + work.candidates_examined += 1; + if player_id == self_pid { + continue; } + let Some(session) = self.session_by_player(player_id) else { + continue; + }; + if !session.cell.is_some_and(|other| { + other.chebyshev_distance(cell) <= self.cfg.aoi_radius_i32() + }) { + continue; + } + players.push(player_snap(session)); + if session.aboard_boat != 0 { + boats.push(boat_snap(session)); + } + if players.len() == SNAPSHOT_ENTITIES_PER_PACKET { + break; + } + } + if let Some(session) = self.sessions.get_mut(&peer) { + session.snapshot_cursor = last_examined; + session.snapshot_remaining = remaining.saturating_sub(examined); } + self.snapshot_recipients.push_back(peer); if players.is_empty() && boats.is_empty() { continue; } let bytes = codec::snapshot_delta(self.next_seq(), server_tick, &players, &boats); - self.send(peer, &bytes); + work.player_states_encoded += players.len(); + if self.send_bounded(peer, &bytes, "snapshot delta") { + work.packets += 1; + work.encoded_bytes += bytes.len(); + } + } + work + } + + fn player_candidates_after(&self, cursor: Option, limit: usize) -> Vec { + let limit = limit.min(self.player_order.len()); + let mut candidates = Vec::with_capacity(limit); + if let Some(cursor) = cursor { + candidates.extend( + self.player_order + .range((Excluded(cursor), Unbounded)) + .chain(self.player_order.range(..=cursor)) + .take(limit) + .copied(), + ); + } else { + candidates.extend(self.player_order.iter().take(limit).copied()); + } + candidates + } + + fn process_aoi_work(&mut self) -> AoiTickWork { + let mut work = AoiTickWork::default(); + let recipients = self.aoi_recipients.len().min(AOI_WORK_ITEMS_PER_TICK); + for _ in 0..recipients { + let Some(peer) = self.aoi_recipients.pop_front() else { + break; + }; + let Some(session) = self.sessions.get_mut(&peer) else { + continue; + }; + session.aoi_queued = false; + work.recipient_visits += 1; + let needs_more = match self.process_one_aoi_work(peer, &mut work) { + Ok(needs_more) => needs_more, + Err(error) => { + tracing::warn!(peer, error = %error, "bounded AoI hydration failed"); + true + } + }; + if needs_more { + self.schedule_aoi(peer); + } + } + work + } + + fn process_one_aoi_work( + &mut self, + peer: PeerId, + work: &mut AoiTickWork, + ) -> anyhow::Result { + if let Some((added, removed)) = self.next_aoi_chunk(peer) { + let bytes = codec::aoi_update(self.next_seq(), &added, &removed); + if self.send_bounded(peer, &bytes, "AoI update") { + work.packets += 1; + } + return Ok(self.session_has_aoi_work(peer)); + } + + let active_players = self.player_order.len(); + let Some((cell, player_cursor, players_remaining, players_complete)) = + self.prepare_cell_hydration(peer, active_players) + else { + return Ok(self.session_has_aoi_work(peer)); + }; + + if !players_complete { + let candidates = self.player_candidates_after( + player_cursor, + players_remaining.min(CELL_ENTITY_SCAN_PER_WORK), + ); + let mut players = Vec::new(); + let mut boats = Vec::new(); + let mut examined = 0usize; + let mut last_examined = None; + for player_id in candidates { + examined += 1; + last_examined = Some(player_id); + let Some(session) = self.session_by_player(player_id) else { + continue; + }; + if session.cell != Some(cell) { + continue; + } + players.push(player_snap(session)); + if session.aboard_boat != 0 { + boats.push(boat_snap(session)); + } + if players.len() == CELL_ENTITIES_PER_PACKET { + break; + } + } + + let mut completed_scan = false; + if let Some(session) = self.sessions.get_mut(&peer) { + if let Some(hydration) = session.active_hydration.as_mut() { + hydration.player_cursor = last_examined.or(hydration.player_cursor); + hydration.players_remaining = + hydration.players_remaining.saturating_sub(examined); + if hydration.players_remaining == 0 || examined == 0 { + hydration.players_complete = true; + completed_scan = true; + } + } + } + + if !players.is_empty() || !boats.is_empty() { + let bytes = codec::cell_snapshot(self.next_seq(), cell, &players, &boats, &[]); + if self.send_bounded(peer, &bytes, "cell player snapshot") { + work.packets += 1; + if let Some(session) = self.sessions.get_mut(&peer) { + if let Some(hydration) = session.active_hydration.as_mut() { + hydration.sent_any = true; + } + } + } + } + if !completed_scan { + return Ok(true); + } + return Ok(self.session_has_aoi_work(peer)); + } + + let mooring_cursor = self + .sessions + .get(&peer) + .and_then(|session| session.active_hydration.as_ref()) + .and_then(|hydration| hydration.mooring_cursor); + let rows = self.db.moorings_in_cell_after( + cell.cx, + cell.cz, + mooring_cursor, + CELL_MOORINGS_PER_PACKET.min(MAX_MOORINGS_PAGE_ROWS), + )?; + work.persisted_queries += 1; + if let Some(row) = rows.into_iter().next() { + let next_cursor = row.boat_id; + let snapshot = + (row.name.len() <= MAX_SNAPSHOT_MOORING_NAME_BYTES).then(|| mooring_snap(row)); + if let Some(session) = self.sessions.get_mut(&peer) { + if let Some(hydration) = session.active_hydration.as_mut() { + hydration.mooring_cursor = Some(next_cursor); + } + } + if let Some(snapshot) = snapshot { + let bytes = codec::cell_snapshot(self.next_seq(), cell, &[], &[], &[snapshot]); + if self.send_bounded(peer, &bytes, "cell mooring snapshot") { + work.packets += 1; + work.moorings_encoded += 1; + if let Some(session) = self.sessions.get_mut(&peer) { + if let Some(hydration) = session.active_hydration.as_mut() { + hydration.sent_any = true; + } + } + } + } else { + work.oversized_moorings_skipped += 1; + } + return Ok(true); + } + + let sent_any = self + .sessions + .get(&peer) + .and_then(|session| session.active_hydration.as_ref()) + .is_some_and(|hydration| hydration.sent_any); + if !sent_any { + let bytes = codec::cell_snapshot(self.next_seq(), cell, &[], &[], &[]); + if self.send_bounded(peer, &bytes, "empty cell snapshot") { + work.packets += 1; + } + } + if let Some(session) = self.sessions.get_mut(&peer) { + session.active_hydration = None; + } + work.cells_completed += 1; + Ok(self.session_has_aoi_work(peer)) + } + + fn next_aoi_chunk(&mut self, peer: PeerId) -> Option<(Vec, Vec)> { + let session = self.sessions.get_mut(&peer)?; + let mut removed: Vec = session + .published_cells + .difference(session.sub.cells()) + .copied() + .collect(); + removed.sort_by_key(|cell| (cell.cz, cell.cx)); + removed.truncate(AOI_CELLS_PER_UPDATE); + + let remaining = AOI_CELLS_PER_UPDATE - removed.len(); + let mut added: Vec = session + .sub + .cells() + .difference(&session.published_cells) + .copied() + .collect(); + added.sort_by_key(|cell| (cell.cz, cell.cx)); + added.truncate(remaining); + if added.is_empty() && removed.is_empty() { + return None; + } + + for cell in &removed { + session.published_cells.remove(cell); + session.hydration_pending.remove(cell); + } + for &cell in &added { + session.published_cells.insert(cell); + if session.hydration_pending.insert(cell) { + session.hydration_cells.push_back(cell); + } + } + Some((added, removed)) + } + + fn prepare_cell_hydration( + &mut self, + peer: PeerId, + active_players: usize, + ) -> Option<(Cell, Option, usize, bool)> { + let session = self.sessions.get_mut(&peer)?; + if let Some(active) = session.active_hydration.as_ref() { + if !session.sub.contains(active.cell) || !session.published_cells.contains(&active.cell) + { + session.hydration_pending.remove(&active.cell); + session.active_hydration = None; + return None; + } + } + if session.active_hydration.is_none() { + let cell = session.hydration_cells.pop_front()?; + if !session.hydration_pending.remove(&cell) + || !session.sub.contains(cell) + || !session.published_cells.contains(&cell) + { + return None; + } + session.active_hydration = Some(CellHydration { + cell, + player_cursor: None, + players_remaining: active_players, + mooring_cursor: None, + players_complete: active_players == 0, + sent_any: false, + }); } + let hydration = session.active_hydration.as_ref()?; + Some(( + hydration.cell, + hydration.player_cursor, + hydration.players_remaining, + hydration.players_complete, + )) + } + + fn session_has_aoi_work(&self, peer: PeerId) -> bool { + self.sessions.get(&peer).is_some_and(|session| { + session.published_cells != *session.sub.cells() + || session.active_hydration.is_some() + || !session.hydration_cells.is_empty() + }) } /// Entity ids visible to a viewer centred on `cell`: everything within the /// configured AoI radius, minus the viewer itself. This bounds a /// recipient's snapshot to AoI density, never the global population. + #[cfg(test)] fn players_in_view(&self, cell: Cell, self_pid: u64) -> Vec { self.world .entities_in_radius(cell, self.cfg.aoi_radius_i32()) @@ -873,29 +1265,6 @@ impl Server { } } - fn gather_cell( - &self, - cell: Cell, - ) -> anyhow::Result<(Vec, Vec, Vec)> { - let mut players = Vec::new(); - let mut boats = Vec::new(); - for eid in self.world.entities_in(cell) { - if let Some(s) = self.session_by_player(eid) { - players.push(player_snap(s)); - if s.aboard_boat != 0 { - boats.push(boat_snap(s)); - } - } - } - let moorings = self - .db - .moorings_in_cell(cell.cx, cell.cz)? - .into_iter() - .map(mooring_snap) - .collect(); - Ok((players, boats, moorings)) - } - fn flush_dirty(&mut self) -> anyhow::Result<()> { let now = now_ms(); let dirty: Vec<(PeerId, u64)> = self @@ -923,7 +1292,10 @@ impl Server { } fn session_by_player(&self, player_id: u64) -> Option<&Session> { - self.sessions.values().find(|s| s.player_id == player_id) + let peer = self.player_peers.get(&player_id)?; + self.sessions + .get(peer) + .filter(|session| session.player_id == player_id) } fn caps(&self) -> Caps { @@ -958,6 +1330,20 @@ impl Server { tracing::warn!(peer, error = %e, "send failed"); } } + + fn send_bounded(&mut self, peer: PeerId, bytes: &[u8], kind: &'static str) -> bool { + if bytes.len() > protocol::MTU - protocol::HEADER_SIZE { + tracing::error!( + peer, + kind, + bytes = bytes.len(), + "bounded packet encoder exceeded transport payload" + ); + return false; + } + self.send(peer, bytes); + true + } } fn player_snap(s: &Session) -> PlayerSnap { @@ -1137,6 +1523,10 @@ mod handshake_tests { db, world, sessions: HashMap::new(), + player_peers: HashMap::new(), + player_order: BTreeSet::new(), + snapshot_recipients: VecDeque::new(), + aoi_recipients: VecDeque::new(), identity_players, seq: 0, snapshot_tick: 0, @@ -1962,6 +2352,8 @@ mod handshake_tests { deliver_hello_at(&mut server, first_peer, &hello, first_admission_ms); assert_eq!(receive_server_hello(&first_client), (true, String::new())); + let original_player_id = server.sessions[&first_peer].player_id; + assert_eq!(server.player_peers[&original_player_id], first_peer); let (replacement_client, replacement_peer) = connect_peer(&mut server); let replacement_admission_ms = first_admission_ms @@ -1981,7 +2373,24 @@ mod handshake_tests { ); let player_id = server.sessions[&replacement_peer].player_id; let world_cell = server.world.cell_of_entity(player_id); + assert_eq!(player_id, original_player_id); + assert_eq!(server.player_peers[&player_id], replacement_peer); + assert_eq!( + server.player_order.iter().copied().collect::>(), + vec![player_id] + ); + assert_eq!( + server + .session_by_player(player_id) + .map(|session| session.peer), + Some(replacement_peer) + ); assert!(!server.sessions.contains_key(&first_peer)); + assert!(!server + .snapshot_recipients + .iter() + .any(|&peer| peer == first_peer)); + assert!(!server.aoi_recipients.iter().any(|&peer| peer == first_peer)); assert!( server.host.peer_addr(first_peer).is_none(), "a superseded logical session must be removed from the transport" @@ -2139,6 +2548,14 @@ mod handshake_tests { server .on_disconnect(first_peer, DisconnectReason::Remote) .unwrap(); + assert!(!server.player_peers.contains_key(&player_id)); + assert!(!server.player_order.contains(&player_id)); + assert!(server.session_by_player(player_id).is_none()); + assert!(!server + .snapshot_recipients + .iter() + .any(|&peer| peer == first_peer)); + assert!(!server.aoi_recipients.iter().any(|&peer| peer == first_peer)); assert_eq!( server.reconnect_limiter.tracked_count(), 1, @@ -2153,6 +2570,7 @@ mod handshake_tests { "source rotation inside the window must not repeat persistence admission" ); assert!(!server.sessions.contains_key(&rotated_peer)); + assert!(!server.player_peers.contains_key(&player_id)); let after_expiry_ms = first_admission_ms + server.cfg.hello_min_interval_ms_i64() + 1; deliver_hello_at(&mut server, rotated_peer, &hello, after_expiry_ms); @@ -2162,6 +2580,13 @@ mod handshake_tests { "the same reconnect must become eligible when the window expires" ); assert_eq!(server.sessions[&rotated_peer].player_id, player_id); + assert_eq!(server.player_peers[&player_id], rotated_peer); + assert_eq!( + server + .session_by_player(player_id) + .map(|session| session.peer), + Some(rotated_peer) + ); } #[test] @@ -2679,6 +3104,10 @@ mod aoi_harden_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), + player_peers: HashMap::new(), + player_order: BTreeSet::new(), + snapshot_recipients: VecDeque::new(), + aoi_recipients: VecDeque::new(), identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -2842,6 +3271,252 @@ mod aoi_harden_tests { assert!(visible.len() < far_ids.len()); } + fn insert_dense_session(server: &mut Server, peer: PeerId, cell: Cell) { + let player_id = u64::from(peer); + let mut sub = Subscription::new(server.cfg.aoi_radius_i32()); + sub.recenter(cell); + server.world.place_in_cell(player_id, cell); + server + .register_session( + peer, + Session { + peer, + player_id, + identity_hash: format!("identity-{player_id}"), + display_name: format!("Player {player_id}"), + aboard_boat: player_id, + pos: [0.0, 0.0, 0.0], + rot: [0.0, 0.0, 0.0, 1.0], + vel: [0.0, 0.0, 0.0], + t_ms: 0, + sub, + cell: Some(cell), + dirty: false, + snapshot_cursor: None, + snapshot_remaining: 0, + published_cells: HashSet::new(), + hydration_cells: VecDeque::new(), + hydration_pending: HashSet::new(), + active_hydration: None, + aoi_queued: false, + }, + ) + .unwrap(); + } + + #[test] + fn dense_snapshot_broadcast_has_a_fixed_global_packet_budget() { + const DENSE_SESSIONS: PeerId = 1_024; + + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + for peer in 1..=DENSE_SESSIONS { + insert_dense_session(&mut server, peer, center); + } + + let recipient_round = (DENSE_SESSIONS as usize).div_ceil(SNAPSHOT_PACKETS_PER_TICK); + for _ in 0..recipient_round { + let work = server.broadcast_snapshots(); + assert!(work.recipient_visits <= SNAPSHOT_PACKETS_PER_TICK); + assert!( + work.candidates_examined + <= SNAPSHOT_PACKETS_PER_TICK * SNAPSHOT_ENTITY_SCAN_PER_PACKET + ); + assert!( + work.player_states_encoded + <= SNAPSHOT_PACKETS_PER_TICK * SNAPSHOT_ENTITIES_PER_PACKET + ); + assert!(work.packets <= SNAPSHOT_PACKETS_PER_TICK); + assert!( + work.encoded_bytes + <= SNAPSHOT_PACKETS_PER_TICK * (protocol::MTU - protocol::HEADER_SIZE) + ); + } + assert!( + server + .sessions + .values() + .all(|session| session.snapshot_cursor.is_some()), + "one bounded recipient round must eventually visit every session" + ); + + server.snapshot_recipients = VecDeque::from([1]); + { + let viewer = server.sessions.get_mut(&1).unwrap(); + viewer.snapshot_cursor = None; + viewer.snapshot_remaining = 0; + } + let mut encoded_for_viewer = 0usize; + let mut viewer_visits = 0usize; + loop { + let work = server.broadcast_snapshots(); + assert!(work.candidates_examined <= SNAPSHOT_ENTITY_SCAN_PER_PACKET); + assert!(work.packets <= 1); + assert!(work.encoded_bytes <= protocol::MTU - protocol::HEADER_SIZE); + encoded_for_viewer += work.player_states_encoded; + viewer_visits += 1; + if server.sessions[&1].snapshot_remaining == 0 { + break; + } + } + assert_eq!( + encoded_for_viewer, + DENSE_SESSIONS as usize - 1, + "one deterministic entity sweep must eventually emit every other co-located player once" + ); + assert!( + viewer_visits <= (DENSE_SESSIONS as usize).div_ceil(SNAPSHOT_ENTITIES_PER_PACKET), + "entity cursor must make fixed positive progress on every dense visit" + ); + } + + #[test] + fn fixed_snapshot_chunks_fit_the_unreliable_payload() { + let players: Vec = (0..SNAPSHOT_ENTITIES_PER_PACKET) + .map(|id| PlayerSnap { + player_id: id as u64, + pos: [f32::MAX; 3], + rot: [f32::MAX; 4], + aboard_boat: id as u64, + t_ms: u32::MAX, + }) + .collect(); + let boats: Vec = (0..SNAPSHOT_ENTITIES_PER_PACKET) + .map(|id| BoatSnap { + boat_id: id as u64, + owner: id as u64, + pos: [f32::MAX; 3], + rot: [f32::MAX; 4], + vel: [f32::MAX; 3], + t_ms: u32::MAX, + }) + .collect(); + let mooring = MooringSnap { + boat_id: u64::MAX, + owner: u64::MAX, + cell: (i32::MIN, i32::MAX), + pos: [f32::MAX; 3], + rot: [f32::MAX; 4], + name: "m".repeat(MAX_SNAPSHOT_MOORING_NAME_BYTES), + created_at: u64::MAX, + }; + let added: Vec = (0..AOI_CELLS_PER_UPDATE / 2) + .map(|cell| Cell::new(cell as i32, i32::MIN)) + .collect(); + let removed: Vec = (0..AOI_CELLS_PER_UPDATE / 2) + .map(|cell| Cell::new(cell as i32, i32::MAX)) + .collect(); + let payload_limit = protocol::MTU - protocol::HEADER_SIZE; + + for (kind, bytes) in [ + ( + "snapshot delta", + codec::snapshot_delta(1, 1, &players, &boats), + ), + ( + "cell player snapshot", + codec::cell_snapshot(1, Cell::new(0, 0), &players, &boats, &[]), + ), + ( + "cell mooring snapshot", + codec::cell_snapshot(1, Cell::new(0, 0), &[], &[], &[mooring]), + ), + ("AoI update", codec::aoi_update(1, &added, &removed)), + ] { + assert!( + bytes.len() <= payload_limit, + "{kind} encoded {} bytes beyond the {payload_limit}-byte unreliable payload", + bytes.len() + ); + } + } + + #[test] + fn radius_sixteen_admission_defers_persisted_cell_hydration() { + let mut server = make_server(Config { + aoi_radius_cells: 16, + ..Config::default() + }); + let hello_bytes = hello_envelope("tok-wide-aoi", "Wide AoI"); + let hello = decode_envelope(&hello_bytes).unwrap(); + let seq_before = server.seq; + + server + .on_hello(1, hello.payload_as_client_hello().unwrap()) + .unwrap(); + + assert_eq!( + server.seq.wrapping_sub(seq_before), + 1, + "admission must send only ServerHello; the 1,089-cell persisted AoI \ + hydration belongs to the bounded fixed-tick scheduler" + ); + } + + #[test] + fn radius_sixteen_hydration_has_fixed_tick_bounds_and_eventual_progress() { + let mut server = make_server(Config { + aoi_radius_cells: 16, + ..Config::default() + }); + let last_cell = Cell::new(16, 16); + for (boat_id, name) in [ + (1, "x".repeat(MAX_SNAPSHOT_MOORING_NAME_BYTES + 1)), + (2, "eventual".to_string()), + ] { + server + .db + .upsert_mooring(&MooringRow { + boat_id, + owner: 1, + cell_x: last_cell.cx, + cell_z: last_cell.cz, + pos: [0.0; 3], + rot: [0.0, 0.0, 0.0, 1.0], + name, + created_at: 1, + }) + .unwrap(); + } + let hello_bytes = hello_envelope("tok-wide-progress", "Wide Progress"); + let hello = decode_envelope(&hello_bytes).unwrap(); + server + .on_hello(1, hello.payload_as_client_hello().unwrap()) + .unwrap(); + + let mut total_queries = 0usize; + let mut total_completed = 0usize; + let mut total_moorings = 0usize; + let mut total_oversized_skips = 0usize; + for _ in 0..5_000 { + let work = server.process_aoi_work(); + assert!(work.recipient_visits <= AOI_WORK_ITEMS_PER_TICK); + assert!(work.persisted_queries <= AOI_WORK_ITEMS_PER_TICK); + assert!(work.packets <= AOI_WORK_ITEMS_PER_TICK); + total_queries += work.persisted_queries; + total_completed += work.cells_completed; + total_moorings += work.moorings_encoded; + total_oversized_skips += work.oversized_moorings_skipped; + if !server.session_has_aoi_work(1) { + break; + } + } + + let session = &server.sessions[&1]; + assert_eq!(session.published_cells, *session.sub.cells()); + assert!(!server.session_has_aoi_work(1)); + assert_eq!(total_completed, 1_089); + assert_eq!( + total_queries, 1_091, + "1,089 cells plus two keyset continuations in the hostile final cell" + ); + assert_eq!(total_oversized_skips, 1); + assert_eq!( + total_moorings, 1, + "the valid row after an oversized legacy row must still progress" + ); + } + #[test] fn caps_advertise_configured_aoi() { let server = make_server(Config { @@ -2896,6 +3571,10 @@ mod market_dispatch_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), + player_peers: HashMap::new(), + player_order: BTreeSet::new(), + snapshot_recipients: VecDeque::new(), + aoi_recipients: VecDeque::new(), identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -3146,6 +3825,10 @@ mod input_hardening_tests { db: Db::open_in_memory().unwrap(), world, sessions: HashMap::new(), + player_peers: HashMap::new(), + player_order: BTreeSet::new(), + snapshot_recipients: VecDeque::new(), + aoi_recipients: VecDeque::new(), identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -3731,19 +4414,17 @@ mod input_hardening_tests { // i.e. the server keeps up with real time at N clients. Timing is // wall-clock, so this is `#[ignore]`d out of the required gate (`cargo test` // skips it) and run only by the non-blocking load job / `make load-test`. - const N: u32 = 200; + const N: u32 = 1_024; const TICKS: u32 = 60; const LOAD_EPOCH_MS: i64 = 1_700_000_000_000; let cfg = Config::default(); let tick_dt = Duration::from_secs_f64(1.0 / cfg.tick_hz as f64); - let cell = cfg.cell_size_m; let mut server = make_server(cfg); let session_step_ms = server.cfg.new_session_min_interval_ms_i64(); - // Join N clients, each seeded into a distinct cell on a roughly square - // grid so AoI density is realistic and bounded, not all stacked together. - let side = (N as f64).sqrt().ceil() as u32; + // Hostile-density load: every authenticated session occupies the same + // cell. Snapshot work must remain bounded at the transport ceiling. for i in 0..N { let peer = (i + 1) as PeerId; let session_offset_ms = i64::from(i) * session_step_ms; @@ -3754,14 +4435,12 @@ mod input_hardening_tests { 1_000 + session_offset_ms, LOAD_EPOCH_MS + session_offset_ms, ); - let cx = (i % side) as f32; - let cz = (i / side) as f32; // The seed time advances per client so the client-state throttle never // drops a placement. send_state( &mut server, peer, - &motion_envelope(cx * cell + 1.0, 0.0, cz * cell + 1.0, 0.0, 0.0, 0.0), + &motion_envelope(1.0, 0.0, 1.0, 0.0, 0.0, 0.0), 1_000 + i as i64, ); } @@ -3777,20 +4456,11 @@ mod input_hardening_tests { let now = 10_000 + (t as i64) * step_ms; for i in 0..N { let peer = (i + 1) as PeerId; - let cx = (i % side) as f32; - let cz = (i / side) as f32; let jitter = (t % 8) as f32; // small in-cell movement send_state( &mut server, peer, - &motion_envelope( - cx * cell + 1.0 + jitter, - 0.0, - cz * cell + 1.0, - 0.0, - 0.0, - 0.0, - ), + &motion_envelope(1.0 + jitter, 0.0, 1.0, 0.0, 0.0, 0.0), now, ); } From dd14b7da06ec2e2dc6ab8bcabae6f489f4f36881 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 22:03:01 -0600 Subject: [PATCH 30/36] fix(server): bound mooring and recipient work Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 427 ++++++++++++++++++++++---- 1 file changed, 374 insertions(+), 53 deletions(-) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 7fbc4a5..5a0e354 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -44,7 +44,13 @@ const AOI_CELLS_PER_UPDATE: usize = 32; const CELL_ENTITY_SCAN_PER_WORK: usize = 8; const CELL_ENTITIES_PER_PACKET: usize = 5; const CELL_MOORINGS_PER_PACKET: usize = 1; -const MAX_SNAPSHOT_MOORING_NAME_BYTES: usize = 512; +/// Largest UTF-8 mooring name that fits both unreliable record responses. +/// +/// `mooring_name_limit_is_the_largest_mtu_safe_record_name` derives this value +/// through the real codecs and proves that the next byte exceeds the 1,023-byte +/// LiteNetLib payload in at least one of `MoorAck` or `CellSnapshot`. +const MAX_MOORING_NAME_BYTES: usize = 835; +const MOOR_NAME_TOO_LONG_REASON: &str = "mooring name exceeds server limit"; struct CellHydration { cell: Cell, @@ -75,7 +81,6 @@ struct Session { hydration_cells: VecDeque, hydration_pending: HashSet, active_hydration: Option, - aoi_queued: bool, } #[derive(Debug, Default, PartialEq, Eq)] @@ -94,7 +99,19 @@ struct AoiTickWork { packets: usize, cells_completed: usize, moorings_encoded: usize, - oversized_moorings_skipped: usize, + corrupt_moorings_skipped: usize, +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct RecipientIndexWork { + recipient_index_operations: usize, +} + +impl RecipientIndexWork { + fn remove(&mut self, recipients: &mut BTreeSet, peer: PeerId) { + self.recipient_index_operations += 1; + recipients.remove(&peer); + } } /// The server. @@ -106,8 +123,10 @@ pub struct Server { sessions: HashMap, player_peers: HashMap, player_order: BTreeSet, - snapshot_recipients: VecDeque, - aoi_recipients: VecDeque, + snapshot_recipients: BTreeSet, + snapshot_recipient_cursor: Option, + aoi_recipients: BTreeSet, + aoi_recipient_cursor: Option, identity_players: HashMap, seq: u32, snapshot_tick: u32, @@ -180,8 +199,10 @@ impl Server { sessions: HashMap::new(), player_peers: HashMap::new(), player_order: BTreeSet::new(), - snapshot_recipients: VecDeque::new(), - aoi_recipients: VecDeque::new(), + snapshot_recipients: BTreeSet::new(), + snapshot_recipient_cursor: None, + aoi_recipients: BTreeSet::new(), + aoi_recipient_cursor: None, identity_players, seq: 0, snapshot_tick: 0, @@ -500,7 +521,6 @@ impl Server { hydration_cells: VecDeque::new(), hydration_pending: HashSet::new(), active_hydration: None, - aoi_queued: false, }, )?; @@ -752,7 +772,13 @@ impl Server { let pos = motion.pos; let rot = motion.rot; let name = req.name().unwrap_or("mooring"); - if !validate::string_within_limit(name, self.cfg.max_wire_string_len_usize()) { + let name_limit = self + .cfg + .max_wire_string_len_usize() + .min(MAX_MOORING_NAME_BYTES); + if !validate::string_within_limit(name, name_limit) { + let bytes = codec::moor_ack(self.next_seq(), false, None, MOOR_NAME_TOO_LONG_REASON); + self.send_bounded(peer, &bytes, "moor rejection"); return Ok(()); } let name = name.to_string(); @@ -882,44 +908,51 @@ impl Server { self.sessions.insert(peer, session); self.player_peers.insert(player_id, peer); self.player_order.insert(player_id); - self.snapshot_recipients.push_back(peer); + self.snapshot_recipients.insert(peer); self.schedule_aoi(peer); Ok(()) } fn unregister_session(&mut self, peer: PeerId) -> Option { + self.unregister_session_with_work(peer) + .map(|(session, _)| session) + } + + fn unregister_session_with_work( + &mut self, + peer: PeerId, + ) -> Option<(Session, RecipientIndexWork)> { let session = self.sessions.remove(&peer)?; if self.player_peers.get(&session.player_id) == Some(&peer) { self.player_peers.remove(&session.player_id); self.player_order.remove(&session.player_id); } - self.snapshot_recipients.retain(|&queued| queued != peer); - self.aoi_recipients.retain(|&queued| queued != peer); - Some(session) + let mut work = RecipientIndexWork::default(); + work.remove(&mut self.snapshot_recipients, peer); + work.remove(&mut self.aoi_recipients, peer); + Some((session, work)) } fn schedule_aoi(&mut self, peer: PeerId) { - let Some(session) = self.sessions.get_mut(&peer) else { + if !self.sessions.contains_key(&peer) { return; - }; - if !session.aoi_queued { - session.aoi_queued = true; - self.aoi_recipients.push_back(peer); } + self.aoi_recipients.insert(peer); } fn broadcast_snapshots(&mut self) -> SnapshotTickWork { self.snapshot_tick = self.snapshot_tick.wrapping_add(1); let server_tick = self.snapshot_tick; let mut work = SnapshotTickWork::default(); - let recipients = self - .snapshot_recipients - .len() - .min(SNAPSHOT_PACKETS_PER_TICK); - for _ in 0..recipients { - let Some(peer) = self.snapshot_recipients.pop_front() else { - break; - }; + let recipients = ordered_peers_after( + &self.snapshot_recipients, + self.snapshot_recipient_cursor, + SNAPSHOT_PACKETS_PER_TICK, + ); + if let Some(&last) = recipients.last() { + self.snapshot_recipient_cursor = Some(last); + } + for peer in recipients { let Some((self_pid, cell, cursor, remaining)) = self.sessions.get(&peer).and_then(|s| { s.cell @@ -967,7 +1000,6 @@ impl Server { session.snapshot_cursor = last_examined; session.snapshot_remaining = remaining.saturating_sub(examined); } - self.snapshot_recipients.push_back(peer); if players.is_empty() && boats.is_empty() { continue; } @@ -1000,15 +1032,19 @@ impl Server { fn process_aoi_work(&mut self) -> AoiTickWork { let mut work = AoiTickWork::default(); - let recipients = self.aoi_recipients.len().min(AOI_WORK_ITEMS_PER_TICK); - for _ in 0..recipients { - let Some(peer) = self.aoi_recipients.pop_front() else { - break; - }; - let Some(session) = self.sessions.get_mut(&peer) else { + let recipients = ordered_peers_after( + &self.aoi_recipients, + self.aoi_recipient_cursor, + AOI_WORK_ITEMS_PER_TICK, + ); + if let Some(&last) = recipients.last() { + self.aoi_recipient_cursor = Some(last); + } + for peer in recipients { + self.aoi_recipients.remove(&peer); + if !self.sessions.contains_key(&peer) { continue; - }; - session.aoi_queued = false; + } work.recipient_visits += 1; let needs_more = match self.process_one_aoi_work(peer, &mut work) { Ok(needs_more) => needs_more, @@ -1115,8 +1151,8 @@ impl Server { work.persisted_queries += 1; if let Some(row) = rows.into_iter().next() { let next_cursor = row.boat_id; - let snapshot = - (row.name.len() <= MAX_SNAPSHOT_MOORING_NAME_BYTES).then(|| mooring_snap(row)); + let corrupt_name_len = row.name.len(); + let snapshot = (corrupt_name_len <= MAX_MOORING_NAME_BYTES).then(|| mooring_snap(row)); if let Some(session) = self.sessions.get_mut(&peer) { if let Some(hydration) = session.active_hydration.as_mut() { hydration.mooring_cursor = Some(next_cursor); @@ -1134,7 +1170,13 @@ impl Server { } } } else { - work.oversized_moorings_skipped += 1; + work.corrupt_moorings_skipped += 1; + tracing::warn!( + boat_id = next_cursor, + name_bytes = corrupt_name_len, + max_name_bytes = MAX_MOORING_NAME_BYTES, + "legacy mooring row exceeds the protocol field limit" + ); } return Ok(true); } @@ -1346,6 +1388,23 @@ impl Server { } } +fn ordered_peers_after( + peers: &BTreeSet, + cursor: Option, + limit: usize, +) -> Vec { + let limit = limit.min(peers.len()); + match cursor { + Some(cursor) => peers + .range((Excluded(cursor), Unbounded)) + .chain(peers.range(..=cursor)) + .take(limit) + .copied() + .collect(), + None => peers.iter().take(limit).copied().collect(), + } +} + fn player_snap(s: &Session) -> PlayerSnap { PlayerSnap { player_id: s.player_id, @@ -1525,8 +1584,10 @@ mod handshake_tests { sessions: HashMap::new(), player_peers: HashMap::new(), player_order: BTreeSet::new(), - snapshot_recipients: VecDeque::new(), - aoi_recipients: VecDeque::new(), + snapshot_recipients: BTreeSet::new(), + snapshot_recipient_cursor: None, + aoi_recipients: BTreeSet::new(), + aoi_recipient_cursor: None, identity_players, seq: 0, snapshot_tick: 0, @@ -3106,8 +3167,10 @@ mod aoi_harden_tests { sessions: HashMap::new(), player_peers: HashMap::new(), player_order: BTreeSet::new(), - snapshot_recipients: VecDeque::new(), - aoi_recipients: VecDeque::new(), + snapshot_recipients: BTreeSet::new(), + snapshot_recipient_cursor: None, + aoi_recipients: BTreeSet::new(), + aoi_recipient_cursor: None, identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -3298,7 +3361,6 @@ mod aoi_harden_tests { hydration_cells: VecDeque::new(), hydration_pending: HashSet::new(), active_hydration: None, - aoi_queued: false, }, ) .unwrap(); @@ -3340,7 +3402,8 @@ mod aoi_harden_tests { "one bounded recipient round must eventually visit every session" ); - server.snapshot_recipients = VecDeque::from([1]); + server.snapshot_recipients = BTreeSet::from([1]); + server.snapshot_recipient_cursor = None; { let viewer = server.sessions.get_mut(&1).unwrap(); viewer.snapshot_cursor = None; @@ -3370,6 +3433,93 @@ mod aoi_harden_tests { ); } + #[test] + fn dense_aoi_scheduler_visits_all_1024_recipients_in_one_bounded_round() { + const DENSE_SESSIONS: PeerId = 1_024; + + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + for peer in 1..=DENSE_SESSIONS { + insert_dense_session(&mut server, peer, center); + } + + let recipient_round = (DENSE_SESSIONS as usize).div_ceil(AOI_WORK_ITEMS_PER_TICK); + for _ in 0..recipient_round { + let work = server.process_aoi_work(); + assert!(work.recipient_visits <= AOI_WORK_ITEMS_PER_TICK); + assert!(work.persisted_queries <= AOI_WORK_ITEMS_PER_TICK); + assert!(work.packets <= AOI_WORK_ITEMS_PER_TICK); + } + + assert!( + server + .sessions + .values() + .all(|session| session.published_cells == *session.sub.cells()), + "the ordered cursor must visit every dirty recipient before wrapping" + ); + assert_eq!( + server.aoi_recipients.len(), + DENSE_SESSIONS as usize, + "unfinished per-session hydration remains one deduplicated work item per live peer" + ); + } + + #[test] + fn reconnect_churn_keeps_recipient_indexes_bounded_without_linear_cleanup() { + const LIVE_PEERS: PeerId = 1_024; + const CHURN_EVENTS: PeerId = 4_096; + + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + let mut live = VecDeque::new(); + for peer in 1..=LIVE_PEERS { + insert_dense_session(&mut server, peer, center); + live.push_back(peer); + } + + for event in 0..CHURN_EVENTS { + let departed = live.pop_front().unwrap(); + let (_, work) = server + .unregister_session_with_work(departed) + .expect("the selected live session must unregister"); + server.world.remove(u64::from(departed)); + assert_eq!( + work.recipient_index_operations, 2, + "disconnect cleanup must perform a fixed pair of ordered-index removals" + ); + + let replacement = LIVE_PEERS + event + 1; + insert_dense_session(&mut server, replacement, center); + live.push_back(replacement); + + assert_eq!(server.sessions.len(), LIVE_PEERS as usize); + assert_eq!( + server.snapshot_recipients.len(), + server.sessions.len(), + "the snapshot index must contain exactly the live peers" + ); + assert!( + server.aoi_recipients.len() <= server.sessions.len(), + "the dirty AoI index must never exceed the live peers" + ); + assert!( + server + .snapshot_recipients + .iter() + .all(|peer| server.sessions.contains_key(peer)), + "disconnect/reconnect churn must not accumulate stale snapshot work" + ); + assert!( + server + .aoi_recipients + .iter() + .all(|peer| server.sessions.contains_key(peer)), + "disconnect/reconnect churn must not accumulate stale AoI work" + ); + } + } + #[test] fn fixed_snapshot_chunks_fit_the_unreliable_payload() { let players: Vec = (0..SNAPSHOT_ENTITIES_PER_PACKET) @@ -3397,7 +3547,7 @@ mod aoi_harden_tests { cell: (i32::MIN, i32::MAX), pos: [f32::MAX; 3], rot: [f32::MAX; 4], - name: "m".repeat(MAX_SNAPSHOT_MOORING_NAME_BYTES), + name: "m".repeat(MAX_MOORING_NAME_BYTES), created_at: u64::MAX, }; let added: Vec = (0..AOI_CELLS_PER_UPDATE / 2) @@ -3461,7 +3611,7 @@ mod aoi_harden_tests { }); let last_cell = Cell::new(16, 16); for (boat_id, name) in [ - (1, "x".repeat(MAX_SNAPSHOT_MOORING_NAME_BYTES + 1)), + (1, "x".repeat(MAX_MOORING_NAME_BYTES + 1)), (2, "eventual".to_string()), ] { server @@ -3487,7 +3637,7 @@ mod aoi_harden_tests { let mut total_queries = 0usize; let mut total_completed = 0usize; let mut total_moorings = 0usize; - let mut total_oversized_skips = 0usize; + let mut total_corrupt_skips = 0usize; for _ in 0..5_000 { let work = server.process_aoi_work(); assert!(work.recipient_visits <= AOI_WORK_ITEMS_PER_TICK); @@ -3496,7 +3646,7 @@ mod aoi_harden_tests { total_queries += work.persisted_queries; total_completed += work.cells_completed; total_moorings += work.moorings_encoded; - total_oversized_skips += work.oversized_moorings_skipped; + total_corrupt_skips += work.corrupt_moorings_skipped; if !server.session_has_aoi_work(1) { break; } @@ -3510,7 +3660,10 @@ mod aoi_harden_tests { total_queries, 1_091, "1,089 cells plus two keyset continuations in the hostile final cell" ); - assert_eq!(total_oversized_skips, 1); + assert_eq!( + total_corrupt_skips, 1, + "a directly injected legacy row above the protocol limit is corruption, not accepted content" + ); assert_eq!( total_moorings, 1, "the valid row after an oversized legacy row must still progress" @@ -3573,8 +3726,10 @@ mod market_dispatch_tests { sessions: HashMap::new(), player_peers: HashMap::new(), player_order: BTreeSet::new(), - snapshot_recipients: VecDeque::new(), - aoi_recipients: VecDeque::new(), + snapshot_recipients: BTreeSet::new(), + snapshot_recipient_cursor: None, + aoi_recipients: BTreeSet::new(), + aoi_recipient_cursor: None, identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -3827,8 +3982,10 @@ mod input_hardening_tests { sessions: HashMap::new(), player_peers: HashMap::new(), player_order: BTreeSet::new(), - snapshot_recipients: VecDeque::new(), - aoi_recipients: VecDeque::new(), + snapshot_recipients: BTreeSet::new(), + snapshot_recipient_cursor: None, + aoi_recipients: BTreeSet::new(), + aoi_recipient_cursor: None, identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -4175,6 +4332,170 @@ mod input_hardening_tests { assert_eq!(server.chat_limiter.tracked_count(), 0); } + #[test] + fn mooring_name_snapshot_boundary_is_enforced_before_persistence() { + let mut server = make_server(Config { + max_wire_string_len: 4_096, + moor_min_interval_ms: 0, + ..Config::default() + }); + let peer: PeerId = 1; + join(&mut server, peer, "tok-moor-name-boundary"); + let cell = server.world.grid().cell_of(0.0, 0.0); + + let too_long = "x".repeat(MAX_MOORING_NAME_BYTES + 1); + let seq_before_rejection = server.seq; + send_moor( + &mut server, + peer, + &moor_envelope(0.0, 0.0, &too_long), + 1_000, + ); + assert_eq!( + server.seq, + seq_before_rejection.wrapping_add(1), + "the invalid request must receive a bounded MoorAck rejection" + ); + assert!( + server + .db + .moorings_in_cell(cell.cx, cell.cz) + .unwrap() + .is_empty(), + "a name that cannot hydrate in one unreliable CellSnapshot must never persist" + ); + assert_eq!( + server.moor_limiter.tracked_count(), + 0, + "field validation must run before the persistent-write rate limiter" + ); + + let boundary = "b".repeat(MAX_MOORING_NAME_BYTES); + send_moor( + &mut server, + peer, + &moor_envelope(0.0, 0.0, &boundary), + 1_001, + ); + let persisted = server + .db + .moorings_in_cell(cell.cx, cell.cz) + .unwrap() + .into_iter() + .next() + .expect("the exact field boundary must persist"); + assert_eq!(persisted.name, boundary); + + let snapshot = mooring_snap(persisted); + let cell_payload = codec::cell_snapshot(1, cell, &[], &[], &[snapshot]); + assert!( + cell_payload.len() <= protocol::MTU - protocol::HEADER_SIZE, + "the exact persisted boundary must hydrate in one unreliable CellSnapshot" + ); + let envelope = decode_envelope(&cell_payload).unwrap(); + let hydrated = envelope + .payload_as_cell_snapshot() + .unwrap() + .moorings() + .unwrap() + .get(0); + assert_eq!(hydrated.name(), Some(boundary.as_str())); + + { + let session = server.sessions.get_mut(&peer).unwrap(); + session.published_cells = session.sub.cells().clone(); + session.hydration_cells.clear(); + session.hydration_pending.clear(); + session.active_hydration = Some(CellHydration { + cell, + player_cursor: None, + players_remaining: 0, + mooring_cursor: None, + players_complete: true, + sent_any: false, + }); + } + server.schedule_aoi(peer); + let hydration_work = server.process_aoi_work(); + assert_eq!(hydration_work.persisted_queries, 1); + assert_eq!(hydration_work.moorings_encoded, 1); + assert_eq!(hydration_work.corrupt_moorings_skipped, 0); + + let rejection = codec::moor_ack(2, false, None, MOOR_NAME_TOO_LONG_REASON); + assert!( + rejection.len() <= protocol::MTU - protocol::HEADER_SIZE, + "the field-specific rejection must itself stay transport bounded" + ); + } + + #[test] + fn mooring_name_keeps_the_lower_configured_wire_limit() { + let mut server = make_server(Config { + max_wire_string_len: 16, + moor_min_interval_ms: 0, + ..Config::default() + }); + let peer: PeerId = 1; + join(&mut server, peer, "tok-config"); + let cell = server.world.grid().cell_of(0.0, 0.0); + + send_moor( + &mut server, + peer, + &moor_envelope(0.0, 0.0, &"x".repeat(17)), + 1_000, + ); + assert!( + server + .db + .moorings_in_cell(cell.cx, cell.cz) + .unwrap() + .is_empty(), + "the field-specific MTU cap must not weaken a lower configured wire cap" + ); + } + + #[test] + fn mooring_name_limit_is_the_largest_mtu_safe_record_name() { + let payload_limit = protocol::MTU - protocol::HEADER_SIZE; + let cell = Cell::new(i32::MIN, i32::MAX); + let encoded_lengths = |name_len: usize| { + let make_snapshot = || MooringSnap { + boat_id: u64::MAX, + owner: u64::MAX, + cell: (i32::MIN, i32::MAX), + pos: [f32::MAX; 3], + rot: [f32::MAX; 4], + name: "n".repeat(name_len), + created_at: u64::MAX, + }; + ( + codec::cell_snapshot(1, cell, &[], &[], &[make_snapshot()]).len(), + codec::moor_ack(1, true, Some(&make_snapshot()), "").len(), + ) + }; + let derived_limit = (0..=4_096) + .take_while(|&name_len| { + let (cell_len, ack_len) = encoded_lengths(name_len); + cell_len <= payload_limit && ack_len <= payload_limit + }) + .last() + .unwrap(); + + assert_eq!( + MAX_MOORING_NAME_BYTES, derived_limit, + "the field limit must be derived from both unreliable record encodings" + ); + let (boundary_cell_len, boundary_ack_len) = encoded_lengths(derived_limit); + assert!(boundary_cell_len <= payload_limit); + assert!(boundary_ack_len <= payload_limit); + let (over_cell_len, over_ack_len) = encoded_lengths(derived_limit + 1); + assert!( + over_cell_len > payload_limit || over_ack_len > payload_limit, + "the next byte must exceed at least one unreliable record encoding" + ); + } + // ---- PART 1b: per-class rate limits ---- #[test] From 4fcca54be0a8b45ffcfc3ead6f9e0ba9bd9afde3 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 22:40:14 -0600 Subject: [PATCH 31/36] fix(server): bound dense population work Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 728 ++++++++++++++++++++++---- 1 file changed, 623 insertions(+), 105 deletions(-) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 5a0e354..0d33117 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -33,10 +33,29 @@ const KEY_WEATHER_SEED: &str = "weather_seed"; /// How often dirty player state is flushed to the database. const FLUSH_INTERVAL: Duration = Duration::from_secs(5); -/// Global snapshot transport and CPU budgets charged on each snapshot tick. -const SNAPSHOT_PACKETS_PER_TICK: usize = 8; -const SNAPSHOT_ENTITY_SCAN_PER_PACKET: usize = 8; +/// Global snapshot transport and CPU budgets charged on every fixed server tick. +/// +/// At the supported 1,024-session ceiling, 32 recipient visits make one round +/// in 32 ticks. A dense recipient's stable 15-player selection takes three +/// five-player packets, so every elected player recurs in at most 96 ticks +/// (3.2 seconds at the production 30 Hz), strictly before the client's +/// five-second stale-sample eviction. +const SNAPSHOT_PACKETS_PER_TICK: usize = 32; +const SNAPSHOT_ENTITY_SCAN_PER_PACKET: usize = sw_net::DEFAULT_MAX_PEERS; const SNAPSHOT_ENTITIES_PER_PACKET: usize = 5; +const SNAPSHOT_VISIBILITY_CEILING: usize = 15; + +/// Queued population fanout budgets. Chat is FIFO and rejects new accepted +/// work at the fixed queue boundary; clock state has one coalescing latest-value +/// slot. Both share this fixed per-tick transport budget. +const CHAT_QUEUE_ITEMS: usize = 256; +const CHAT_QUEUE_BYTES: usize = CHAT_QUEUE_ITEMS * (protocol::MTU - protocol::HEADER_SIZE); +const FANOUT_RECIPIENT_SCANS_PER_TICK: usize = sw_net::DEFAULT_MAX_PEERS; +const FANOUT_SENDS_PER_TICK: usize = sw_net::DEFAULT_MAX_PEERS; + +/// Persistence work is spread across fixed ticks after each five-second flush +/// boundary. At 1,024 sessions this drains in 128 ticks (about 4.27 seconds). +const DIRTY_DB_UPDATES_PER_TICK: usize = 8; /// Global AoI delivery budget charged on every fixed server tick. const AOI_WORK_ITEMS_PER_TICK: usize = 8; @@ -61,9 +80,15 @@ struct CellHydration { sent_any: bool, } +struct FanoutJob { + bytes: Vec, + sender_cell: Option, + recipient_cursor: Option, + recipients_remaining: usize, +} + /// Per-connection state, created on ClientHello. struct Session { - peer: PeerId, player_id: u64, identity_hash: String, display_name: String, @@ -77,9 +102,10 @@ struct Session { dirty: bool, snapshot_cursor: Option, snapshot_remaining: usize, + snapshot_last_visit_tick: u32, published_cells: HashSet, - hydration_cells: VecDeque, - hydration_pending: HashSet, + hydration_cells: HashSet, + hydration_cursor: Option, active_hydration: Option, } @@ -90,6 +116,8 @@ struct SnapshotTickWork { player_states_encoded: usize, packets: usize, encoded_bytes: usize, + #[cfg(test)] + advertised: Vec<(PeerId, u64)>, } #[derive(Debug, Default, PartialEq, Eq)] @@ -107,6 +135,19 @@ struct RecipientIndexWork { recipient_index_operations: usize, } +#[derive(Debug, Default, PartialEq, Eq)] +struct FanoutTickWork { + recipient_scans: usize, + sends: usize, + encoded_bytes: usize, + jobs_completed: usize, +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct DirtyFlushTickWork { + db_updates: usize, +} + impl RecipientIndexWork { fn remove(&mut self, recipients: &mut BTreeSet, peer: PeerId) { self.recipient_index_operations += 1; @@ -127,6 +168,13 @@ pub struct Server { snapshot_recipient_cursor: Option, aoi_recipients: BTreeSet, aoi_recipient_cursor: Option, + chat_fanout: VecDeque, + chat_fanout_bytes: usize, + clock_fanout: Option, + fanout_prefer_clock: bool, + dirty_sessions: BTreeSet, + flush_sessions: BTreeSet, + flush_paused: bool, identity_players: HashMap, seq: u32, snapshot_tick: u32, @@ -203,6 +251,13 @@ impl Server { snapshot_recipient_cursor: None, aoi_recipients: BTreeSet::new(), aoi_recipient_cursor: None, + chat_fanout: VecDeque::new(), + chat_fanout_bytes: 0, + clock_fanout: None, + fanout_prefer_clock: true, + dirty_sessions: BTreeSet::new(), + flush_sessions: BTreeSet::new(), + flush_paused: false, identity_players, seq: 0, snapshot_tick: 0, @@ -229,10 +284,14 @@ impl Server { // Readiness line on stdout (the protocol-smoke harness waits for this). println!("listening on {addr}"); std::io::stdout().flush().ok(); - tracing::info!(%addr, server = %self.cfg.server_name, "server started"); + tracing::info!( + %addr, + server = %self.cfg.server_name, + nominal_snapshot_cadence_ticks = self.cfg.ticks_per_snapshot(), + "server started" + ); let tick_dt = Duration::from_secs_f64(1.0 / self.cfg.tick_hz as f64); - let ticks_per_snapshot = self.cfg.ticks_per_snapshot(); let ticks_per_clock_broadcast = self.cfg.ticks_per_clock_broadcast(); let mut tick: u64 = 0; let mut last_flush = Instant::now(); @@ -248,9 +307,10 @@ impl Server { } self.process_aoi_work(); - - if tick % ticks_per_snapshot == 0 { - self.broadcast_snapshots(); + self.broadcast_snapshots(); + self.process_fanout_work(); + if let Err(e) = self.process_dirty_flush_at(now_ms()) { + tracing::warn!(error = %e, "bounded dirty flush paused until next cadence"); } if tick % ticks_per_clock_broadcast == 0 { @@ -258,9 +318,7 @@ impl Server { } if frame_start.duration_since(last_flush) >= FLUSH_INTERVAL { - if let Err(e) = self.flush_dirty() { - tracing::warn!(error = %e, "dirty flush failed"); - } + self.begin_dirty_flush(); last_flush = frame_start; } @@ -503,7 +561,6 @@ impl Server { self.register_session( peer, Session { - peer, player_id, identity_hash, display_name: name, @@ -517,9 +574,10 @@ impl Server { dirty: true, snapshot_cursor: None, snapshot_remaining: 0, + snapshot_last_visit_tick: 0, published_cells: HashSet::new(), - hydration_cells: VecDeque::new(), - hydration_pending: HashSet::new(), + hydration_cells: HashSet::new(), + hydration_cursor: None, active_hydration: None, }, )?; @@ -611,6 +669,7 @@ impl Server { aoi = s.sub.recenter(cell); s.cell = Some(cell); } + self.dirty_sessions.insert(peer); self.emit_aoi(peer, &aoi); } @@ -847,9 +906,14 @@ impl Server { if !validate::string_within_limit(text, self.cfg.max_wire_string_len_usize()) { return; } + if self.chat_fanout.len() >= CHAT_QUEUE_ITEMS + || self.chat_fanout_bytes > CHAT_QUEUE_BYTES - (protocol::MTU - protocol::HEADER_SIZE) + { + return; + } // Per-player chat throttle: a flood beyond the configured rate is dropped - // before it fans out to every AoI subscriber. + // before it enters the bounded fanout queue. if !self.chat_limiter.allow(sender_player, now_ms) { return; } @@ -863,17 +927,13 @@ impl Server { return; } - // Deliver to every session whose AoI currently includes the sender's cell. - let recipients: Vec = self - .sessions - .values() - .filter(|r| r.sub.contains(cell)) - .map(|r| r.peer) - .collect(); - - for target in recipients { - self.send(target, &bytes); - } + self.chat_fanout_bytes += bytes.len(); + self.chat_fanout.push_back(FanoutJob { + bytes, + sender_cell: Some(cell), + recipient_cursor: None, + recipients_remaining: self.snapshot_recipients.len(), + }); } fn on_disconnect(&mut self, peer: PeerId, reason: DisconnectReason) -> anyhow::Result<()> { @@ -905,10 +965,14 @@ impl Server { )); } let player_id = session.player_id; + let dirty = session.dirty; self.sessions.insert(peer, session); self.player_peers.insert(player_id, peer); self.player_order.insert(player_id); self.snapshot_recipients.insert(peer); + if dirty { + self.dirty_sessions.insert(peer); + } self.schedule_aoi(peer); Ok(()) } @@ -930,6 +994,8 @@ impl Server { let mut work = RecipientIndexWork::default(); work.remove(&mut self.snapshot_recipients, peer); work.remove(&mut self.aoi_recipients, peer); + self.dirty_sessions.remove(&peer); + self.flush_sessions.remove(&peer); Some((session, work)) } @@ -953,52 +1019,80 @@ impl Server { self.snapshot_recipient_cursor = Some(last); } for peer in recipients { - let Some((self_pid, cell, cursor, remaining)) = + let Some((self_pid, cell, cursor, remaining, last_visit_tick)) = self.sessions.get(&peer).and_then(|s| { - s.cell - .map(|cell| (s.player_id, cell, s.snapshot_cursor, s.snapshot_remaining)) + s.cell.map(|cell| { + ( + s.player_id, + cell, + s.snapshot_cursor, + s.snapshot_remaining, + s.snapshot_last_visit_tick, + ) + }) }) else { continue; }; + let nominal_cadence = self.cfg.ticks_per_snapshot().min(u64::from(u32::MAX)) as u32; + if last_visit_tick != 0 && server_tick.wrapping_sub(last_visit_tick) < nominal_cadence { + continue; + } + if let Some(session) = self.sessions.get_mut(&peer) { + session.snapshot_last_visit_tick = server_tick; + } work.recipient_visits += 1; - let remaining = if remaining == 0 { - self.player_order.len() + let candidates = + self.player_candidates_after(Some(self_pid), SNAPSHOT_ENTITY_SCAN_PER_PACKET); + work.candidates_examined += candidates.len(); + let selected: Vec = candidates + .into_iter() + .filter(|&player_id| player_id != self_pid) + .filter(|&player_id| { + self.session_by_player(player_id) + .and_then(|session| session.cell) + .is_some_and(|other| { + other.chebyshev_distance(cell) <= self.cfg.aoi_radius_i32() + }) + }) + .take(SNAPSHOT_VISIBILITY_CEILING) + .collect(); + if selected.is_empty() { + if let Some(session) = self.sessions.get_mut(&peer) { + session.snapshot_cursor = None; + session.snapshot_remaining = 0; + } + continue; + } + + let cursor_position = cursor + .and_then(|cursor| selected.iter().position(|&player_id| player_id == cursor)); + let remaining = if remaining == 0 || cursor_position.is_none() { + selected.len() } else { - remaining + remaining.min(selected.len()) }; - let candidates = self - .player_candidates_after(cursor, remaining.min(SNAPSHOT_ENTITY_SCAN_PER_PACKET)); + let start = cursor_position.map_or(0, |position| (position + 1) % selected.len()); + let emit_count = remaining.min(SNAPSHOT_ENTITIES_PER_PACKET); let mut players = Vec::new(); let mut boats = Vec::new(); let mut last_examined = None; - let mut examined = 0usize; - for player_id in candidates { + for offset in 0..emit_count { + let player_id = selected[(start + offset) % selected.len()]; last_examined = Some(player_id); - examined += 1; - work.candidates_examined += 1; - if player_id == self_pid { - continue; - } let Some(session) = self.session_by_player(player_id) else { continue; }; - if !session.cell.is_some_and(|other| { - other.chebyshev_distance(cell) <= self.cfg.aoi_radius_i32() - }) { - continue; - } players.push(player_snap(session)); if session.aboard_boat != 0 { boats.push(boat_snap(session)); } - if players.len() == SNAPSHOT_ENTITIES_PER_PACKET { - break; - } + #[cfg(test)] + work.advertised.push((peer, player_id)); } if let Some(session) = self.sessions.get_mut(&peer) { session.snapshot_cursor = last_examined; - session.snapshot_remaining = remaining.saturating_sub(examined); + session.snapshot_remaining = remaining.saturating_sub(emit_count); } if players.is_empty() && boats.is_empty() { continue; @@ -1224,13 +1318,18 @@ impl Server { for cell in &removed { session.published_cells.remove(cell); - session.hydration_pending.remove(cell); + session.hydration_cells.remove(cell); + if session + .active_hydration + .as_ref() + .is_some_and(|active| active.cell == *cell) + { + session.active_hydration = None; + } } for &cell in &added { session.published_cells.insert(cell); - if session.hydration_pending.insert(cell) { - session.hydration_cells.push_back(cell); - } + session.hydration_cells.insert(cell); } Some((added, removed)) } @@ -1244,19 +1343,13 @@ impl Server { if let Some(active) = session.active_hydration.as_ref() { if !session.sub.contains(active.cell) || !session.published_cells.contains(&active.cell) { - session.hydration_pending.remove(&active.cell); session.active_hydration = None; - return None; } } if session.active_hydration.is_none() { - let cell = session.hydration_cells.pop_front()?; - if !session.hydration_pending.remove(&cell) - || !session.sub.contains(cell) - || !session.published_cells.contains(&cell) - { - return None; - } + let cell = next_hydration_cell(&session.hydration_cells, session.hydration_cursor)?; + session.hydration_cells.remove(&cell); + session.hydration_cursor = Some(cell); session.active_hydration = Some(CellHydration { cell, player_cursor: None, @@ -1295,33 +1388,123 @@ impl Server { .collect() } - /// Broadcast the current world clock to every connected session. The clock - /// is derived authority (see [`clock_from_epoch`]); the weather seed is - /// join-only in `ServerHello` and is deliberately not rebroadcast here. + /// Queue the current derived world clock for bounded delivery. Repeated + /// cadence events coalesce to the latest clock instead of accumulating one + /// population-wide job per cadence. fn broadcast_clock(&mut self) { let clock = self.clock_now(); - let peers: Vec = self.sessions.keys().copied().collect(); - for peer in peers { - let bytes = codec::world_clock(self.next_seq(), clock); - self.send(peer, &bytes); + let bytes = codec::world_clock(self.next_seq(), clock); + if self.snapshot_recipients.is_empty() { + self.clock_fanout = None; + return; } + self.clock_fanout = Some(FanoutJob { + bytes, + sender_cell: None, + recipient_cursor: None, + recipients_remaining: self.snapshot_recipients.len(), + }); } - fn flush_dirty(&mut self) -> anyhow::Result<()> { - let now = now_ms(); - let dirty: Vec<(PeerId, u64)> = self - .sessions + fn process_fanout_work(&mut self) -> FanoutTickWork { + let mut work = FanoutTickWork::default(); + while work.recipient_scans < FANOUT_RECIPIENT_SCANS_PER_TICK + && work.sends < FANOUT_SENDS_PER_TICK + { + let from_clock = self.clock_fanout.is_some() + && (self.fanout_prefer_clock || self.chat_fanout.is_empty()); + let Some(mut job) = (if from_clock { + self.clock_fanout.take() + } else { + self.chat_fanout.pop_front() + }) else { + if self.clock_fanout.is_none() && self.chat_fanout.is_empty() { + break; + } + self.fanout_prefer_clock = !self.fanout_prefer_clock; + continue; + }; + + while job.recipients_remaining > 0 + && work.recipient_scans < FANOUT_RECIPIENT_SCANS_PER_TICK + && work.sends < FANOUT_SENDS_PER_TICK + { + let Some(peer) = next_peer_after(&self.snapshot_recipients, job.recipient_cursor) + else { + job.recipients_remaining = 0; + break; + }; + job.recipient_cursor = Some(peer); + job.recipients_remaining -= 1; + work.recipient_scans += 1; + + let should_send = job.sender_cell.is_none_or(|cell| { + self.sessions + .get(&peer) + .is_some_and(|session| session.sub.contains(cell)) + }); + if should_send { + self.send(peer, &job.bytes); + work.sends += 1; + work.encoded_bytes += job.bytes.len(); + } + } + + if job.recipients_remaining == 0 { + work.jobs_completed += 1; + if from_clock { + self.fanout_prefer_clock = false; + } else { + self.chat_fanout_bytes = self.chat_fanout_bytes.saturating_sub(job.bytes.len()); + self.fanout_prefer_clock = true; + } + } else { + if from_clock { + self.clock_fanout = Some(job); + } else { + self.chat_fanout.push_front(job); + } + break; + } + } + work + } + + fn begin_dirty_flush(&mut self) { + self.flush_paused = false; + if self.flush_sessions.is_empty() { + std::mem::swap(&mut self.flush_sessions, &mut self.dirty_sessions); + } + } + + fn process_dirty_flush_at(&mut self, now: i64) -> anyhow::Result { + let mut work = DirtyFlushTickWork::default(); + if self.flush_paused { + return Ok(work); + } + let peers: Vec = self + .flush_sessions .iter() - .filter(|(_, s)| s.dirty) - .map(|(&peer, s)| (peer, s.player_id)) + .take(DIRTY_DB_UPDATES_PER_TICK) + .copied() .collect(); - for (peer, player_id) in dirty { - self.db.touch_last_seen(player_id as i64, now)?; - if let Some(s) = self.sessions.get_mut(&peer) { - s.dirty = false; + for peer in peers { + let Some(player_id) = self.sessions.get(&peer).map(|session| session.player_id) else { + self.flush_sessions.remove(&peer); + continue; + }; + if let Err(error) = self.db.touch_last_seen(player_id as i64, now) { + self.flush_paused = true; + return Err(error.into()); + } + self.flush_sessions.remove(&peer); + work.db_updates += 1; + let still_dirty = self.dirty_sessions.contains(&peer); + if let Some(session) = self.sessions.get_mut(&peer) { + session.dirty = still_dirty; } } - Ok(()) + Ok(work) } fn flush_all(&mut self) -> anyhow::Result<()> { @@ -1438,6 +1621,30 @@ fn mooring_snap(row: MooringRow) -> MooringSnap { } } +fn next_peer_after(peers: &BTreeSet, cursor: Option) -> Option { + match cursor { + Some(cursor) => peers + .range((Excluded(cursor), Unbounded)) + .next() + .or_else(|| peers.iter().next()) + .copied(), + None => peers.iter().next().copied(), + } +} + +fn next_hydration_cell(cells: &HashSet, cursor: Option) -> Option { + let key = |cell: &Cell| (cell.cz, cell.cx); + cursor + .and_then(|cursor| { + cells + .iter() + .filter(|cell| key(cell) > key(&cursor)) + .min_by_key(|cell| key(cell)) + .copied() + }) + .or_else(|| cells.iter().min_by_key(|cell| key(cell)).copied()) +} + fn vec3_of(v: Option<&p::Vec3>) -> [f32; 3] { v.map(|v| [v.x(), v.y(), v.z()]).unwrap_or([0.0, 0.0, 0.0]) } @@ -1588,6 +1795,13 @@ mod handshake_tests { snapshot_recipient_cursor: None, aoi_recipients: BTreeSet::new(), aoi_recipient_cursor: None, + chat_fanout: VecDeque::new(), + chat_fanout_bytes: 0, + clock_fanout: None, + fanout_prefer_clock: true, + dirty_sessions: BTreeSet::new(), + flush_sessions: BTreeSet::new(), + flush_paused: false, identity_players, seq: 0, snapshot_tick: 0, @@ -2337,6 +2551,10 @@ mod handshake_tests { .unwrap(); assert_eq!(server.seq, seq_before_chat.wrapping_add(1)); + let fanout = server.process_fanout_work(); + assert_eq!(fanout.recipient_scans, 2); + assert_eq!(fanout.sends, 2); + assert_eq!(fanout.jobs_completed, 1); let sender_payload = receive_payload(&sender, p::Payload::ChatBroadcast); let observer_payload = receive_payload(&observer, p::Payload::ChatBroadcast); assert_eq!( @@ -2443,8 +2661,8 @@ mod handshake_tests { assert_eq!( server .session_by_player(player_id) - .map(|session| session.peer), - Some(replacement_peer) + .map(|session| session.player_id), + Some(player_id) ); assert!(!server.sessions.contains_key(&first_peer)); assert!(!server @@ -2645,8 +2863,8 @@ mod handshake_tests { assert_eq!( server .session_by_player(player_id) - .map(|session| session.peer), - Some(rotated_peer) + .map(|session| session.player_id), + Some(player_id) ); } @@ -3133,7 +3351,7 @@ mod handshake_tests { mod aoi_harden_tests { use super::*; use flatbuffers::FlatBufferBuilder; - use std::collections::HashSet; + use std::collections::{HashMap, HashSet}; use sw_contracts::{decode_envelope, finish_envelope}; use sw_world::{cells_in_radius, EntityId, Grid}; @@ -3171,6 +3389,13 @@ mod aoi_harden_tests { snapshot_recipient_cursor: None, aoi_recipients: BTreeSet::new(), aoi_recipient_cursor: None, + chat_fanout: VecDeque::new(), + chat_fanout_bytes: 0, + clock_fanout: None, + fanout_prefer_clock: true, + dirty_sessions: BTreeSet::new(), + flush_sessions: BTreeSet::new(), + flush_paused: false, identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -3343,7 +3568,6 @@ mod aoi_harden_tests { .register_session( peer, Session { - peer, player_id, identity_hash: format!("identity-{player_id}"), display_name: format!("Player {player_id}"), @@ -3357,9 +3581,10 @@ mod aoi_harden_tests { dirty: false, snapshot_cursor: None, snapshot_remaining: 0, + snapshot_last_visit_tick: 0, published_cells: HashSet::new(), - hydration_cells: VecDeque::new(), - hydration_pending: HashSet::new(), + hydration_cells: HashSet::new(), + hydration_cursor: None, active_hydration: None, }, ) @@ -3408,6 +3633,7 @@ mod aoi_harden_tests { let viewer = server.sessions.get_mut(&1).unwrap(); viewer.snapshot_cursor = None; viewer.snapshot_remaining = 0; + viewer.snapshot_last_visit_tick = 0; } let mut encoded_for_viewer = 0usize; let mut viewer_visits = 0usize; @@ -3417,22 +3643,98 @@ mod aoi_harden_tests { assert!(work.packets <= 1); assert!(work.encoded_bytes <= protocol::MTU - protocol::HEADER_SIZE); encoded_for_viewer += work.player_states_encoded; - viewer_visits += 1; + viewer_visits += work.recipient_visits; if server.sessions[&1].snapshot_remaining == 0 { break; } } assert_eq!( encoded_for_viewer, - DENSE_SESSIONS as usize - 1, - "one deterministic entity sweep must eventually emit every other co-located player once" + SNAPSHOT_VISIBILITY_CEILING, + "one deterministic entity sweep must emit the recipient's stable dense visibility ceiling" ); assert!( - viewer_visits <= (DENSE_SESSIONS as usize).div_ceil(SNAPSHOT_ENTITIES_PER_PACKET), + viewer_visits <= SNAPSHOT_VISIBILITY_CEILING.div_ceil(SNAPSHOT_ENTITIES_PER_PACKET), "entity cursor must make fixed positive progress on every dense visit" ); } + #[test] + fn dense_snapshot_elected_visibility_refreshes_before_client_expiry() { + const DENSE_SESSIONS: PeerId = 1_024; + const CLIENT_STALE_MS: usize = 5_000; + const SIMULATION_TICKS: usize = 600; + + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + for peer in 1..=DENSE_SESSIONS { + insert_dense_session(&mut server, peer, center); + } + + let stale_ticks = CLIENT_STALE_MS * server.cfg.tick_hz as usize / 1_000; + let mut last_advertised = HashMap::<(PeerId, u64), usize>::new(); + let mut elected = HashMap::>::new(); + let mut repeated = 0usize; + + for tick in 0..SIMULATION_TICKS { + let work = server.broadcast_snapshots(); + assert!(work.recipient_visits <= SNAPSHOT_PACKETS_PER_TICK); + assert!( + work.candidates_examined + <= SNAPSHOT_PACKETS_PER_TICK * SNAPSHOT_ENTITY_SCAN_PER_PACKET + ); + assert!(work.packets <= SNAPSHOT_PACKETS_PER_TICK); + assert!( + work.encoded_bytes + <= SNAPSHOT_PACKETS_PER_TICK * (protocol::MTU - protocol::HEADER_SIZE) + ); + + for &(peer, player_id) in &work.advertised { + elected.entry(peer).or_default().insert(player_id); + if let Some(previous) = last_advertised.insert((peer, player_id), tick) { + repeated += 1; + assert!( + tick - previous < stale_ticks, + "peer {peer}'s elected player {player_id} went {} ticks without a refresh, \ + but the real client expires it after {stale_ticks} production ticks", + tick - previous + ); + } + } + } + + assert_eq!(elected.len(), DENSE_SESSIONS as usize); + assert!( + elected + .values() + .all(|players| players.len() == SNAPSHOT_VISIBILITY_CEILING), + "every dense recipient must retain one stable, full visibility ceiling" + ); + assert!( + repeated >= DENSE_SESSIONS as usize * SNAPSHOT_VISIBILITY_CEILING, + "every elected pair must be observed often enough to prove recurrence" + ); + } + + #[test] + fn sparse_snapshots_keep_the_configured_nominal_cadence() { + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + insert_dense_session(&mut server, 1, center); + insert_dense_session(&mut server, 2, center); + + let cadence = server.cfg.ticks_per_snapshot() as usize; + let mut packet_ticks = Vec::new(); + for tick in 0..=cadence * 3 { + let work = server.broadcast_snapshots(); + if work.packets != 0 { + assert_eq!(work.packets, 2); + packet_ticks.push(tick); + } + } + assert_eq!(packet_ticks, vec![0, cadence, cadence * 2, cadence * 3]); + } + #[test] fn dense_aoi_scheduler_visits_all_1024_recipients_in_one_bounded_round() { const DENSE_SESSIONS: PeerId = 1_024; @@ -3465,6 +3767,160 @@ mod aoi_harden_tests { ); } + #[test] + fn hostile_movement_churn_cannot_accumulate_stale_hydration_cells() { + const MOVES: usize = 20_000; + + let mut server = make_server(Config::default()); + insert_dense_session(&mut server, 1, Cell::new(0, 0)); + + for step in 0..MOVES { + let x = if step % 2 == 0 { 10_000.0 } else { 0.0 }; + let cell = server.world.grid().cell_of(x, 0.0); + let aoi = { + let session = server.sessions.get_mut(&1).unwrap(); + let update = session.sub.recenter(cell); + session.cell = Some(cell); + update + }; + server.emit_aoi(1, &aoi); + server.process_aoi_work(); + + let session = &server.sessions[&1]; + assert!( + session.hydration_cells.len() + usize::from(session.active_hydration.is_some()) + <= session.published_cells.len(), + "stale movement history escaped the current published-interest bound" + ); + assert!( + session + .hydration_cells + .iter() + .all(|cell| session.published_cells.contains(cell)), + "removed cells must leave hydration storage immediately" + ); + } + + let final_cell = Cell::new(0, 0); + let final_update = { + let session = server.sessions.get_mut(&1).unwrap(); + let update = session.sub.recenter(final_cell); + session.cell = Some(final_cell); + update + }; + server.emit_aoi(1, &final_update); + + let max_visits = server.sessions[&1].sub.cells().len() * 3 + 2; + for _ in 0..max_visits { + server.process_aoi_work(); + if !server.session_has_aoi_work(1) { + break; + } + } + assert!( + !server.session_has_aoi_work(1), + "current desired cells must hydrate within a fixed number of visits independent of movement history" + ); + } + + #[test] + fn hostile_population_fanout_and_flush_stay_capped_and_make_progress() { + const DENSE_SESSIONS: PeerId = 1_024; + const SIMULATION_TICKS: usize = 603; + const FLUSH_TICKS: usize = 5 * 30; + const CLOCK_TICKS: usize = 10 * 30; + + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + for peer in 1..=DENSE_SESSIONS { + insert_dense_session(&mut server, peer, center); + let session = server.sessions.get_mut(&peer).unwrap(); + session.dirty = true; + server.dirty_sessions.insert(peer); + } + + let seq_before_chat = server.seq; + for peer in 1..=DENSE_SESSIONS { + let mut fbb = FlatBufferBuilder::new(); + let text = fbb.create_string("transport-sized hostile batch"); + let chat = p::ChatSend::create( + &mut fbb, + &p::ChatSendArgs { + text: Some(text), + channel: 0, + }, + ); + let bytes = finish_envelope(&mut fbb, 1, p::Payload::ChatSend, chat.as_union_value()); + let envelope = decode_envelope(&bytes).unwrap(); + server.on_chat( + peer, + envelope.payload_as_chat_send().unwrap(), + i64::from(peer), + ); + } + assert_eq!( + server.seq.wrapping_sub(seq_before_chat) as usize, + CHAT_QUEUE_ITEMS, + "one real maximum transport batch is accepted and excess distinct-player chat is rejected" + ); + assert_eq!(server.chat_fanout.len(), CHAT_QUEUE_ITEMS); + assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); + + let mut total_jobs_completed = 0usize; + let mut total_db_updates = 0usize; + for tick in 0..SIMULATION_TICKS { + let fanout = server.process_fanout_work(); + assert!(fanout.recipient_scans <= FANOUT_RECIPIENT_SCANS_PER_TICK); + assert!(fanout.sends <= FANOUT_SENDS_PER_TICK); + assert!( + fanout.encoded_bytes + <= FANOUT_SENDS_PER_TICK * (protocol::MTU - protocol::HEADER_SIZE) + ); + total_jobs_completed += fanout.jobs_completed; + + let flush = server.process_dirty_flush_at(1_700_000_000_000 + tick as i64); + let flush = flush.unwrap(); + assert!(flush.db_updates <= DIRTY_DB_UPDATES_PER_TICK); + total_db_updates += flush.db_updates; + + if tick > 0 && tick % FLUSH_TICKS == 0 { + server.begin_dirty_flush(); + } + if tick % CLOCK_TICKS == 0 { + server.broadcast_clock(); + server.broadcast_clock(); + assert!( + server.clock_fanout.is_some(), + "repeated clock cadence coalesces into one latest-value job" + ); + } + + assert!(server.chat_fanout.len() <= CHAT_QUEUE_ITEMS); + assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); + assert!(server.clock_fanout.iter().count() <= 1); + assert!(server.dirty_sessions.len() <= server.sessions.len()); + assert!(server.flush_sessions.len() <= server.sessions.len()); + assert!( + server.dirty_sessions.len() + server.flush_sessions.len() + <= server.sessions.len() * 2 + ); + } + + assert!(server.chat_fanout.is_empty()); + assert_eq!(server.chat_fanout_bytes, 0); + assert!( + server.clock_fanout.is_none(), + "the last coalesced clock must eventually reach its bounded audience" + ); + assert!(server.dirty_sessions.is_empty()); + assert!(server.flush_sessions.is_empty()); + assert_eq!(total_db_updates, DENSE_SESSIONS as usize); + assert!( + total_jobs_completed >= CHAT_QUEUE_ITEMS + 3, + "all accepted chats and the three coalesced clock jobs must complete fairly" + ); + } + #[test] fn reconnect_churn_keeps_recipient_indexes_bounded_without_linear_cleanup() { const LIVE_PEERS: PeerId = 1_024; @@ -3730,6 +4186,13 @@ mod market_dispatch_tests { snapshot_recipient_cursor: None, aoi_recipients: BTreeSet::new(), aoi_recipient_cursor: None, + chat_fanout: VecDeque::new(), + chat_fanout_bytes: 0, + clock_fanout: None, + fanout_prefer_clock: true, + dirty_sessions: BTreeSet::new(), + flush_sessions: BTreeSet::new(), + flush_paused: false, identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -3986,6 +4449,13 @@ mod input_hardening_tests { snapshot_recipient_cursor: None, aoi_recipients: BTreeSet::new(), aoi_recipient_cursor: None, + chat_fanout: VecDeque::new(), + chat_fanout_bytes: 0, + clock_fanout: None, + fanout_prefer_clock: true, + dirty_sessions: BTreeSet::new(), + flush_sessions: BTreeSet::new(), + flush_paused: false, identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -4405,7 +4875,6 @@ mod input_hardening_tests { let session = server.sessions.get_mut(&peer).unwrap(); session.published_cells = session.sub.cells().clone(); session.hydration_cells.clear(); - session.hydration_pending.clear(); session.active_hydration = Some(CellHydration { cell, player_cursor: None, @@ -4729,14 +5198,14 @@ mod input_hardening_tests { #[test] #[ignore = "load/perf test: wall-clock timed; run via `make load-test` or the non-blocking CI load job"] fn load_n_clients_stay_within_the_tick_budget() { - // Headless load: N simulated clients drive the *real* client-state handler - // and snapshot broadcast every tick against the in-memory server. The - // per-tick server work must stay under the fixed-tick budget (1 / tick_hz), - // i.e. the server keeps up with real time at N clients. Timing is - // wall-clock, so this is `#[ignore]`d out of the required gate (`cargo test` - // skips it) and run only by the non-blocking load job / `make load-test`. + // Headless load: N simulated clients drive the real client-state, AoI, + // snapshot, chat/clock fanout, and persistence-flush paths. The per-tick + // server work must stay under the fixed-tick budget (1 / tick_hz), i.e. + // the server keeps up with real time at N clients. Timing is wall-clock, + // so this is `#[ignore]`d out of the required gate (`cargo test` skips it) + // and run only by the non-blocking load job / `make load-test`. const N: u32 = 1_024; - const TICKS: u32 = 60; + const TICKS: u32 = 450; const LOAD_EPOCH_MS: i64 = 1_700_000_000_000; let cfg = Config::default(); @@ -4768,6 +5237,19 @@ mod input_hardening_tests { assert_eq!(server.sessions.len(), N as usize); assert_eq!(server.world.len(), N as usize); + // One maximum transport poll worth of accepted dense chat fanout; the + // remaining distinct-player inputs are rejected at the fixed queue cap. + for peer in 1..=N as PeerId { + send_chat( + &mut server, + peer, + "release hostile fanout", + 100_000 + i64::from(peer), + ); + } + assert_eq!(server.chat_fanout.len(), CHAT_QUEUE_ITEMS); + assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); + // Drive TICKS simulated ticks and measure the wall-clock server work. The // simulated clock advances by a full tick each round so every client's // per-tick update clears the throttle window (worst-case load). @@ -4785,11 +5267,47 @@ mod input_hardening_tests { now, ); } - server.broadcast_snapshots(); + let aoi = server.process_aoi_work(); + assert!(aoi.recipient_visits <= AOI_WORK_ITEMS_PER_TICK); + let snapshots = server.broadcast_snapshots(); + assert!(snapshots.recipient_visits <= SNAPSHOT_PACKETS_PER_TICK); + assert!( + snapshots.candidates_examined + <= SNAPSHOT_PACKETS_PER_TICK * SNAPSHOT_ENTITY_SCAN_PER_PACKET + ); + let fanout = server.process_fanout_work(); + assert!(fanout.recipient_scans <= FANOUT_RECIPIENT_SCANS_PER_TICK); + assert!(fanout.sends <= FANOUT_SENDS_PER_TICK); + assert!( + fanout.encoded_bytes + <= FANOUT_SENDS_PER_TICK * (protocol::MTU - protocol::HEADER_SIZE) + ); + let flush = server + .process_dirty_flush_at(LOAD_EPOCH_MS + i64::from(t)) + .unwrap(); + assert!(flush.db_updates <= DIRTY_DB_UPDATES_PER_TICK); + + if t > 0 && t % (server.cfg.tick_hz * FLUSH_INTERVAL.as_secs() as u32) == 0 { + server.begin_dirty_flush(); + } + if t % server.cfg.ticks_per_clock_broadcast() as u32 == 0 { + server.broadcast_clock(); + server.broadcast_clock(); + assert!(server.clock_fanout.is_some()); + } + assert!(server.chat_fanout.len() <= CHAT_QUEUE_ITEMS); + assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); + assert!(server.clock_fanout.iter().count() <= 1); + assert!(server.dirty_sessions.len() <= server.sessions.len()); + assert!(server.flush_sessions.len() <= server.sessions.len()); } let elapsed = start.elapsed(); let per_tick = elapsed / TICKS; + assert!(server.chat_fanout.is_empty()); + assert_eq!(server.chat_fanout_bytes, 0); + assert!(server.clock_fanout.is_none()); + println!( "load: {N} clients x {TICKS} ticks in {elapsed:?} => {per_tick:?}/tick (real-time budget {tick_dt:?})" ); From 4518e54a160f3087070e34b24848f6e7cbd3914c Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 23:05:13 -0600 Subject: [PATCH 32/36] test(server): prove bounded scheduler recovery Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 339 +++++++++++++++++++++++--- 1 file changed, 301 insertions(+), 38 deletions(-) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 0d33117..79d321b 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -148,6 +148,13 @@ struct DirtyFlushTickWork { db_updates: usize, } +#[cfg(test)] +#[derive(Debug, Default)] +struct DirtyFlushTestHook { + failures_remaining: usize, + attempts: usize, +} + impl RecipientIndexWork { fn remove(&mut self, recipients: &mut BTreeSet, peer: PeerId) { self.recipient_index_operations += 1; @@ -175,6 +182,8 @@ pub struct Server { dirty_sessions: BTreeSet, flush_sessions: BTreeSet, flush_paused: bool, + #[cfg(test)] + dirty_flush_test: DirtyFlushTestHook, identity_players: HashMap, seq: u32, snapshot_tick: u32, @@ -258,6 +267,8 @@ impl Server { dirty_sessions: BTreeSet::new(), flush_sessions: BTreeSet::new(), flush_paused: false, + #[cfg(test)] + dirty_flush_test: DirtyFlushTestHook::default(), identity_players, seq: 0, snapshot_tick: 0, @@ -1472,9 +1483,7 @@ impl Server { fn begin_dirty_flush(&mut self) { self.flush_paused = false; - if self.flush_sessions.is_empty() { - std::mem::swap(&mut self.flush_sessions, &mut self.dirty_sessions); - } + self.flush_sessions.append(&mut self.dirty_sessions); } fn process_dirty_flush_at(&mut self, now: i64) -> anyhow::Result { @@ -1493,9 +1502,9 @@ impl Server { self.flush_sessions.remove(&peer); continue; }; - if let Err(error) = self.db.touch_last_seen(player_id as i64, now) { + if let Err(error) = self.touch_last_seen_for_flush(player_id as i64, now) { self.flush_paused = true; - return Err(error.into()); + return Err(error); } self.flush_sessions.remove(&peer); work.db_updates += 1; @@ -1507,6 +1516,29 @@ impl Server { Ok(work) } + fn touch_last_seen_for_flush(&mut self, player_id: i64, now: i64) -> anyhow::Result<()> { + #[cfg(test)] + { + self.dirty_flush_test.attempts += 1; + if self.dirty_flush_test.failures_remaining > 0 { + self.dirty_flush_test.failures_remaining -= 1; + return Err(anyhow::anyhow!("injected dirty flush failure")); + } + } + self.db.touch_last_seen(player_id, now)?; + Ok(()) + } + + #[cfg(test)] + fn inject_dirty_flush_failures(&mut self, failures: usize) { + self.dirty_flush_test.failures_remaining = failures; + } + + #[cfg(test)] + fn dirty_flush_attempts(&self) -> usize { + self.dirty_flush_test.attempts + } + fn flush_all(&mut self) -> anyhow::Result<()> { let now = now_ms(); let ids: Vec = self.sessions.values().map(|s| s.player_id).collect(); @@ -1802,6 +1834,7 @@ mod handshake_tests { dirty_sessions: BTreeSet::new(), flush_sessions: BTreeSet::new(), flush_paused: false, + dirty_flush_test: DirtyFlushTestHook::default(), identity_players, seq: 0, snapshot_tick: 0, @@ -1976,6 +2009,19 @@ mod handshake_tests { finish_envelope(&mut fbb, 2, p::Payload::ClientState, state.as_union_value()) } + fn chat_envelope(seq: u32, text: &str, channel: u8) -> Vec { + let mut fbb = FlatBufferBuilder::new(); + let text = fbb.create_string(text); + let chat = p::ChatSend::create( + &mut fbb, + &p::ChatSendArgs { + text: Some(text), + channel, + }, + ); + finish_envelope(&mut fbb, seq, p::Payload::ChatSend, chat.as_union_value()) + } + fn deliver_hello(server: &mut Server, peer: PeerId, bytes: &[u8]) { let env = decode_envelope(bytes).unwrap(); server @@ -2498,7 +2544,7 @@ mod handshake_tests { } #[test] - fn valid_chat_is_preencoded_once_and_fanned_out_through_connected_peers() { + fn valid_chat_is_preencoded_once_and_fanned_out_fifo_through_connected_peers() { let mut server = make_server(); let (sender, sender_peer) = connect_peer(&mut server); let (observer, observer_peer) = connect_peer(&mut server); @@ -2534,41 +2580,110 @@ mod handshake_tests { assert_eq!(receive_server_hello(&observer), (true, String::new())); let sender_player = server.sessions[&sender_peer].player_id; - let mut fbb = FlatBufferBuilder::new(); - let text = fbb.create_string("fair winds"); - let chat = p::ChatSend::create( - &mut fbb, - &p::ChatSendArgs { - text: Some(text), - channel: 2, - }, - ); - let bytes = finish_envelope(&mut fbb, 3, p::Payload::ChatSend, chat.as_union_value()); + let expected = [("first watch", 1), ("second watch", 2), ("third watch", 3)]; let seq_before_chat = server.seq; - server - .handle_data_at(sender_peer, &bytes, 1_000, 1_000) - .unwrap(); + for (index, &(text, channel)) in expected.iter().enumerate() { + let bytes = chat_envelope(index as u32 + 3, text, channel); + let admission_ms = 1_000 + index as i64 * server.cfg.chat_min_interval_ms_i64(); + server + .handle_data_at(sender_peer, &bytes, admission_ms, admission_ms) + .unwrap(); + } - assert_eq!(server.seq, seq_before_chat.wrapping_add(1)); + assert_eq!( + server.seq, + seq_before_chat.wrapping_add(expected.len() as u32) + ); let fanout = server.process_fanout_work(); - assert_eq!(fanout.recipient_scans, 2); - assert_eq!(fanout.sends, 2); - assert_eq!(fanout.jobs_completed, 1); - let sender_payload = receive_payload(&sender, p::Payload::ChatBroadcast); - let observer_payload = receive_payload(&observer, p::Payload::ChatBroadcast); + assert_eq!(fanout.recipient_scans, expected.len() * 2); + assert_eq!(fanout.sends, expected.len() * 2); + assert_eq!(fanout.jobs_completed, expected.len()); + assert!(fanout.recipient_scans <= FANOUT_RECIPIENT_SCANS_PER_TICK); + assert!(fanout.sends <= FANOUT_SENDS_PER_TICK); + + for (index, &(expected_text, expected_channel)) in expected.iter().enumerate() { + let sender_payload = receive_payload(&sender, p::Payload::ChatBroadcast); + let observer_payload = receive_payload(&observer, p::Payload::ChatBroadcast); + assert_eq!( + sender_payload, observer_payload, + "every recipient must receive the same pre-encoded FIFO item" + ); + + let env = decode_envelope(&observer_payload).unwrap(); + assert_eq!(env.seq(), seq_before_chat.wrapping_add(index as u32 + 1)); + let broadcast = env.payload_as_chat_broadcast().unwrap(); + assert_eq!(broadcast.player_id(), sender_player); + assert_eq!(broadcast.display_name(), Some("Skipper")); + assert_eq!(broadcast.text(), Some(expected_text)); + assert_eq!(broadcast.channel(), expected_channel); + } + } + + #[test] + fn chat_queue_accepts_exactly_256_fifo_items_and_rejects_only_over_cap_inputs() { + assert_eq!(CHAT_QUEUE_ITEMS, 256); + let mut server = make_server_with_config(Config { + chat_min_interval_ms: 0, + ..Config::default() + }); + let hello = hello_envelope( + "queue-boundary-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, 1, &hello, 1_000); + let seq_before_chat = server.seq; + + for index in 0..CHAT_QUEUE_ITEMS { + let message = format!("queue-{index:03}"); + let bytes = chat_envelope(index as u32 + 1, &message, 4); + server + .handle_data_at(1, &bytes, 2_000 + index as i64, 2_000 + index as i64) + .unwrap(); + assert_eq!(server.seq, seq_before_chat.wrapping_add(index as u32 + 1)); + assert_eq!(server.chat_fanout.len(), index + 1); + } + + let mut decoded_bytes = 0usize; + for (index, job) in server.chat_fanout.iter().enumerate() { + decoded_bytes += job.bytes.len(); + let envelope = decode_envelope(&job.bytes).unwrap(); + assert_eq!( + envelope.seq(), + seq_before_chat.wrapping_add(index as u32 + 1) + ); + let broadcast = envelope.payload_as_chat_broadcast().unwrap(); + assert_eq!(broadcast.text(), Some(format!("queue-{index:03}").as_str())); + } + assert_eq!(server.chat_fanout.len(), CHAT_QUEUE_ITEMS); assert_eq!( - sender_payload, observer_payload, - "every recipient must receive the one pre-encoded broadcast" + server.chat_fanout_bytes, decoded_bytes, + "the byte counter must exactly equal all accepted FIFO payloads" ); + assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); - let env = decode_envelope(&sender_payload).unwrap(); - assert_eq!(env.seq(), seq_before_chat.wrapping_add(1)); - let broadcast = env.payload_as_chat_broadcast().unwrap(); - assert_eq!(broadcast.player_id(), sender_player); - assert_eq!(broadcast.display_name(), Some("Skipper")); - assert_eq!(broadcast.text(), Some("fair winds")); - assert_eq!(broadcast.channel(), 2); + let boundary = ( + server.seq, + server.chat_fanout.len(), + server.chat_fanout_bytes, + ); + for index in CHAT_QUEUE_ITEMS..CHAT_QUEUE_ITEMS + 3 { + let message = format!("queue-{index:03}"); + let bytes = chat_envelope(index as u32 + 1, &message, 4); + server + .handle_data_at(1, &bytes, 2_000 + index as i64, 2_000 + index as i64) + .unwrap(); + assert_eq!( + ( + server.seq, + server.chat_fanout.len(), + server.chat_fanout_bytes, + ), + boundary, + "the first and every later over-cap input must not consume a sequence or grow either queue bound" + ); + } } #[test] @@ -3396,6 +3511,7 @@ mod aoi_harden_tests { dirty_sessions: BTreeSet::new(), flush_sessions: BTreeSet::new(), flush_paused: false, + dirty_flush_test: DirtyFlushTestHook::default(), identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -3662,7 +3778,6 @@ mod aoi_harden_tests { #[test] fn dense_snapshot_elected_visibility_refreshes_before_client_expiry() { const DENSE_SESSIONS: PeerId = 1_024; - const CLIENT_STALE_MS: usize = 5_000; const SIMULATION_TICKS: usize = 600; let mut server = make_server(Config::default()); @@ -3671,10 +3786,14 @@ mod aoi_harden_tests { insert_dense_session(&mut server, peer, center); } - let stale_ticks = CLIENT_STALE_MS * server.cfg.tick_hz as usize / 1_000; + let recipient_round_ticks = (DENSE_SESSIONS as usize).div_ceil(SNAPSHOT_PACKETS_PER_TICK); + let elected_packets = SNAPSHOT_VISIBILITY_CEILING.div_ceil(SNAPSHOT_ENTITIES_PER_PACKET); + let exact_recurrence_bound = recipient_round_ticks * elected_packets; + assert_eq!(exact_recurrence_bound, 96); let mut last_advertised = HashMap::<(PeerId, u64), usize>::new(); let mut elected = HashMap::>::new(); let mut repeated = 0usize; + let mut maximum_recurrence_gap = 0usize; for tick in 0..SIMULATION_TICKS { let work = server.broadcast_snapshots(); @@ -3693,10 +3812,11 @@ mod aoi_harden_tests { elected.entry(peer).or_default().insert(player_id); if let Some(previous) = last_advertised.insert((peer, player_id), tick) { repeated += 1; + maximum_recurrence_gap = maximum_recurrence_gap.max(tick - previous); assert!( - tick - previous < stale_ticks, + tick - previous <= exact_recurrence_bound, "peer {peer}'s elected player {player_id} went {} ticks without a refresh, \ - but the real client expires it after {stale_ticks} production ticks", + exceeding the scheduler's derived {exact_recurrence_bound}-tick bound", tick - previous ); } @@ -3714,6 +3834,10 @@ mod aoi_harden_tests { repeated >= DENSE_SESSIONS as usize * SNAPSHOT_VISIBILITY_CEILING, "every elected pair must be observed often enough to prove recurrence" ); + assert_eq!( + maximum_recurrence_gap, exact_recurrence_bound, + "the dense proof must observe the exact worst-case gap, not merely a looser expiry threshold" + ); } #[test] @@ -3823,6 +3947,143 @@ mod aoi_harden_tests { ); } + #[test] + fn dirty_flush_error_preserves_old_and_new_work_until_the_next_cadence() { + const INITIAL_LAST_SEEN: i64 = 100; + const FAILED_FLUSH_AT: i64 = 1_000; + const RECOVERED_FLUSH_AT: i64 = 6_000; + + let mut server = make_server(Config::default()); + for peer in 1..=2 { + let player = server + .db + .upsert_player_by_token( + &format!("{peer:016x}"), + &format!("Player {peer}"), + INITIAL_LAST_SEEN, + ) + .unwrap(); + assert_eq!(player.id, i64::from(peer)); + insert_dense_session(&mut server, peer, Cell::new(0, 0)); + } + + let first_state = state_envelope(1.0, 1.0); + let envelope = decode_envelope(&first_state).unwrap(); + server.on_client_state( + 1, + envelope.payload_as_client_state().unwrap(), + FAILED_FLUSH_AT, + ); + server.begin_dirty_flush(); + assert_eq!(server.flush_sessions, BTreeSet::from([1])); + assert!(server.dirty_sessions.is_empty()); + + server.inject_dirty_flush_failures(1); + let error = server.process_dirty_flush_at(FAILED_FLUSH_AT).unwrap_err(); + assert!(error.to_string().contains("injected dirty flush failure")); + assert_eq!(server.dirty_flush_attempts(), 1); + assert!(server.flush_paused); + assert_eq!(server.flush_sessions, BTreeSet::from([1])); + assert!(server.sessions[&1].dirty); + + let second_state = state_envelope(2.0, 2.0); + let envelope = decode_envelope(&second_state).unwrap(); + server.on_client_state( + 2, + envelope.payload_as_client_state().unwrap(), + FAILED_FLUSH_AT + 1, + ); + assert_eq!(server.flush_sessions, BTreeSet::from([1])); + assert_eq!(server.dirty_sessions, BTreeSet::from([2])); + assert!(server.sessions[&2].dirty); + + for tick in 0..100 { + assert_eq!( + server + .process_dirty_flush_at(FAILED_FLUSH_AT + tick) + .unwrap(), + DirtyFlushTickWork::default(), + "paused ticks must neither retry nor report persistence work" + ); + } + assert_eq!( + server.dirty_flush_attempts(), + 1, + "one database error must produce one failed attempt until the next five-second cadence" + ); + for player_id in 1..=2 { + assert_eq!( + server.db.player(player_id).unwrap().unwrap().last_seen, + INITIAL_LAST_SEEN + ); + } + + server.begin_dirty_flush(); + assert!(!server.flush_paused); + assert_eq!( + server.flush_sessions, + BTreeSet::from([1, 2]), + "the recovery cadence must preserve the active retry and merge newly dirty work" + ); + assert!(server.dirty_sessions.is_empty()); + + let recovered = server.process_dirty_flush_at(RECOVERED_FLUSH_AT).unwrap(); + assert_eq!(recovered.db_updates, 2); + assert!(recovered.db_updates <= DIRTY_DB_UPDATES_PER_TICK); + assert_eq!(server.dirty_flush_attempts(), 3); + assert!(server.flush_sessions.is_empty()); + assert!(server.dirty_sessions.is_empty()); + for player_id in 1..=2 { + assert_eq!( + server.db.player(player_id).unwrap().unwrap().last_seen, + RECOVERED_FLUSH_AT + ); + assert!(!server.sessions[&(player_id as PeerId)].dirty); + } + } + + #[test] + fn dirty_flush_disconnect_and_shutdown_paths_still_persist_live_sessions() { + const INITIAL_LAST_SEEN: i64 = 100; + + let mut server = make_server(Config::default()); + for peer in 1..=2 { + let player = server + .db + .upsert_player_by_token( + &format!("{peer:016x}"), + &format!("Player {peer}"), + INITIAL_LAST_SEEN, + ) + .unwrap(); + assert_eq!(player.id, i64::from(peer)); + insert_dense_session(&mut server, peer, Cell::new(0, 0)); + let state = state_envelope(peer as f32, peer as f32); + let envelope = decode_envelope(&state).unwrap(); + server.on_client_state( + peer, + envelope.payload_as_client_state().unwrap(), + i64::from(peer), + ); + } + server.begin_dirty_flush(); + assert_eq!(server.flush_sessions, BTreeSet::from([1, 2])); + + server.on_disconnect(1, DisconnectReason::Remote).unwrap(); + assert!(!server.flush_sessions.contains(&1)); + assert!(!server.sessions.contains_key(&1)); + assert!( + server.db.player(1).unwrap().unwrap().last_seen > INITIAL_LAST_SEEN, + "disconnect must synchronously persist the departing session" + ); + + server.flush_all().unwrap(); + assert!( + server.db.player(2).unwrap().unwrap().last_seen > INITIAL_LAST_SEEN, + "shutdown flush must synchronously persist every remaining live session" + ); + } + #[test] fn hostile_population_fanout_and_flush_stay_capped_and_make_progress() { const DENSE_SESSIONS: PeerId = 1_024; @@ -4193,6 +4454,7 @@ mod market_dispatch_tests { dirty_sessions: BTreeSet::new(), flush_sessions: BTreeSet::new(), flush_paused: false, + dirty_flush_test: DirtyFlushTestHook::default(), identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, @@ -4456,6 +4718,7 @@ mod input_hardening_tests { dirty_sessions: BTreeSet::new(), flush_sessions: BTreeSet::new(), flush_paused: false, + dirty_flush_test: DirtyFlushTestHook::default(), identity_players: HashMap::new(), seq: 0, snapshot_tick: 0, From 5c4514de841dfdea86a21f45fe03ee5cf3046056 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Thu, 23 Jul 2026 23:39:08 -0600 Subject: [PATCH 33/36] fix(server): harden fanout and persistence retries Co-Authored-By: Claude --- server/crates/sw-server/src/config.rs | 86 +++- server/crates/sw-server/src/server.rs | 554 +++++++++++++++++++++----- 2 files changed, 539 insertions(+), 101 deletions(-) diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index 66192b1..8d9fe4d 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -45,8 +45,24 @@ const MIN_HELLO_MIN_INTERVAL_MS: u32 = 1; /// operator misconfiguration from wedging fresh identities indefinitely. pub const MAX_NEW_SESSION_MIN_INTERVAL_MS: u32 = 1_000; -/// Highest configurable live transport-peer ceiling. -pub const MAX_TRANSPORT_PEERS: u32 = 65_535; +/// Highest configurable live transport-peer ceiling. This is the single +/// population ceiling exercised by the transport, scheduler, load, and +/// recurrence tests. +pub const MAX_TRANSPORT_PEERS: u32 = sw_net::DEFAULT_MAX_PEERS as u32; + +/// Snapshot scheduler constants. Configuration validation and the production +/// scheduler share these values so the freshness proof cannot drift from the +/// code that performs the work. +pub(crate) const SNAPSHOT_PACKETS_PER_TICK: usize = 32; +pub(crate) const SNAPSHOT_ENTITIES_PER_PACKET: usize = 5; +pub(crate) const SNAPSHOT_VISIBILITY_CEILING: usize = 15; +pub(crate) const SNAPSHOT_CLIENT_EXPIRY_SECS: u64 = 5; + +/// Worst-case ticks between advertisements of one elected dense player. +pub(crate) fn snapshot_recurrence_ticks(peers: u32) -> u64 { + u64::from(peers).div_ceil(SNAPSHOT_PACKETS_PER_TICK as u64) + * (SNAPSHOT_VISIBILITY_CEILING as u64).div_ceil(SNAPSHOT_ENTITIES_PER_PACKET as u64) +} /// Highest configurable persistent player-row ceiling. The server still uses /// the operator's lower configured value; this only prevents an accidental @@ -262,6 +278,14 @@ impl Config { "max_transport_peers must be in 1..={MAX_TRANSPORT_PEERS}" )); } + let recurrence_ticks = snapshot_recurrence_ticks(self.max_transport_peers); + let expiry_ticks = u64::from(self.tick_hz) * SNAPSHOT_CLIENT_EXPIRY_SECS; + if recurrence_ticks >= expiry_ticks { + return Err(anyhow::anyhow!( + "snapshot recurrence ({recurrence_ticks} ticks) must be strictly less than \ + the {SNAPSHOT_CLIENT_EXPIRY_SECS}-second client expiry ({expiry_ticks} ticks)" + )); + } if self.max_transport_peers_per_ip == 0 || self.max_transport_peers_per_ip > self.max_transport_peers { @@ -425,6 +449,60 @@ mod tests { assert_eq!(cfg.ticks_per_snapshot(), 30 / 4); } + #[test] + fn snapshot_recurrence_is_fresh_for_every_accepted_capacity_and_tick_pair() { + let default = Config::default(); + default.validate().unwrap(); + assert_eq!(snapshot_recurrence_ticks(default.max_transport_peers), 96); + assert!( + snapshot_recurrence_ticks(default.max_transport_peers) + < u64::from(default.tick_hz) * SNAPSHOT_CLIENT_EXPIRY_SECS + ); + + let boundary = Config { + tick_hz: 20, + max_transport_peers: sw_net::DEFAULT_MAX_PEERS as u32, + ..Config::default() + }; + boundary.validate().unwrap(); + + let stale = Config { + tick_hz: 19, + max_transport_peers: sw_net::DEFAULT_MAX_PEERS as u32, + ..Config::default() + }; + let error = stale.validate().unwrap_err(); + assert!( + error.to_string().contains("snapshot recurrence"), + "low-tick rejection must explain the freshness invariant: {error}" + ); + } + + #[test] + fn snapshot_recurrence_formula_tracks_the_scheduler_constants() { + for peers in [1, 32, 33, sw_net::DEFAULT_MAX_PEERS as u32] { + assert_eq!( + snapshot_recurrence_ticks(peers), + u64::from(peers).div_ceil(SNAPSHOT_PACKETS_PER_TICK as u64) + * (SNAPSHOT_VISIBILITY_CEILING as u64) + .div_ceil(SNAPSHOT_ENTITIES_PER_PACKET as u64) + ); + } + } + + #[test] + fn transport_capacity_above_the_tested_scheduler_ceiling_is_rejected() { + let cfg = Config { + max_transport_peers: 2_048, + ..Config::default() + }; + let error = cfg.validate().unwrap_err(); + assert!( + error.to_string().contains("max_transport_peers"), + "capacity rejection must name the offending setting: {error}" + ); + } + #[test] fn clock_broadcast_cadence() { let cfg = Config::default(); @@ -775,7 +853,7 @@ mod tests { fn parses_new_hardening_keys() { let toml_text = r#" new_session_min_interval_ms = 125 - max_transport_peers = 2048 + max_transport_peers = 512 max_transport_peers_per_ip = 24 max_player_rows = 5000 client_state_min_interval_ms = 33 @@ -786,7 +864,7 @@ mod tests { "#; let cfg: Config = toml::from_str(toml_text).unwrap(); assert_eq!(cfg.new_session_min_interval_ms, 125); - assert_eq!(cfg.max_transport_peers, 2048); + assert_eq!(cfg.max_transport_peers, 512); assert_eq!(cfg.max_transport_peers_per_ip, 24); assert_eq!(cfg.max_player_rows, 5000); assert_eq!(cfg.client_state_min_interval_ms, 33); diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 79d321b..e9e497e 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -2,7 +2,12 @@ use crate::clock::{clock_from_epoch, WorldClock}; use crate::codec::{self, BoatSnap, Caps, MooringSnap, PlayerSnap}; -use crate::config::{Config, MAX_PLAYER_ROWS}; +#[cfg(test)] +use crate::config::snapshot_recurrence_ticks; +use crate::config::{ + Config, MAX_PLAYER_ROWS, SNAPSHOT_ENTITIES_PER_PACKET, SNAPSHOT_PACKETS_PER_TICK, + SNAPSHOT_VISIBILITY_CEILING, +}; use crate::econ_store::{DbLedgerStore, DbMarketStore}; use crate::ratelimit::{BoundedRateLimiter, GlobalRateLimiter, RateLimiter}; use crate::validate; @@ -40,22 +45,22 @@ const FLUSH_INTERVAL: Duration = Duration::from_secs(5); /// five-player packets, so every elected player recurs in at most 96 ticks /// (3.2 seconds at the production 30 Hz), strictly before the client's /// five-second stale-sample eviction. -const SNAPSHOT_PACKETS_PER_TICK: usize = 32; const SNAPSHOT_ENTITY_SCAN_PER_PACKET: usize = sw_net::DEFAULT_MAX_PEERS; -const SNAPSHOT_ENTITIES_PER_PACKET: usize = 5; -const SNAPSHOT_VISIBILITY_CEILING: usize = 15; /// Queued population fanout budgets. Chat is FIFO and rejects new accepted /// work at the fixed queue boundary; clock state has one coalescing latest-value /// slot. Both share this fixed per-tick transport budget. const CHAT_QUEUE_ITEMS: usize = 256; const CHAT_QUEUE_BYTES: usize = CHAT_QUEUE_ITEMS * (protocol::MTU - protocol::HEADER_SIZE); +const CHAT_QUEUE_RECIPIENTS: usize = CHAT_QUEUE_ITEMS * sw_net::DEFAULT_MAX_PEERS; +const FANOUT_QUEUE_RECIPIENTS: usize = CHAT_QUEUE_RECIPIENTS + sw_net::DEFAULT_MAX_PEERS; const FANOUT_RECIPIENT_SCANS_PER_TICK: usize = sw_net::DEFAULT_MAX_PEERS; const FANOUT_SENDS_PER_TICK: usize = sw_net::DEFAULT_MAX_PEERS; /// Persistence work is spread across fixed ticks after each five-second flush /// boundary. At 1,024 sessions this drains in 128 ticks (about 4.27 seconds). const DIRTY_DB_UPDATES_PER_TICK: usize = 8; +const SHUTDOWN_FLUSH_ATTEMPTS: usize = 3; /// Global AoI delivery budget charged on every fixed server tick. const AOI_WORK_ITEMS_PER_TICK: usize = 8; @@ -83,12 +88,13 @@ struct CellHydration { struct FanoutJob { bytes: Vec, sender_cell: Option, - recipient_cursor: Option, - recipients_remaining: usize, + recipients: Vec<(PeerId, u64)>, + next_recipient: usize, } /// Per-connection state, created on ClientHello. struct Session { + generation: u64, player_id: u64, identity_hash: String, display_name: String, @@ -141,6 +147,8 @@ struct FanoutTickWork { sends: usize, encoded_bytes: usize, jobs_completed: usize, + #[cfg(test)] + delivered: Vec<(PeerId, u64)>, } #[derive(Debug, Default, PartialEq, Eq)] @@ -177,14 +185,16 @@ pub struct Server { aoi_recipient_cursor: Option, chat_fanout: VecDeque, chat_fanout_bytes: usize, + chat_fanout_recipients: usize, clock_fanout: Option, fanout_prefer_clock: bool, - dirty_sessions: BTreeSet, - flush_sessions: BTreeSet, + dirty_players: BTreeSet, + flush_players: BTreeSet, flush_paused: bool, #[cfg(test)] dirty_flush_test: DirtyFlushTestHook, identity_players: HashMap, + next_session_generation: u64, seq: u32, snapshot_tick: u32, boot: Instant, @@ -262,14 +272,16 @@ impl Server { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, + chat_fanout_recipients: 0, clock_fanout: None, fanout_prefer_clock: true, - dirty_sessions: BTreeSet::new(), - flush_sessions: BTreeSet::new(), + dirty_players: BTreeSet::new(), + flush_players: BTreeSet::new(), flush_paused: false, #[cfg(test)] dirty_flush_test: DirtyFlushTestHook::default(), identity_players, + next_session_generation: 0, seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -341,9 +353,9 @@ impl Server { } tracing::info!("shutting down"); - self.flush_all()?; + let flush_result = self.flush_all(); let _ = self.host.shutdown(); - Ok(()) + flush_result } fn handle_event(&mut self, ev: Event) -> anyhow::Result<()> { @@ -572,6 +584,7 @@ impl Server { self.register_session( peer, Session { + generation: 0, player_id, identity_hash, display_name: name, @@ -680,7 +693,7 @@ impl Server { aoi = s.sub.recenter(cell); s.cell = Some(cell); } - self.dirty_sessions.insert(peer); + self.dirty_players.insert(player_id); self.emit_aoi(peer, &aoi); } @@ -937,13 +950,28 @@ impl Server { if bytes.len() > protocol::MTU - protocol::HEADER_SIZE { return; } + let recipients = self.capture_fanout_recipients(); + if recipients.is_empty() + || recipients.len() > CHAT_QUEUE_RECIPIENTS - self.chat_fanout_recipients + { + return; + } self.chat_fanout_bytes += bytes.len(); + self.chat_fanout_recipients += recipients.len(); + debug_assert!( + self.chat_fanout_recipients + + self + .clock_fanout + .as_ref() + .map_or(0, |job| job.recipients.len()) + <= FANOUT_QUEUE_RECIPIENTS + ); self.chat_fanout.push_back(FanoutJob { bytes, sender_cell: Some(cell), - recipient_cursor: None, - recipients_remaining: self.snapshot_recipients.len(), + recipients, + next_recipient: 0, }); } @@ -960,13 +988,19 @@ impl Server { self.chat_limiter.clear(s.player_id); self.econ_limiter.clear(s.player_id); self.moor_limiter.clear(s.player_id); - self.db.touch_last_seen(s.player_id as i64, now_ms())?; + self.flush_players.insert(s.player_id); + if let Err(error) = self.touch_last_seen_for_flush(s.player_id as i64, now_ms()) { + self.flush_paused = true; + return Err(error); + } + self.flush_players.remove(&s.player_id); + self.dirty_players.remove(&s.player_id); tracing::info!(peer, player_id = s.player_id, ?reason, "peer disconnected"); } Ok(()) } - fn register_session(&mut self, peer: PeerId, session: Session) -> anyhow::Result<()> { + fn register_session(&mut self, peer: PeerId, mut session: Session) -> anyhow::Result<()> { if self.sessions.contains_key(&peer) { return Err(anyhow::anyhow!("peer already owns a session")); } @@ -975,6 +1009,7 @@ impl Server { "player already owns session peer {existing_peer}" )); } + session.generation = self.allocate_session_generation(); let player_id = session.player_id; let dirty = session.dirty; self.sessions.insert(peer, session); @@ -982,7 +1017,7 @@ impl Server { self.player_order.insert(player_id); self.snapshot_recipients.insert(peer); if dirty { - self.dirty_sessions.insert(peer); + self.dirty_players.insert(player_id); } self.schedule_aoi(peer); Ok(()) @@ -1005,11 +1040,25 @@ impl Server { let mut work = RecipientIndexWork::default(); work.remove(&mut self.snapshot_recipients, peer); work.remove(&mut self.aoi_recipients, peer); - self.dirty_sessions.remove(&peer); - self.flush_sessions.remove(&peer); Some((session, work)) } + fn allocate_session_generation(&mut self) -> u64 { + if self.next_session_generation == u64::MAX { + self.invalidate_fanout_jobs(); + self.next_session_generation = 0; + } + self.next_session_generation += 1; + self.next_session_generation + } + + fn invalidate_fanout_jobs(&mut self) { + self.chat_fanout.clear(); + self.chat_fanout_bytes = 0; + self.chat_fanout_recipients = 0; + self.clock_fanout = None; + } + fn schedule_aoi(&mut self, peer: PeerId) { if !self.sessions.contains_key(&peer) { return; @@ -1405,23 +1454,43 @@ impl Server { fn broadcast_clock(&mut self) { let clock = self.clock_now(); let bytes = codec::world_clock(self.next_seq(), clock); - if self.snapshot_recipients.is_empty() { + let recipients = self.capture_fanout_recipients(); + if recipients.is_empty() { self.clock_fanout = None; return; } + debug_assert!(self.chat_fanout_recipients + recipients.len() <= FANOUT_QUEUE_RECIPIENTS); self.clock_fanout = Some(FanoutJob { bytes, sender_cell: None, - recipient_cursor: None, - recipients_remaining: self.snapshot_recipients.len(), + recipients, + next_recipient: 0, }); } + fn capture_fanout_recipients(&self) -> Vec<(PeerId, u64)> { + self.snapshot_recipients + .iter() + .filter_map(|&peer| { + self.sessions + .get(&peer) + .map(|session| (peer, session.generation)) + }) + .take(self.cfg.max_transport_peers_usize()) + .collect() + } + fn process_fanout_work(&mut self) -> FanoutTickWork { + self.process_fanout_work_with_budget(FANOUT_RECIPIENT_SCANS_PER_TICK, FANOUT_SENDS_PER_TICK) + } + + fn process_fanout_work_with_budget( + &mut self, + recipient_scan_budget: usize, + send_budget: usize, + ) -> FanoutTickWork { let mut work = FanoutTickWork::default(); - while work.recipient_scans < FANOUT_RECIPIENT_SCANS_PER_TICK - && work.sends < FANOUT_SENDS_PER_TICK - { + while work.recipient_scans < recipient_scan_budget && work.sends < send_budget { let from_clock = self.clock_fanout.is_some() && (self.fanout_prefer_clock || self.chat_fanout.is_empty()); let Some(mut job) = (if from_clock { @@ -1436,37 +1505,38 @@ impl Server { continue; }; - while job.recipients_remaining > 0 - && work.recipient_scans < FANOUT_RECIPIENT_SCANS_PER_TICK - && work.sends < FANOUT_SENDS_PER_TICK + while job.next_recipient < job.recipients.len() + && work.recipient_scans < recipient_scan_budget + && work.sends < send_budget { - let Some(peer) = next_peer_after(&self.snapshot_recipients, job.recipient_cursor) - else { - job.recipients_remaining = 0; - break; - }; - job.recipient_cursor = Some(peer); - job.recipients_remaining -= 1; + let (peer, generation) = job.recipients[job.next_recipient]; + job.next_recipient += 1; work.recipient_scans += 1; - let should_send = job.sender_cell.is_none_or(|cell| { - self.sessions - .get(&peer) - .is_some_and(|session| session.sub.contains(cell)) + let should_send = self.sessions.get(&peer).is_some_and(|session| { + session.generation == generation + && job + .sender_cell + .is_none_or(|cell| session.sub.contains(cell)) }); if should_send { self.send(peer, &job.bytes); work.sends += 1; work.encoded_bytes += job.bytes.len(); + #[cfg(test)] + work.delivered.push((peer, generation)); } } - if job.recipients_remaining == 0 { + if job.next_recipient == job.recipients.len() { work.jobs_completed += 1; if from_clock { self.fanout_prefer_clock = false; } else { self.chat_fanout_bytes = self.chat_fanout_bytes.saturating_sub(job.bytes.len()); + self.chat_fanout_recipients = self + .chat_fanout_recipients + .saturating_sub(job.recipients.len()); self.fanout_prefer_clock = true; } } else { @@ -1483,7 +1553,7 @@ impl Server { fn begin_dirty_flush(&mut self) { self.flush_paused = false; - self.flush_sessions.append(&mut self.dirty_sessions); + self.flush_players.append(&mut self.dirty_players); } fn process_dirty_flush_at(&mut self, now: i64) -> anyhow::Result { @@ -1491,26 +1561,26 @@ impl Server { if self.flush_paused { return Ok(work); } - let peers: Vec = self - .flush_sessions + let player_ids: Vec = self + .flush_players .iter() .take(DIRTY_DB_UPDATES_PER_TICK) .copied() .collect(); - for peer in peers { - let Some(player_id) = self.sessions.get(&peer).map(|session| session.player_id) else { - self.flush_sessions.remove(&peer); - continue; - }; + for player_id in player_ids { if let Err(error) = self.touch_last_seen_for_flush(player_id as i64, now) { self.flush_paused = true; return Err(error); } - self.flush_sessions.remove(&peer); + self.flush_players.remove(&player_id); work.db_updates += 1; - let still_dirty = self.dirty_sessions.contains(&peer); - if let Some(session) = self.sessions.get_mut(&peer) { - session.dirty = still_dirty; + let still_dirty = self.dirty_players.contains(&player_id); + if let Some(peer) = self.player_peers.get(&player_id).copied() { + if let Some(session) = self.sessions.get_mut(&peer) { + if session.player_id == player_id { + session.dirty = still_dirty; + } + } } } Ok(work) @@ -1541,11 +1611,45 @@ impl Server { fn flush_all(&mut self) -> anyhow::Result<()> { let now = now_ms(); - let ids: Vec = self.sessions.values().map(|s| s.player_id).collect(); - for player_id in ids { - self.db.touch_last_seen(player_id as i64, now)?; + let mut pending: BTreeSet = self.sessions.values().map(|s| s.player_id).collect(); + pending.extend(self.dirty_players.iter().copied()); + pending.extend(self.flush_players.iter().copied()); + self.flush_players.extend(pending.iter().copied()); + + let mut last_error = None; + for _ in 0..SHUTDOWN_FLUSH_ATTEMPTS { + if pending.is_empty() { + return Ok(()); + } + let attempt: Vec = pending.iter().copied().collect(); + for player_id in attempt { + match self.touch_last_seen_for_flush(player_id as i64, now) { + Ok(()) => { + pending.remove(&player_id); + self.flush_players.remove(&player_id); + self.dirty_players.remove(&player_id); + if let Some(peer) = self.player_peers.get(&player_id).copied() { + if let Some(session) = self.sessions.get_mut(&peer) { + if session.player_id == player_id { + session.dirty = false; + } + } + } + } + Err(error) => { + last_error = Some(error); + } + } + } } - Ok(()) + let failed = pending.len(); + let error = last_error + .map(|error| error.to_string()) + .unwrap_or_else(|| "unknown persistence failure".to_string()); + Err(anyhow::anyhow!( + "shutdown persistence failed for {failed} player(s) after \ + {SHUTDOWN_FLUSH_ATTEMPTS} attempts: {error}" + )) } fn session_by_player(&self, player_id: u64) -> Option<&Session> { @@ -1653,17 +1757,6 @@ fn mooring_snap(row: MooringRow) -> MooringSnap { } } -fn next_peer_after(peers: &BTreeSet, cursor: Option) -> Option { - match cursor { - Some(cursor) => peers - .range((Excluded(cursor), Unbounded)) - .next() - .or_else(|| peers.iter().next()) - .copied(), - None => peers.iter().next().copied(), - } -} - fn next_hydration_cell(cells: &HashSet, cursor: Option) -> Option { let key = |cell: &Cell| (cell.cz, cell.cx); cursor @@ -1829,13 +1922,15 @@ mod handshake_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, + chat_fanout_recipients: 0, clock_fanout: None, fanout_prefer_clock: true, - dirty_sessions: BTreeSet::new(), - flush_sessions: BTreeSet::new(), + dirty_players: BTreeSet::new(), + flush_players: BTreeSet::new(), flush_paused: false, dirty_flush_test: DirtyFlushTestHook::default(), identity_players, + next_session_generation: 0, seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -3416,6 +3511,41 @@ mod handshake_tests { assert_eq!(receive_server_hello(&client), (true, String::new())); } + #[test] + fn permanent_shutdown_flush_failure_still_sends_transport_shutdown() { + let mut server = make_server(); + let (client, peer) = connect_peer(&mut server); + let hello = hello_envelope( + "shutdown-failure-token", + sw_contracts::PROTOCOL_VERSION, + Some("surface-hash"), + ); + deliver_hello_at(&mut server, peer, &hello, 1_000); + assert_eq!(receive_server_hello(&client), (true, String::new())); + + server.inject_dirty_flush_failures(usize::MAX); + server.running.store(false, Ordering::SeqCst); + let error = server.run().unwrap_err(); + assert!( + error.to_string().contains("injected dirty flush failure"), + "the bounded persistence failure must be returned after shutdown: {error}" + ); + let player_id = server.sessions[&peer].player_id; + assert!( + server.flush_players.contains(&player_id) && server.dirty_players.contains(&player_id), + "a permanent shutdown failure must retain the dirty player id for diagnosis or retry" + ); + + loop { + let mut packet = [0u8; protocol::MTU]; + let received = client.recv(&mut packet).unwrap(); + if protocol::Header::from_byte(packet[0]).property == protocol::property::DISCONNECT { + assert_eq!(received, protocol::DISCONNECT_SIZE); + break; + } + } + } + #[test] fn same_peer_cannot_replace_an_established_session_with_a_different_identity() { let mut server = make_server(); @@ -3506,13 +3636,15 @@ mod aoi_harden_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, + chat_fanout_recipients: 0, clock_fanout: None, fanout_prefer_clock: true, - dirty_sessions: BTreeSet::new(), - flush_sessions: BTreeSet::new(), + dirty_players: BTreeSet::new(), + flush_players: BTreeSet::new(), flush_paused: false, dirty_flush_test: DirtyFlushTestHook::default(), identity_players: HashMap::new(), + next_session_generation: 0, seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -3676,7 +3808,15 @@ mod aoi_harden_tests { } fn insert_dense_session(server: &mut Server, peer: PeerId, cell: Cell) { - let player_id = u64::from(peer); + insert_dense_session_for_player(server, peer, u64::from(peer), cell); + } + + fn insert_dense_session_for_player( + server: &mut Server, + peer: PeerId, + player_id: u64, + cell: Cell, + ) { let mut sub = Subscription::new(server.cfg.aoi_radius_i32()); sub.recenter(cell); server.world.place_in_cell(player_id, cell); @@ -3684,6 +3824,7 @@ mod aoi_harden_tests { .register_session( peer, Session { + generation: 0, player_id, identity_hash: format!("identity-{player_id}"), display_name: format!("Player {player_id}"), @@ -3707,6 +3848,109 @@ mod aoi_harden_tests { .unwrap(); } + #[test] + fn queued_fanout_never_wraps_or_delivers_to_a_reused_connection_identity() { + const RECIPIENTS: PeerId = sw_net::DEFAULT_MAX_PEERS as PeerId; + + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + for peer in 1..=RECIPIENTS { + insert_dense_session(&mut server, peer, center); + } + let original_last_generation = server.sessions[&RECIPIENTS].generation; + + let mut fbb = FlatBufferBuilder::new(); + let text = fbb.create_string("captured audience"); + let chat = p::ChatSend::create( + &mut fbb, + &p::ChatSendArgs { + text: Some(text), + channel: 0, + }, + ); + let bytes = finish_envelope(&mut fbb, 1, p::Payload::ChatSend, chat.as_union_value()); + let envelope = decode_envelope(&bytes).unwrap(); + server.on_chat(1, envelope.payload_as_chat_send().unwrap(), 1_000); + + let first = server + .process_fanout_work_with_budget(RECIPIENTS as usize - 1, RECIPIENTS as usize - 1); + assert_eq!(first.delivered.len(), RECIPIENTS as usize - 1); + + let departed = server.unregister_session(RECIPIENTS).unwrap(); + server.world.remove(departed.player_id); + insert_dense_session_for_player( + &mut server, + RECIPIENTS, + u64::from(RECIPIENTS) + 10_000, + center, + ); + let replacement_generation = server.sessions[&RECIPIENTS].generation; + assert_ne!(replacement_generation, original_last_generation); + + let last = server.process_fanout_work_with_budget(1, 1); + assert_eq!(last.recipient_scans, 1); + assert!(last.delivered.is_empty()); + assert_eq!(last.jobs_completed, 1); + + let delivered: HashSet<_> = first.delivered.iter().copied().collect(); + assert_eq!(delivered.len(), RECIPIENTS as usize - 1); + for peer in 1..RECIPIENTS { + assert!( + delivered + .iter() + .any(|(delivered_peer, _)| *delivered_peer == peer), + "each remaining original recipient must receive exactly once" + ); + } + assert!( + !delivered + .iter() + .any(|(peer, generation)| *peer == RECIPIENTS + && *generation == replacement_generation), + "a queued message must never reach a later connection reusing the peer slot" + ); + } + + #[test] + fn clock_fanout_uses_the_same_captured_connection_identity_path() { + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + for peer in 1..=2 { + insert_dense_session(&mut server, peer, center); + } + server.broadcast_clock(); + let first = server.process_fanout_work_with_budget(1, 1); + assert_eq!(first.delivered.len(), 1); + + let old_generation = server.sessions[&2].generation; + let departed = server.unregister_session(2).unwrap(); + server.world.remove(departed.player_id); + insert_dense_session_for_player(&mut server, 2, 22, center); + assert_ne!(server.sessions[&2].generation, old_generation); + + let last = server.process_fanout_work_with_budget(1, 1); + assert!(last.delivered.is_empty()); + assert_eq!(last.jobs_completed, 1); + } + + #[test] + fn fanout_generation_wrap_invalidates_every_queued_identity() { + let mut server = make_server(Config::default()); + let center = Cell::new(0, 0); + insert_dense_session(&mut server, 1, center); + server.broadcast_clock(); + server.next_session_generation = u64::MAX; + + insert_dense_session(&mut server, 2, center); + + assert!( + server.clock_fanout.is_none() && server.chat_fanout.is_empty(), + "generation wrap must invalidate all captured pre-wrap identities" + ); + assert_eq!(server.chat_fanout_bytes, 0); + assert_eq!(server.chat_fanout_recipients, 0); + } + #[test] fn dense_snapshot_broadcast_has_a_fixed_global_packet_budget() { const DENSE_SESSIONS: PeerId = 1_024; @@ -3789,6 +4033,11 @@ mod aoi_harden_tests { let recipient_round_ticks = (DENSE_SESSIONS as usize).div_ceil(SNAPSHOT_PACKETS_PER_TICK); let elected_packets = SNAPSHOT_VISIBILITY_CEILING.div_ceil(SNAPSHOT_ENTITIES_PER_PACKET); let exact_recurrence_bound = recipient_round_ticks * elected_packets; + assert_eq!( + exact_recurrence_bound as u64, + snapshot_recurrence_ticks(DENSE_SESSIONS), + "config validation and the production scheduler must share one recurrence proof" + ); assert_eq!(exact_recurrence_bound, 96); let mut last_advertised = HashMap::<(PeerId, u64), usize>::new(); let mut elected = HashMap::>::new(); @@ -3975,15 +4224,15 @@ mod aoi_harden_tests { FAILED_FLUSH_AT, ); server.begin_dirty_flush(); - assert_eq!(server.flush_sessions, BTreeSet::from([1])); - assert!(server.dirty_sessions.is_empty()); + assert_eq!(server.flush_players, BTreeSet::from([1])); + assert!(server.dirty_players.is_empty()); server.inject_dirty_flush_failures(1); let error = server.process_dirty_flush_at(FAILED_FLUSH_AT).unwrap_err(); assert!(error.to_string().contains("injected dirty flush failure")); assert_eq!(server.dirty_flush_attempts(), 1); assert!(server.flush_paused); - assert_eq!(server.flush_sessions, BTreeSet::from([1])); + assert_eq!(server.flush_players, BTreeSet::from([1])); assert!(server.sessions[&1].dirty); let second_state = state_envelope(2.0, 2.0); @@ -3993,8 +4242,8 @@ mod aoi_harden_tests { envelope.payload_as_client_state().unwrap(), FAILED_FLUSH_AT + 1, ); - assert_eq!(server.flush_sessions, BTreeSet::from([1])); - assert_eq!(server.dirty_sessions, BTreeSet::from([2])); + assert_eq!(server.flush_players, BTreeSet::from([1])); + assert_eq!(server.dirty_players, BTreeSet::from([2])); assert!(server.sessions[&2].dirty); for tick in 0..100 { @@ -4021,18 +4270,18 @@ mod aoi_harden_tests { server.begin_dirty_flush(); assert!(!server.flush_paused); assert_eq!( - server.flush_sessions, + server.flush_players, BTreeSet::from([1, 2]), "the recovery cadence must preserve the active retry and merge newly dirty work" ); - assert!(server.dirty_sessions.is_empty()); + assert!(server.dirty_players.is_empty()); let recovered = server.process_dirty_flush_at(RECOVERED_FLUSH_AT).unwrap(); assert_eq!(recovered.db_updates, 2); assert!(recovered.db_updates <= DIRTY_DB_UPDATES_PER_TICK); assert_eq!(server.dirty_flush_attempts(), 3); - assert!(server.flush_sessions.is_empty()); - assert!(server.dirty_sessions.is_empty()); + assert!(server.flush_players.is_empty()); + assert!(server.dirty_players.is_empty()); for player_id in 1..=2 { assert_eq!( server.db.player(player_id).unwrap().unwrap().last_seen, @@ -4067,10 +4316,10 @@ mod aoi_harden_tests { ); } server.begin_dirty_flush(); - assert_eq!(server.flush_sessions, BTreeSet::from([1, 2])); + assert_eq!(server.flush_players, BTreeSet::from([1, 2])); server.on_disconnect(1, DisconnectReason::Remote).unwrap(); - assert!(!server.flush_sessions.contains(&1)); + assert!(!server.flush_players.contains(&1)); assert!(!server.sessions.contains_key(&1)); assert!( server.db.player(1).unwrap().unwrap().last_seen > INITIAL_LAST_SEEN, @@ -4084,6 +4333,88 @@ mod aoi_harden_tests { ); } + #[test] + fn failed_disconnect_persistence_retries_the_player_id_after_peer_reuse() { + const INITIAL_LAST_SEEN: i64 = 100; + const RETRY_AT: i64 = 6_000; + + let mut server = make_server(Config::default()); + for player_id in 1..=2 { + server + .db + .upsert_player_by_token( + &format!("{player_id:016x}"), + &format!("Player {player_id}"), + INITIAL_LAST_SEEN, + ) + .unwrap(); + } + insert_dense_session_for_player(&mut server, 1, 1, Cell::new(0, 0)); + + server.inject_dirty_flush_failures(1); + let error = server + .on_disconnect(1, DisconnectReason::Remote) + .unwrap_err(); + assert!(error.to_string().contains("injected dirty flush failure")); + assert_eq!(server.flush_players, BTreeSet::from([1])); + assert!(server.flush_paused); + + insert_dense_session_for_player(&mut server, 1, 2, Cell::new(0, 0)); + for tick in 0..100 { + assert_eq!( + server.process_dirty_flush_at(1_000 + tick).unwrap(), + DirtyFlushTickWork::default(), + "a disconnect failure must not retry on every server tick" + ); + } + + server.begin_dirty_flush(); + let work = server.process_dirty_flush_at(RETRY_AT).unwrap(); + assert_eq!(work.db_updates, 1); + assert_eq!(server.db.player(1).unwrap().unwrap().last_seen, RETRY_AT); + assert_eq!( + server.db.player(2).unwrap().unwrap().last_seen, + INITIAL_LAST_SEEN, + "peer-slot reuse must not redirect orphaned persistence work" + ); + assert!(server.flush_players.is_empty()); + } + + #[test] + fn shutdown_flush_attempts_all_players_and_retries_only_failures() { + const INITIAL_LAST_SEEN: i64 = 100; + + let mut server = make_server(Config::default()); + for peer in 1..=2 { + server + .db + .upsert_player_by_token( + &format!("{peer:016x}"), + &format!("Player {peer}"), + INITIAL_LAST_SEEN, + ) + .unwrap(); + insert_dense_session(&mut server, peer, Cell::new(0, 0)); + } + server.inject_dirty_flush_failures(1); + + server.flush_all().unwrap(); + + assert_eq!( + server.dirty_flush_attempts(), + 3, + "the first pass must continue after row one fails, then retry only that row" + ); + for player_id in 1..=2 { + assert!( + server.db.player(player_id).unwrap().unwrap().last_seen > INITIAL_LAST_SEEN, + "shutdown must persist player {player_id}" + ); + } + assert!(server.dirty_players.is_empty()); + assert!(server.flush_players.is_empty()); + } + #[test] fn hostile_population_fanout_and_flush_stay_capped_and_make_progress() { const DENSE_SESSIONS: PeerId = 1_024; @@ -4097,7 +4428,7 @@ mod aoi_harden_tests { insert_dense_session(&mut server, peer, center); let session = server.sessions.get_mut(&peer).unwrap(); session.dirty = true; - server.dirty_sessions.insert(peer); + server.dirty_players.insert(session.player_id); } let seq_before_chat = server.seq; @@ -4126,6 +4457,11 @@ mod aoi_harden_tests { ); assert_eq!(server.chat_fanout.len(), CHAT_QUEUE_ITEMS); assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); + assert_eq!( + server.chat_fanout_recipients, + CHAT_QUEUE_ITEMS * DENSE_SESSIONS as usize + ); + assert!(server.chat_fanout_recipients <= CHAT_QUEUE_RECIPIENTS); let mut total_jobs_completed = 0usize; let mut total_db_updates = 0usize; @@ -4158,23 +4494,33 @@ mod aoi_harden_tests { assert!(server.chat_fanout.len() <= CHAT_QUEUE_ITEMS); assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); + assert!(server.chat_fanout_recipients <= CHAT_QUEUE_RECIPIENTS); assert!(server.clock_fanout.iter().count() <= 1); - assert!(server.dirty_sessions.len() <= server.sessions.len()); - assert!(server.flush_sessions.len() <= server.sessions.len()); assert!( - server.dirty_sessions.len() + server.flush_sessions.len() - <= server.sessions.len() * 2 + server.chat_fanout_recipients + + server + .clock_fanout + .as_ref() + .map_or(0, |job| job.recipients.len()) + <= FANOUT_QUEUE_RECIPIENTS + ); + assert!(server.dirty_players.len() <= server.cfg.max_player_rows_u32() as usize); + assert!(server.flush_players.len() <= server.cfg.max_player_rows_u32() as usize); + assert!( + server.dirty_players.len() + server.flush_players.len() + <= server.cfg.max_player_rows_u32() as usize * 2 ); } assert!(server.chat_fanout.is_empty()); assert_eq!(server.chat_fanout_bytes, 0); + assert_eq!(server.chat_fanout_recipients, 0); assert!( server.clock_fanout.is_none(), "the last coalesced clock must eventually reach its bounded audience" ); - assert!(server.dirty_sessions.is_empty()); - assert!(server.flush_sessions.is_empty()); + assert!(server.dirty_players.is_empty()); + assert!(server.flush_players.is_empty()); assert_eq!(total_db_updates, DENSE_SESSIONS as usize); assert!( total_jobs_completed >= CHAT_QUEUE_ITEMS + 3, @@ -4449,13 +4795,15 @@ mod market_dispatch_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, + chat_fanout_recipients: 0, clock_fanout: None, fanout_prefer_clock: true, - dirty_sessions: BTreeSet::new(), - flush_sessions: BTreeSet::new(), + dirty_players: BTreeSet::new(), + flush_players: BTreeSet::new(), flush_paused: false, dirty_flush_test: DirtyFlushTestHook::default(), identity_players: HashMap::new(), + next_session_generation: 0, seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -4713,13 +5061,15 @@ mod input_hardening_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, + chat_fanout_recipients: 0, clock_fanout: None, fanout_prefer_clock: true, - dirty_sessions: BTreeSet::new(), - flush_sessions: BTreeSet::new(), + dirty_players: BTreeSet::new(), + flush_players: BTreeSet::new(), flush_paused: false, dirty_flush_test: DirtyFlushTestHook::default(), identity_players: HashMap::new(), + next_session_generation: 0, seq: 0, snapshot_tick: 0, boot: Instant::now(), @@ -5560,15 +5910,25 @@ mod input_hardening_tests { } assert!(server.chat_fanout.len() <= CHAT_QUEUE_ITEMS); assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); + assert!(server.chat_fanout_recipients <= CHAT_QUEUE_RECIPIENTS); assert!(server.clock_fanout.iter().count() <= 1); - assert!(server.dirty_sessions.len() <= server.sessions.len()); - assert!(server.flush_sessions.len() <= server.sessions.len()); + assert!( + server.chat_fanout_recipients + + server + .clock_fanout + .as_ref() + .map_or(0, |job| job.recipients.len()) + <= FANOUT_QUEUE_RECIPIENTS + ); + assert!(server.dirty_players.len() <= server.cfg.max_player_rows_u32() as usize); + assert!(server.flush_players.len() <= server.cfg.max_player_rows_u32() as usize); } let elapsed = start.elapsed(); let per_tick = elapsed / TICKS; assert!(server.chat_fanout.is_empty()); assert_eq!(server.chat_fanout_bytes, 0); + assert_eq!(server.chat_fanout_recipients, 0); assert!(server.clock_fanout.is_none()); println!( From 850aced7ed940fe9420b4a66435e68737fb9777d Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Fri, 24 Jul 2026 00:15:20 -0600 Subject: [PATCH 34/36] fix(server): validate cadence and share fanout audiences Co-Authored-By: Claude --- server/crates/sw-server/src/config.rs | 194 +++++++++++++-- server/crates/sw-server/src/server.rs | 324 ++++++++++++++++++-------- 2 files changed, 406 insertions(+), 112 deletions(-) diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index 8d9fe4d..0d238a3 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -2,6 +2,7 @@ use serde::Deserialize; use std::path::Path; +use std::time::Duration; /// Upper bound on the AoI radius, in cells. A radius drives a `(2r+1)^2` block /// allocation ([`sw_world::cells_in_radius`]); bounding it here keeps that math @@ -58,10 +59,38 @@ pub(crate) const SNAPSHOT_ENTITIES_PER_PACKET: usize = 5; pub(crate) const SNAPSHOT_VISIBILITY_CEILING: usize = 15; pub(crate) const SNAPSHOT_CLIENT_EXPIRY_SECS: u64 = 5; -/// Worst-case ticks between advertisements of one elected dense player. -pub(crate) fn snapshot_recurrence_ticks(peers: u32) -> u64 { - u64::from(peers).div_ceil(SNAPSHOT_PACKETS_PER_TICK as u64) - * (SNAPSHOT_VISIBILITY_CEILING as u64).div_ceil(SNAPSHOT_ENTITIES_PER_PACKET as u64) +fn checked_div_ceil(value: u64, divisor: u64) -> Option { + value + .checked_add(divisor.checked_sub(1)?)? + .checked_div(divisor) +} + +/// Worst-case ticks between advertisements of one player in a recipient's +/// stable dense selection. +/// +/// A recipient is considered once per `ceil(peers / packets_per_tick)` round. +/// Its nominal cadence can only become due on one of those visits, so the +/// cadence is rounded up to a whole recipient round. The stable selection then +/// takes `ceil(min(peers - 1, visibility_ceiling) / entities_per_packet)` due +/// visits to repeat. Every operation is checked even though validated +/// configuration is much smaller, keeping this proof safe for hostile direct +/// callers as well as the normal validation path. +pub(crate) fn snapshot_recurrence_ticks(peers: u32, nominal_cadence_ticks: u64) -> Option { + if peers <= 1 { + return Some(0); + } + if nominal_cadence_ticks == 0 { + return None; + } + let packets_per_tick = u64::try_from(SNAPSHOT_PACKETS_PER_TICK).ok()?; + let entities_per_packet = u64::try_from(SNAPSHOT_ENTITIES_PER_PACKET).ok()?; + let visibility_ceiling = u64::try_from(SNAPSHOT_VISIBILITY_CEILING).ok()?; + let recipient_round = checked_div_ceil(u64::from(peers), packets_per_tick)?; + let visible_players = u64::from(peers.checked_sub(1)?).min(visibility_ceiling); + let stable_chunks = checked_div_ceil(visible_players, entities_per_packet)?; + let aligned_due_interval = + checked_div_ceil(nominal_cadence_ticks, recipient_round)?.checked_mul(recipient_round)?; + aligned_due_interval.checked_mul(stable_chunks) } /// Highest configurable persistent player-row ceiling. The server still uses @@ -241,9 +270,26 @@ impl Config { fn validate(&self) -> anyhow::Result<()> { if self.tick_hz == 0 { - return Err(anyhow::anyhow!("tick_hz must be > 0")); + return Err(anyhow::anyhow!("tick_hz must be in 1..={}", u8::MAX)); + } + if Duration::from_secs_f64(1.0 / f64::from(self.tick_hz)).is_zero() { + return Err(anyhow::anyhow!( + "tick_hz produces a zero fixed-tick duration" + )); } - if self.snapshot_hz == 0 || self.snapshot_hz > self.tick_hz { + if self.tick_hz > u32::from(u8::MAX) { + return Err(anyhow::anyhow!( + "tick_hz must be faithfully representable on the wire in 1..={}", + u8::MAX + )); + } + if self.snapshot_hz == 0 || self.snapshot_hz > u32::from(u8::MAX) { + return Err(anyhow::anyhow!( + "snapshot_hz must be faithfully representable on the wire in 1..={}", + u8::MAX + )); + } + if self.snapshot_hz > self.tick_hz { return Err(anyhow::anyhow!("snapshot_hz must be in 1..=tick_hz")); } if self.aoi_radius_cells == 0 || self.aoi_radius_cells > MAX_AOI_RADIUS_CELLS { @@ -278,8 +324,12 @@ impl Config { "max_transport_peers must be in 1..={MAX_TRANSPORT_PEERS}" )); } - let recurrence_ticks = snapshot_recurrence_ticks(self.max_transport_peers); - let expiry_ticks = u64::from(self.tick_hz) * SNAPSHOT_CLIENT_EXPIRY_SECS; + let recurrence_ticks = + snapshot_recurrence_ticks(self.max_transport_peers, self.ticks_per_snapshot()) + .ok_or_else(|| anyhow::anyhow!("snapshot recurrence calculation overflowed"))?; + let expiry_ticks = u64::from(self.tick_hz) + .checked_mul(SNAPSHOT_CLIENT_EXPIRY_SECS) + .ok_or_else(|| anyhow::anyhow!("snapshot expiry calculation overflowed"))?; if recurrence_ticks >= expiry_ticks { return Err(anyhow::anyhow!( "snapshot recurrence ({recurrence_ticks} ticks) must be strictly less than \ @@ -419,6 +469,17 @@ impl Config { (self.tick_hz / self.snapshot_hz).max(1) as u64 } + /// Fixed tick rate as the exact handshake representation. + pub fn tick_hz_u8(&self) -> u8 { + u8::try_from(self.tick_hz).expect("Config::validate guarantees tick_hz fits the wire") + } + + /// Snapshot rate as the exact handshake representation. + pub fn snapshot_hz_u8(&self) -> u8 { + u8::try_from(self.snapshot_hz) + .expect("Config::validate guarantees snapshot_hz fits the wire") + } + /// Number of ticks between standalone world-clock broadcasts. Uses a /// saturating multiply for parity with the other cadence accessors, so the /// `tick_hz * clock_broadcast_secs` product can never overflow even if a @@ -453,9 +514,13 @@ mod tests { fn snapshot_recurrence_is_fresh_for_every_accepted_capacity_and_tick_pair() { let default = Config::default(); default.validate().unwrap(); - assert_eq!(snapshot_recurrence_ticks(default.max_transport_peers), 96); + assert_eq!( + snapshot_recurrence_ticks(default.max_transport_peers, default.ticks_per_snapshot()), + Some(96) + ); assert!( - snapshot_recurrence_ticks(default.max_transport_peers) + snapshot_recurrence_ticks(default.max_transport_peers, default.ticks_per_snapshot()) + .unwrap() < u64::from(default.tick_hz) * SNAPSHOT_CLIENT_EXPIRY_SECS ); @@ -478,15 +543,67 @@ mod tests { ); } + fn simulate_snapshot_recurrence(peers: u32, nominal_cadence: u64) -> u64 { + if peers <= 1 { + return 0; + } + let recipient_round = u64::from(peers).div_ceil(SNAPSHOT_PACKETS_PER_TICK as u64); + let stable_chunks = u64::from(peers - 1) + .min(SNAPSHOT_VISIBILITY_CEILING as u64) + .div_ceil(SNAPSHOT_ENTITIES_PER_PACKET as u64); + let mut last_due_tick = 0; + let mut chunk = 0; + let mut first_target_tick = None; + + for recipient_visit in 0..=nominal_cadence * stable_chunks * 2 { + let visit_tick = 1 + recipient_visit * recipient_round; + if last_due_tick != 0 && visit_tick - last_due_tick < nominal_cadence { + continue; + } + last_due_tick = visit_tick; + if chunk == 0 { + if let Some(first) = first_target_tick { + return visit_tick - first; + } + first_target_tick = Some(visit_tick); + } + chunk = (chunk + 1) % stable_chunks; + } + panic!("the bounded scheduler simulation did not repeat its target chunk"); + } + #[test] - fn snapshot_recurrence_formula_tracks_the_scheduler_constants() { - for peers in [1, 32, 33, sw_net::DEFAULT_MAX_PEERS as u32] { - assert_eq!( - snapshot_recurrence_ticks(peers), - u64::from(peers).div_ceil(SNAPSHOT_PACKETS_PER_TICK as u64) - * (SNAPSHOT_VISIBILITY_CEILING as u64) - .div_ceil(SNAPSHOT_ENTITIES_PER_PACKET as u64) - ); + fn snapshot_recurrence_formula_matches_scheduler_for_all_accepted_rates_and_boundaries() { + let peer_boundaries = [1, 2, 5, 6, 16, 31, 32, 33, 64, 1_023, 1_024]; + let mut simulated = [[0u64; u8::MAX as usize + 1]; 11]; + for (peer_index, peers) in peer_boundaries.into_iter().enumerate() { + for nominal_cadence in 1..=u64::from(u8::MAX) { + simulated[peer_index][nominal_cadence as usize] = + simulate_snapshot_recurrence(peers, nominal_cadence); + } + } + + for tick_hz in 1..=u32::from(u8::MAX) { + for snapshot_hz in 1..=tick_hz { + for (peer_index, peers) in peer_boundaries.into_iter().enumerate() { + let cfg = Config { + tick_hz, + snapshot_hz, + max_transport_peers: peers, + max_transport_peers_per_ip: peers.min(32), + ..Config::default() + }; + if cfg.validate().is_err() { + continue; + } + let nominal_cadence = cfg.ticks_per_snapshot(); + assert_eq!( + snapshot_recurrence_ticks(peers, nominal_cadence), + Some(simulated[peer_index][nominal_cadence as usize]), + "tick_hz={tick_hz}, snapshot_hz={snapshot_hz}, peers={peers}" + ); + } + } } } @@ -533,9 +650,46 @@ mod tests { } #[test] - fn rejects_bad_snapshot_rate() { + fn wire_rates_are_exactly_one_through_255() { + let boundary = Config { + tick_hz: u32::from(u8::MAX), + snapshot_hz: u32::from(u8::MAX), + ..Config::default() + }; + boundary.validate().unwrap(); + + for tick_hz in [0, 256, 1_000, u32::MAX] { + let cfg = Config { + tick_hz, + snapshot_hz: 1, + ..Config::default() + }; + let error = cfg.validate().unwrap_err(); + assert!( + error.to_string().contains("tick_hz"), + "tick_hz={tick_hz} must fail as an unrepresentable wire rate: {error}" + ); + } + + for snapshot_hz in [0, 256, 1_000, u32::MAX] { + let cfg = Config { + tick_hz: u32::from(u8::MAX), + snapshot_hz, + ..Config::default() + }; + let error = cfg.validate().unwrap_err(); + assert!( + error.to_string().contains("snapshot_hz"), + "snapshot_hz={snapshot_hz} must fail as an unrepresentable wire rate: {error}" + ); + } + } + + #[test] + fn rejects_snapshot_rate_above_tick_rate() { let cfg = Config { - snapshot_hz: 100, + tick_hz: 30, + snapshot_hz: 31, ..Config::default() }; assert!(cfg.validate().is_err()); diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index e9e497e..26f097b 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -40,11 +40,8 @@ const FLUSH_INTERVAL: Duration = Duration::from_secs(5); /// Global snapshot transport and CPU budgets charged on every fixed server tick. /// -/// At the supported 1,024-session ceiling, 32 recipient visits make one round -/// in 32 ticks. A dense recipient's stable 15-player selection takes three -/// five-player packets, so every elected player recurs in at most 96 ticks -/// (3.2 seconds at the production 30 Hz), strictly before the client's -/// five-second stale-sample eviction. +/// Configuration validation derives the exact worst case from these shared +/// constants, including the nominal cadence rounded up to a recipient round. const SNAPSHOT_ENTITY_SCAN_PER_PACKET: usize = sw_net::DEFAULT_MAX_PEERS; /// Queued population fanout budgets. Chat is FIFO and rejects new accepted @@ -52,8 +49,7 @@ const SNAPSHOT_ENTITY_SCAN_PER_PACKET: usize = sw_net::DEFAULT_MAX_PEERS; /// slot. Both share this fixed per-tick transport budget. const CHAT_QUEUE_ITEMS: usize = 256; const CHAT_QUEUE_BYTES: usize = CHAT_QUEUE_ITEMS * (protocol::MTU - protocol::HEADER_SIZE); -const CHAT_QUEUE_RECIPIENTS: usize = CHAT_QUEUE_ITEMS * sw_net::DEFAULT_MAX_PEERS; -const FANOUT_QUEUE_RECIPIENTS: usize = CHAT_QUEUE_RECIPIENTS + sw_net::DEFAULT_MAX_PEERS; +const FANOUT_QUEUE_RECIPIENTS: usize = (CHAT_QUEUE_ITEMS + 1) * sw_net::DEFAULT_MAX_PEERS; const FANOUT_RECIPIENT_SCANS_PER_TICK: usize = sw_net::DEFAULT_MAX_PEERS; const FANOUT_SENDS_PER_TICK: usize = sw_net::DEFAULT_MAX_PEERS; @@ -85,10 +81,16 @@ struct CellHydration { sent_any: bool, } +#[derive(Clone, Copy)] +struct FanoutRecipient { + peer: PeerId, + generation: u64, +} + struct FanoutJob { bytes: Vec, sender_cell: Option, - recipients: Vec<(PeerId, u64)>, + recipients: Arc<[FanoutRecipient]>, next_recipient: usize, } @@ -185,8 +187,8 @@ pub struct Server { aoi_recipient_cursor: Option, chat_fanout: VecDeque, chat_fanout_bytes: usize, - chat_fanout_recipients: usize, clock_fanout: Option, + fanout_audience_cache: Option>, fanout_prefer_clock: bool, dirty_players: BTreeSet, flush_players: BTreeSet, @@ -272,8 +274,8 @@ impl Server { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, - chat_fanout_recipients: 0, clock_fanout: None, + fanout_audience_cache: None, fanout_prefer_clock: true, dirty_players: BTreeSet::new(), flush_players: BTreeSet::new(), @@ -950,23 +952,12 @@ impl Server { if bytes.len() > protocol::MTU - protocol::HEADER_SIZE { return; } - let recipients = self.capture_fanout_recipients(); - if recipients.is_empty() - || recipients.len() > CHAT_QUEUE_RECIPIENTS - self.chat_fanout_recipients - { + let Some(recipients) = self.capture_fanout_recipients() else { return; - } + }; self.chat_fanout_bytes += bytes.len(); - self.chat_fanout_recipients += recipients.len(); - debug_assert!( - self.chat_fanout_recipients - + self - .clock_fanout - .as_ref() - .map_or(0, |job| job.recipients.len()) - <= FANOUT_QUEUE_RECIPIENTS - ); + debug_assert!(self.retained_fanout_audience_entries() <= FANOUT_QUEUE_RECIPIENTS); self.chat_fanout.push_back(FanoutJob { bytes, sender_cell: Some(cell), @@ -1016,6 +1007,7 @@ impl Server { self.player_peers.insert(player_id, peer); self.player_order.insert(player_id); self.snapshot_recipients.insert(peer); + self.fanout_audience_cache = None; if dirty { self.dirty_players.insert(player_id); } @@ -1040,6 +1032,7 @@ impl Server { let mut work = RecipientIndexWork::default(); work.remove(&mut self.snapshot_recipients, peer); work.remove(&mut self.aoi_recipients, peer); + self.fanout_audience_cache = None; Some((session, work)) } @@ -1055,8 +1048,8 @@ impl Server { fn invalidate_fanout_jobs(&mut self) { self.chat_fanout.clear(); self.chat_fanout_bytes = 0; - self.chat_fanout_recipients = 0; self.clock_fanout = None; + self.fanout_audience_cache = None; } fn schedule_aoi(&mut self, peer: PeerId) { @@ -1454,12 +1447,13 @@ impl Server { fn broadcast_clock(&mut self) { let clock = self.clock_now(); let bytes = codec::world_clock(self.next_seq(), clock); - let recipients = self.capture_fanout_recipients(); - if recipients.is_empty() { - self.clock_fanout = None; + // Coalescing replaces the previous clock job, so release its captured + // audience before checking whether the current membership snapshot fits. + self.clock_fanout = None; + let Some(recipients) = self.capture_fanout_recipients() else { return; - } - debug_assert!(self.chat_fanout_recipients + recipients.len() <= FANOUT_QUEUE_RECIPIENTS); + }; + debug_assert!(self.retained_fanout_audience_entries() <= FANOUT_QUEUE_RECIPIENTS); self.clock_fanout = Some(FanoutJob { bytes, sender_cell: None, @@ -1468,16 +1462,43 @@ impl Server { }); } - fn capture_fanout_recipients(&self) -> Vec<(PeerId, u64)> { - self.snapshot_recipients + fn capture_fanout_recipients(&mut self) -> Option> { + if let Some(cached) = self.fanout_audience_cache.as_ref() { + return Some(Arc::clone(cached)); + } + let recipients: Vec = self + .snapshot_recipients .iter() .filter_map(|&peer| { - self.sessions - .get(&peer) - .map(|session| (peer, session.generation)) + self.sessions.get(&peer).map(|session| FanoutRecipient { + peer, + generation: session.generation, + }) }) .take(self.cfg.max_transport_peers_usize()) - .collect() + .collect(); + if recipients.is_empty() + || self + .retained_fanout_audience_entries() + .checked_add(recipients.len()) + .is_none_or(|entries| entries > FANOUT_QUEUE_RECIPIENTS) + { + return None; + } + let recipients = Arc::<[FanoutRecipient]>::from(recipients); + self.fanout_audience_cache = Some(Arc::clone(&recipients)); + Some(recipients) + } + + fn retained_fanout_audience_entries(&self) -> usize { + let mut seen = HashSet::new(); + self.fanout_audience_cache + .iter() + .chain(self.chat_fanout.iter().map(|job| &job.recipients)) + .chain(self.clock_fanout.iter().map(|job| &job.recipients)) + .filter(|audience| seen.insert(audience.as_ptr() as usize)) + .map(|audience| audience.len()) + .sum() } fn process_fanout_work(&mut self) -> FanoutTickWork { @@ -1509,22 +1530,22 @@ impl Server { && work.recipient_scans < recipient_scan_budget && work.sends < send_budget { - let (peer, generation) = job.recipients[job.next_recipient]; + let recipient = job.recipients[job.next_recipient]; job.next_recipient += 1; work.recipient_scans += 1; - let should_send = self.sessions.get(&peer).is_some_and(|session| { - session.generation == generation + let should_send = self.sessions.get(&recipient.peer).is_some_and(|session| { + session.generation == recipient.generation && job .sender_cell .is_none_or(|cell| session.sub.contains(cell)) }); if should_send { - self.send(peer, &job.bytes); + self.send(recipient.peer, &job.bytes); work.sends += 1; work.encoded_bytes += job.bytes.len(); #[cfg(test)] - work.delivered.push((peer, generation)); + work.delivered.push((recipient.peer, recipient.generation)); } } @@ -1534,9 +1555,6 @@ impl Server { self.fanout_prefer_clock = false; } else { self.chat_fanout_bytes = self.chat_fanout_bytes.saturating_sub(job.bytes.len()); - self.chat_fanout_recipients = self - .chat_fanout_recipients - .saturating_sub(job.recipients.len()); self.fanout_prefer_clock = true; } } else { @@ -1661,8 +1679,8 @@ impl Server { fn caps(&self) -> Caps { Caps { - tick_hz: self.cfg.tick_hz.min(u8::MAX as u32) as u8, - snapshot_hz: self.cfg.snapshot_hz.min(u8::MAX as u32) as u8, + tick_hz: self.cfg.tick_hz_u8(), + snapshot_hz: self.cfg.snapshot_hz_u8(), aoi_radius_cells: self.cfg.aoi_radius_cells.min(u8::MAX as u32) as u8, cell_size_m: self.world.grid().cell_size_m, features: FEATURES, @@ -1922,8 +1940,8 @@ mod handshake_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, - chat_fanout_recipients: 0, clock_fanout: None, + fanout_audience_cache: None, fanout_prefer_clock: true, dirty_players: BTreeSet::new(), flush_players: BTreeSet::new(), @@ -3636,8 +3654,8 @@ mod aoi_harden_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, - chat_fanout_recipients: 0, clock_fanout: None, + fanout_audience_cache: None, fanout_prefer_clock: true, dirty_players: BTreeSet::new(), flush_players: BTreeSet::new(), @@ -3848,6 +3866,108 @@ mod aoi_harden_tests { .unwrap(); } + fn retained_chat_audience_entries(server: &Server) -> usize { + let mut audiences = HashSet::new(); + server + .chat_fanout + .iter() + .filter(|job| audiences.insert(job.recipients.as_ptr() as usize)) + .map(|job| job.recipients.len()) + .sum() + } + + fn chat_bytes(text: &str) -> Vec { + let mut fbb = FlatBufferBuilder::new(); + let text = fbb.create_string(text); + let chat = p::ChatSend::create( + &mut fbb, + &p::ChatSendArgs { + text: Some(text), + channel: 0, + }, + ); + finish_envelope(&mut fbb, 1, p::Payload::ChatSend, chat.as_union_value()) + } + + #[test] + fn stable_membership_shares_one_immutable_fanout_audience() { + const LIVE_PEERS: PeerId = 32; + const CHAT_BURST: usize = 128; + + let mut server = make_server(Config { + chat_min_interval_ms: 0, + ..Config::default() + }); + let center = Cell::new(0, 0); + for peer in 1..=LIVE_PEERS { + insert_dense_session(&mut server, peer, center); + } + let bytes = chat_bytes("shared audience"); + for index in 0..CHAT_BURST { + let peer = index as PeerId % LIVE_PEERS + 1; + server + .handle_data_at(peer, &bytes, 1_000 + index as i64, 1_000 + index as i64) + .unwrap(); + } + + assert_eq!(server.chat_fanout.len(), CHAT_BURST); + assert_eq!( + retained_chat_audience_entries(&server), + LIVE_PEERS as usize, + "Arc references must count one immutable retained audience allocation" + ); + let first = server.chat_fanout.front().unwrap().recipients.as_ptr(); + assert!( + server + .chat_fanout + .iter() + .all(|job| std::ptr::eq(job.recipients.as_ptr(), first)), + "unchanged membership must reuse the exact audience allocation" + ); + } + + #[test] + fn membership_churn_retains_only_one_bounded_snapshot_per_enqueued_generation() { + const LIVE_PEERS: PeerId = 32; + const CHURNED_JOBS: usize = 64; + + let mut server = make_server(Config { + chat_min_interval_ms: 0, + ..Config::default() + }); + let center = Cell::new(0, 0); + for peer in 1..=LIVE_PEERS { + insert_dense_session(&mut server, peer, center); + } + let bytes = chat_bytes("generation snapshot"); + for generation in 0..CHURNED_JOBS { + server + .handle_data_at( + 1, + &bytes, + 2_000 + generation as i64, + 2_000 + generation as i64, + ) + .unwrap(); + let departed = server.unregister_session(LIVE_PEERS).unwrap(); + server.world.remove(departed.player_id); + insert_dense_session_for_player( + &mut server, + LIVE_PEERS, + 10_000 + generation as u64, + center, + ); + } + + let retained = retained_chat_audience_entries(&server); + assert_eq!( + retained, + LIVE_PEERS as usize * CHURNED_JOBS, + "each membership generation must retain one unique frozen audience, not one allocation per Arc reference" + ); + assert!(retained <= FANOUT_QUEUE_RECIPIENTS); + } + #[test] fn queued_fanout_never_wraps_or_delivers_to_a_reused_connection_identity() { const RECIPIENTS: PeerId = sw_net::DEFAULT_MAX_PEERS as PeerId; @@ -3948,7 +4068,7 @@ mod aoi_harden_tests { "generation wrap must invalidate all captured pre-wrap identities" ); assert_eq!(server.chat_fanout_bytes, 0); - assert_eq!(server.chat_fanout_recipients, 0); + assert_eq!(server.retained_fanout_audience_entries(), 0); } #[test] @@ -4035,7 +4155,7 @@ mod aoi_harden_tests { let exact_recurrence_bound = recipient_round_ticks * elected_packets; assert_eq!( exact_recurrence_bound as u64, - snapshot_recurrence_ticks(DENSE_SESSIONS), + snapshot_recurrence_ticks(DENSE_SESSIONS, server.cfg.ticks_per_snapshot()).unwrap(), "config validation and the production scheduler must share one recurrence proof" ); assert_eq!(exact_recurrence_bound, 96); @@ -4091,7 +4211,13 @@ mod aoi_harden_tests { #[test] fn sparse_snapshots_keep_the_configured_nominal_cadence() { - let mut server = make_server(Config::default()); + let mut server = make_server(Config { + tick_hz: 30, + snapshot_hz: 1, + max_transport_peers: 2, + max_transport_peers_per_ip: 2, + ..Config::default() + }); let center = Cell::new(0, 0); insert_dense_session(&mut server, 1, center); insert_dense_session(&mut server, 2, center); @@ -4458,10 +4584,10 @@ mod aoi_harden_tests { assert_eq!(server.chat_fanout.len(), CHAT_QUEUE_ITEMS); assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); assert_eq!( - server.chat_fanout_recipients, - CHAT_QUEUE_ITEMS * DENSE_SESSIONS as usize + server.retained_fanout_audience_entries(), + DENSE_SESSIONS as usize, + "all stable-membership jobs and the cache share one audience allocation" ); - assert!(server.chat_fanout_recipients <= CHAT_QUEUE_RECIPIENTS); let mut total_jobs_completed = 0usize; let mut total_db_updates = 0usize; @@ -4494,16 +4620,8 @@ mod aoi_harden_tests { assert!(server.chat_fanout.len() <= CHAT_QUEUE_ITEMS); assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); - assert!(server.chat_fanout_recipients <= CHAT_QUEUE_RECIPIENTS); assert!(server.clock_fanout.iter().count() <= 1); - assert!( - server.chat_fanout_recipients - + server - .clock_fanout - .as_ref() - .map_or(0, |job| job.recipients.len()) - <= FANOUT_QUEUE_RECIPIENTS - ); + assert!(server.retained_fanout_audience_entries() <= FANOUT_QUEUE_RECIPIENTS); assert!(server.dirty_players.len() <= server.cfg.max_player_rows_u32() as usize); assert!(server.flush_players.len() <= server.cfg.max_player_rows_u32() as usize); assert!( @@ -4514,7 +4632,11 @@ mod aoi_harden_tests { assert!(server.chat_fanout.is_empty()); assert_eq!(server.chat_fanout_bytes, 0); - assert_eq!(server.chat_fanout_recipients, 0); + assert_eq!( + server.retained_fanout_audience_entries(), + DENSE_SESSIONS as usize, + "the current membership cache retains one reusable bounded audience" + ); assert!( server.clock_fanout.is_none(), "the last coalesced clock must eventually reach its bounded audience" @@ -4736,11 +4858,15 @@ mod aoi_harden_tests { #[test] fn caps_advertise_configured_aoi() { let server = make_server(Config { + tick_hz: u32::from(u8::MAX), + snapshot_hz: u32::from(u8::MAX), aoi_radius_cells: 5, cell_size_m: 2048.0, ..Config::default() }); let caps = server.caps(); + assert_eq!(caps.tick_hz, u8::MAX); + assert_eq!(caps.snapshot_hz, u8::MAX); assert_eq!(caps.aoi_radius_cells, 5); assert_eq!(caps.cell_size_m, 2048.0); } @@ -4795,8 +4921,8 @@ mod market_dispatch_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, - chat_fanout_recipients: 0, clock_fanout: None, + fanout_audience_cache: None, fanout_prefer_clock: true, dirty_players: BTreeSet::new(), flush_players: BTreeSet::new(), @@ -5061,8 +5187,8 @@ mod input_hardening_tests { aoi_recipient_cursor: None, chat_fanout: VecDeque::new(), chat_fanout_bytes: 0, - chat_fanout_recipients: 0, clock_fanout: None, + fanout_audience_cache: None, fanout_prefer_clock: true, dirty_players: BTreeSet::new(), flush_players: BTreeSet::new(), @@ -5819,6 +5945,7 @@ mod input_hardening_tests { // and run only by the non-blocking load job / `make load-test`. const N: u32 = 1_024; const TICKS: u32 = 450; + const TRANSPORT_CHAT_BURST: PeerId = 128; const LOAD_EPOCH_MS: i64 = 1_700_000_000_000; let cfg = Config::default(); @@ -5850,26 +5977,38 @@ mod input_hardening_tests { assert_eq!(server.sessions.len(), N as usize); assert_eq!(server.world.len(), N as usize); - // One maximum transport poll worth of accepted dense chat fanout; the - // remaining distinct-player inputs are rejected at the fixed queue cap. - for peer in 1..=N as PeerId { - send_chat( - &mut server, - peer, - "release hostile fanout", - 100_000 + i64::from(peer), - ); - } - assert_eq!(server.chat_fanout.len(), CHAT_QUEUE_ITEMS); - assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); - // Drive TICKS simulated ticks and measure the wall-clock server work. The // simulated clock advances by a full tick each round so every client's // per-tick update clears the throttle window (worst-case load). let step_ms = tick_dt.as_millis() as i64 + 1; + let chat = chat_envelope("release transport burst"); let start = Instant::now(); + let mut maximum_tick = Duration::ZERO; + let mut burst_tick = Duration::ZERO; for t in 0..TICKS { + let tick_start = Instant::now(); let now = 10_000 + (t as i64) * step_ms; + if t == 0 { + for peer in 1..=TRANSPORT_CHAT_BURST { + server + .handle_data_at(peer, &chat, now + i64::from(peer), LOAD_EPOCH_MS) + .unwrap(); + } + assert_eq!(server.chat_fanout.len(), TRANSPORT_CHAT_BURST as usize); + assert_eq!( + server.retained_fanout_audience_entries(), + N as usize, + "the burst must retain one shared audience allocation, not 128 copies" + ); + let first = server.chat_fanout.front().unwrap().recipients.as_ptr(); + assert!( + server + .chat_fanout + .iter() + .all(|job| std::ptr::eq(job.recipients.as_ptr(), first)), + "all jobs captured under stable membership must share one audience" + ); + } for i in 0..N { let peer = (i + 1) as PeerId; let jitter = (t % 8) as f32; // small in-cell movement @@ -5910,33 +6049,34 @@ mod input_hardening_tests { } assert!(server.chat_fanout.len() <= CHAT_QUEUE_ITEMS); assert!(server.chat_fanout_bytes <= CHAT_QUEUE_BYTES); - assert!(server.chat_fanout_recipients <= CHAT_QUEUE_RECIPIENTS); assert!(server.clock_fanout.iter().count() <= 1); - assert!( - server.chat_fanout_recipients - + server - .clock_fanout - .as_ref() - .map_or(0, |job| job.recipients.len()) - <= FANOUT_QUEUE_RECIPIENTS - ); + assert!(server.retained_fanout_audience_entries() <= FANOUT_QUEUE_RECIPIENTS); assert!(server.dirty_players.len() <= server.cfg.max_player_rows_u32() as usize); assert!(server.flush_players.len() <= server.cfg.max_player_rows_u32() as usize); + + let tick_elapsed = tick_start.elapsed(); + maximum_tick = maximum_tick.max(tick_elapsed); + if t == 0 { + burst_tick = tick_elapsed; + } } let elapsed = start.elapsed(); let per_tick = elapsed / TICKS; assert!(server.chat_fanout.is_empty()); assert_eq!(server.chat_fanout_bytes, 0); - assert_eq!(server.chat_fanout_recipients, 0); + assert_eq!(server.retained_fanout_audience_entries(), N as usize); assert!(server.clock_fanout.is_none()); println!( - "load: {N} clients x {TICKS} ticks in {elapsed:?} => {per_tick:?}/tick (real-time budget {tick_dt:?})" + "load: {N} clients x {TICKS} ticks in {elapsed:?} => {per_tick:?}/tick, \ + max {maximum_tick:?}, 128-chat burst tick {burst_tick:?} \ + (real-time budget {tick_dt:?})" ); assert!( - per_tick < tick_dt, - "per-tick server work {per_tick:?} exceeded the {tick_dt:?} real-time budget at {N} clients" + maximum_tick < tick_dt, + "whole-tick server work {maximum_tick:?} (burst {burst_tick:?}) exceeded the \ + {tick_dt:?} real-time budget at {N} clients" ); } } From b6699e1eb73e33d5089bde8909489848a8cba287 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Fri, 24 Jul 2026 00:37:47 -0600 Subject: [PATCH 35/36] fix(server): preserve fanout identities across wrap Co-Authored-By: Claude --- server/crates/sw-server/src/server.rs | 157 +++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 6 deletions(-) diff --git a/server/crates/sw-server/src/server.rs b/server/crates/sw-server/src/server.rs index 26f097b..4687fc3 100644 --- a/server/crates/sw-server/src/server.rs +++ b/server/crates/sw-server/src/server.rs @@ -1000,7 +1000,7 @@ impl Server { "player already owns session peer {existing_peer}" )); } - session.generation = self.allocate_session_generation(); + session.generation = self.allocate_session_generation()?; let player_id = session.player_id; let dirty = session.dirty; self.sessions.insert(peer, session); @@ -1036,13 +1036,50 @@ impl Server { Some((session, work)) } - fn allocate_session_generation(&mut self) -> u64 { + fn allocate_session_generation(&mut self) -> anyhow::Result { if self.next_session_generation == u64::MAX { - self.invalidate_fanout_jobs(); - self.next_session_generation = 0; + self.rekey_live_session_generations()?; } - self.next_session_generation += 1; - self.next_session_generation + self.next_session_generation = self + .next_session_generation + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("session generation exhausted"))?; + Ok(self.next_session_generation) + } + + fn rekey_live_session_generations(&mut self) -> anyhow::Result<()> { + let live_session_count = self.sessions.len(); + if live_session_count > self.cfg.max_transport_peers_usize() { + return Err(anyhow::anyhow!( + "live session count exceeds configured transport limit" + )); + } + + let mut peers: Vec<_> = self.sessions.keys().copied().collect(); + peers.sort_unstable(); + let rekeyed: Vec<_> = peers + .into_iter() + .enumerate() + .map(|(index, peer)| { + let generation = u64::try_from(index) + .ok() + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| anyhow::anyhow!("live session generation overflow"))?; + Ok((peer, generation)) + }) + .collect::>()?; + let next_generation = u64::try_from(live_session_count) + .map_err(|_| anyhow::anyhow!("live session count exceeds generation range"))?; + + self.invalidate_fanout_jobs(); + for (peer, generation) in rekeyed { + self.sessions + .get_mut(&peer) + .ok_or_else(|| anyhow::anyhow!("live session disappeared during re-key"))? + .generation = generation; + } + self.next_session_generation = next_generation; + Ok(()) } fn invalidate_fanout_jobs(&mut self) { @@ -4071,6 +4108,114 @@ mod aoi_harden_tests { assert_eq!(server.retained_fanout_audience_entries(), 0); } + #[test] + fn post_wrap_fanout_never_reaches_a_replacement_with_a_colliding_generation() { + let mut server = make_server(Config { + chat_min_interval_ms: 0, + ..Config::default() + }); + let center = Cell::new(0, 0); + insert_dense_session(&mut server, 1, center); + insert_dense_session(&mut server, 2, center); + assert_eq!(server.sessions[&1].generation, 1); + assert_eq!(server.sessions[&2].generation, 2); + + server.next_session_generation = u64::MAX; + insert_dense_session(&mut server, 3, center); + + let bytes = chat_bytes("post-wrap audience"); + let envelope = decode_envelope(&bytes).unwrap(); + server.on_chat(1, envelope.payload_as_chat_send().unwrap(), 1_000); + server.broadcast_clock(); + let captured_generation = server.sessions[&2].generation; + + let departed = server.unregister_session(2).unwrap(); + server.world.remove(departed.player_id); + insert_dense_session_for_player(&mut server, 2, 22, center); + let replacement_generation = server.sessions[&2].generation; + assert_ne!( + replacement_generation, captured_generation, + "a replacement must never reuse a live identity captured after generation wrap" + ); + + let work = server.process_fanout_work_with_budget(6, 6); + assert_eq!(work.recipient_scans, 6); + assert_eq!(work.sends, 4); + assert_eq!(work.jobs_completed, 2); + let delivered_peers: Vec<_> = work + .delivered + .iter() + .map(|(peer, _generation)| *peer) + .collect(); + assert_eq!( + delivered_peers.iter().filter(|&&peer| peer == 1).count(), + 2, + "the first captured connection must receive chat and clock" + ); + assert_eq!( + delivered_peers.iter().filter(|&&peer| peer == 3).count(), + 2, + "the post-wrap connection must receive chat and clock" + ); + assert!( + delivered_peers.iter().all(|&peer| peer != 2), + "the replacement must receive neither stale chat nor stale clock" + ); + } + + #[test] + fn repeated_generation_wraps_rekey_all_1024_live_sessions_without_index_drift() { + const WRAPS: PeerId = 3; + const MAX_SESSIONS: PeerId = sw_net::DEFAULT_MAX_PEERS as PeerId; + + let mut server = make_server(Config::default()); + assert_eq!( + server.cfg.max_transport_peers_usize(), + MAX_SESSIONS as usize + ); + let center = Cell::new(0, 0); + for peer in 1..=(MAX_SESSIONS - WRAPS) { + insert_dense_session(&mut server, peer, center); + } + + for peer in (MAX_SESSIONS - WRAPS + 1)..=MAX_SESSIONS { + server.next_session_generation = u64::MAX; + insert_dense_session(&mut server, peer, center); + + let generations: BTreeSet<_> = server + .sessions + .values() + .map(|session| session.generation) + .collect(); + assert_eq!(generations.len(), server.sessions.len()); + assert!(!generations.contains(&0)); + assert_eq!(server.next_session_generation, server.sessions.len() as u64); + for expected_peer in 1..=peer { + let session = &server.sessions[&expected_peer]; + assert_eq!( + session.generation, + u64::from(expected_peer), + "wrap re-keying must be deterministic in PeerId order" + ); + assert_eq!( + server.player_peers.get(&session.player_id), + Some(&expected_peer) + ); + assert!(server.player_order.contains(&session.player_id)); + assert!(server.snapshot_recipients.contains(&expected_peer)); + assert!(server.aoi_recipients.contains(&expected_peer)); + assert_eq!(server.world.cell_of_entity(session.player_id), Some(center)); + } + } + + assert_eq!(server.sessions.len(), MAX_SESSIONS as usize); + assert_eq!(server.player_peers.len(), MAX_SESSIONS as usize); + assert_eq!(server.player_order.len(), MAX_SESSIONS as usize); + assert_eq!(server.snapshot_recipients.len(), MAX_SESSIONS as usize); + assert_eq!(server.aoi_recipients.len(), MAX_SESSIONS as usize); + assert_eq!(server.world.len(), MAX_SESSIONS as usize); + } + #[test] fn dense_snapshot_broadcast_has_a_fixed_global_packet_budget() { const DENSE_SESSIONS: PeerId = 1_024; From d16ff43a49beddd3dec98dd0c4766bbddd6560a3 Mon Sep 17 00:00:00 2001 From: aram-devdocs Date: Fri, 24 Jul 2026 01:07:07 -0600 Subject: [PATCH 36/36] test(server): exhaust snapshot recurrence proof Co-Authored-By: Claude --- server/crates/sw-server/src/config.rs | 122 ++++++++++++++++---------- 1 file changed, 77 insertions(+), 45 deletions(-) diff --git a/server/crates/sw-server/src/config.rs b/server/crates/sw-server/src/config.rs index 0d238a3..25449a6 100644 --- a/server/crates/sw-server/src/config.rs +++ b/server/crates/sw-server/src/config.rs @@ -511,7 +511,7 @@ mod tests { } #[test] - fn snapshot_recurrence_is_fresh_for_every_accepted_capacity_and_tick_pair() { + fn snapshot_recurrence_validation_matches_every_wire_rate_pair() { let default = Config::default(); default.validate().unwrap(); assert_eq!( @@ -541,70 +541,102 @@ mod tests { error.to_string().contains("snapshot recurrence"), "low-tick rejection must explain the freshness invariant: {error}" ); + + // The scheduler proof below exhausts every `(peers, nominal_cadence)` + // input. Here one peer-capacity boundary and every wire-rate pair prove + // that `Config::validate` maps `tick_hz / snapshot_hz` to that proof and + // accepts exactly the pairs whose recurrence fits before client expiry. + // Keeping these as orthogonal matrices avoids 1024 * 32,640 redundant + // configurations, or 66,846,720 duplicate formula/simulation checks. + let mut rate_pairs = 0usize; + let mut accepted_pairs = 0usize; + for tick_hz in 1..=u32::from(u8::MAX) { + for snapshot_hz in 1..=tick_hz { + rate_pairs += 1; + let cfg = Config { + tick_hz, + snapshot_hz, + max_transport_peers: MAX_TRANSPORT_PEERS, + ..Config::default() + }; + let recurrence = + snapshot_recurrence_ticks(MAX_TRANSPORT_PEERS, cfg.ticks_per_snapshot()) + .unwrap(); + let expiry = u64::from(tick_hz) * SNAPSHOT_CLIENT_EXPIRY_SECS; + let validation = cfg.validate(); + assert_eq!( + validation.is_ok(), + recurrence < expiry, + "tick_hz={tick_hz}, snapshot_hz={snapshot_hz}, \ + nominal_cadence={}, recurrence={recurrence}, expiry={expiry}, \ + validation={validation:?}", + cfg.ticks_per_snapshot() + ); + accepted_pairs += usize::from(validation.is_ok()); + } + } + assert_eq!(rate_pairs, 32_640); + assert!(accepted_pairs > 0); + assert!(accepted_pairs < rate_pairs); } fn simulate_snapshot_recurrence(peers: u32, nominal_cadence: u64) -> u64 { if peers <= 1 { return 0; } - let recipient_round = u64::from(peers).div_ceil(SNAPSHOT_PACKETS_PER_TICK as u64); - let stable_chunks = u64::from(peers - 1) - .min(SNAPSHOT_VISIBILITY_CEILING as u64) - .div_ceil(SNAPSHOT_ENTITIES_PER_PACKET as u64); - let mut last_due_tick = 0; - let mut chunk = 0; + let mut tick = 0u64; + let mut recipients_remaining = peers; + let visible_players = (peers - 1).min(SNAPSHOT_VISIBILITY_CEILING as u32); + let mut next_player = 0u32; + let mut last_due_tick = None; let mut first_target_tick = None; - for recipient_visit in 0..=nominal_cadence * stable_chunks * 2 { - let visit_tick = 1 + recipient_visit * recipient_round; - if last_due_tick != 0 && visit_tick - last_due_tick < nominal_cadence { + loop { + tick += 1; + recipients_remaining = + recipients_remaining.saturating_sub(SNAPSHOT_PACKETS_PER_TICK as u32); + if recipients_remaining != 0 { + continue; + } + recipients_remaining = peers; + + if last_due_tick.is_some_and(|last| tick - last < nominal_cadence) { continue; } - last_due_tick = visit_tick; - if chunk == 0 { + last_due_tick = Some(tick); + if next_player == 0 { if let Some(first) = first_target_tick { - return visit_tick - first; + return tick - first; } - first_target_tick = Some(visit_tick); + first_target_tick = Some(tick); + } + next_player = (next_player + SNAPSHOT_ENTITIES_PER_PACKET as u32).min(visible_players); + if next_player == visible_players { + next_player = 0; } - chunk = (chunk + 1) % stable_chunks; } - panic!("the bounded scheduler simulation did not repeat its target chunk"); } #[test] - fn snapshot_recurrence_formula_matches_scheduler_for_all_accepted_rates_and_boundaries() { - let peer_boundaries = [1, 2, 5, 6, 16, 31, 32, 33, 64, 1_023, 1_024]; - let mut simulated = [[0u64; u8::MAX as usize + 1]; 11]; - for (peer_index, peers) in peer_boundaries.into_iter().enumerate() { + fn snapshot_recurrence_formula_matches_every_scheduler_opportunity_simulation() { + let started = std::time::Instant::now(); + let mut cases = 0usize; + for peers in 1..=MAX_TRANSPORT_PEERS { for nominal_cadence in 1..=u64::from(u8::MAX) { - simulated[peer_index][nominal_cadence as usize] = - simulate_snapshot_recurrence(peers, nominal_cadence); - } - } - - for tick_hz in 1..=u32::from(u8::MAX) { - for snapshot_hz in 1..=tick_hz { - for (peer_index, peers) in peer_boundaries.into_iter().enumerate() { - let cfg = Config { - tick_hz, - snapshot_hz, - max_transport_peers: peers, - max_transport_peers_per_ip: peers.min(32), - ..Config::default() - }; - if cfg.validate().is_err() { - continue; - } - let nominal_cadence = cfg.ticks_per_snapshot(); - assert_eq!( - snapshot_recurrence_ticks(peers, nominal_cadence), - Some(simulated[peer_index][nominal_cadence as usize]), - "tick_hz={tick_hz}, snapshot_hz={snapshot_hz}, peers={peers}" - ); - } + cases += 1; + assert_eq!( + snapshot_recurrence_ticks(peers, nominal_cadence), + Some(simulate_snapshot_recurrence(peers, nominal_cadence)), + "peers={peers}, nominal_cadence={nominal_cadence}" + ); } } + let elapsed = started.elapsed(); + assert_eq!(cases, MAX_TRANSPORT_PEERS as usize * u8::MAX as usize); + assert!( + elapsed < Duration::from_secs(10), + "{cases} exhaustive scheduler cases took {elapsed:?}" + ); } #[test]