From 2cee6b6c42033f34342d22da51ad1d3a6e45f788 Mon Sep 17 00:00:00 2001 From: MDA2AV Date: Sun, 21 Jun 2026 19:02:56 +0100 Subject: [PATCH] Fix reactor crash on Connection: close with fragmented headers A request whose headers span multiple recvs followed by a server-initiated close (Connection: close) raced the connection teardown: MarkClosed() disarmed and completed the flush, then the in-flight SEND's CQE called CompleteFlush() which completed the value-task source a second time, throwing InvalidOperationException on the reactor thread and aborting the whole process. - ioxide core: CompleteFlush now only signals when it is the call that disarms the flush (Interlocked.Exchange(_flushArmed,0)==1), mirroring MarkClosed and the recv path. Prevents the double completion. - ioxide.Kestrel: HopDuplexPipe.DisposeAsync drains the send pump before MarkClosed (so the response is fully flushed and no live flush is completed twice), then half-closes (FIN) the write side so EOF-delimited Connection: close responses terminate for the client. - Bump packages to 0.0.14. --- ioxide.Kestrel/HopDuplexPipe.cs | 25 +++++++++++++++------ ioxide.Kestrel/ioxide.Kestrel.csproj | 2 +- ioxide.file/ioxide.file.csproj | 2 +- ioxide.pg/ioxide.pg.csproj | 2 +- ioxide.redis/ioxide.redis.csproj | 2 +- ioxide.tls/ioxide.tls.csproj | 2 +- ioxide/Connection/Connection.Write.Flush.cs | 11 +++++++-- ioxide/ioxide.csproj | 2 +- 8 files changed, 33 insertions(+), 15 deletions(-) diff --git a/ioxide.Kestrel/HopDuplexPipe.cs b/ioxide.Kestrel/HopDuplexPipe.cs index 1397a5f..c7d330b 100644 --- a/ioxide.Kestrel/HopDuplexPipe.cs +++ b/ioxide.Kestrel/HopDuplexPipe.cs @@ -23,6 +23,10 @@ internal sealed class HopDuplexPipe : IDuplexPipe, IAsyncDisposable private Task _sendPump = Task.CompletedTask; private int _started; + [System.Runtime.InteropServices.DllImport("libc", EntryPoint = "shutdown")] + private static extern int Shutdown(int sockfd, int how); + private const int ShutWr = 1; // SHUT_WR + public PipeReader Input => _inbound.Reader; public PipeWriter Output => _outbound.Writer; @@ -139,14 +143,21 @@ private async Task SendPumpAsync() public async ValueTask DisposeAsync() { - // Kestrel has completed its ends; wake the pumps and unwind. MarkClosed wakes a recv parked in - // conn.ReadAsync — schedule it ON the reactor so the recv continuation (which touches reactor-owned - // recv state) runs there, not on Kestrel's dispose thread. The pipe cancels resume via the pipes' - // reactor reader/writer schedulers, so they're reactor-safe too. + // Quiesce the send side BEFORE closing. Wake the send pump if it's parked on the output reader and + // await it, so any in-flight SEND completes normally and the full response is flushed. Draining here + // — rather than concurrently with MarkClosed — is what makes the response reliably reach the client + // and avoids completing a live flush twice (MarkClosed racing the SEND CQE's CompleteFlush). + _outbound.Reader.CancelPendingRead(); + try { await _sendPump.ConfigureAwait(false); } catch { } + + // Half-close the write side so EOF-delimited clients (Connection: close / upgrade) see the end of + // the response — ioxide's refcounted teardown does not FIN a server-initiated close on its own. + Shutdown(_conn.ClientFd, ShutWr); + + // Now wake and unwind the recv side. MarkClosed wakes a recv parked in conn.ReadAsync — schedule it + // on the reactor so the recv continuation (reactor-owned state) runs there, not the dispose thread. _reactor.ScheduleOnReactor(static c => ((Connection)c!).MarkClosed(), _conn); _inbound.Writer.CancelPendingFlush(); - _outbound.Reader.CancelPendingRead(); - try { await _recvPump; } catch { } - try { await _sendPump; } catch { } + try { await _recvPump.ConfigureAwait(false); } catch { } } } diff --git a/ioxide.Kestrel/ioxide.Kestrel.csproj b/ioxide.Kestrel/ioxide.Kestrel.csproj index d5f08ad..6b0a358 100644 --- a/ioxide.Kestrel/ioxide.Kestrel.csproj +++ b/ioxide.Kestrel/ioxide.Kestrel.csproj @@ -8,7 +8,7 @@ ioxide.Kestrel ioxide.Kestrel - 0.0.13 + 0.0.14 MDA2AV ASP.NET Core Kestrel transport backed by the ioxide io_uring runtime: one reactor (ring) per core, SO_REUSEPORT load-balanced, with Kestrel's HTTP request loop pinned to the reactor thread. Drop-in via UseIoxide(). MIT diff --git a/ioxide.file/ioxide.file.csproj b/ioxide.file/ioxide.file.csproj index b76e443..667ff97 100644 --- a/ioxide.file/ioxide.file.csproj +++ b/ioxide.file/ioxide.file.csproj @@ -8,7 +8,7 @@ ioxide.file ioxide.file - 0.0.13 + 0.0.14 MDA2AV File serving for the ioxide io_uring runtime: immutable asset snapshots with baked responses, pooled positional ring reads, atomic reloads. MIT diff --git a/ioxide.pg/ioxide.pg.csproj b/ioxide.pg/ioxide.pg.csproj index aa40215..1d09ffb 100644 --- a/ioxide.pg/ioxide.pg.csproj +++ b/ioxide.pg/ioxide.pg.csproj @@ -8,7 +8,7 @@ ioxide.pg ioxide.pg - 0.0.13 + 0.0.14 MDA2AV Postgres driver for the ioxide io_uring runtime: pooled ring-native connections per reactor, ring-native connect and handshake, inline completion resume. MIT diff --git a/ioxide.redis/ioxide.redis.csproj b/ioxide.redis/ioxide.redis.csproj index aa975a1..b7668de 100644 --- a/ioxide.redis/ioxide.redis.csproj +++ b/ioxide.redis/ioxide.redis.csproj @@ -8,7 +8,7 @@ ioxide.redis ioxide.redis - 0.0.13 + 0.0.14 MDA2AV Redis client for the ioxide io_uring runtime: pooled ring-native connections per reactor, full RESP2 protocol, a generic command API plus typed helpers (strings, keys, hashes, lists, sets, sorted sets, pub/sub, transactions, scripting), and pipelining. Inline completion resume. MIT diff --git a/ioxide.tls/ioxide.tls.csproj b/ioxide.tls/ioxide.tls.csproj index ffe2c56..1a677a0 100644 --- a/ioxide.tls/ioxide.tls.csproj +++ b/ioxide.tls/ioxide.tls.csproj @@ -8,7 +8,7 @@ ioxide.tls ioxide.tls - 0.0.13 + 0.0.14 MDA2AV TLS for the ioxide io_uring runtime: OpenSSL handshake driven over the ring, then kernel TLS (kTLS) transmit offload - handlers keep writing plaintext through the same connection API. Requires Linux kTLS (tls module) and OpenSSL 3. MIT diff --git a/ioxide/Connection/Connection.Write.Flush.cs b/ioxide/Connection/Connection.Write.Flush.cs index 8b3b6df..e53f300 100644 --- a/ioxide/Connection/Connection.Write.Flush.cs +++ b/ioxide/Connection/Connection.Write.Flush.cs @@ -89,9 +89,16 @@ internal void CompleteFlush() WriteInFlight = 0; ZcNotifPending = 0; Volatile.Write(ref _flushInProgress, 0); - Interlocked.Exchange(ref _flushArmed, 0); - _flushSignal.SetResult(true); + // Guard against a double completion. During teardown MarkClosed() may have already disarmed and + // completed this flush (e.g. a Connection: close response whose SEND CQE lands after the close), + // which Resets/invalidates the value-task source. Only the call that actually disarms the flush + // signals — mirroring MarkClosed and the recv path's _armed check. Without this, the late CQE's + // SetResult throws InvalidOperationException on the reactor thread and crashes the process. + if (Interlocked.Exchange(ref _flushArmed, 0) == 1) + { + _flushSignal.SetResult(true); + } } #region IValueTaskSource diff --git a/ioxide/ioxide.csproj b/ioxide/ioxide.csproj index 17ab46e..c1d0bfa 100644 --- a/ioxide/ioxide.csproj +++ b/ioxide/ioxide.csproj @@ -8,7 +8,7 @@ ioxide ioxide - 0.0.13 + 0.0.14 MDA2AV A shared-nothing io_uring runtime for .NET: one ring per reactor thread, inline completions, zero native dependencies. The engine - reactor, connection, and the IRingHost client seam. MIT