ioxide.Kestrel: TLS termination via kTLS (HTTP/1.1)

ioxide.Kestrel/HopDuplexPipe.cs

@@ -1,6 +1,7 @@
 using System.Buffers;
 using System.IO.Pipelines;
 using ioxide;
+using ioxide.tls;
 using ioxide.utils;
 
 namespace ioxide.Kestrel;
@@ -16,6 +17,7 @@ internal sealed class HopDuplexPipe : IDuplexPipe, IAsyncDisposable
 {
     private readonly Connection _conn;
     private readonly Reactor _reactor;
+    private readonly TlsSession? _tls;   // non-null on a kTLS-terminated connection: inbound is decrypted here
     private readonly Pipe _inbound;    // recv pump writes; Kestrel reads (Transport.Input)
     private readonly Pipe _outbound;   // Kestrel writes (Transport.Output); send pump reads
 
@@ -30,10 +32,11 @@ internal sealed class HopDuplexPipe : IDuplexPipe, IAsyncDisposable
     public PipeReader Input => _inbound.Reader;
     public PipeWriter Output => _outbound.Writer;
 
-    public HopDuplexPipe(Connection conn, Reactor reactor)
+    public HopDuplexPipe(Connection conn, Reactor reactor, TlsSession? tls = null)
     {
         _conn = conn;
         _reactor = reactor;
+        _tls = tls;
         var scheduler = new ReactorPipeScheduler(reactor);
 
         // Reader schedulers = the reactor: Kestrel's HTTP parse (inbound reader) and the send pump
@@ -64,12 +67,29 @@ public void Start()
         _sendPump = SendPumpAsync();
     }
 
-    // Reactor → inbound pipe. Copies each recv slice into the pipe and flushes; Kestrel reads it.
+    // Reactor → inbound pipe. Copies (or, for kTLS, decrypts) each recv slice into the pipe and flushes;
+    // Kestrel reads it. kTLS has no RX offload, so inbound stays ciphertext and is decrypted here.
     private async Task RecvPumpAsync()
     {
         PipeWriter writer = _inbound.Writer;
         try
         {
+            // TLS: the client's first request can ride in bundled with its Finished flight (already
+            // decrypted during the handshake). Hand it to Kestrel before the first recv.
+            if (_tls is not null)
+            {
+                ReadOnlySpan<byte> initial = _tls.DrainPlaintext();
+                if (!initial.IsEmpty)
+                {
+                    writer.Write(initial);
+                    FlushResult ifr = await writer.FlushAsync();
+                    if (ifr.IsCompleted || ifr.IsCanceled)
+                    {
+                        return;
+                    }
+                }
+            }
+
             while (true)
             {
                 RecvSnapshot snap = await _conn.ReadAsync();
@@ -78,13 +98,21 @@ private async Task RecvPumpAsync()
                 {
                     if (item.HasBuffer && item.Len > 0)
                     {
-                        CopySlice(in item, writer);
+                        if (_tls is null)
+                        {
+                            CopySlice(in item, writer);
+                        }
+                        else
+                        {
+                            DecryptSlice(in item, writer, _tls);
+                        }
                     }
                     _conn.ReturnBuffer(in item);
                 }
                 _conn.ResetRead();
 
-                if (snap.IsClosed)
+                // Peer close: a TCP FIN, or (TLS) a clean close_notify decoded by the session.
+                if (snap.IsClosed || (_tls is not null && _tls.Closed))
                 {
                     break;
                 }
@@ -107,6 +135,16 @@ private static unsafe void CopySlice(in SpscRecvRing.Item item, PipeWriter write
         writer.Advance(item.Len);
     }
 
+    // kTLS RX stays in userspace: feed the ciphertext slice through OpenSSL and write the plaintext.
+    private static unsafe void DecryptSlice(in SpscRecvRing.Item item, PipeWriter writer, TlsSession tls)
+    {
+        ReadOnlySpan<byte> plain = tls.Decrypt(item.Ptr, item.Len);
+        if (!plain.IsEmpty)
+        {
+            writer.Write(plain);
+        }
+    }
+
     // Outbound pipe → connection send. Drains Kestrel's response into the slab and submits one SEND.
     private async Task SendPumpAsync()
     {
@@ -150,14 +188,27 @@ public async ValueTask DisposeAsync()
         _outbound.Reader.CancelPendingRead();
         try { await _sendPump.ConfigureAwait(false); } catch { }
 
-        // Half-close the write side so EOF-delimited clients (Connection: close / upgrade) see the end of
-        // the response — ioxide's refcounted teardown does not FIN a server-initiated close on its own.
-        Shutdown(_conn.ClientFd, ShutWr);
-
-        // Now wake and unwind the recv side. MarkClosed wakes a recv parked in conn.ReadAsync — schedule it
-        // on the reactor so the recv continuation (reactor-owned state) runs there, not the dispose thread.
-        _reactor.ScheduleOnReactor(static c => ((Connection)c!).MarkClosed(), _conn);
-        _inbound.Writer.CancelPendingFlush();
-        try { await _recvPump.ConfigureAwait(false); } catch { }
+        if (_tls is null)
+        {
+            // Plaintext: half-close the write side so EOF-delimited clients (Connection: close / upgrade)
+            // see the end of the response — ioxide's refcounted teardown does not FIN a server-initiated
+            // close on its own. Then wake and unwind the recv side (MarkClosed wakes a recv parked in
+            // conn.ReadAsync — schedule it on the reactor so the continuation runs there, not the dispose thread).
+            Shutdown(_conn.ClientFd, ShutWr);
+            _reactor.ScheduleOnReactor(static c => ((Connection)c!).MarkClosed(), _conn);
+            _inbound.Writer.CancelPendingFlush();
+            try { await _recvPump.ConfigureAwait(false); } catch { }
+        }
+        else
+        {
+            // TLS: unwind the recv side FIRST so the recv pump stops touching the session, then dispose it
+            // — that sends close_notify (a raw kTLS control send, which must precede the FIN while the write
+            // side is still open) and frees the SSL — then FIN.
+            _reactor.ScheduleOnReactor(static c => ((Connection)c!).MarkClosed(), _conn);
+            _inbound.Writer.CancelPendingFlush();
+            try { await _recvPump.ConfigureAwait(false); } catch { }
+            _tls.Dispose();
+            Shutdown(_conn.ClientFd, ShutWr);
+        }
     }
 }

ioxide.Kestrel/IoxideConnectionContext.cs

@@ -1,9 +1,12 @@
 using System.IO.Pipelines;
 using System.Net;
+using System.Text;
 using Microsoft.AspNetCore.Connections;
 using Microsoft.AspNetCore.Connections.Features;
 using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Features;
 using ioxide;
+using ioxide.tls;
 
 namespace ioxide.Kestrel;
 
@@ -29,9 +32,9 @@ internal sealed class IoxideConnectionContext : ConnectionContext,
 
     private int _disposed;
 
-    public IoxideConnectionContext(Connection connection, Reactor reactor, EndPoint localEndPoint, long id)
+    public IoxideConnectionContext(Connection connection, Reactor reactor, EndPoint localEndPoint, long id, TlsSession? session = null, string? alpn = null)
     {
-        _pipe = new HopDuplexPipe(connection, reactor);
+        _pipe = new HopDuplexPipe(connection, reactor, session);
 
         ConnectionId = $"ioxide-{id:x}";
         LocalEndPoint = localEndPoint;
@@ -44,6 +47,16 @@ public IoxideConnectionContext(Connection connection, Reactor reactor, EndPoint
         _features.Set<IConnectionItemsFeature>(this);
         _features.Set<IConnectionLifetimeFeature>(this);
         _features.Set<IConnectionEndPointFeature>(this);
+
+        if (session is not null)
+        {
+            // TLS terminated in the transport (kTLS): present the connection to Kestrel as HTTPS with the
+            // negotiated ALPN, in place of UseHttps()/SslStream.
+            var tlsFeature = new IoxideTlsFeature(Encoding.ASCII.GetBytes(alpn ?? "http/1.1"));
+            _features.Set<ITlsConnectionFeature>(tlsFeature);
+            _features.Set<ITlsHandshakeFeature>(tlsFeature);
+            _features.Set<ITlsApplicationProtocolFeature>(tlsFeature);
+        }
     }
 
     /// <summary>Resolves once Kestrel has finished with this connection; the reactor's Handle callback awaits it.</summary>

ioxide.Kestrel/IoxideConnectionListener.cs

@@ -3,6 +3,7 @@
 using Microsoft.AspNetCore.Connections;
 using Microsoft.Extensions.Logging;
 using ioxide;
+using ioxide.tls;
 
 namespace ioxide.Kestrel;
 
@@ -19,15 +20,17 @@ internal sealed class IoxideConnectionListener : IConnectionListener
     private readonly Channel<ConnectionContext> _accepted;
     private readonly Reactor[] _reactors;
     private readonly Thread[] _threads;
+    private readonly TlsOptions? _tlsOptions;
     private long _connectionCounter;
     private int _stopped;
 
     public EndPoint EndPoint { get; }
 
-    public IoxideConnectionListener(IPEndPoint endpoint, IoxideTransportOptions options, ILogger<IoxideConnectionListener> logger)
+    public IoxideConnectionListener(IPEndPoint endpoint, IoxideTransportOptions options, TlsOptions? tlsOptions, ILogger<IoxideConnectionListener> logger)
     {
         EndPoint = endpoint;
         _logger = logger;
+        _tlsOptions = tlsOptions;
 
         _accepted = Channel.CreateUnbounded<ConnectionContext>(new UnboundedChannelOptions
         {
@@ -54,6 +57,12 @@ public IoxideConnectionListener(IPEndPoint endpoint, IoxideTransportOptions opti
             {
                 Handle = HandleConnectionAsync,
             };
+            if (_tlsOptions is not null)
+            {
+                // Start the per-reactor TLS service on the reactor's own thread (OpenSSL ctx + cert load).
+                var tls = _tlsOptions;
+                reactor.OnStart = r => TlsService.Start(r, tls);
+            }
             _reactors[i] = reactor;
             _threads[i] = new Thread(reactor.Run)
             {
@@ -74,7 +83,26 @@ public IoxideConnectionListener(IPEndPoint endpoint, IoxideTransportOptions opti
     private async Task HandleConnectionAsync(Reactor reactor, Connection conn)
     {
         var id = Interlocked.Increment(ref _connectionCounter);
-        var ctx = new IoxideConnectionContext(conn, reactor, EndPoint, id);
+
+        // kTLS handshake (TLS endpoints only) runs here on the reactor thread, before Kestrel sees the
+        // connection. It awaits the ring recv/send and resumes inline. On success the kernel does TX
+        // encryption from here on; the returned session decrypts inbound records — Kestrel gets plaintext.
+        TlsSession? session = null;
+        if (_tlsOptions is not null)
+        {
+            try
+            {
+                session = await reactor.GetService<TlsService>().AcceptAsync(conn).ConfigureAwait(false);
+            }
+            catch
+            {
+                // Handshake failed (bad client, peer closed mid-handshake): drop the connection.
+                conn.DecRef();
+                return;
+            }
+        }
+
+        var ctx = new IoxideConnectionContext(conn, reactor, EndPoint, id, session, _tlsOptions?.Alpn);
 
         if (!_accepted.Writer.TryWrite(ctx))
         {

ioxide.Kestrel/IoxideTlsFeature.cs

@@ -0,0 +1,47 @@
+using System.Net.Security;
+using System.Security.Authentication;
+using System.Security.Cryptography.X509Certificates;
+using Microsoft.AspNetCore.Connections.Features;
+using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Features;
+
+namespace ioxide.Kestrel;
+
+/// <summary>
+/// The TLS connection features Kestrel reads when a connection is already TLS-terminated by the transport
+/// (kTLS via ioxide.tls) instead of by <c>UseHttps()</c>/SslStream. Setting these on the
+/// <see cref="IoxideConnectionContext"/> makes Kestrel treat the connection as HTTPS (scheme, IsHttps) and
+/// pick the application protocol from ALPN. Values are fixed by ioxide.tls (TLS 1.3, AES-128-GCM-SHA256,
+/// no client certificates / mTLS).
+/// </summary>
+internal sealed class IoxideTlsFeature : ITlsConnectionFeature, ITlsHandshakeFeature, ITlsApplicationProtocolFeature
+{
+    public IoxideTlsFeature(ReadOnlyMemory<byte> applicationProtocol)
+    {
+        ApplicationProtocol = applicationProtocol;
+    }
+
+    // ITlsApplicationProtocolFeature — the negotiated ALPN (HTTP/1.1 in Phase 1).
+    public ReadOnlyMemory<byte> ApplicationProtocol { get; }
+
+    // ITlsConnectionFeature — ioxide.tls never requests a client certificate.
+    public X509Certificate2? ClientCertificate { get; set; }
+
+    public Task<X509Certificate2?> GetClientCertificateAsync(CancellationToken cancellationToken)
+        => Task.FromResult(ClientCertificate);
+
+    // ITlsHandshakeFeature — fixed by ioxide.tls's single TLS 1.3 ciphersuite. NegotiatedCipherSuite is the
+    // modern accessor; the legacy CipherAlgorithm/HashAlgorithm/KeyExchangeAlgorithm (+ *Strength) properties
+    // are obsolete (SYSLIB0058) but still required interface members, so implement and suppress the warning.
+    public SslProtocols Protocol => SslProtocols.Tls13;
+    public TlsCipherSuite NegotiatedCipherSuite => TlsCipherSuite.TLS_AES_128_GCM_SHA256;
+    public string HostName => string.Empty;
+#pragma warning disable SYSLIB0058 // legacy ITlsHandshakeFeature cipher properties are obsolete but required
+    public CipherAlgorithmType CipherAlgorithm => CipherAlgorithmType.Aes128;
+    public int CipherStrength => 128;
+    public HashAlgorithmType HashAlgorithm => HashAlgorithmType.Sha256;
+    public int HashStrength => 256;
+    public ExchangeAlgorithmType KeyExchangeAlgorithm => ExchangeAlgorithmType.None;
+    public int KeyExchangeStrength => 0;
+#pragma warning restore SYSLIB0058
+}

ioxide.Kestrel/IoxideTransportFactory.cs

@@ -27,8 +27,20 @@ public ValueTask<IConnectionListener> BindAsync(EndPoint endpoint, CancellationT
             throw new NotSupportedException($"ioxide transport only supports {nameof(IPEndPoint)} (got {endpoint.GetType().Name}).");
         }
 
+        // If this port is configured for kTLS, build the ioxide.tls options the listener hands to its reactors.
+        ioxide.tls.TlsOptions? tlsOptions = null;
+        if (_options.Tls is { } tls && tls.Ports.Contains(ipEndpoint.Port))
+        {
+            tlsOptions = new ioxide.tls.TlsOptions
+            {
+                CertificatePath = tls.CertificatePath,
+                KeyPath = tls.KeyPath,
+                Alpn = tls.Alpn,
+            };
+        }
+
         var logger = _loggerFactory.CreateLogger<IoxideConnectionListener>();
-        IConnectionListener listener = new IoxideConnectionListener(ipEndpoint, _options, logger);
+        IConnectionListener listener = new IoxideConnectionListener(ipEndpoint, _options, tlsOptions, logger);
         return ValueTask.FromResult(listener);
     }
 }

ioxide.Kestrel/IoxideTransportOptions.cs

@@ -17,4 +17,48 @@ public sealed class IoxideTransportOptions
     /// overridden afterwards by the bound endpoint and <see cref="ReactorCount"/>.
     /// </summary>
     public Func<ServerConfig, ServerConfig>? ConfigureServer { get; set; }
+
+    /// <summary>
+    /// TLS termination via kTLS (kernel TLS), done in the transport on the listed ports. When set, the
+    /// reactor runs the TLS 1.3 handshake on accept, installs kTLS TX, and hands Kestrel a plaintext
+    /// connection with the TLS connection features set — so the endpoint must NOT use <c>UseHttps()</c>.
+    /// Null = no TLS (every port is plaintext). Currently HTTP/1.1 only (see <see cref="IoxideTlsOptions.Alpn"/>).
+    /// </summary>
+    public IoxideTlsOptions? Tls { get; set; }
+
+    /// <summary>
+    /// Convenience over assigning <see cref="Tls"/>: terminate kTLS on <paramref name="ports"/> with one
+    /// certificate/key. Example: <c>o.UseTls("/certs/server.crt", "/certs/server.key", new[] { 8081 });</c>.
+    /// </summary>
+    public void UseTls(string certificatePath, string keyPath, IEnumerable<int> ports, string alpn = "http/1.1")
+    {
+        var tls = new IoxideTlsOptions
+        {
+            CertificatePath = certificatePath,
+            KeyPath = keyPath,
+            Alpn = alpn,
+        };
+        foreach (var p in ports)
+        {
+            tls.Ports.Add(p);
+        }
+        Tls = tls;
+    }
+}
+
+/// <summary>kTLS termination settings for the ioxide Kestrel transport.</summary>
+public sealed class IoxideTlsOptions
+{
+    /// <summary>PEM certificate chain file.</summary>
+    public required string CertificatePath { get; set; }
+
+    /// <summary>PEM private key file.</summary>
+    public required string KeyPath { get; set; }
+
+    /// <summary>The single ALPN protocol to advertise/select (HTTP/1.1 for now). HTTP/2-over-TLS needs
+    /// dynamic ALPN, which is not yet exposed by ioxide.tls.</summary>
+    public string Alpn { get; set; } = "http/1.1";
+
+    /// <summary>Listen ports that terminate TLS in the transport. Connections on other ports stay plaintext.</summary>
+    public HashSet<int> Ports { get; set; } = new();
 }

ioxide.Kestrel/ioxide.Kestrel.csproj

@@ -8,7 +8,7 @@
     <RootNamespace>ioxide.Kestrel</RootNamespace>
 
     <PackageId>ioxide.Kestrel</PackageId>
-    <Version>0.0.14</Version>
+    <Version>0.0.15</Version>
     <Authors>MDA2AV</Authors>
     <Description>ASP.NET Core Kestrel transport backed by the ioxide io_uring runtime: one reactor (ring) per core, SO_REUSEPORT load-balanced, with Kestrel's HTTP request loop pinned to the reactor thread. Drop-in via UseIoxide().</Description>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
@@ -26,6 +26,7 @@
   <ItemGroup>
     <None Include="README.md" Pack="true" PackagePath="/" />
     <ProjectReference Include="../ioxide/ioxide.csproj" />
+    <ProjectReference Include="../ioxide.tls/ioxide.tls.csproj" />
   </ItemGroup>
 
 </Project>

ioxide.file/ioxide.file.csproj

@@ -8,7 +8,7 @@
     <RootNamespace>ioxide.file</RootNamespace>
 
     <PackageId>ioxide.file</PackageId>
-    <Version>0.0.14</Version>
+    <Version>0.0.15</Version>
     <Authors>MDA2AV</Authors>
     <Description>File serving for the ioxide io_uring runtime: immutable asset snapshots with baked responses, pooled positional ring reads, atomic reloads.</Description>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>