From a625f32a596efa995302dc7bb1a23e04f92743cc Mon Sep 17 00:00:00 2001 From: Migorithm Date: Mon, 27 Jul 2026 23:46:18 +0400 Subject: [PATCH 01/56] feat: config added for secure mode - Secure is the default. - Trusted development requires --security-mode trusted-development or SECURITY_MODE=trusted-development. - Integration environments opt in explicitly. - The example YAML documents the mode. --- eastguard.yaml.example | 4 ++++ src/config.rs | 26 +++++++++++++++++++++++++- src/it/helpers.rs | 3 ++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/eastguard.yaml.example b/eastguard.yaml.example index ef670971..bc18c7db 100644 --- a/eastguard.yaml.example +++ b/eastguard.yaml.example @@ -1,5 +1,9 @@ # EastGuard Configuration +# Secure is the production default. Use trusted-development only on isolated +# development and test networks until secure transports are configured. +security_mode: "secure" + # Sockets and networking client_port: 2921 cluster_port: 2922 diff --git a/src/config.rs b/src/config.rs index 11cd88ae..7e1e041f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, LazyLock}; use std::fs::{self, OpenOptions}; -use clap::Parser; +use clap::{Parser, ValueEnum}; use uuid::Uuid; use crate::control_plane::membership::peer_discovery::JoinAttempt; @@ -15,9 +15,20 @@ use crate::data_plane::sparse_index::SparseIndex; use crate::schedulers::ticker::TICK_PERIOD_100_MS; pub static ENV: LazyLock = LazyLock::new(Environment::init); +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum SecurityMode { + Secure, + TrustedDevelopment, +} + #[derive(Parser, Debug, Clone)] #[command(version, about, long_about = None)] pub struct Environment { + /// Secure mode requires authenticated encrypted transports. Trusted-development + /// mode keeps the existing plaintext protocols for isolated tests and local work. + #[arg(long, env = "SECURITY_MODE", value_enum, default_value = "secure")] + pub security_mode: SecurityMode, + #[arg(long, env = "CONFIG_DIR", default_value = "./eastguard/config")] pub config_dir: String, @@ -428,6 +439,7 @@ mod tests { fn make_env() -> Environment { Environment { + security_mode: SecurityMode::TrustedDevelopment, config_dir: "./eastguard/config".to_string(), config_file: None, data_dir: "./eastguard/data".to_string(), @@ -494,6 +506,18 @@ mod tests { assert_eq!(env.host, "0.0.0.0"); assert_eq!(env.data_dir, "/tmp/test"); assert_eq!(env.vnodes_per_node, 8); + assert_eq!(env.security_mode, SecurityMode::Secure); + } + + #[test] + fn security_mode_requires_explicit_trusted_development_opt_in() { + let secure = Environment::try_parse_from(["eastguard"]).unwrap(); + let trusted = + Environment::try_parse_from(["eastguard", "--security-mode", "trusted-development"]) + .unwrap(); + + assert_eq!(secure.security_mode, SecurityMode::Secure); + assert_eq!(trusted.security_mode, SecurityMode::TrustedDevelopment); } #[test] diff --git a/src/it/helpers.rs b/src/it/helpers.rs index 9ad2cc3c..4fb89b85 100644 --- a/src/it/helpers.rs +++ b/src/it/helpers.rs @@ -1,5 +1,5 @@ use crate::client::ClientSuccess; -use crate::config::Environment; +use crate::config::{Environment, SecurityMode}; use crate::connections::protocol::{AdminRequest, ClientRequest, ClientResponse, NodeState}; use crate::connections::reader::ClientStreamReader; use crate::connections::writer::ClientRawWriter; @@ -9,6 +9,7 @@ use crate::net::TcpStream; pub fn default_env(idx: u32, node_id: String, client_port: u16, cluster_port: u16) -> Environment { Environment { + security_mode: SecurityMode::TrustedDevelopment, config_dir: std::env::temp_dir() .join(format!("eastguard-config-{}-{}", idx, uuid::Uuid::new_v4())) .to_string_lossy() From dd5b0a46fad4c9b30240ebd0a71daed9b7771bf7 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 00:08:58 +0400 Subject: [PATCH 02/56] security config --- Cargo.lock | 217 +++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 + docs/security/roadmap.md | 40 ++++++-- eastguard.yaml.example | 3 + src/config.rs | 15 +++ src/it/helpers.rs | 3 + src/lib.rs | 1 + src/security.rs | 137 ++++++++++++++++++++++++ 8 files changed, 396 insertions(+), 22 deletions(-) create mode 100644 src/security.rs diff --git a/Cargo.lock b/Cargo.lock index e4634bd1..636e2bf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -58,7 +58,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -82,6 +82,29 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -256,6 +279,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -300,6 +332,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "east-guard" version = "0.1.0" @@ -318,6 +356,8 @@ dependencies = [ "murmur3", "rand 0.10.1", "rocksdb", + "rustls", + "rustls-pemfile", "rustyline", "serde", "serde_json", @@ -359,7 +399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -401,6 +441,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -567,7 +613,7 @@ version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -758,7 +804,7 @@ checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -804,7 +850,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1020,6 +1066,20 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rocksdb" version = "0.24.0" @@ -1046,7 +1106,51 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1074,7 +1178,7 @@ dependencies = [ "unicode-segmentation", "unicode-width", "utf8parse", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1243,7 +1347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1261,6 +1365,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -1282,7 +1392,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1328,7 +1438,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1484,6 +1594,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1624,6 +1740,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1633,6 +1758,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "1.0.3" @@ -1750,6 +1939,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index a36c9b8e..5062736b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,8 @@ serde_yaml = "0.9.34" serde = "1" serde_json = "1" futures = "0.3" +rustls = { version = "0.23.42", default-features = false, features = ["aws_lc_rs", "std"] } +rustls-pemfile = "2.2.0" [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index c9525ce8..74962767 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -10,7 +10,7 @@ EastGuard operates in two distinct security modes: -- **Secure Mode (Default):** Mutual TLS 1.3 / DTLS 1.3 enforced across all listeners. Unauthenticated or unauthorized traffic is immediately rejected. Plaintext connections and protocol downgrades are forbidden; invalid configuration prevents startup. +- **Secure Mode (Default):** Mutual TLS 1.3 protects TCP listeners, and QUIC v1 datagrams protect SWIM UDP. Unauthenticated or unauthorized traffic is immediately rejected. Plaintext connections and protocol downgrades are forbidden; invalid configuration prevents startup. - **Trusted Development Mode:** Plaintext protocols enabled strictly via explicit opt-in configuration for isolated test environments. ### Threat Model @@ -25,7 +25,7 @@ The production boundary defends against external network attackers attempting ea | **Client** | TCP 2921 | TLS 1.3 | Mutual X.509 | Metadata queries, administration, produce, fetch | | **Raft** | TCP 2922 | TLS 1.3 | Mutual X.509 | Metadata shard consensus log replication | | **Data** | TCP 2923 | TLS 1.3 | Mutual X.509 | Segment replication, repair, and coordination | -| **SWIM** | UDP 2922 | DTLS 1.3 | Mutual X.509 | Membership gossip & failure detection | +| **SWIM** | UDP 2922 | QUIC v1 datagrams (TLS 1.3) | Mutual X.509 | Membership gossip and failure detection | --- @@ -34,7 +34,7 @@ The production boundary defends against external network attackers attempting ea Security checks are split between the transport layer and application state machines to keep state machines free of security I/O: ``` - [ TLS 1.3 / DTLS 1.3 Transport Layer ] + [ TLS 1.3 / QUIC v1 Transport Layer ] - Authenticate peer X.509 certificates - Enforce framing, datagram MTU, and resource limits - Bind connection envelope sender to verified identity @@ -116,9 +116,9 @@ SWIM liveness gossip is decoupled from cluster admission authority to prevent ne [ Raft Reconciliation ] ──► Commit AddPeer / RemovePeer ``` -1. **Process Proof:** A node connection proves possession of the process private key bound to its admitted epoch. The process key signs the admitted identity and a connection-specific value produced by TLS or DTLS, so the proof cannot be replayed on another connection. The reusable node certificate alone cannot create or claim a newer epoch. +1. **Process Proof:** A node connection proves possession of the process private key bound to its admitted epoch. The process key signs the admitted identity and a connection-specific value produced by TLS or QUIC, so the proof cannot be replayed on another connection. The reusable node certificate alone cannot create or claim a newer epoch. 2. **Admission Gate:** The SWIM actor checks incoming packets against a local admission cache backed by metadata Raft before passing facts to the SWIM state machine. -3. **Gossip Rule:** DTLS authenticates the immediate sender. Every relayed membership fact is separately accepted only when its subject `NodeId` and `Admission Epoch` match an active admission record. +3. **Gossip Rule:** The QUIC connection authenticates the immediate sender. Every relayed membership fact is separately accepted only when its subject `NodeId` and `Admission Epoch` match an active admission record. 4. **Cache Policy:** Admission records are cached locally with a maximum TTL of 60 seconds. If a cache entry expires while the owning metadata shard is unreachable, the gate fails closed. --- @@ -178,7 +178,7 @@ Security records (`security/node/{id}`, `security/acl/{resource}`, `security/rev ### Online Credential Rotation - **Zero-Downtime CA Rotation:** Brokers support dual trust chain loading. New root CAs can be added and leaf certificates reloaded online without restarting brokers or changing `Admission Epoch` / `NodeId`. -- **Revocation & Expiry:** Certificate revocations commit to metadata Raft records. Active TLS/DTLS sessions are terminated within the cache enforcement window. Expired certificates are rejected with clock-skew tolerance. +- **Revocation & Expiry:** Certificate revocations commit to metadata Raft records. Active TLS and QUIC connections are terminated within the cache enforcement window. Expired certificates are rejected with clock-skew tolerance. - **Recovery:** Runbooks cover lost authorized-operator access, lost issuing keys, expiry, accidental revocation, trust-root replacement, and cold-cluster restart. --- @@ -187,7 +187,24 @@ Security records (`security/node/{id}`, `security/acl/{resource}`, `security/rev ### Rate & Memory Bounds - Every listener enforces strict limits on unauthenticated handshakes, concurrent connections, in-flight frames, memory allocations, and per-source request rates. -- **DTLS Datagram Size:** Encrypted SWIM datagram payloads are capped at **1200 bytes** to ensure the total packet fits within the 1280-byte minimum IPv6 MTU without IP fragmentation. +- **QUIC Datagram Size:** QUIC UDP payloads are capped at **1200 bytes**, which fits within the 1280-byte minimum IPv6 MTU after IPv6 and UDP headers. The SWIM payload budget is what remains after QUIC packet protection and framing. + +### Deterministic Simulation + +Secure SWIM must run under EastGuard's turmoil-based deterministic simulation +tests. Quinn's abstract UDP socket and runtime boundaries must use +`crate::net::UdpSocket` and Tokio's virtualized clock in test builds: + +``` +production deterministic test +real UDP + real Tokio time turmoil UDP + virtual Tokio time + \ / + same Quinn protocol +``` + +A production-only QUIC path does not pass S0. Feasibility requires a multi-node +turmoil test that completes mutual authentication and exchanges QUIC datagrams +under virtual time. ### Audit Subsystem - **Non-Blocking Execution:** Security audit events (authentication success/failure, ACL denials, admissions) are queued asynchronously. Audit backpressure never blocks protocol execution or consensus. @@ -201,15 +218,15 @@ Security records (`security/node/{id}`, `security/acl/{resource}`, `security/rev ``` S0 ──► S1 ──► S2 ──► S3 ──► S4 ──► S5 ──► S6 config records TCP SWIM clients operations production - mTLS DTLS + ACLs + audit gate + mTLS QUIC + ACLs + audit gate ``` | Phase | Target Scope | Key Deliverable | Exit Criteria | | :--- | :--- | :--- | :--- | -| **S0** | Configuration | Security modes, cert loader, DTLS 1.3 feasibility | Secure mode opens no plaintext listeners; a maintained DTLS 1.3 implementation satisfies certificate, replay, cookie, and resource-bound requirements | +| **S0** | Configuration | Security modes, certificate loader, Quinn feasibility | Secure mode opens no plaintext listeners; Quinn proves mutual certificate authentication, datagram delivery, address validation, replay protection, disabled 0-RTT, bounded connection state, and deterministic operation under turmoil | | **S1** | Metadata Storage | Security record schema, sharded Raft state | Security records survive snapshot & recovery | | **S2** | Cluster Transport | TLS 1.3 on TCP 2922/2923, Raft D8 RPC authorization | Authenticated and authorized cluster TCP traffic | -| **S3** | Membership | DTLS 1.3 on UDP 2922 & SWIM admission gate | Secure gossip & partition-safe admission fencing | +| **S3** | Membership | QUIC v1 datagrams on UDP 2922 and SWIM admission gate | Secure gossip and partition-safe admission fencing | | **S4** | Client API | Client mTLS on TCP 2921, principal binding, ACLs | Default-deny enforcement on all client APIs | | **S5** | Operations | Certificate rotation, revocation, expiry, recovery, audit logging | Online credential operations and recovery runbooks | | **S6** | Production Gate | Adversarial testing, fuzzing, partition stress | Passes all production readiness checks | @@ -217,7 +234,8 @@ config records TCP SWIM clients operations production S6 must verify node and client impersonation, stale-process replay, unauthorized operations, protocol downgrade, rotation under live traffic, expired and revoked credentials, cold-cluster restart, handshake and datagram fuzzing, resource -bounds, and secret-free diagnostics. +bounds, secret-free diagnostics, and reproducible secure-SWIM behavior under +turmoil with pinned randomness and node identities. --- diff --git a/eastguard.yaml.example b/eastguard.yaml.example index bc18c7db..5a821630 100644 --- a/eastguard.yaml.example +++ b/eastguard.yaml.example @@ -3,6 +3,9 @@ # Secure is the production default. Use trusted-development only on isolated # development and test networks until secure transports are configured. security_mode: "secure" +certificate_chain_path: "./eastguard/config/node-chain.pem" +private_key_path: "./eastguard/config/node-key.pem" +trust_root_path: "./eastguard/config/cluster-ca.pem" # Sockets and networking client_port: 2921 diff --git a/src/config.rs b/src/config.rs index 7e1e041f..b795d50d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -29,6 +29,18 @@ pub struct Environment { #[arg(long, env = "SECURITY_MODE", value_enum, default_value = "secure")] pub security_mode: SecurityMode, + /// PEM certificate chain presented by this node in secure mode. + #[arg(long, env = "CERTIFICATE_CHAIN_PATH")] + pub certificate_chain_path: Option, + + /// PEM private key matching this node's leaf certificate. + #[arg(long, env = "PRIVATE_KEY_PATH")] + pub private_key_path: Option, + + /// PEM certificate authorities trusted for node and client authentication. + #[arg(long, env = "TRUST_ROOT_PATH")] + pub trust_root_path: Option, + #[arg(long, env = "CONFIG_DIR", default_value = "./eastguard/config")] pub config_dir: String, @@ -440,6 +452,9 @@ mod tests { fn make_env() -> Environment { Environment { security_mode: SecurityMode::TrustedDevelopment, + certificate_chain_path: None, + private_key_path: None, + trust_root_path: None, config_dir: "./eastguard/config".to_string(), config_file: None, data_dir: "./eastguard/data".to_string(), diff --git a/src/it/helpers.rs b/src/it/helpers.rs index 4fb89b85..4b1d7d84 100644 --- a/src/it/helpers.rs +++ b/src/it/helpers.rs @@ -10,6 +10,9 @@ use crate::net::TcpStream; pub fn default_env(idx: u32, node_id: String, client_port: u16, cluster_port: u16) -> Environment { Environment { security_mode: SecurityMode::TrustedDevelopment, + certificate_chain_path: None, + private_key_path: None, + trust_root_path: None, config_dir: std::env::temp_dir() .join(format!("eastguard-config-{}-{}", idx, uuid::Uuid::new_v4())) .to_string_lossy() diff --git a/src/lib.rs b/src/lib.rs index a45dd1d7..eb10a624 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ mod data_plane; mod net; pub(crate) mod schedulers; +mod security; pub(crate) mod impls; #[cfg(test)] diff --git a/src/security.rs b/src/security.rs new file mode 100644 index 00000000..8729e349 --- /dev/null +++ b/src/security.rs @@ -0,0 +1,137 @@ +#![allow(dead_code)] +use std::fs::File; +use std::io::BufReader; +use std::path::Path; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use rustls::server::WebPkiClientVerifier; +use rustls::{ClientConfig, RootCertStore, ServerConfig}; + +use crate::config::{Environment, SecurityMode}; + +pub(crate) struct SecureTransportConfig { + pub(crate) server: Arc, + pub(crate) client: Arc, +} + +impl SecureTransportConfig { + pub(crate) fn load(env: &Environment) -> Result> { + match env.security_mode { + SecurityMode::Secure => { + let certificate_chain = env + .certificate_chain_path + .as_deref() + .context("certificate_chain_path is required in secure mode")?; + let private_key_path = env + .private_key_path + .as_deref() + .context("private_key_path is required in secure mode")?; + let trust_roots = env + .trust_root_path + .as_deref() + .context("trust_root_path is required in secure mode")?; + Self::load_from_paths(certificate_chain, private_key_path, trust_roots).map(Some) + } + SecurityMode::TrustedDevelopment => Ok(None), + } + } + + fn load_from_paths( + certificate_chain_path: &Path, + private_key_path: &Path, + trust_root_path: &Path, + ) -> Result { + let certificate_chain = + Self::load_certificates(certificate_chain_path, "certificate chain")?; + let private_key = Self::load_private_key(private_key_path)?; + let trust_roots = Arc::new(Self::load_trust_roots(trust_root_path)?); + + let client_verifier = WebPkiClientVerifier::builder(trust_roots.clone()).build()?; + let server = ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_client_cert_verifier(client_verifier) + .with_single_cert(certificate_chain.clone(), private_key.clone_key())?; + let client = ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_root_certificates((*trust_roots).clone()) + .with_client_auth_cert(certificate_chain, private_key)?; + + Ok(Self { + server: Arc::new(server), + client: Arc::new(client), + }) + } + + fn load_certificates(path: &Path, kind: &'static str) -> Result>> { + let file = + File::open(path).context(format!("failed to open {kind} file {}", path.display()))?; + + let certificates = rustls_pemfile::certs(&mut BufReader::new(file)) + .collect::, _>>() + .context(format!("failed to parse {kind} file {}", path.display()))?; + + anyhow::ensure!( + !certificates.is_empty(), + "{kind} file {} contains no certificates", + path.display() + ); + Ok(certificates) + } + + fn load_private_key(path: &Path) -> Result> { + let file = File::open(path).context(format!( + "failed to open private key file {}", + path.display() + ))?; + + rustls_pemfile::private_key(&mut BufReader::new(file)) + .context(format!( + "failed to parse private key file {}", + path.display() + ))? + .context(format!( + "private key file {} contains no supported key", + path.display() + )) + } + + fn load_trust_roots(path: &Path) -> Result { + let certificates = Self::load_certificates(path, "trust root")?; + let mut roots = RootCertStore::empty(); + for certificate in certificates { + roots + .add(certificate) + .with_context(|| format!("invalid trust root in {}", path.display()))?; + } + Ok(roots) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[test] + fn secure_mode_requires_every_credential_path() { + let env = Environment::try_parse_from(["eastguard"]).unwrap(); + + let error = SecureTransportConfig::load(&env) + .err() + .expect("secure mode without credential paths must fail"); + + assert_eq!( + error.to_string(), + "certificate_chain_path is required in secure mode" + ); + } + + #[test] + fn trusted_development_does_not_load_credentials() { + let env = + Environment::try_parse_from(["eastguard", "--security-mode", "trusted-development"]) + .unwrap(); + + assert!(SecureTransportConfig::load(&env).unwrap().is_none()); + } +} From ab388ab37efb26d65bea664fe3e689d9d7f1ed7d Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 00:16:49 +0400 Subject: [PATCH 03/56] add quinn, update lock --- Cargo.lock | 174 +++++++++++++++++++++++++++++++++++++++-------------- Cargo.toml | 3 +- 2 files changed, 132 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 636e2bf0..317aea75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,29 +82,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "aws-lc-rs" -version = "1.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", - "pkg-config", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -279,15 +256,6 @@ dependencies = [ "error-code", ] -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -332,12 +300,6 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "east-guard" version = "0.1.0" @@ -354,6 +316,7 @@ dependencies = [ "libc", "lz4_flex", "murmur3", + "quinn", "rand 0.10.1", "rocksdb", "rustls", @@ -408,6 +371,18 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash 0.2.0", + "libm", + "portable-atomic", + "siphasher", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -442,10 +417,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] -name = "fs_extra" -version = "1.3.0" +name = "foldhash" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "futures" @@ -567,11 +542,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -592,7 +569,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -756,6 +733,12 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" @@ -910,6 +893,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -947,6 +936,63 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "fastbloom", + "getrandom 0.4.1", + "lru-slab", + "rand 0.10.1", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.45" @@ -1028,6 +1074,15 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1115,8 +1170,8 @@ version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ - "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1138,6 +1193,7 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ + "web-time", "zeroize", ] @@ -1147,7 +1203,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1328,6 +1383,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -1424,6 +1485,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -1734,6 +1810,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 5062736b..ec734956 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,8 @@ serde_yaml = "0.9.34" serde = "1" serde_json = "1" futures = "0.3" -rustls = { version = "0.23.42", default-features = false, features = ["aws_lc_rs", "std"] } +quinn = { version = "0.11.11", default-features = false, features = ["bloom", "runtime-tokio", "rustls-ring"] } +rustls = { version = "0.23.42", default-features = false, features = ["ring", "std"] } rustls-pemfile = "2.2.0" [target.'cfg(target_os = "linux")'.dependencies] From c629309a9adbd60ff0e1326fa0d37ef4cb637d65 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 00:30:31 +0400 Subject: [PATCH 04/56] infra: quinn udp socket QuinnUdpSocket and its turmoil test --- Cargo.lock | 270 ++++++++++++++++++++++++++++++++++--- Cargo.toml | 5 +- src/net/mod.rs | 22 +++ src/{net.rs => net/tcp.rs} | 60 +++------ src/net/udp.rs | 252 ++++++++++++++++++++++++++++++++++ 5 files changed, 546 insertions(+), 63 deletions(-) create mode 100644 src/net/mod.rs rename src/{net.rs => net/tcp.rs} (65%) create mode 100644 src/net/udp.rs diff --git a/Cargo.lock b/Cargo.lock index 317aea75..cd3b9595 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,45 @@ dependencies = [ "rustversion", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -97,7 +136,16 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", ] [[package]] @@ -127,7 +175,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -238,7 +286,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -300,6 +348,43 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "east-guard" version = "0.1.0" @@ -318,6 +403,7 @@ dependencies = [ "murmur3", "quinn", "rand 0.10.1", + "rcgen", "rocksdb", "rustls", "rustls-pemfile", @@ -478,7 +564,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -751,9 +837,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -836,6 +922,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -846,6 +957,15 @@ dependencies = [ "libm", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -899,6 +1019,12 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -915,7 +1041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -1083,6 +1209,19 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rcgen" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +dependencies = [ + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1151,6 +1290,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1244,7 +1392,7 @@ checksum = "64e5587417a3c4e16a4415e8d7d07f80998ed835ade621d19dfbe9fbe3205b0f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1298,7 +1446,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1329,9 +1477,9 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.5.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" +checksum = "a6df5ed973ad8d834e09f824f9e9f449af6b9a3745f78dec7cc752770bd3bf11" dependencies = [ "futures-executor", "futures-util", @@ -1343,13 +1491,13 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.5.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" +checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -1443,6 +1591,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -1473,7 +1643,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1485,6 +1655,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.12.0" @@ -1525,7 +1725,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1577,7 +1777,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1763,7 +1963,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1947,7 +2147,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -1963,7 +2163,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -2005,6 +2205,34 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + [[package]] name = "zerocopy" version = "0.8.40" @@ -2022,7 +2250,7 @@ checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ec734956..1c178195 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ dashmap = "6.2.1" crc32fast = "1.4" arc-swap = "1" flume = "0.12" -lz4_flex = "0.13.1" +lz4_flex = "0.14.0" zstd = "0.13" rustyline = { version = "18.0.1", features = ["derive"] } shlex = "2.0.1" @@ -39,8 +39,9 @@ rustls-pemfile = "2.2.0" libc = "0.2" [dev-dependencies] +rcgen = { version = "0.14.8", default-features = false, features = ["crypto", "ring"] } tempfile = "3.27.0" -serial_test = "3.5.0" +serial_test = "4.0.1" turmoil = { version = "0.7.2", package = "turmoil" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/src/net/mod.rs b/src/net/mod.rs new file mode 100644 index 00000000..4e460088 --- /dev/null +++ b/src/net/mod.rs @@ -0,0 +1,22 @@ +#![allow(clippy::disallowed_types)] + +mod tcp; +pub(crate) mod udp; + +pub use tcp::{OwnedReadHalf, OwnedWriteHalf, TcpListener, TcpStream}; +pub use udp::UdpSocket; + +#[cfg(not(test))] +pub(super) mod inner { + pub use tokio::net::{ + TcpListener, TcpStream, ToSocketAddrs, UdpSocket, tcp::OwnedReadHalf, tcp::OwnedWriteHalf, + }; +} + +#[cfg(test)] +pub(super) mod inner { + pub use turmoil::ToSocketAddrs; + pub use turmoil::net::{ + TcpListener, TcpStream, UdpSocket, tcp::OwnedReadHalf, tcp::OwnedWriteHalf, + }; +} diff --git a/src/net.rs b/src/net/tcp.rs similarity index 65% rename from src/net.rs rename to src/net/tcp.rs index be50afb9..48dbc251 100644 --- a/src/net.rs +++ b/src/net/tcp.rs @@ -1,34 +1,23 @@ -#![allow(clippy::disallowed_types)] use std::ops::{Deref, DerefMut}; use std::pin::Pin; use std::task::{Context, Poll}; + use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; -#[cfg(not(test))] -pub mod inner { - pub use tokio::net::{ - TcpListener, TcpStream, ToSocketAddrs, UdpSocket, tcp::OwnedReadHalf, tcp::OwnedWriteHalf, - }; -} +use super::inner; -#[cfg(test)] -pub mod inner { - pub use turmoil::ToSocketAddrs; - pub use turmoil::net::{ - TcpListener, TcpStream, UdpSocket, tcp::OwnedReadHalf, tcp::OwnedWriteHalf, - }; -} -// 2. A single macro handles ALL types now! -macro_rules! wrap_type { +macro_rules! tcp_wrapper { ($name:ident) => { - pub struct $name(inner::$name); + pub struct $name(pub(super) inner::$name); impl Deref for $name { type Target = inner::$name; + fn deref(&self) -> &Self::Target { &self.0 } } + impl DerefMut for $name { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 @@ -37,14 +26,10 @@ macro_rules! wrap_type { }; } -// 3. Generate the wrappers -wrap_type!(TcpListener); -wrap_type!(UdpSocket); -wrap_type!(TcpStream); -wrap_type!(OwnedReadHalf); -wrap_type!(OwnedWriteHalf); - -// 4. Implement AsyncRead / AsyncWrite safely using `Unpin` +tcp_wrapper!(TcpListener); +tcp_wrapper!(TcpStream); +tcp_wrapper!(OwnedReadHalf); +tcp_wrapper!(OwnedWriteHalf); impl AsyncRead for TcpStream { fn poll_read( @@ -64,9 +49,11 @@ impl AsyncWrite for TcpStream { ) -> Poll> { Pin::new(&mut self.0).poll_write(cx, buf) } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.0).poll_flush(cx) } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.0).poll_shutdown(cx) } @@ -90,9 +77,11 @@ impl AsyncWrite for OwnedWriteHalf { ) -> Poll> { Pin::new(&mut self.0).poll_write(cx, buf) } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.0).poll_flush(cx) } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.0).poll_shutdown(cx) } @@ -100,34 +89,25 @@ impl AsyncWrite for OwnedWriteHalf { impl TcpListener { pub async fn bind(addr: A) -> std::io::Result { - let inner_listener = inner::TcpListener::bind(addr).await?; - Ok(Self(inner_listener)) + let listener = inner::TcpListener::bind(addr).await?; + Ok(Self(listener)) } pub async fn accept(&self) -> std::io::Result<(TcpStream, std::net::SocketAddr)> { - // We intercept accept() so it returns our wrapper TcpStream instead of the inner one - let (inner_stream, addr) = self.0.accept().await?; - Ok((TcpStream(inner_stream), addr)) + let (stream, addr) = self.0.accept().await?; + Ok((TcpStream(stream), addr)) } } impl TcpStream { #[allow(dead_code)] pub async fn connect(addr: A) -> std::io::Result { - let inner_stream = inner::TcpStream::connect(addr).await?; - Ok(Self(inner_stream)) + let stream = inner::TcpStream::connect(addr).await?; + Ok(Self(stream)) } pub fn into_split(self) -> (OwnedReadHalf, OwnedWriteHalf) { - // We intercept into_split() so it returns our wrapped halves let (read_half, write_half) = self.0.into_split(); (OwnedReadHalf(read_half), OwnedWriteHalf(write_half)) } } - -impl UdpSocket { - pub async fn bind(addr: A) -> std::io::Result { - let inner_socket = inner::UdpSocket::bind(addr).await?; - Ok(Self(inner_socket)) - } -} diff --git a/src/net/udp.rs b/src/net/udp.rs new file mode 100644 index 00000000..b863808e --- /dev/null +++ b/src/net/udp.rs @@ -0,0 +1,252 @@ +#![allow(dead_code)] +use super::inner; +use quinn::udp::{RecvMeta, Transmit}; +use quinn::{AsyncUdpSocket, UdpPoller}; +use std::fmt; +use std::future::Future; +use std::io::{self, IoSliceMut}; +use std::ops::{Deref, DerefMut}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +type Readiness = Pin> + Send>>; + +pub struct UdpSocket(inner::UdpSocket); + +impl Deref for UdpSocket { + type Target = inner::UdpSocket; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for UdpSocket { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl UdpSocket { + pub async fn bind(addr: A) -> std::io::Result { + let socket = inner::UdpSocket::bind(addr).await?; + Ok(Self(socket)) + } +} + +pub struct QuinnUdpSocket { + socket: Arc, + readable: Mutex>, +} + +impl QuinnUdpSocket { + pub fn new(socket: UdpSocket) -> Self { + Self { + socket: Arc::new(socket), + readable: Mutex::new(None), + } + } +} + +impl fmt::Debug for QuinnUdpSocket { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("QuinnUdpSocket") + .field("local_addr", &self.socket.local_addr()) + .finish() + } +} + +impl AsyncUdpSocket for QuinnUdpSocket { + fn create_io_poller(self: Arc) -> Pin> { + Box::pin(QuinnUdpPoller { + socket: self.socket.clone(), + writable: Mutex::new(None), + }) + } + + fn try_send(&self, transmit: &Transmit<'_>) -> io::Result<()> { + debug_assert!(transmit.segment_size.is_none()); + self.socket + .try_send_to(transmit.contents, transmit.destination) + .map(|_| ()) + } + + fn poll_recv( + &self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll> { + if bufs.is_empty() || meta.is_empty() { + return Poll::Ready(Ok(0)); + } + + loop { + match self.socket.try_recv_from(&mut bufs[0]) { + Ok((len, addr)) => { + meta[0] = RecvMeta { + addr, + len, + stride: len, + ecn: None, + dst_ip: None, + }; + self.readable + .lock() + .expect("readable mutex poisoned") + .take(); + return Poll::Ready(Ok(1)); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => return Poll::Ready(Err(error)), + } + + let mut readable = self.readable.lock().expect("readable mutex poisoned"); + let socket = self.socket.clone(); + let readiness = readable.get_or_insert_with(|| { + Box::pin(async move { socket.readable().await }) as Readiness + }); + + match readiness.as_mut().poll(cx) { + Poll::Ready(Ok(())) => { + readable.take(); + } + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + Poll::Pending => return Poll::Pending, + } + } + } + + fn local_addr(&self) -> io::Result { + self.socket.local_addr() + } +} + +struct QuinnUdpPoller { + socket: Arc, + writable: Mutex>, +} + +impl fmt::Debug for QuinnUdpPoller { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.debug_struct("QuinnUdpPoller").finish() + } +} + +impl UdpPoller for QuinnUdpPoller { + fn poll_writable(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut writable = self.writable.lock().expect("writable mutex poisoned"); + let socket = self.socket.clone(); + let readiness = + writable.get_or_insert_with(|| Box::pin(async move { socket.writable().await })); + + let result = readiness.as_mut().poll(cx); + if result.is_ready() { + writable.take(); + } + result + } +} + +#[cfg(test)] +mod tests { + use std::net::SocketAddr; + use std::time::Duration; + + use bytes::Bytes; + use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig}; + use rcgen::{CertifiedKey, generate_simple_self_signed}; + use rustls::RootCertStore; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + use rustls::server::WebPkiClientVerifier; + use turmoil::Builder; + + use super::*; + + fn configs() -> (quinn::ServerConfig, quinn::ClientConfig) { + let CertifiedKey { cert, signing_key } = + generate_simple_self_signed(["server".to_string()]).unwrap(); + let certificate = cert.der().clone(); + let private_key: PrivateKeyDer<'static> = + PrivatePkcs8KeyDer::from(signing_key.serialize_der()).into(); + + let mut roots = RootCertStore::empty(); + roots.add(certificate.clone()).unwrap(); + let roots = Arc::new(roots); + + let client_verifier = WebPkiClientVerifier::builder(roots.clone()) + .build() + .unwrap(); + let server_crypto = + rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_client_cert_verifier(client_verifier) + .with_single_cert(vec![certificate.clone()], private_key.clone_key()) + .unwrap(); + let client_crypto = + rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_root_certificates((*roots).clone()) + .with_client_auth_cert(vec![certificate], private_key) + .unwrap(); + + ( + quinn::ServerConfig::with_crypto(Arc::new( + QuicServerConfig::try_from(server_crypto).unwrap(), + )), + quinn::ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto).unwrap())), + ) + } + + #[test] + fn quinn_exchanges_datagram_under_turmoil() -> turmoil::Result { + let (server_config, client_config) = configs(); + let mut sim = Builder::new() + .simulation_duration(Duration::from_secs(10)) + .rng_seed(1) + .build(); + + sim.host("server", move || { + let server_config = server_config.clone(); + async move { + let socket = UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], 4433))) + .await + .unwrap(); + let endpoint = quinn::Endpoint::new_with_abstract_socket( + quinn::EndpointConfig::default(), + Some(server_config), + Arc::new(QuinnUdpSocket::new(socket)), + Arc::new(quinn::TokioRuntime), + ) + .unwrap(); + + let connection = endpoint.accept().await.unwrap().await.unwrap(); + assert_eq!(connection.read_datagram().await.unwrap(), b"hello"[..]); + Ok(()) + } + }); + + sim.client("client", async move { + let socket = UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], 4434))) + .await + .unwrap(); + let mut endpoint = quinn::Endpoint::new_with_abstract_socket( + quinn::EndpointConfig::default(), + None, + Arc::new(QuinnUdpSocket::new(socket)), + Arc::new(quinn::TokioRuntime), + ) + .unwrap(); + endpoint.set_default_client_config(client_config); + + let server = SocketAddr::new(turmoil::lookup("server"), 4433); + let connection = endpoint.connect(server, "server").unwrap().await.unwrap(); + connection + .send_datagram(Bytes::from_static(b"hello")) + .unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + Ok(()) + }); + + sim.run() + } +} From 6344070cfd31e36706174420af0b687a9bfdf048 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 00:35:21 +0400 Subject: [PATCH 05/56] temporary assertion --- src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index eb10a624..0361576e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,6 +40,7 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, UdpSocket}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; +use crate::security::SecureTransportConfig; use crate::{ config::ENV, control_plane::membership::{actor::SwimActor, transport::SwimTransportActor}, @@ -66,6 +67,10 @@ impl StartUp { } pub async fn run(self) -> Result<()> { + if SecureTransportConfig::load(&self.env)?.is_some() { + anyhow::bail!("secure transport listeners are not implemented"); + } + // Bind sockets before spawning — fail fast on port conflicts let udp_socket = UdpSocket::bind(self.env.peer_bind_addr()).await?; let tcp_listener = TcpListener::bind(self.env.peer_bind_addr()).await?; From b05979e32306896dc61c9233e54d27745fa6f5b9 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 00:37:43 +0400 Subject: [PATCH 06/56] fix: address validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deterministic turmoil test now proves: client Initial │ ▼ server Retry token │ ▼ client retries │ ▼ server confirms address validated │ ▼ mutual TLS handshake + QUIC datagram --- src/net/udp.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/net/udp.rs b/src/net/udp.rs index b863808e..ed654d1a 100644 --- a/src/net/udp.rs +++ b/src/net/udp.rs @@ -198,7 +198,7 @@ mod tests { } #[test] - fn quinn_exchanges_datagram_under_turmoil() -> turmoil::Result { + fn quinn_validates_address_and_exchanges_datagram_under_turmoil() -> turmoil::Result { let (server_config, client_config) = configs(); let mut sim = Builder::new() .simulation_duration(Duration::from_secs(10)) @@ -219,7 +219,13 @@ mod tests { ) .unwrap(); - let connection = endpoint.accept().await.unwrap().await.unwrap(); + let initial = endpoint.accept().await.unwrap(); + assert!(!initial.remote_address_validated()); + initial.retry().unwrap(); + + let validated = endpoint.accept().await.unwrap(); + assert!(validated.remote_address_validated()); + let connection = validated.await.unwrap(); assert_eq!(connection.read_datagram().await.unwrap(), b"hello"[..]); Ok(()) } From 9421d7d724e36e73d21064f9fd065c5a2191fe11 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 00:42:25 +0400 Subject: [PATCH 07/56] text: turmoil with quinn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Maximum 128 pending handshakes. - No bidirectional or unidirectional QUIC streams. - 64 × 1200-byte send and receive datagram buffers. - 1200-byte maximum UDP payload. - TLS early data/0-RTT confirmed disabled. - Required Retry address validation. --- src/net/udp.rs | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/net/udp.rs b/src/net/udp.rs index ed654d1a..82180ab1 100644 --- a/src/net/udp.rs +++ b/src/net/udp.rs @@ -164,6 +164,25 @@ mod tests { use super::*; + const MAX_INCOMING_HANDSHAKES: usize = 128; + const DATAGRAM_BUFFER_BYTES: usize = 64 * 1200; + + fn endpoint_config() -> quinn::EndpointConfig { + let mut config = quinn::EndpointConfig::default(); + config.max_udp_payload_size(1200).unwrap(); + config + } + + fn transport_config() -> Arc { + let mut config = quinn::TransportConfig::default(); + config + .max_concurrent_bidi_streams(0_u8.into()) + .max_concurrent_uni_streams(0_u8.into()) + .datagram_receive_buffer_size(Some(DATAGRAM_BUFFER_BYTES)) + .datagram_send_buffer_size(DATAGRAM_BUFFER_BYTES); + Arc::new(config) + } + fn configs() -> (quinn::ServerConfig, quinn::ClientConfig) { let CertifiedKey { cert, signing_key } = generate_simple_self_signed(["server".to_string()]).unwrap(); @@ -188,13 +207,21 @@ mod tests { .with_root_certificates((*roots).clone()) .with_client_auth_cert(vec![certificate], private_key) .unwrap(); - - ( - quinn::ServerConfig::with_crypto(Arc::new( - QuicServerConfig::try_from(server_crypto).unwrap(), - )), - quinn::ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto).unwrap())), - ) + assert_eq!(server_crypto.max_early_data_size, 0); + assert!(!client_crypto.enable_early_data); + + let transport = transport_config(); + let mut server = quinn::ServerConfig::with_crypto(Arc::new( + QuicServerConfig::try_from(server_crypto).unwrap(), + )); + server + .transport_config(transport.clone()) + .max_incoming(MAX_INCOMING_HANDSHAKES); + let mut client = + quinn::ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto).unwrap())); + client.transport_config(transport); + + (server, client) } #[test] @@ -212,7 +239,7 @@ mod tests { .await .unwrap(); let endpoint = quinn::Endpoint::new_with_abstract_socket( - quinn::EndpointConfig::default(), + endpoint_config(), Some(server_config), Arc::new(QuinnUdpSocket::new(socket)), Arc::new(quinn::TokioRuntime), @@ -236,7 +263,7 @@ mod tests { .await .unwrap(); let mut endpoint = quinn::Endpoint::new_with_abstract_socket( - quinn::EndpointConfig::default(), + endpoint_config(), None, Arc::new(QuinnUdpSocket::new(socket)), Arc::new(quinn::TokioRuntime), From a2561499329d19444e043643058b84f5d31a46a2 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 08:14:49 +0400 Subject: [PATCH 08/56] simplify smart p --- src/net/udp.rs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/net/udp.rs b/src/net/udp.rs index 82180ab1..2805d956 100644 --- a/src/net/udp.rs +++ b/src/net/udp.rs @@ -1,11 +1,12 @@ #![allow(dead_code)] +use crate::impl_new_struct_wrapper; + use super::inner; use quinn::udp::{RecvMeta, Transmit}; use quinn::{AsyncUdpSocket, UdpPoller}; use std::fmt; use std::future::Future; use std::io::{self, IoSliceMut}; -use std::ops::{Deref, DerefMut}; use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; @@ -13,19 +14,7 @@ type Readiness = Pin> + Send>>; pub struct UdpSocket(inner::UdpSocket); -impl Deref for UdpSocket { - type Target = inner::UdpSocket; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for UdpSocket { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} +impl_new_struct_wrapper!(UdpSocket, inner::UdpSocket); impl UdpSocket { pub async fn bind(addr: A) -> std::io::Result { From 6f8f8d95ac281217348ece6975efa011ce47d605 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 08:23:24 +0400 Subject: [PATCH 09/56] clarify ambiguity --- docs/security/roadmap.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 74962767..51fe415c 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -63,8 +63,8 @@ To allow safe node restarts and hardware replacement without exposing the cluste | Term | Scope | Lifetime / Ordering | Function | | :--- | :--- | :--- | :--- | -| **Certificate Node ID** | Configuration | Long-lived / Reused | Operator-assigned node name embedded in the X.509 certificate. | -| **Admission Epoch** | Metadata Shard | Monotonically increasing `u64` | Assigned by metadata Raft upon restart; higher epoch **fences** older instances. | +| **Certificate Node ID** | X.509 Certificate | Long-lived / Reused | Operator-assigned node name and stable admission-record key. | +| **Admission Epoch** | Certificate Node ID | Monotonically increasing `u64` | Assigned by metadata Raft upon restart; higher epoch **fences** older instances. | | **NodeId** | Running Process | Single process lifetime | Unique ID generated on startup; used by SWIM, topology ring, Raft, and data placement. | | **Process Key** | Running Process | Single process lifetime | Proves that the connection belongs to the process admitted for this epoch. | | **SWIM Incarnation** | Running Process | Monotonically increasing counter | Incremented by the *same* process instance to refute false `Suspect`/`Dead` gossip. | @@ -149,13 +149,23 @@ A security record is one durable admission, ACL, or revocation entry. Its record path selects one metadata shard; its revision lets brokers detect stale cached copies. -Security records (`security/node/{id}`, `security/acl/{resource}`, `security/revocation/{issuer}/{serial}`) do not rely on a centralized security controller. Instead, they hash to standard metadata shards and replicate via Raft: +Security records (`security/node/{certificate-node-id}`, `security/acl/{resource}`, `security/revocation/{issuer}/{serial}`) do not rely on a centralized security controller. Instead, they hash to standard metadata shards and replicate via Raft: ``` Client/Node Request ──► Any Broker ──► Hash Record Path ──► Hosts Shard? ─┬─► Yes ──► Commit via Raft └─► No ──► Return Owner Redirect ``` +- **Stable Admission Key:** A restart changes the process `NodeId` and key, but + not the Certificate Node ID. The same record and metadata shard therefore + replace the old admitted process atomically: + +``` +security/node/{certificate-node-id} + │ + └── Admission Epoch + NodeId + Process Public Key +``` + - **Local Authorization:** Brokers evaluate ACLs against local cached security records. - **Freshness & Expiry:** Cached records include a monotonic deadline (max 60s) and revision counter. Expired entries require re-validation from the owner metadata shard; if the owner is offline, authorization fails closed. @@ -172,7 +182,9 @@ Security records (`security/node/{id}`, `security/acl/{resource}`, `security/rev 3. A later joining node generates a new `NodeId` and process key. 4. An authorized operator approves that exact `NodeId` and process public key. The reusable node certificate alone cannot authorize replacement. 5. The joining node connects to a **limited admission endpoint** using its X.509 certificate. -6. The endpoint routes to the owning metadata shard, which atomically commits the next `Admission Epoch`, `NodeId`, and process public key. +6. The endpoint uses the authenticated Certificate Node ID to route to its + admission record. The owning metadata shard atomically replaces the prior + process with the next `Admission Epoch`, `NodeId`, and process public key. 7. The joining node proves possession of the process private key before entering SWIM gossip and Raft membership reconciliation. ### Online Credential Rotation From 57ed87590c7002a9263c45478bd600489034a8eb Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 08:30:14 +0400 Subject: [PATCH 10/56] security record --- src/control_plane/metadata/mod.rs | 2 + src/control_plane/metadata/security.rs | 75 ++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 src/control_plane/metadata/security.rs diff --git a/src/control_plane/metadata/mod.rs b/src/control_plane/metadata/mod.rs index 3b5c1693..27d7f454 100644 --- a/src/control_plane/metadata/mod.rs +++ b/src/control_plane/metadata/mod.rs @@ -7,6 +7,8 @@ pub(crate) mod event; pub(crate) mod range; mod producer_sessions; +#[allow(dead_code)] +pub(crate) mod security; pub mod strategy; pub(crate) mod topic; diff --git a/src/control_plane/metadata/security.rs b/src/control_plane/metadata/security.rs new file mode 100644 index 00000000..6e8a9193 --- /dev/null +++ b/src/control_plane/metadata/security.rs @@ -0,0 +1,75 @@ +use borsh::{BorshDeserialize, BorshSerialize}; + +use crate::control_plane::NodeId; + +/// Current process admitted for `security/node/{certificate_node_id}`. +/// +/// A restart replaces this record through its metadata shard. Admission checks +/// accept SWIM facts only when the epoch, node ID, and process key match it. +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub(crate) struct AdmissionRecord { + pub certificate_node_id: String, + pub revision: u64, + pub epoch: u64, + pub node_id: NodeId, + pub process_public_key: Box<[u8]>, +} + +/// Principals granted the permissions of one exact +/// `security/acl/{resource}` entry. +/// +/// Authorization caches use the revision to reject stale copies. Missing +/// principals and missing records deny access. +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub(crate) struct AclRecord { + pub resource: String, + pub revision: u64, + pub principals: Box<[String]>, +} + +/// Certificate blocked by `security/revocation/{issuer}/{serial}`. +/// +/// Brokers cache these records and terminate or reject matching authenticated +/// connections within the cache enforcement window. +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub(crate) struct RevocationRecord { + pub issuer: String, + pub serial: Box<[u8]>, + pub revision: u64, + pub revoked_at: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(value: &T) + where + T: BorshSerialize + BorshDeserialize + PartialEq + std::fmt::Debug, + { + let bytes = borsh::to_vec(value).unwrap(); + assert_eq!(&borsh::from_slice::(&bytes).unwrap(), value); + } + + #[test] + fn security_records_round_trip() { + round_trip(&AdmissionRecord { + certificate_node_id: "broker-a".to_string(), + revision: 3, + epoch: 2, + node_id: NodeId::new("broker-a::process-2"), + process_public_key: vec![1, 2, 3].into_boxed_slice(), + }); + round_trip(&AclRecord { + resource: "security/cluster".to_string(), + revision: 4, + principals: vec!["operator".to_string()].into_boxed_slice(), + }); + round_trip(&RevocationRecord { + issuer: "cluster-ca".to_string(), + serial: vec![0x12, 0x34].into_boxed_slice(), + revision: 5, + revoked_at: 100, + }); + } +} From e21f3699f6e525ac6369de99a837df9eb7ec60ca Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 08:48:46 +0400 Subject: [PATCH 11/56] embed security state into metadata state --- .claude/rules/metadata-state-machine.md | 8 ++- .../consensus/raft/states/metadata_state.rs | 69 +++++++++++++++++++ .../consensus/raft/states/mod.rs | 2 + .../raft/states}/security.rs | 41 +++++++++++ src/control_plane/metadata/mod.rs | 3 +- 5 files changed, 120 insertions(+), 3 deletions(-) rename src/control_plane/{metadata => consensus/raft/states}/security.rs (62%) diff --git a/.claude/rules/metadata-state-machine.md b/.claude/rules/metadata-state-machine.md index 956d58c5..2af12f03 100644 --- a/.claude/rules/metadata-state-machine.md +++ b/.claude/rules/metadata-state-machine.md @@ -21,7 +21,11 @@ MetadataStateMachine (one per shard group) │ │ SegmentMeta (nested inside RangeMeta) │ -└── topic_name_index: HashMap +├── topic_name_index: HashMap +└── security + ├── admissions: Certificate Node ID → Admission Record + ├── ACLs: Resource → ACL Record + └── revocations: (Issuer, Serial) → Revocation Record ``` ## Commands @@ -80,3 +84,5 @@ MetadataStateMachine (one per shard group) 21. **`ReassignSegment` only re-points a sealed segment.** `apply_reassign_segment()` accepts only a `Sealed` segment, swaps `replica_set`, and changes nothing else — state stays `Sealed`; data, offsets, lineage, and timestamps stay frozen (invariant 3). An active, deleting, or unknown segment is rejected (`SegmentNotSealed` / `SegmentNotFound`), logged but not fatal (invariant 11). Re-applying with the same `replica_set` succeeds without raising a metadata event, tolerating duplicate death detection and no-leader re-proposals (cf. invariant 19). The swap runs through `apply`, so the umbrella `assert_invariants` re-checks every other invariant afterward — a reassignment cannot leave the machine inconsistent. 22. **A committed consumer-group generation assigns each active range exactly once.** When a group has members, its assignment keys exactly equal the topic's active ranges and every assignment names a current member. When it has no members, it has no assignments. Membership or range-topology changes advance the generation and recompute the full desired assignment through the Raft log; heartbeat refreshes that do not change membership leave the generation unchanged. + +23. **Security map keys match their records.** Admission keys equal the record's Certificate Node ID, ACL keys equal the record's resource, and revocation keys equal the record's issuer and serial. A map therefore contains at most one current admission per Certificate Node ID and snapshot restore cannot silently associate a security record with a different lookup key. diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index ec4b19fa..4e0cb13e 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -1,3 +1,4 @@ +use crate::control_plane::consensus::raft::states::security::SecurityState; use crate::control_plane::metadata::SegmentMeta; use crate::control_plane::metadata::command::*; use crate::control_plane::metadata::event::*; @@ -6,6 +7,7 @@ use crate::control_plane::NodeId; use crate::control_plane::Replicas; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::ConsumerGroupAssignment; + use crate::control_plane::metadata::topic::{TopicMeta, TopicState, TopicStats}; use crate::control_plane::metadata::{EntryId, RangeId, SegmentId, TopicId, error::MetadataError}; use crate::data_plane::SegmentKey; @@ -19,11 +21,13 @@ use uuid::Uuid; #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct MetadataStateSnapshot { topics: HashMap, + security: Box, next_topic_id: u64, } pub struct MetadataState { pub(crate) topics: HashMap, + security: SecurityState, pub(crate) last_applied_index: u64, topic_name_index: HashMap, next_topic_id: u64, @@ -35,6 +39,7 @@ impl MetadataState { pub(crate) fn new(shard_group_id: ShardGroupId) -> Self { MetadataState { topics: HashMap::new(), + security: SecurityState::default(), last_applied_index: 0, topic_name_index: HashMap::new(), next_topic_id: shard_group_id.0 << 32, @@ -46,6 +51,7 @@ impl MetadataState { pub(crate) fn snapshot(&self) -> MetadataStateSnapshot { MetadataStateSnapshot { topics: self.topics.clone(), + security: Box::new(self.security.clone()), next_topic_id: self.next_topic_id, } } @@ -58,6 +64,7 @@ impl MetadataState { .collect(); Self { topics, + security: *snapshot.security, last_applied_index, topic_name_index, next_topic_id: snapshot.next_topic_id, @@ -582,6 +589,8 @@ impl crate::test_traits::TAssertInvariant for MetadataState { assert!(id.0 < self.next_topic_id, "topic ID >= next_topic_id"); assert_eq!(*id, topic.id, "topic map key does not match topic identity"); } + + self.security.assert_invariants(); for topic in self.topics.values() { topic.assert_invariants(); } @@ -592,6 +601,9 @@ impl crate::test_traits::TAssertInvariant for MetadataState { mod tests { use super::*; use crate::connections::protocol::ConsumerGroupSyncAction; + use crate::control_plane::consensus::raft::states::security::{ + AclRecord, AdmissionRecord, RevocationRecord, + }; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::constants::*; use crate::control_plane::metadata::range::*; @@ -619,6 +631,61 @@ mod tests { } } + #[test] + fn security_records_survive_snapshot_restore() { + let mut state = MetadataState::new(ShardGroupId(1)); + let admission = AdmissionRecord { + certificate_node_id: "broker-a".to_string(), + revision: 3, + epoch: 2, + node_id: NodeId::new("broker-a::process-2"), + process_public_key: vec![1, 2, 3].into_boxed_slice(), + }; + let acl = AclRecord { + resource: "security/cluster".to_string(), + revision: 4, + principals: vec!["operator".to_string()].into_boxed_slice(), + }; + let revocation = RevocationRecord { + issuer: "cluster-ca".to_string(), + serial: vec![0x12, 0x34].into_boxed_slice(), + revision: 5, + revoked_at: 100, + }; + + state + .security + .admissions + .insert(admission.certificate_node_id.clone(), admission.clone()); + state + .security + .acls + .insert(acl.resource.clone(), acl.clone()); + state.security.revocations.insert( + (revocation.issuer.clone(), revocation.serial.clone()), + revocation.clone(), + ); + + let bytes = borsh::to_vec(&state.snapshot()).unwrap(); + let snapshot = borsh::from_slice(&bytes).unwrap(); + let restored = MetadataState::from_snapshot(snapshot, 9); + + assert_eq!( + restored.security.admissions.get("broker-a"), + Some(&admission) + ); + assert_eq!(restored.security.acls.get("security/cluster"), Some(&acl)); + assert_eq!( + restored.security.revocations.get(&( + "cluster-ca".to_string(), + vec![0x12, 0x34].into_boxed_slice() + )), + Some(&revocation) + ); + assert_eq!(restored.last_applied_index, 9); + restored.assert_invariants(); + } + fn replica_set() -> Replicas { Replicas::new(vec![ NodeId::new("node-1"), @@ -670,10 +737,12 @@ mod tests { let a = MetadataStateSnapshot { topics: HashMap::from([(first.id, first.clone()), (second.id, second.clone())]), next_topic_id: 3, + security: Box::default(), }; let b = MetadataStateSnapshot { topics: HashMap::from([(second.id, second), (first.id, first)]), next_topic_id: 3, + security: Box::default(), }; assert_eq!(borsh::to_vec(&a).unwrap(), borsh::to_vec(&b).unwrap()); diff --git a/src/control_plane/consensus/raft/states/mod.rs b/src/control_plane/consensus/raft/states/mod.rs index fd907485..5fe6f6c8 100644 --- a/src/control_plane/consensus/raft/states/mod.rs +++ b/src/control_plane/consensus/raft/states/mod.rs @@ -43,3 +43,5 @@ boundaries describe the control-plane Raft metadata log. */ pub(crate) mod consensus; pub(crate) mod metadata_state; + +pub(crate) mod security; diff --git a/src/control_plane/metadata/security.rs b/src/control_plane/consensus/raft/states/security.rs similarity index 62% rename from src/control_plane/metadata/security.rs rename to src/control_plane/consensus/raft/states/security.rs index 6e8a9193..3aa948c0 100644 --- a/src/control_plane/metadata/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -1,7 +1,20 @@ +use std::collections::HashMap; + use borsh::{BorshDeserialize, BorshSerialize}; use crate::control_plane::NodeId; +/// Security records replicated by one metadata shard. +/// +/// The live metadata state holds this directly. Snapshots box it so security +/// indexes do not enlarge every variant of the Raft snapshot state. +#[derive(Debug, Clone, Default, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub(crate) struct SecurityState { + pub(super) admissions: HashMap, + pub(super) acls: HashMap, + pub(super) revocations: HashMap<(String, Box<[u8]>), RevocationRecord>, +} + /// Current process admitted for `security/node/{certificate_node_id}`. /// /// A restart replaces this record through its metadata shard. Admission checks @@ -39,6 +52,34 @@ pub(crate) struct RevocationRecord { pub revoked_at: u64, } +#[cfg(any(test, debug_assertions))] +impl crate::test_traits::TAssertInvariant for SecurityState { + fn assert_invariants(&self) { + for (certificate_node_id, admission) in &self.admissions { + assert_eq!( + certificate_node_id, &admission.certificate_node_id, + "admission map key does not match Certificate Node ID" + ); + } + for (resource, acl) in &self.acls { + assert_eq!( + resource, &acl.resource, + "ACL map key does not match resource" + ); + } + for ((issuer, serial), revocation) in &self.revocations { + assert_eq!( + issuer, &revocation.issuer, + "revocation map key does not match issuer" + ); + assert_eq!( + serial, &revocation.serial, + "revocation map key does not match serial" + ); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/control_plane/metadata/mod.rs b/src/control_plane/metadata/mod.rs index 27d7f454..a3806137 100644 --- a/src/control_plane/metadata/mod.rs +++ b/src/control_plane/metadata/mod.rs @@ -7,8 +7,7 @@ pub(crate) mod event; pub(crate) mod range; mod producer_sessions; -#[allow(dead_code)] -pub(crate) mod security; + pub mod strategy; pub(crate) mod topic; From 23f5c148b80f0341b89fc6914a6d82b519c61e40 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 10:12:08 +0400 Subject: [PATCH 12/56] rename Cert Node Id -> node cert principal --- .claude/rules/metadata-state-machine.md | 4 ++-- docs/security/roadmap.md | 16 ++++++++-------- .../consensus/raft/states/metadata_state.rs | 10 +++++----- .../consensus/raft/states/security.rs | 12 ++++++------ 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.claude/rules/metadata-state-machine.md b/.claude/rules/metadata-state-machine.md index 2af12f03..91386e54 100644 --- a/.claude/rules/metadata-state-machine.md +++ b/.claude/rules/metadata-state-machine.md @@ -23,7 +23,7 @@ MetadataStateMachine (one per shard group) │ ├── topic_name_index: HashMap └── security - ├── admissions: Certificate Node ID → Admission Record + ├── admissions: Node Certificate Principal → Admission Record ├── ACLs: Resource → ACL Record └── revocations: (Issuer, Serial) → Revocation Record ``` @@ -85,4 +85,4 @@ MetadataStateMachine (one per shard group) 22. **A committed consumer-group generation assigns each active range exactly once.** When a group has members, its assignment keys exactly equal the topic's active ranges and every assignment names a current member. When it has no members, it has no assignments. Membership or range-topology changes advance the generation and recompute the full desired assignment through the Raft log; heartbeat refreshes that do not change membership leave the generation unchanged. -23. **Security map keys match their records.** Admission keys equal the record's Certificate Node ID, ACL keys equal the record's resource, and revocation keys equal the record's issuer and serial. A map therefore contains at most one current admission per Certificate Node ID and snapshot restore cannot silently associate a security record with a different lookup key. +23. **Security map keys match their records.** Admission keys equal the record's Node Certificate Principal, ACL keys equal the record's resource, and revocation keys equal the record's issuer and serial. A map therefore contains at most one current admission per Node Certificate Principal and snapshot restore cannot silently associate a security record with a different lookup key. diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 51fe415c..95e01cc5 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -63,8 +63,8 @@ To allow safe node restarts and hardware replacement without exposing the cluste | Term | Scope | Lifetime / Ordering | Function | | :--- | :--- | :--- | :--- | -| **Certificate Node ID** | X.509 Certificate | Long-lived / Reused | Operator-assigned node name and stable admission-record key. | -| **Admission Epoch** | Certificate Node ID | Monotonically increasing `u64` | Assigned by metadata Raft upon restart; higher epoch **fences** older instances. | +| **Node Certificate Principal** | X.509 Certificate | Long-lived / Reused | Operator-assigned node principal read from the authenticated certificate and used as the stable admission-record key. | +| **Admission Epoch** | Node Certificate Principal | Monotonically increasing `u64` | Assigned by metadata Raft upon restart; higher epoch **fences** older instances. | | **NodeId** | Running Process | Single process lifetime | Unique ID generated on startup; used by SWIM, topology ring, Raft, and data placement. | | **Process Key** | Running Process | Single process lifetime | Proves that the connection belongs to the process admitted for this epoch. | | **SWIM Incarnation** | Running Process | Monotonically increasing counter | Incremented by the *same* process instance to refute false `Suspect`/`Dead` gossip. | @@ -149,7 +149,7 @@ A security record is one durable admission, ACL, or revocation entry. Its record path selects one metadata shard; its revision lets brokers detect stale cached copies. -Security records (`security/node/{certificate-node-id}`, `security/acl/{resource}`, `security/revocation/{issuer}/{serial}`) do not rely on a centralized security controller. Instead, they hash to standard metadata shards and replicate via Raft: +Security records (`security/node/{node-certificate-principal}`, `security/acl/{resource}`, `security/revocation/{issuer}/{serial}`) do not rely on a centralized security controller. Instead, they hash to standard metadata shards and replicate via Raft: ``` Client/Node Request ──► Any Broker ──► Hash Record Path ──► Hosts Shard? ─┬─► Yes ──► Commit via Raft @@ -157,11 +157,11 @@ Security records (`security/node/{certificate-node-id}`, `security/acl/{resource ``` - **Stable Admission Key:** A restart changes the process `NodeId` and key, but - not the Certificate Node ID. The same record and metadata shard therefore + not the Node Certificate Principal. The same record and metadata shard therefore replace the old admitted process atomically: ``` -security/node/{certificate-node-id} +security/node/{node-certificate-principal} │ └── Admission Epoch + NodeId + Process Public Key ``` @@ -182,7 +182,7 @@ security/node/{certificate-node-id} 3. A later joining node generates a new `NodeId` and process key. 4. An authorized operator approves that exact `NodeId` and process public key. The reusable node certificate alone cannot authorize replacement. 5. The joining node connects to a **limited admission endpoint** using its X.509 certificate. -6. The endpoint uses the authenticated Certificate Node ID to route to its +6. The endpoint uses the authenticated Node Certificate Principal to route to its admission record. The owning metadata shard atomically replaces the prior process with the next `Admission Epoch`, `NodeId`, and process public key. 7. The joining node proves possession of the process private key before entering SWIM gossip and Raft membership reconciliation. @@ -255,8 +255,8 @@ turmoil with pinned randomness and node identities. ### System Invariants -1. **Single Connection Identity:** Every established client connection has exactly one authenticated principal; every node connection has exactly one `(Certificate Node ID, Admission Epoch, NodeId, Process Public Key)`. -2. **Unique Active Node Admission:** Metadata state maintains at most one active `(Admission Epoch, NodeId, Process Public Key)` per Certificate Node ID. +1. **Single Connection Identity:** Every established client connection has exactly one authenticated principal; every node connection has exactly one `(Node Certificate Principal, Admission Epoch, NodeId, Process Public Key)`. +2. **Unique Active Node Admission:** Metadata state maintains at most one active `(Admission Epoch, NodeId, Process Public Key)` per Node Certificate Principal. 3. **Immutable Producer Session Principal:** Every producer session is immutably bound to the principal that created it. 4. **Explicit Cache Bounding:** Every cached security entry specifies its source metadata shard, revision, and expiry measured with a monotonic clock (≤ 60 seconds). 5. **Bounded Audit Footprint:** Audit queues and aggregate rate counters remain within configured capacity. diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index 4e0cb13e..d59ebfe7 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -635,7 +635,7 @@ mod tests { fn security_records_survive_snapshot_restore() { let mut state = MetadataState::new(ShardGroupId(1)); let admission = AdmissionRecord { - certificate_node_id: "broker-a".to_string(), + node_certificate_principal: "broker-a".to_string(), revision: 3, epoch: 2, node_id: NodeId::new("broker-a::process-2"), @@ -653,10 +653,10 @@ mod tests { revoked_at: 100, }; - state - .security - .admissions - .insert(admission.certificate_node_id.clone(), admission.clone()); + state.security.admissions.insert( + admission.node_certificate_principal.clone(), + admission.clone(), + ); state .security .acls diff --git a/src/control_plane/consensus/raft/states/security.rs b/src/control_plane/consensus/raft/states/security.rs index 3aa948c0..5b8901ca 100644 --- a/src/control_plane/consensus/raft/states/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -15,13 +15,13 @@ pub(crate) struct SecurityState { pub(super) revocations: HashMap<(String, Box<[u8]>), RevocationRecord>, } -/// Current process admitted for `security/node/{certificate_node_id}`. +/// Current process admitted for `security/node/{node_certificate_principal}`. /// /// A restart replaces this record through its metadata shard. Admission checks /// accept SWIM facts only when the epoch, node ID, and process key match it. #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct AdmissionRecord { - pub certificate_node_id: String, + pub node_certificate_principal: String, pub revision: u64, pub epoch: u64, pub node_id: NodeId, @@ -55,10 +55,10 @@ pub(crate) struct RevocationRecord { #[cfg(any(test, debug_assertions))] impl crate::test_traits::TAssertInvariant for SecurityState { fn assert_invariants(&self) { - for (certificate_node_id, admission) in &self.admissions { + for (node_certificate_principal, admission) in &self.admissions { assert_eq!( - certificate_node_id, &admission.certificate_node_id, - "admission map key does not match Certificate Node ID" + node_certificate_principal, &admission.node_certificate_principal, + "admission map key does not match Node Certificate Principal" ); } for (resource, acl) in &self.acls { @@ -95,7 +95,7 @@ mod tests { #[test] fn security_records_round_trip() { round_trip(&AdmissionRecord { - certificate_node_id: "broker-a".to_string(), + node_certificate_principal: "broker-a".to_string(), revision: 3, epoch: 2, node_id: NodeId::new("broker-a::process-2"), From e492f264c6d6f0039fba782ffddb05c5d0bcdcd3 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 12:45:19 +0400 Subject: [PATCH 13/56] principal extraction from certificate -> SAN -> uri --- Cargo.lock | 1 + Cargo.toml | 1 + docs/security/roadmap.md | 4 ++ src/security.rs | 82 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index cd3b9595..c9482e9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -420,6 +420,7 @@ dependencies = [ "tracing-subscriber", "turmoil", "uuid", + "x509-parser", "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index 1c178195..8dbcf2c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ futures = "0.3" quinn = { version = "0.11.11", default-features = false, features = ["bloom", "runtime-tokio", "rustls-ring"] } rustls = { version = "0.23.42", default-features = false, features = ["ring", "std"] } rustls-pemfile = "2.2.0" +x509-parser = "0.18.1" [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 95e01cc5..e6f95871 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -69,6 +69,10 @@ To allow safe node restarts and hardware replacement without exposing the cluste | **Process Key** | Running Process | Single process lifetime | Proves that the connection belongs to the process admitted for this epoch. | | **SWIM Incarnation** | Running Process | Monotonically increasing counter | Incremented by the *same* process instance to refute false `Suspect`/`Dead` gossip. | +The Node Certificate Principal is the value after `urn:eastguard:node:` in one +URI Subject Alternative Name. A node certificate must contain exactly one such +URI; missing or repeated values fail authentication. + ### Resolution Rules Node identity conflicts and stale gossip are resolved in this order: diff --git a/src/security.rs b/src/security.rs index 8729e349..7c5ed63e 100644 --- a/src/security.rs +++ b/src/security.rs @@ -8,6 +8,8 @@ use anyhow::{Context, Result}; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls::server::WebPkiClientVerifier; use rustls::{ClientConfig, RootCertStore, ServerConfig}; +use x509_parser::extensions::GeneralName; +use x509_parser::prelude::{FromDer, X509Certificate}; use crate::config::{Environment, SecurityMode}; @@ -107,10 +109,59 @@ impl SecureTransportConfig { } } +/// Reads the stable node principal from a leaf certificate's URI Subject +/// Alternative Name. +/// +/// The certificate must contain exactly one URI beginning with +/// `urn:eastguard:node:`. The text after that prefix is the principal used as +/// the admission-record key. This function only parses the certificate; callers +/// must use it after rustls has authenticated the peer's certificate chain. +fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Result { + const NODE_PRINCIPAL_URI_PREFIX: &str = "urn:eastguard:node:"; + + let (_, certificate) = + X509Certificate::from_der(certificate.as_ref()).context("invalid X.509 certificate")?; + let subject_alt_name = certificate + .subject_alternative_name() + .context("invalid X.509 subject alternative name")? + .context("node certificate has no subject alternative name")?; + + let mut principals = + subject_alt_name + .value + .general_names + .iter() + .filter_map(|name| match name { + GeneralName::URI(uri) => uri.strip_prefix(NODE_PRINCIPAL_URI_PREFIX), + _ => None, + }); + let principal = principals + .next() + .filter(|principal| !principal.is_empty()) + .context("node certificate has no Node Certificate Principal")?; + anyhow::ensure!( + principals.next().is_none(), + "node certificate has multiple Node Certificate Principals" + ); + Ok(principal.to_string()) +} + #[cfg(test)] mod tests { use super::*; use clap::Parser; + use rcgen::string::Ia5String; + use rcgen::{CertificateParams, KeyPair, SanType}; + + fn certificate_with_uris(uris: &[&str]) -> CertificateDer<'static> { + let mut params = CertificateParams::default(); + params.subject_alt_names = uris + .iter() + .map(|uri| SanType::URI(Ia5String::try_from(*uri).unwrap())) + .collect(); + let key = KeyPair::generate().unwrap(); + params.self_signed(&key).unwrap().der().clone() + } #[test] fn secure_mode_requires_every_credential_path() { @@ -134,4 +185,35 @@ mod tests { assert!(SecureTransportConfig::load(&env).unwrap().is_none()); } + + #[test] + fn reads_node_principal_from_uri_subject_alternative_name() { + let certificate = + certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:node:broker-a"]); + + assert_eq!( + node_certificate_principal(&certificate).unwrap(), + "broker-a" + ); + } + + #[test] + fn requires_exactly_one_node_principal() { + let missing = certificate_with_uris(&["urn:example:unrelated"]); + let ambiguous = + certificate_with_uris(&["urn:eastguard:node:broker-a", "urn:eastguard:node:broker-b"]); + + assert_eq!( + node_certificate_principal(&missing) + .unwrap_err() + .to_string(), + "node certificate has no Node Certificate Principal" + ); + assert_eq!( + node_certificate_principal(&ambiguous) + .unwrap_err() + .to_string(), + "node certificate has multiple Node Certificate Principals" + ); + } } From d7f9362ef3d21ee7045a42f8392a22b713a9a31d Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 12:57:38 +0400 Subject: [PATCH 14/56] server cert verifier --- src/security.rs | 126 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 3 deletions(-) diff --git a/src/security.rs b/src/security.rs index 7c5ed63e..5a29de3b 100644 --- a/src/security.rs +++ b/src/security.rs @@ -1,18 +1,110 @@ #![allow(dead_code)] +use std::fmt; use std::fs::File; use std::io::BufReader; use std::path::Path; use std::sync::Arc; use anyhow::{Context, Result}; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::client::verify_server_cert_signed_by_trust_anchor; +use rustls::crypto::{WebPkiSupportedAlgorithms, verify_tls12_signature, verify_tls13_signature}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime}; +use rustls::server::ParsedCertificate; use rustls::server::WebPkiClientVerifier; -use rustls::{ClientConfig, RootCertStore, ServerConfig}; +use rustls::{ + CertificateError, ClientConfig, DigitallySignedStruct, Error as RustlsError, OtherError, + RootCertStore, ServerConfig, SignatureScheme, +}; use x509_parser::extensions::GeneralName; use x509_parser::prelude::{FromDer, X509Certificate}; use crate::config::{Environment, SecurityMode}; +/// Verifies certificates presented to EastGuard's outbound node connections. +/// +/// The shared client config uses this verifier when Raft or data transport +/// connects to another broker. It retains certificate-chain, validity, +/// server-usage, and TLS handshake-signature verification. It does not compare +/// the certificate with a DNS name because brokers are identified by the Node +/// Certificate Principal carried in the certificate; admission later binds +/// that stable principal to the process-specific `NodeId`. +/// +/// Peer certificate +// ├── trusted CA chain? ── no → reject +// ├── valid lifetime and server usage? ── no → reject +// ├── valid TLS handshake signature? ── no → reject +// └── exactly one Node Certificate Principal? ── no → reject +struct NodeServerCertVerifier { + roots: Arc, + supported: WebPkiSupportedAlgorithms, +} + +impl NodeServerCertVerifier { + fn new(roots: Arc) -> Self { + Self { + roots, + supported: rustls::crypto::ring::default_provider().signature_verification_algorithms, + } + } +} + +impl fmt::Debug for NodeServerCertVerifier { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("NodeServerCertVerifier") + .finish_non_exhaustive() + } +} + +impl ServerCertVerifier for NodeServerCertVerifier { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + now: UnixTime, + ) -> std::result::Result { + let certificate = ParsedCertificate::try_from(end_entity)?; + verify_server_cert_signed_by_trust_anchor( + &certificate, + &self.roots, + intermediates, + now, + self.supported.all, + )?; + node_certificate_principal(end_entity).map_err(|error| { + CertificateError::Other(OtherError(Arc::new(std::io::Error::other( + error.to_string(), + )))) + })?; + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + certificate: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls12_signature(message, certificate, signature, &self.supported) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + certificate: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls13_signature(message, certificate, signature, &self.supported) + } + + fn supported_verify_schemes(&self) -> Vec { + self.supported.supported_schemes() + } +} + pub(crate) struct SecureTransportConfig { pub(crate) server: Arc, pub(crate) client: Arc, @@ -55,7 +147,8 @@ impl SecureTransportConfig { .with_client_cert_verifier(client_verifier) .with_single_cert(certificate_chain.clone(), private_key.clone_key())?; let client = ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .with_root_certificates((*trust_roots).clone()) + .dangerous() + .with_custom_certificate_verifier(Arc::new(NodeServerCertVerifier::new(trust_roots))) .with_client_auth_cert(certificate_chain, private_key)?; Ok(Self { @@ -216,4 +309,31 @@ mod tests { "node certificate has multiple Node Certificate Principals" ); } + + #[test] + fn server_verifier_requires_trust_and_node_principal() { + let trusted = certificate_with_uris(&["urn:eastguard:node:broker-a"]); + let missing_principal = certificate_with_uris(&["urn:example:unrelated"]); + let mut roots = RootCertStore::empty(); + roots.add(trusted.clone()).unwrap(); + roots.add(missing_principal.clone()).unwrap(); + let verifier = NodeServerCertVerifier::new(Arc::new(roots)); + let server_name = ServerName::try_from("unused.eastguard").unwrap(); + + verifier + .verify_server_cert(&trusted, &[], &server_name, &[], UnixTime::now()) + .unwrap(); + assert!( + verifier + .verify_server_cert(&missing_principal, &[], &server_name, &[], UnixTime::now(),) + .is_err() + ); + + let untrusted = certificate_with_uris(&["urn:eastguard:node:broker-b"]); + assert!( + verifier + .verify_server_cert(&untrusted, &[], &server_name, &[], UnixTime::now()) + .is_err() + ); + } } From 8d6f617f7c33248749830cb4d045bff4b2f2f37a Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 13:18:04 +0400 Subject: [PATCH 15/56] =?UTF-8?q?feat:=20authenticate=20TCP=20prims=20=20?= =?UTF-8?q?=20turmoil=20TCP=20socket=20=20=20=20=20=20=20=20=20=E2=86=93?= =?UTF-8?q?=20=20=20TLS=201.3=20mutual=20authentication=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=E2=86=93=20=20=20AuthenticatedTcpStream=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=E2=94=9C=E2=94=80=E2=94=80=20encrypted=20byte?= =?UTF-8?q?=20stream=20=20=20=20=20=20=20=20=20=E2=94=94=E2=94=80=E2=94=80?= =?UTF-8?q?=20authenticated=20peer=20principal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added tokio-rustls. - Supports inbound and outbound mutual TLS. - Extracts both peers’ Node Certificate Principals. - Deterministic turmoil test exchanges ping/pong. - Not yet used by production actors; the explicit dead_code allowance will be removed when Raft/data transports adopt it in the next slice. - Focused turmoil test and all-target/all-feature clippy pass. - Formatting and diff checks pass. --- Cargo.lock | 11 +++ Cargo.toml | 1 + src/net/tcp.rs | 180 ++++++++++++++++++++++++++++++++++++++++++++++++ src/security.rs | 2 +- 4 files changed, 193 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c9482e9b..c4604a69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -416,6 +416,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-rustls", "tracing", "tracing-subscriber", "turmoil", @@ -1729,6 +1730,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 8dbcf2c9..38e6b213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ futures = "0.3" quinn = { version = "0.11.11", default-features = false, features = ["bloom", "runtime-tokio", "rustls-ring"] } rustls = { version = "0.23.42", default-features = false, features = ["ring", "std"] } rustls-pemfile = "2.2.0" +tokio-rustls = { version = "0.26.4", default-features = false, features = ["ring"] } x509-parser = "0.18.1" [target.'cfg(target_os = "linux")'.dependencies] diff --git a/src/net/tcp.rs b/src/net/tcp.rs index 48dbc251..c8936273 100644 --- a/src/net/tcp.rs +++ b/src/net/tcp.rs @@ -1,10 +1,15 @@ use std::ops::{Deref, DerefMut}; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; +use anyhow::{Context as _, Result}; +use rustls::pki_types::ServerName; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio_rustls::{TlsAcceptor, TlsConnector, TlsStream}; use super::inner; +use crate::security::node_certificate_principal; macro_rules! tcp_wrapper { ($name:ident) => { @@ -31,6 +36,85 @@ tcp_wrapper!(TcpStream); tcp_wrapper!(OwnedReadHalf); tcp_wrapper!(OwnedWriteHalf); +/// Mutually authenticated node connection used by the Raft and data transports. +/// +/// TLS authenticates the certificate chain before this stream exposes the peer's +/// Node Certificate Principal. Admission later binds that stable principal to +/// the process-specific `NodeId` carried by the transport handshake. +#[allow(dead_code)] +pub struct AuthenticatedTcpStream { + peer_principal: String, + stream: TlsStream, +} + +#[allow(dead_code)] +impl AuthenticatedTcpStream { + pub async fn accept(stream: TcpStream, config: Arc) -> Result { + let stream = TlsAcceptor::from(config).accept(stream).await?; + Self::from_tls_stream(stream.into()) + } + + pub async fn connect( + addr: A, + config: Arc, + ) -> Result { + let stream = TcpStream::connect(addr).await?; + // NodeServerCertVerifier authenticates the certificate chain and node + // principal. It intentionally does not use DNS-name matching. + let server_name = ServerName::try_from("unused.eastguard")?; + let stream = TlsConnector::from(config) + .connect(server_name, stream) + .await?; + Self::from_tls_stream(stream.into()) + } + + fn from_tls_stream(stream: TlsStream) -> Result { + let certificate = stream + .get_ref() + .1 + .peer_certificates() + .and_then(|certificates| certificates.first()) + .context("authenticated TLS peer supplied no certificate")?; + let peer_principal = node_certificate_principal(certificate)?; + Ok(Self { + peer_principal, + stream, + }) + } + + pub fn peer_principal(&self) -> &str { + &self.peer_principal + } +} + +impl AsyncRead for AuthenticatedTcpStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.stream).poll_read(cx, buf) + } +} + +impl AsyncWrite for AuthenticatedTcpStream { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.stream).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } +} + impl AsyncRead for TcpStream { fn poll_read( mut self: Pin<&mut Self>, @@ -111,3 +195,99 @@ impl TcpStream { (OwnedReadHalf(read_half), OwnedWriteHalf(write_half)) } } + +#[cfg(test)] +mod tests { + use super::*; + use rcgen::string::Ia5String; + use rcgen::{CertificateParams, KeyPair, SanType}; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + use rustls::server::WebPkiClientVerifier; + use rustls::{ClientConfig, RootCertStore, ServerConfig}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use turmoil::Builder; + + fn certificate( + principal: &str, + dns_name: Option<&str>, + ) -> ( + rustls::pki_types::CertificateDer<'static>, + PrivateKeyDer<'static>, + ) { + let mut params = CertificateParams::default(); + params.subject_alt_names.push(SanType::URI( + Ia5String::try_from(format!("urn:eastguard:node:{principal}")).unwrap(), + )); + if let Some(dns_name) = dns_name { + params + .subject_alt_names + .push(SanType::DnsName(Ia5String::try_from(dns_name).unwrap())); + } + let key = KeyPair::generate().unwrap(); + let certificate = params.self_signed(&key).unwrap().der().clone(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(key.serialize_der())); + (certificate, key) + } + + fn tls_configs() -> (Arc, Arc) { + let (server_certificate, server_key) = + certificate("broker-server", Some("unused.eastguard")); + let (client_certificate, client_key) = certificate("broker-client", None); + + let mut client_roots = RootCertStore::empty(); + client_roots.add(client_certificate.clone()).unwrap(); + let client_verifier = WebPkiClientVerifier::builder(Arc::new(client_roots)) + .build() + .unwrap(); + let server = ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_client_cert_verifier(client_verifier) + .with_single_cert(vec![server_certificate.clone()], server_key) + .unwrap(); + + let mut server_roots = RootCertStore::empty(); + server_roots.add(server_certificate).unwrap(); + let client = ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_root_certificates(server_roots) + .with_client_auth_cert(vec![client_certificate], client_key) + .unwrap(); + (Arc::new(server), Arc::new(client)) + } + + #[test] + fn mutual_tls_exposes_peer_principals_under_turmoil() -> turmoil::Result { + let (server_config, client_config) = tls_configs(); + let mut sim = Builder::new().build(); + + sim.host("server", move || { + let server_config = server_config.clone(); + async move { + let listener = TcpListener::bind("0.0.0.0:9000").await?; + let (stream, _) = listener.accept().await?; + let mut stream = AuthenticatedTcpStream::accept(stream, server_config) + .await + .unwrap(); + assert_eq!(stream.peer_principal(), "broker-client"); + let mut message = [0; 4]; + stream.read_exact(&mut message).await?; + assert_eq!(&message, b"ping"); + stream.write_all(b"pong").await?; + Ok(()) + } + }); + + sim.client("client", async move { + let mut stream = + AuthenticatedTcpStream::connect((turmoil::lookup("server"), 9000), client_config) + .await + .unwrap(); + assert_eq!(stream.peer_principal(), "broker-server"); + stream.write_all(b"ping").await?; + let mut message = [0; 4]; + stream.read_exact(&mut message).await?; + assert_eq!(&message, b"pong"); + Ok(()) + }); + + sim.run() + } +} diff --git a/src/security.rs b/src/security.rs index 5a29de3b..cccd6a26 100644 --- a/src/security.rs +++ b/src/security.rs @@ -209,7 +209,7 @@ impl SecureTransportConfig { /// `urn:eastguard:node:`. The text after that prefix is the principal used as /// the admission-record key. This function only parses the certificate; callers /// must use it after rustls has authenticated the peer's certificate chain. -fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Result { +pub(crate) fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Result { const NODE_PRINCIPAL_URI_PREFIX: &str = "urn:eastguard:node:"; let (_, certificate) = From 02e784f12bceac4133dcc4f998f168f803240b29 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 13:33:49 +0400 Subject: [PATCH 16/56] feat: NodeTcpStream - RaftRpcListener now has one constructor taking the read half and explicit transport identity. - Raft transport uses shared node-stream halves. - Secure streams retain the certificate principal at the connection boundary. - Trusted-development tests name their mode explicitly. - Large TLS stream variant is boxed. - Mutual-TLS turmoil test and five Raft transport tests pass. - All-target/all-feature clippy, formatting, and diff checks pass. --- .../consensus/transport/inbound.rs | 31 ++- src/control_plane/consensus/transport/mod.rs | 44 +++- .../consensus/transport/outbound.rs | 40 +++- src/it/raft/election.rs | 2 + src/it/raft/leader_event.rs | 2 + src/it/raft/membership_change.rs | 2 + src/it/raft/snapshot.rs | 2 + src/lib.rs | 6 +- src/net/mod.rs | 5 +- src/net/tcp.rs | 191 +++++++++++++++++- src/security.rs | 37 +++- 11 files changed, 319 insertions(+), 43 deletions(-) diff --git a/src/control_plane/consensus/transport/inbound.rs b/src/control_plane/consensus/transport/inbound.rs index 9df4b3ad..e7862fb6 100644 --- a/src/control_plane/consensus/transport/inbound.rs +++ b/src/control_plane/consensus/transport/inbound.rs @@ -2,33 +2,52 @@ use crate::control_plane::NodeId; use crate::control_plane::consensus::actor::MutlRaftSender; use crate::control_plane::consensus::messages::InboundRaftRpc; use crate::control_plane::consensus::messages::WireRaftMessage; -use crate::net::OwnedReadHalf; +use crate::net::NodeReadHalf; +use crate::security::NodeTransportIdentity; use tokio::io::AsyncReadExt; -pub(super) struct RaftRpcListener(pub(super) OwnedReadHalf); +pub(super) struct RaftRpcListener { + read_half: NodeReadHalf, + transport_identity: NodeTransportIdentity, +} impl RaftRpcListener { + pub(super) fn new( + read_half: impl Into, + transport_identity: NodeTransportIdentity, + ) -> Self { + Self { + read_half: read_half.into(), + transport_identity, + } + } + pub(super) async fn read_node_id(&mut self) -> anyhow::Result { - let len = self.0.read_u32().await? as usize; + let len = self.read_half.read_u32().await? as usize; anyhow::ensure!(len <= 1024, "NodeId frame too large: {len} bytes"); let mut buf = vec![0u8; len]; - self.0.read_exact(&mut buf).await?; + self.read_half.read_exact(&mut buf).await?; let id = borsh::from_slice::(&buf)?; Ok(id) } pub(super) async fn read_message(&mut self) -> anyhow::Result { - let len = self.0.read_u32().await? as usize; + let len = self.read_half.read_u32().await? as usize; anyhow::ensure!( len <= 4 * 1024 * 1024, "Raft message frame too large: {len} bytes" ); let mut buf = vec![0u8; len]; - self.0.read_exact(&mut buf).await?; + self.read_half.read_exact(&mut buf).await?; let msg = borsh::from_slice::(&buf)?; Ok(msg) } + #[tracing::instrument( + level = "trace", + skip_all, + fields(peer = %peer, transport_identity = ?self.transport_identity) + )] pub(super) async fn run(mut self, tx: MutlRaftSender, peer: NodeId) { loop { match self.read_message().await { diff --git a/src/control_plane/consensus/transport/mod.rs b/src/control_plane/consensus/transport/mod.rs index 072d0bf5..84ec681a 100644 --- a/src/control_plane/consensus/transport/mod.rs +++ b/src/control_plane/consensus/transport/mod.rs @@ -12,7 +12,11 @@ use crate::control_plane::consensus::actor::MutlRaftSender; use crate::control_plane::NodeId; use crate::control_plane::consensus::messages::RaftTransportCommand; use crate::control_plane::membership::actor::SwimSender; +use crate::net::NodeTcpStream; use crate::net::TcpListener; +#[cfg(test)] +use crate::security::NodeTransportIdentity; +use crate::security::NodeTransportSecurity; const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); @@ -26,16 +30,20 @@ impl RaftTransportActor { raft_tx: MutlRaftSender, mut from_actor: mpsc::Receiver>, swim_tx: SwimSender, + security: NodeTransportSecurity, ) { let (dial_tx, mut dial_rx) = mpsc::channel(256); - let mut dispatcher = RaftRpcDispatcher::new(node_id, dial_tx); + let mut dispatcher = RaftRpcDispatcher::new(node_id, dial_tx, security.clone()); let mut cleanup_interval = tokio::time::interval(std::time::Duration::from_secs(300)); cleanup_interval.tick().await; // consume immediate first tick loop { tokio::select! { Ok((stream, _)) = listener.accept() => { - dispatcher.accept(stream, &raft_tx).await; + match NodeTcpStream::accept(stream, &security).await { + Ok(stream) => dispatcher.accept(stream, &raft_tx).await, + Err(error) => tracing::debug!("Raft TLS accept rejected: {error}"), + } } Some(batch) = from_actor.recv() => { // Disconnects are applied first so same-batch sends already skip removed peers, then @@ -101,7 +109,8 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = RaftRpcListener(read_half); + let mut reader = + RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); let peer_id = reader.read_node_id().await.unwrap(); assert_eq!(peer_id, NodeId::new("node-abc")); @@ -130,7 +139,8 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = RaftRpcListener(read_half); + let mut reader = + RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); let msg = reader.read_message().await.unwrap(); assert_eq!(msg.shard_group_id, ShardGroupId(42)); @@ -180,7 +190,8 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = RaftRpcListener(read_half); + let mut reader = + RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); let peer = reader.read_node_id().await?; let (raft_tx, mut raft_rx) = MultiRaftActor::channel(8); @@ -236,10 +247,16 @@ mod tests { let (raft_tx, _raft_rx) = MultiRaftActor::channel(16); let listener = TcpListener::bind("0.0.0.0:9000").await?; let (dial_tx, _dial_rx) = tokio::sync::mpsc::channel(8); - let mut state = RaftRpcDispatcher::new(NodeId::new("node-b"), dial_tx); + let mut state = RaftRpcDispatcher::new( + NodeId::new("node-b"), + dial_tx, + NodeTransportSecurity::TrustedDevelopment, + ); let (stream, _) = listener.accept().await?; - state.accept(stream, &raft_tx).await; + state + .accept(NodeTcpStream::TrustedDevelopment(stream), &raft_tx) + .await; assert!( state.contains(&NodeId::new("node-a")), @@ -274,17 +291,24 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let dummy_listener = TcpListener::bind("0.0.0.0:9001").await?; let (dial_tx, _dial_rx) = tokio::sync::mpsc::channel(8); - let mut state = RaftRpcDispatcher::new(NodeId::new("node-b"), dial_tx); + let mut state = RaftRpcDispatcher::new( + NodeId::new("node-b"), + dial_tx, + NodeTransportSecurity::TrustedDevelopment, + ); // First connection from node-a let (stream, _) = listener.accept().await?; - state.accept(stream, &raft_tx).await; + state + .accept(NodeTcpStream::TrustedDevelopment(stream), &raft_tx) + .await; assert!(state.contains(&NodeId::new("node-a"))); // Second connection from node-a (simulating simultaneous connect) let (stream2, _) = dummy_listener.accept().await?; let (read_half, _write_half) = stream2.into_split(); - let mut reader = RaftRpcListener(read_half); + let mut reader = + RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); let peer_id = reader.read_node_id().await.unwrap(); assert_eq!(peer_id, NodeId::new("node-a")); diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index 35530f5f..a313ae72 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -11,7 +11,8 @@ use crate::control_plane::consensus::messages::{OutboundRaftPacket, WireRaftMess use crate::control_plane::NodeId; use crate::control_plane::consensus::transport::RaftRpcListener; use crate::control_plane::membership::actor::SwimSender; -use crate::net::{OwnedWriteHalf, TcpStream}; +use crate::net::{NodeTcpStream, NodeWriteHalf}; +use crate::security::NodeTransportSecurity; const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); /// Upper bound on messages buffered per peer while its dial is in flight; @@ -28,7 +29,7 @@ const PENDING_DIAL_BUFFER_CAP: usize = 256; /// outbound connect), the tie is broken by NodeId ordering. pub(super) struct RaftRpcDispatcher { node_id: NodeId, - writers: HashMap, + writers: HashMap, /// Peers explicitly disconnected via DisconnectPeer. Outbound RPCs /// to these peers are silently dropped until a new connection is /// accepted (peer restart with new UUID won't hit this — different NodeId). @@ -41,16 +42,21 @@ pub(super) struct RaftRpcDispatcher { /// background task; flushed (or dropped on failure) in `on_dial_result`. pending_dials: HashMap>, dial_tx: mpsc::Sender, + security: NodeTransportSecurity, } /// Result of a background dial attempt, delivered back to the transport loop. pub(super) struct DialOutcome { target: NodeId, - outcome: anyhow::Result<(RaftRpcListener, OwnedWriteHalf)>, + outcome: anyhow::Result<(RaftRpcListener, NodeWriteHalf)>, } impl RaftRpcDispatcher { - pub(super) fn new(node_id: NodeId, dial_tx: mpsc::Sender) -> Self { + pub(super) fn new( + node_id: NodeId, + dial_tx: mpsc::Sender, + security: NodeTransportSecurity, + ) -> Self { Self { node_id, writers: HashMap::new(), @@ -58,12 +64,14 @@ impl RaftRpcDispatcher { connect_backoffs: HashMap::new(), pending_dials: HashMap::new(), dial_tx, + security, } } - pub(super) async fn accept(&mut self, stream: TcpStream, raft_tx: &MutlRaftSender) { + pub(super) async fn accept(&mut self, stream: NodeTcpStream, raft_tx: &MutlRaftSender) { + let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); - let mut reader = RaftRpcListener(read_half); + let mut reader = RaftRpcListener::new(read_half, transport_identity); let Ok(peer_id) = reader.read_node_id().await else { tracing::error!("Failed to read peer NodeId during accept"); @@ -139,7 +147,12 @@ impl RaftRpcDispatcher { return; } self.pending_dials.insert(target_id.clone(), msgs); - let dial_task = dial(self.node_id.clone(), target_id.clone(), swim_tx.clone()); + let dial_task = dial( + self.node_id.clone(), + target_id.clone(), + swim_tx.clone(), + self.security.clone(), + ); let dial_tx = self.dial_tx.clone(); tokio::spawn(async move { let outcome = dial_task.await; @@ -235,7 +248,7 @@ impl RaftRpcDispatcher { } } -/// Resolve, connect (3s cap), and handshake — on a spawned task, so a hung +/// Resolve, connect (3secs cap), and handshake — on a spawned task, so a hung /// connect can never block the transport select loop. The loop /// installs the writer and flushes buffered messages in `on_dial_result`. // ! never inline this. Actor Model should onkly do work whose duration it controls. @@ -244,22 +257,27 @@ async fn dial( node_id: NodeId, target_id: NodeId, swim_tx: SwimSender, -) -> anyhow::Result<(RaftRpcListener, OwnedWriteHalf)> { + security: NodeTransportSecurity, +) -> anyhow::Result<(RaftRpcListener, NodeWriteHalf)> { let Some(addr) = swim_tx.resolve_address(target_id.clone()).await? else { anyhow::bail!("[{}] Cannot resolve address for {:?}", node_id, target_id); }; let stream = tokio::time::timeout( std::time::Duration::from_secs(3), - TcpStream::connect(addr.cluster_addr()), + NodeTcpStream::connect(addr.cluster_addr(), &security), ) .await??; + let transport_identity = stream.peer_identity(); let (read_half, mut write_half) = stream.into_split(); let bytes = borsh::to_vec(&node_id) .map_err(|e| anyhow::anyhow!("[{}] Handshake encode failed: {e}", node_id))?; let len = bytes.len() as u32; write_half.write_all(&len.to_be_bytes()).await?; write_half.write_all(&bytes).await?; - Ok((RaftRpcListener(read_half), write_half)) + Ok(( + RaftRpcListener::new(read_half, transport_identity), + write_half, + )) } diff --git a/src/it/raft/election.rs b/src/it/raft/election.rs index ae901097..2424350c 100644 --- a/src/it/raft/election.rs +++ b/src/it/raft/election.rs @@ -19,6 +19,7 @@ use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; use crate::schedulers::ticker_message::{SchedulerSender, TickerCommand}; +use crate::security::NodeTransportSecurity; use super::{CLUSTER_PORT, QUERY_PORT, mock_swim_handler}; @@ -109,6 +110,7 @@ async fn run_raft_node( raft_tx.clone(), transport_rx, swim_tx.clone(), + NodeTransportSecurity::TrustedDevelopment, )); let db = MetadataStorage::open(std::env::temp_dir().join(uuid::Uuid::new_v4().to_string())); let election_jitter_seed = { diff --git a/src/it/raft/leader_event.rs b/src/it/raft/leader_event.rs index bc71ce25..97d6261b 100644 --- a/src/it/raft/leader_event.rs +++ b/src/it/raft/leader_event.rs @@ -19,6 +19,7 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; +use crate::security::NodeTransportSecurity; use super::CLUSTER_PORT; @@ -120,6 +121,7 @@ fn leader_election_emits_leader_change_event() -> turmoil::Result { raft_tx.clone(), transport_rx, swim_tx.clone(), + NodeTransportSecurity::TrustedDevelopment, )); let db = MetadataStorage::open( std::env::temp_dir().join(uuid::Uuid::new_v4().to_string()), diff --git a/src/it/raft/membership_change.rs b/src/it/raft/membership_change.rs index 0e968c96..e24b9196 100644 --- a/src/it/raft/membership_change.rs +++ b/src/it/raft/membership_change.rs @@ -20,6 +20,7 @@ use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; use crate::schedulers::ticker_message::{SchedulerSender, TickerCommand}; +use crate::security::NodeTransportSecurity; use super::{CLUSTER_PORT, QUERY_PORT, mock_swim_handler}; @@ -74,6 +75,7 @@ async fn start_raft_node( raft_tx.clone(), transport_rx, swim_tx.clone(), + NodeTransportSecurity::TrustedDevelopment, )); let db = MetadataStorage::open(std::env::temp_dir().join(uuid::Uuid::new_v4().to_string())); let election_jitter_seed = { diff --git a/src/it/raft/snapshot.rs b/src/it/raft/snapshot.rs index 9de420fb..aaa49a9c 100644 --- a/src/it/raft/snapshot.rs +++ b/src/it/raft/snapshot.rs @@ -19,6 +19,7 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; +use crate::security::NodeTransportSecurity; use super::{CLUSTER_PORT, mock_swim_handler}; @@ -55,6 +56,7 @@ async fn run_node(name: &'static str, ordinal: u16) -> Result<(), Box Result<()> { - if SecureTransportConfig::load(&self.env)?.is_some() { + let security = NodeTransportSecurity::load(&self.env)?; + if security.is_secure() { anyhow::bail!("secure transport listeners are not implemented"); } @@ -112,6 +113,7 @@ impl StartUp { raft_tx.clone(), raft_transport_rx, swim_sender.clone(), + security, )); // Protocol actors (each spawns its own scheduler internally) diff --git a/src/net/mod.rs b/src/net/mod.rs index 4e460088..6dbfba31 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -3,7 +3,10 @@ mod tcp; pub(crate) mod udp; -pub use tcp::{OwnedReadHalf, OwnedWriteHalf, TcpListener, TcpStream}; +pub use tcp::{ + NodeReadHalf, NodeTcpStream, NodeWriteHalf, OwnedReadHalf, OwnedWriteHalf, TcpListener, + TcpStream, +}; pub use udp::UdpSocket; #[cfg(not(test))] diff --git a/src/net/tcp.rs b/src/net/tcp.rs index c8936273..d1620b48 100644 --- a/src/net/tcp.rs +++ b/src/net/tcp.rs @@ -9,7 +9,7 @@ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio_rustls::{TlsAcceptor, TlsConnector, TlsStream}; use super::inner; -use crate::security::node_certificate_principal; +use crate::security::{NodeTransportIdentity, NodeTransportSecurity, node_certificate_principal}; macro_rules! tcp_wrapper { ($name:ident) => { @@ -41,13 +41,11 @@ tcp_wrapper!(OwnedWriteHalf); /// TLS authenticates the certificate chain before this stream exposes the peer's /// Node Certificate Principal. Admission later binds that stable principal to /// the process-specific `NodeId` carried by the transport handshake. -#[allow(dead_code)] pub struct AuthenticatedTcpStream { peer_principal: String, stream: TlsStream, } -#[allow(dead_code)] impl AuthenticatedTcpStream { pub async fn accept(stream: TcpStream, config: Arc) -> Result { let stream = TlsAcceptor::from(config).accept(stream).await?; @@ -85,6 +83,193 @@ impl AuthenticatedTcpStream { pub fn peer_principal(&self) -> &str { &self.peer_principal } + + fn into_split( + self, + ) -> ( + tokio::io::ReadHalf, + tokio::io::WriteHalf, + ) { + tokio::io::split(self) + } +} + +/// Cluster TCP connection. Secure mode carries an authenticated certificate +/// principal; trusted-development mode preserves the existing plaintext path. +pub enum NodeTcpStream { + TrustedDevelopment(TcpStream), + Secure(Box), +} + +pub enum NodeReadHalf { + TrustedDevelopment(OwnedReadHalf), + Secure(tokio::io::ReadHalf), +} + +pub enum NodeWriteHalf { + TrustedDevelopment(OwnedWriteHalf), + Secure(tokio::io::WriteHalf), +} + +impl From for NodeReadHalf { + fn from(value: OwnedReadHalf) -> Self { + Self::TrustedDevelopment(value) + } +} + +impl From for NodeWriteHalf { + fn from(value: OwnedWriteHalf) -> Self { + Self::TrustedDevelopment(value) + } +} + +impl NodeTcpStream { + pub async fn accept(stream: TcpStream, security: &NodeTransportSecurity) -> Result { + match security { + NodeTransportSecurity::Secure { server, .. } => { + AuthenticatedTcpStream::accept(stream, server.clone()) + .await + .map(Box::new) + .map(Self::Secure) + } + NodeTransportSecurity::TrustedDevelopment => Ok(Self::TrustedDevelopment(stream)), + } + } + + pub async fn connect( + addr: A, + security: &NodeTransportSecurity, + ) -> Result { + match security { + NodeTransportSecurity::Secure { client, .. } => { + AuthenticatedTcpStream::connect(addr, client.clone()) + .await + .map(Box::new) + .map(Self::Secure) + } + NodeTransportSecurity::TrustedDevelopment => TcpStream::connect(addr) + .await + .map(Self::TrustedDevelopment) + .map_err(Into::into), + } + } + + pub fn peer_identity(&self) -> NodeTransportIdentity { + match self { + Self::Secure(stream) => { + NodeTransportIdentity::CertificatePrincipal(stream.peer_principal().to_string()) + } + Self::TrustedDevelopment(_) => NodeTransportIdentity::TrustedDevelopment, + } + } + + pub fn into_split(self) -> (NodeReadHalf, NodeWriteHalf) { + match self { + Self::TrustedDevelopment(stream) => { + let (read, write) = TcpStream::into_split(stream); + ( + NodeReadHalf::TrustedDevelopment(read), + NodeWriteHalf::TrustedDevelopment(write), + ) + } + Self::Secure(stream) => { + let (read, write) = AuthenticatedTcpStream::into_split(*stream); + (NodeReadHalf::Secure(read), NodeWriteHalf::Secure(write)) + } + } + } +} + +macro_rules! impl_node_io { + ($type:ty) => { + impl AsyncRead for $type { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_read(cx, buf), + Self::Secure(stream) => Pin::new(stream).poll_read(cx, buf), + } + } + } + + impl AsyncWrite for $type { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_write(cx, buf), + Self::Secure(stream) => Pin::new(stream).poll_write(cx, buf), + } + } + + fn poll_flush( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_flush(cx), + Self::Secure(stream) => Pin::new(stream).poll_flush(cx), + } + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_shutdown(cx), + Self::Secure(stream) => Pin::new(stream).poll_shutdown(cx), + } + } + } + }; +} + +impl_node_io!(NodeTcpStream); + +impl AsyncRead for NodeReadHalf { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_read(cx, buf), + Self::Secure(stream) => Pin::new(stream).poll_read(cx, buf), + } + } +} + +impl AsyncWrite for NodeWriteHalf { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_write(cx, buf), + Self::Secure(stream) => Pin::new(stream).poll_write(cx, buf), + } + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_flush(cx), + Self::Secure(stream) => Pin::new(stream).poll_flush(cx), + } + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match &mut *self { + Self::TrustedDevelopment(stream) => Pin::new(stream).poll_shutdown(cx), + Self::Secure(stream) => Pin::new(stream).poll_shutdown(cx), + } + } } impl AsyncRead for AuthenticatedTcpStream { diff --git a/src/security.rs b/src/security.rs index cccd6a26..0e26f076 100644 --- a/src/security.rs +++ b/src/security.rs @@ -105,13 +105,23 @@ impl ServerCertVerifier for NodeServerCertVerifier { } } -pub(crate) struct SecureTransportConfig { - pub(crate) server: Arc, - pub(crate) client: Arc, +#[derive(Clone)] +pub(crate) enum NodeTransportSecurity { + Secure { + server: Arc, + client: Arc, + }, + TrustedDevelopment, } -impl SecureTransportConfig { - pub(crate) fn load(env: &Environment) -> Result> { +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum NodeTransportIdentity { + CertificatePrincipal(String), + TrustedDevelopment, +} + +impl NodeTransportSecurity { + pub(crate) fn load(env: &Environment) -> Result { match env.security_mode { SecurityMode::Secure => { let certificate_chain = env @@ -126,9 +136,9 @@ impl SecureTransportConfig { .trust_root_path .as_deref() .context("trust_root_path is required in secure mode")?; - Self::load_from_paths(certificate_chain, private_key_path, trust_roots).map(Some) + Self::load_from_paths(certificate_chain, private_key_path, trust_roots) } - SecurityMode::TrustedDevelopment => Ok(None), + SecurityMode::TrustedDevelopment => Ok(Self::TrustedDevelopment), } } @@ -151,12 +161,16 @@ impl SecureTransportConfig { .with_custom_certificate_verifier(Arc::new(NodeServerCertVerifier::new(trust_roots))) .with_client_auth_cert(certificate_chain, private_key)?; - Ok(Self { + Ok(Self::Secure { server: Arc::new(server), client: Arc::new(client), }) } + pub(crate) fn is_secure(&self) -> bool { + matches!(self, Self::Secure { .. }) + } + fn load_certificates(path: &Path, kind: &'static str) -> Result>> { let file = File::open(path).context(format!("failed to open {kind} file {}", path.display()))?; @@ -260,7 +274,7 @@ mod tests { fn secure_mode_requires_every_credential_path() { let env = Environment::try_parse_from(["eastguard"]).unwrap(); - let error = SecureTransportConfig::load(&env) + let error = NodeTransportSecurity::load(&env) .err() .expect("secure mode without credential paths must fail"); @@ -276,7 +290,10 @@ mod tests { Environment::try_parse_from(["eastguard", "--security-mode", "trusted-development"]) .unwrap(); - assert!(SecureTransportConfig::load(&env).unwrap().is_none()); + assert!(matches!( + NodeTransportSecurity::load(&env).unwrap(), + NodeTransportSecurity::TrustedDevelopment + )); } #[test] From 0024867fb6eccbe175a0a78a33eb7d1b1d1836c2 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 13:40:57 +0400 Subject: [PATCH 17/56] data-transport adopt NodeTransportSecurity, NodeTcpStream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NodeTransportSecurity │ ├── Raft TCP └── Data TCP ↓ NodeTcpStream ├── Secure: mutual TLS + certificate principal └── TrustedDevelopment: plaintext Changes: - Data readers/writers now use shared node-stream halves. - Inbound and outbound data connections explicitly select the configured security mode. - Connection identity stays attached to each reader for later admission authorization. - No optional TLS configuration. - Secure startup remains fail-closed because SWIM and client listeners are not secured yet. --- src/data_plane/transport/mod.rs | 13 +++++++++++-- src/data_plane/transport/reader.rs | 24 ++++++++++++++++++++---- src/data_plane/transport/writers.rs | 22 ++++++++++++---------- src/lib.rs | 3 ++- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/data_plane/transport/mod.rs b/src/data_plane/transport/mod.rs index 480277ec..5cbc8da4 100644 --- a/src/data_plane/transport/mod.rs +++ b/src/data_plane/transport/mod.rs @@ -9,7 +9,8 @@ use crate::control_plane::membership::TopologyReader; use crate::control_plane::membership::actor::SwimSender; use crate::data_plane::actor::DataPlaneSender; -use crate::net::TcpListener; +use crate::net::{NodeTcpStream, TcpListener}; +use crate::security::NodeTransportSecurity; use command::DataTransportCommand; use writers::TransportState; @@ -24,8 +25,9 @@ impl DataTransportActor { mut from_actor: mpsc::Receiver>, swim_tx: SwimSender, topology: TopologyReader, + security: NodeTransportSecurity, ) { - let mut state = TransportState::new(node_id); + let mut state = TransportState::new(node_id, security.clone()); let mut cleanup_interval = tokio::time::interval(std::time::Duration::from_secs(300)); cleanup_interval.tick().await; @@ -67,6 +69,13 @@ impl DataTransportActor { } Ok((stream, _)) = listener.accept() => { + let stream = match NodeTcpStream::accept(stream, &security).await { + Ok(stream) => stream, + Err(error) => { + tracing::debug!("Data TLS accept rejected: {error}"); + continue; + } + }; match state.accept(stream).await { Ok((peer, reader)) => { tokio::spawn(reader.run(data_plane_tx.clone(), peer, disconnect_tx.clone())); diff --git a/src/data_plane/transport/reader.rs b/src/data_plane/transport/reader.rs index b6a62a58..1dc7b839 100644 --- a/src/data_plane/transport/reader.rs +++ b/src/data_plane/transport/reader.rs @@ -4,19 +4,30 @@ use tokio::sync::mpsc; use crate::control_plane::NodeId; use crate::data_plane::actor::DataPlaneSender; use crate::data_plane::messages::command::{DataPlaneCommand, ReceivePeerMessage}; -use crate::net::OwnedReadHalf; +use crate::net::NodeReadHalf; +use crate::security::NodeTransportIdentity; const NODE_ID_FRAME_MAX: usize = 1024; const DATA_FRAME_MAX: usize = 64 * 1024 * 1024; -pub(super) struct DataReader(pub OwnedReadHalf); +pub(super) struct DataReader { + read_half: NodeReadHalf, + transport_identity: NodeTransportIdentity, +} impl DataReader { + pub(super) fn new(read_half: NodeReadHalf, transport_identity: NodeTransportIdentity) -> Self { + Self { + read_half, + transport_identity, + } + } + async fn read_frame(&mut self, max: usize) -> anyhow::Result { - let len = self.0.read_u32().await? as usize; + let len = self.read_half.read_u32().await? as usize; anyhow::ensure!(len <= max, "frame too large: {len} bytes (max {max})"); let mut buf = vec![0u8; len]; - self.0.read_exact(&mut buf).await?; + self.read_half.read_exact(&mut buf).await?; let val = borsh::from_slice::(&buf)?; Ok(val) } @@ -25,6 +36,11 @@ impl DataReader { self.read_frame(NODE_ID_FRAME_MAX).await } + #[tracing::instrument( + level = "trace", + skip_all, + fields(peer = %peer, transport_identity = ?self.transport_identity) + )] pub(crate) async fn run( mut self, data_plane_tx: DataPlaneSender, diff --git a/src/data_plane/transport/writers.rs b/src/data_plane/transport/writers.rs index 22725ae1..b3e63ff3 100644 --- a/src/data_plane/transport/writers.rs +++ b/src/data_plane/transport/writers.rs @@ -11,7 +11,8 @@ use crate::data_plane::actor::DataPlaneSender; use crate::data_plane::messages::command::{ DataPlaneCommand, DataPlanePeerMessage, ReceivePeerMessage, }; -use crate::net::{OwnedWriteHalf, TcpStream}; +use crate::net::{NodeTcpStream, NodeWriteHalf}; +use crate::security::NodeTransportSecurity; use super::reader::DataReader; @@ -19,30 +20,30 @@ const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); pub(super) struct TransportState { node_id: NodeId, - writers: HashMap, + writers: HashMap, dead_peers: HashSet, /// Tracks when the last connect attempt to a peer failed. Skips retry /// for CONNECT_BACKOFF (2s) to avoid blocking the select loop on repeated /// 3s TCP timeouts to unreachable peers. Cleared by periodic cleanup (300s). connect_backoffs: HashMap, + security: NodeTransportSecurity, } impl TransportState { - pub fn new(node_id: NodeId) -> Self { + pub fn new(node_id: NodeId, security: NodeTransportSecurity) -> Self { Self { node_id, writers: HashMap::new(), dead_peers: HashSet::new(), connect_backoffs: HashMap::new(), + security, } } - pub async fn accept( - &mut self, - stream: crate::net::TcpStream, - ) -> anyhow::Result<(NodeId, DataReader)> { + pub async fn accept(&mut self, stream: NodeTcpStream) -> anyhow::Result<(NodeId, DataReader)> { + let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); - let mut reader = DataReader(read_half); + let mut reader = DataReader::new(read_half, transport_identity); let peer_id = reader.read_node_id().await?; @@ -132,12 +133,13 @@ impl TransportState { let stream = tokio::time::timeout( std::time::Duration::from_secs(3), - TcpStream::connect(node_addr.data_addr()), + NodeTcpStream::connect(node_addr.data_addr(), &self.security), ) .await .context("connect timed out")? .context("TCP connect failed")?; + let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); self.writers.insert(target_id.clone(), write_half); @@ -151,7 +153,7 @@ impl TransportState { return Err(e).context("initial write failed"); } - Ok(DataReader(read_half)) + Ok(DataReader::new(read_half, transport_identity)) } pub fn disconnect(&mut self, peer_id: NodeId) { diff --git a/src/lib.rs b/src/lib.rs index 3f596e44..32ad86e6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -113,7 +113,7 @@ impl StartUp { raft_tx.clone(), raft_transport_rx, swim_sender.clone(), - security, + security.clone(), )); // Protocol actors (each spawns its own scheduler internally) @@ -145,6 +145,7 @@ impl StartUp { data_transport_rx, swim_sender.clone(), topology_reader.clone(), + security.clone(), )); MultiRaftActor::spawn( From 6b071e9664a9cbea2ce038fa7b018577f1767980 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 13:44:43 +0400 Subject: [PATCH 18/56] d8 doc update --- docs/security/roadmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index e6f95871..cff3fbc1 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -241,7 +241,7 @@ config records TCP SWIM clients operations production | :--- | :--- | :--- | :--- | | **S0** | Configuration | Security modes, certificate loader, Quinn feasibility | Secure mode opens no plaintext listeners; Quinn proves mutual certificate authentication, datagram delivery, address validation, replay protection, disabled 0-RTT, bounded connection state, and deterministic operation under turmoil | | **S1** | Metadata Storage | Security record schema, sharded Raft state | Security records survive snapshot & recovery | -| **S2** | Cluster Transport | TLS 1.3 on TCP 2922/2923, Raft D8 RPC authorization | Authenticated and authorized cluster TCP traffic | +| **S2** | Cluster Transport | TLS 1.3 on TCP 2922/2923, Raft sender and role authorization | Authenticated and authorized cluster TCP traffic | | **S3** | Membership | QUIC v1 datagrams on UDP 2922 and SWIM admission gate | Secure gossip and partition-safe admission fencing | | **S4** | Client API | Client mTLS on TCP 2921, principal binding, ACLs | Default-deny enforcement on all client APIs | | **S5** | Operations | Certificate rotation, revocation, expiry, recovery, audit logging | Online credential operations and recovery runbooks | From a55db5714ded4fee3dae09db8a717950f2fb3b2c Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 15:30:59 +0400 Subject: [PATCH 19/56] defer udp /quic --- Cargo.lock | 133 +---------- Cargo.toml | 1 - docs/security/roadmap.md | 75 +++++-- src/control_plane/membership/transport.rs | 9 +- src/net/udp.rs | 260 ---------------------- 5 files changed, 57 insertions(+), 421 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4604a69..676ebce7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,7 +401,6 @@ dependencies = [ "libc", "lz4_flex", "murmur3", - "quinn", "rand 0.10.1", "rcgen", "rocksdb", @@ -459,18 +458,6 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" -[[package]] -name = "fastbloom" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" -dependencies = [ - "foldhash 0.2.0", - "libm", - "portable-atomic", - "siphasher", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -504,12 +491,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - [[package]] name = "futures" version = "0.3.32" @@ -630,13 +611,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "rand_core 0.10.1", "wasip2", "wasip3", - "wasm-bindgen", ] [[package]] @@ -657,7 +636,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash 0.1.5", + "foldhash", ] [[package]] @@ -821,12 +800,6 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" @@ -1015,12 +988,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "portable-atomic" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" - [[package]] name = "powerfmt" version = "0.2.0" @@ -1064,63 +1031,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "quinn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" -dependencies = [ - "bytes", - "fastbloom", - "getrandom 0.4.1", - "lru-slab", - "rand 0.10.1", - "rand_pcg", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.61.2", -] - [[package]] name = "quote" version = "1.0.45" @@ -1202,15 +1112,6 @@ dependencies = [ "rand 0.9.2", ] -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", -] - [[package]] name = "rcgen" version = "0.14.8" @@ -1343,7 +1244,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ - "web-time", "zeroize", ] @@ -1533,12 +1433,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - [[package]] name = "slab" version = "0.4.12" @@ -1687,21 +1581,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.52.3" @@ -2022,16 +1901,6 @@ dependencies = [ "semver", ] -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 38e6b213..1b29c8c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ serde_yaml = "0.9.34" serde = "1" serde_json = "1" futures = "0.3" -quinn = { version = "0.11.11", default-features = false, features = ["bloom", "runtime-tokio", "rustls-ring"] } rustls = { version = "0.23.42", default-features = false, features = ["ring", "std"] } rustls-pemfile = "2.2.0" tokio-rustls = { version = "0.26.4", default-features = false, features = ["ring"] } diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index cff3fbc1..baf0b238 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -10,7 +10,7 @@ EastGuard operates in two distinct security modes: -- **Secure Mode (Default):** Mutual TLS 1.3 protects TCP listeners, and QUIC v1 datagrams protect SWIM UDP. Unauthenticated or unauthorized traffic is immediately rejected. Plaintext connections and protocol downgrades are forbidden; invalid configuration prevents startup. +- **Secure Mode (Default):** Mutual TLS 1.3 protects TCP listeners. Secure SWIM UDP is intentionally deferred until EastGuard can adopt a suitable datagram security implementation. Until then, secure mode fails startup rather than exposing plaintext SWIM. Unauthenticated or unauthorized traffic is immediately rejected. Plaintext connections and protocol downgrades are forbidden; invalid configuration prevents startup. - **Trusted Development Mode:** Plaintext protocols enabled strictly via explicit opt-in configuration for isolated test environments. ### Threat Model @@ -25,7 +25,7 @@ The production boundary defends against external network attackers attempting ea | **Client** | TCP 2921 | TLS 1.3 | Mutual X.509 | Metadata queries, administration, produce, fetch | | **Raft** | TCP 2922 | TLS 1.3 | Mutual X.509 | Metadata shard consensus log replication | | **Data** | TCP 2923 | TLS 1.3 | Mutual X.509 | Segment replication, repair, and coordination | -| **SWIM** | UDP 2922 | QUIC v1 datagrams (TLS 1.3) | Mutual X.509 | Membership gossip and failure detection | +| **SWIM** | UDP 2922 | Secure datagrams (deferred) | Mutual X.509 | Membership gossip and failure detection | --- @@ -34,7 +34,9 @@ The production boundary defends against external network attackers attempting ea Security checks are split between the transport layer and application state machines to keep state machines free of security I/O: ``` - [ TLS 1.3 / QUIC v1 Transport Layer ] + [ Authenticated Transport Layer ] + TCP: TLS 1.3 + UDP: secure datagrams (deferred) - Authenticate peer X.509 certificates - Enforce framing, datagram MTU, and resource limits - Bind connection envelope sender to verified identity @@ -120,9 +122,9 @@ SWIM liveness gossip is decoupled from cluster admission authority to prevent ne [ Raft Reconciliation ] ──► Commit AddPeer / RemovePeer ``` -1. **Process Proof:** A node connection proves possession of the process private key bound to its admitted epoch. The process key signs the admitted identity and a connection-specific value produced by TLS or QUIC, so the proof cannot be replayed on another connection. The reusable node certificate alone cannot create or claim a newer epoch. +1. **Process Proof:** A node session proves possession of the process private key bound to its admitted epoch. The process key signs the admitted identity and a session-specific value produced by the authenticated transport, so the proof cannot be replayed in another session. The reusable node certificate alone cannot create or claim a newer epoch. 2. **Admission Gate:** The SWIM actor checks incoming packets against a local admission cache backed by metadata Raft before passing facts to the SWIM state machine. -3. **Gossip Rule:** The QUIC connection authenticates the immediate sender. Every relayed membership fact is separately accepted only when its subject `NodeId` and `Admission Epoch` match an active admission record. +3. **Gossip Rule:** The secure datagram session authenticates the immediate sender. Every relayed membership fact is separately accepted only when its subject `NodeId` and `Admission Epoch` match an active admission record. 4. **Cache Policy:** Admission records are cached locally with a maximum TTL of 60 seconds. If a cache entry expires while the owning metadata shard is unreachable, the gate fails closed. --- @@ -194,7 +196,7 @@ security/node/{node-certificate-principal} ### Online Credential Rotation - **Zero-Downtime CA Rotation:** Brokers support dual trust chain loading. New root CAs can be added and leaf certificates reloaded online without restarting brokers or changing `Admission Epoch` / `NodeId`. -- **Revocation & Expiry:** Certificate revocations commit to metadata Raft records. Active TLS and QUIC connections are terminated within the cache enforcement window. Expired certificates are rejected with clock-skew tolerance. +- **Revocation & Expiry:** Certificate revocations commit to metadata Raft records. Active authenticated sessions are terminated within the cache enforcement window. Expired certificates are rejected with clock-skew tolerance. - **Recovery:** Runbooks cover lost authorized-operator access, lost issuing keys, expiry, accidental revocation, trust-root replacement, and cold-cluster restart. --- @@ -203,24 +205,52 @@ security/node/{node-certificate-principal} ### Rate & Memory Bounds - Every listener enforces strict limits on unauthenticated handshakes, concurrent connections, in-flight frames, memory allocations, and per-source request rates. -- **QUIC Datagram Size:** QUIC UDP payloads are capped at **1200 bytes**, which fits within the 1280-byte minimum IPv6 MTU after IPv6 and UDP headers. The SWIM payload budget is what remains after QUIC packet protection and framing. +- The future secure UDP transport must define a payload budget that avoids IP fragmentation after authentication and encryption overhead. -### Deterministic Simulation +### Secure UDP Decision -Secure SWIM must run under EastGuard's turmoil-based deterministic simulation -tests. Quinn's abstract UDP socket and runtime boundaries must use -`crate::net::UdpSocket` and Tokio's virtualized clock in test builds: +EastGuard retains UDP for SWIM because connection-oriented transport does not +fit membership at cluster scale: ``` -production deterministic test -real UDP + real Tokio time turmoil UDP + virtual Tokio time - \ / - same Quinn protocol + SWIM probes one peer per interval + │ + ▼ + stateless UDP + │ + ┌────────────┴────────────┐ + ▼ ▼ +constant socket count packet loss remains +per node visible to SWIM ``` -A production-only QUIC path does not pass S0. Feasibility requires a multi-node -turmoil test that completes mutual authentication and exchanges QUIC datagrams -under virtual time. +TCP with bounded connection reuse was rejected because large clusters would +trade an unbounded connection mesh for continuous handshakes, connection churn, +kernel connection tracking, and head-of-line blocking. QUIC datagrams preserve +UDP delivery semantics, but their per-peer connection state and implementation +complexity are not justified for SWIM's sparse traffic. Current DTLS options do +not meet the combined requirements for maturity, permissive licensing, Rust +integration, and deterministic simulation. + +Secure SWIM therefore remains deferred. Trusted development mode may use +plaintext UDP in isolated environments. Secure mode must fail startup until a +secure datagram transport is selected and implemented; it must never fall back +to plaintext UDP. + +### Acceptance Criteria for a Future Secure UDP Transport + +The selected transport must: + +| Requirement | Reason | +| :--- | :--- | +| Preserve datagram boundaries and loss | SWIM timeouts and indirect probes must observe loss rather than transport retransmission delays | +| Keep per-node transport state bounded independently of cluster size | Membership must remain viable for clusters with thousands of nodes | +| Authenticate node certificates and expose the certificate principal | Admission must bind each packet source to a verified node identity | +| Reject replay and spoofed source traffic | Old or forged membership packets must not alter liveness | +| Avoid IP fragmentation | One lost fragment must not discard an oversized protected packet | +| Run over EastGuard's UDP abstraction | Production and turmoil must exercise the same protocol state machine | +| Use virtual time in deterministic tests | Handshake retry, expiry, and packet loss must be reproducible | +| Use a mature, maintainable, permissively licensed dependency | Cluster security must not rely on an unaudited or incompatible implementation | ### Audit Subsystem - **Non-Blocking Execution:** Security audit events (authentication success/failure, ACL denials, admissions) are queued asynchronously. Audit backpressure never blocks protocol execution or consensus. @@ -234,15 +264,15 @@ under virtual time. ``` S0 ──► S1 ──► S2 ──► S3 ──► S4 ──► S5 ──► S6 config records TCP SWIM clients operations production - mTLS QUIC + ACLs + audit gate + mTLS deferred + ACLs + audit gate ``` | Phase | Target Scope | Key Deliverable | Exit Criteria | | :--- | :--- | :--- | :--- | -| **S0** | Configuration | Security modes, certificate loader, Quinn feasibility | Secure mode opens no plaintext listeners; Quinn proves mutual certificate authentication, datagram delivery, address validation, replay protection, disabled 0-RTT, bounded connection state, and deterministic operation under turmoil | +| **S0** | Configuration | Security modes and certificate loader | Secure mode opens no plaintext listeners and fails startup while any required secure listener is unavailable | | **S1** | Metadata Storage | Security record schema, sharded Raft state | Security records survive snapshot & recovery | | **S2** | Cluster Transport | TLS 1.3 on TCP 2922/2923, Raft sender and role authorization | Authenticated and authorized cluster TCP traffic | -| **S3** | Membership | QUIC v1 datagrams on UDP 2922 and SWIM admission gate | Secure gossip and partition-safe admission fencing | +| **S3** | Membership (deferred) | Secure datagrams on UDP 2922 and SWIM admission gate | A transport meeting the secure UDP acceptance criteria provides authenticated gossip and partition-safe admission fencing | | **S4** | Client API | Client mTLS on TCP 2921, principal binding, ACLs | Default-deny enforcement on all client APIs | | **S5** | Operations | Certificate rotation, revocation, expiry, recovery, audit logging | Online credential operations and recovery runbooks | | **S6** | Production Gate | Adversarial testing, fuzzing, partition stress | Passes all production readiness checks | @@ -251,7 +281,8 @@ S6 must verify node and client impersonation, stale-process replay, unauthorized operations, protocol downgrade, rotation under live traffic, expired and revoked credentials, cold-cluster restart, handshake and datagram fuzzing, resource bounds, secret-free diagnostics, and reproducible secure-SWIM behavior under -turmoil with pinned randomness and node identities. +turmoil with pinned randomness and node identities. S6 cannot pass while S3 is +deferred. --- diff --git a/src/control_plane/membership/transport.rs b/src/control_plane/membership/transport.rs index 7b10cdd4..a021fe60 100644 --- a/src/control_plane/membership/transport.rs +++ b/src/control_plane/membership/transport.rs @@ -1,10 +1,7 @@ -use crate::control_plane::membership::{OutboundPacket, SwimCommand, actor::SwimSender}; +use tokio::sync::mpsc; -// ========================================== -// TRANSPORT LAYER (Presentation) -// ========================================== +use crate::control_plane::membership::{OutboundPacket, SwimCommand, actor::SwimSender}; use crate::net::UdpSocket; -use tokio::sync::mpsc; pub struct SwimTransportActor; @@ -15,7 +12,7 @@ impl SwimTransportActor { mut from_actor: mpsc::Receiver>, ) { tracing::info!( - "Transport Layer listening on {}", + "SWIM trusted-development transport listening on {}", socket.local_addr().unwrap() ); diff --git a/src/net/udp.rs b/src/net/udp.rs index 2805d956..d067ef19 100644 --- a/src/net/udp.rs +++ b/src/net/udp.rs @@ -1,16 +1,6 @@ -#![allow(dead_code)] use crate::impl_new_struct_wrapper; use super::inner; -use quinn::udp::{RecvMeta, Transmit}; -use quinn::{AsyncUdpSocket, UdpPoller}; -use std::fmt; -use std::future::Future; -use std::io::{self, IoSliceMut}; -use std::pin::Pin; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll}; -type Readiness = Pin> + Send>>; pub struct UdpSocket(inner::UdpSocket); @@ -22,253 +12,3 @@ impl UdpSocket { Ok(Self(socket)) } } - -pub struct QuinnUdpSocket { - socket: Arc, - readable: Mutex>, -} - -impl QuinnUdpSocket { - pub fn new(socket: UdpSocket) -> Self { - Self { - socket: Arc::new(socket), - readable: Mutex::new(None), - } - } -} - -impl fmt::Debug for QuinnUdpSocket { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("QuinnUdpSocket") - .field("local_addr", &self.socket.local_addr()) - .finish() - } -} - -impl AsyncUdpSocket for QuinnUdpSocket { - fn create_io_poller(self: Arc) -> Pin> { - Box::pin(QuinnUdpPoller { - socket: self.socket.clone(), - writable: Mutex::new(None), - }) - } - - fn try_send(&self, transmit: &Transmit<'_>) -> io::Result<()> { - debug_assert!(transmit.segment_size.is_none()); - self.socket - .try_send_to(transmit.contents, transmit.destination) - .map(|_| ()) - } - - fn poll_recv( - &self, - cx: &mut Context<'_>, - bufs: &mut [IoSliceMut<'_>], - meta: &mut [RecvMeta], - ) -> Poll> { - if bufs.is_empty() || meta.is_empty() { - return Poll::Ready(Ok(0)); - } - - loop { - match self.socket.try_recv_from(&mut bufs[0]) { - Ok((len, addr)) => { - meta[0] = RecvMeta { - addr, - len, - stride: len, - ecn: None, - dst_ip: None, - }; - self.readable - .lock() - .expect("readable mutex poisoned") - .take(); - return Poll::Ready(Ok(1)); - } - Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} - Err(error) => return Poll::Ready(Err(error)), - } - - let mut readable = self.readable.lock().expect("readable mutex poisoned"); - let socket = self.socket.clone(); - let readiness = readable.get_or_insert_with(|| { - Box::pin(async move { socket.readable().await }) as Readiness - }); - - match readiness.as_mut().poll(cx) { - Poll::Ready(Ok(())) => { - readable.take(); - } - Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), - Poll::Pending => return Poll::Pending, - } - } - } - - fn local_addr(&self) -> io::Result { - self.socket.local_addr() - } -} - -struct QuinnUdpPoller { - socket: Arc, - writable: Mutex>, -} - -impl fmt::Debug for QuinnUdpPoller { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.debug_struct("QuinnUdpPoller").finish() - } -} - -impl UdpPoller for QuinnUdpPoller { - fn poll_writable(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut writable = self.writable.lock().expect("writable mutex poisoned"); - let socket = self.socket.clone(); - let readiness = - writable.get_or_insert_with(|| Box::pin(async move { socket.writable().await })); - - let result = readiness.as_mut().poll(cx); - if result.is_ready() { - writable.take(); - } - result - } -} - -#[cfg(test)] -mod tests { - use std::net::SocketAddr; - use std::time::Duration; - - use bytes::Bytes; - use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig}; - use rcgen::{CertifiedKey, generate_simple_self_signed}; - use rustls::RootCertStore; - use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; - use rustls::server::WebPkiClientVerifier; - use turmoil::Builder; - - use super::*; - - const MAX_INCOMING_HANDSHAKES: usize = 128; - const DATAGRAM_BUFFER_BYTES: usize = 64 * 1200; - - fn endpoint_config() -> quinn::EndpointConfig { - let mut config = quinn::EndpointConfig::default(); - config.max_udp_payload_size(1200).unwrap(); - config - } - - fn transport_config() -> Arc { - let mut config = quinn::TransportConfig::default(); - config - .max_concurrent_bidi_streams(0_u8.into()) - .max_concurrent_uni_streams(0_u8.into()) - .datagram_receive_buffer_size(Some(DATAGRAM_BUFFER_BYTES)) - .datagram_send_buffer_size(DATAGRAM_BUFFER_BYTES); - Arc::new(config) - } - - fn configs() -> (quinn::ServerConfig, quinn::ClientConfig) { - let CertifiedKey { cert, signing_key } = - generate_simple_self_signed(["server".to_string()]).unwrap(); - let certificate = cert.der().clone(); - let private_key: PrivateKeyDer<'static> = - PrivatePkcs8KeyDer::from(signing_key.serialize_der()).into(); - - let mut roots = RootCertStore::empty(); - roots.add(certificate.clone()).unwrap(); - let roots = Arc::new(roots); - - let client_verifier = WebPkiClientVerifier::builder(roots.clone()) - .build() - .unwrap(); - let server_crypto = - rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .with_client_cert_verifier(client_verifier) - .with_single_cert(vec![certificate.clone()], private_key.clone_key()) - .unwrap(); - let client_crypto = - rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .with_root_certificates((*roots).clone()) - .with_client_auth_cert(vec![certificate], private_key) - .unwrap(); - assert_eq!(server_crypto.max_early_data_size, 0); - assert!(!client_crypto.enable_early_data); - - let transport = transport_config(); - let mut server = quinn::ServerConfig::with_crypto(Arc::new( - QuicServerConfig::try_from(server_crypto).unwrap(), - )); - server - .transport_config(transport.clone()) - .max_incoming(MAX_INCOMING_HANDSHAKES); - let mut client = - quinn::ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto).unwrap())); - client.transport_config(transport); - - (server, client) - } - - #[test] - fn quinn_validates_address_and_exchanges_datagram_under_turmoil() -> turmoil::Result { - let (server_config, client_config) = configs(); - let mut sim = Builder::new() - .simulation_duration(Duration::from_secs(10)) - .rng_seed(1) - .build(); - - sim.host("server", move || { - let server_config = server_config.clone(); - async move { - let socket = UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], 4433))) - .await - .unwrap(); - let endpoint = quinn::Endpoint::new_with_abstract_socket( - endpoint_config(), - Some(server_config), - Arc::new(QuinnUdpSocket::new(socket)), - Arc::new(quinn::TokioRuntime), - ) - .unwrap(); - - let initial = endpoint.accept().await.unwrap(); - assert!(!initial.remote_address_validated()); - initial.retry().unwrap(); - - let validated = endpoint.accept().await.unwrap(); - assert!(validated.remote_address_validated()); - let connection = validated.await.unwrap(); - assert_eq!(connection.read_datagram().await.unwrap(), b"hello"[..]); - Ok(()) - } - }); - - sim.client("client", async move { - let socket = UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], 4434))) - .await - .unwrap(); - let mut endpoint = quinn::Endpoint::new_with_abstract_socket( - endpoint_config(), - None, - Arc::new(QuinnUdpSocket::new(socket)), - Arc::new(quinn::TokioRuntime), - ) - .unwrap(); - endpoint.set_default_client_config(client_config); - - let server = SocketAddr::new(turmoil::lookup("server"), 4433); - let connection = endpoint.connect(server, "server").unwrap().await.unwrap(); - connection - .send_datagram(Bytes::from_static(b"hello")) - .unwrap(); - tokio::time::sleep(Duration::from_secs(1)).await; - Ok(()) - }); - - sim.run() - } -} From de673ca31403ab2b42cae6cc3aed65acab54ff45 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 15:46:02 +0400 Subject: [PATCH 20/56] fix: asymmetric bootstrap voter sets - Fixed asymmetric bootstrap voter sets by reasserting known voters through the Raft log during takeover. - Kept strict committed-voter RPC authorization. - Added a focused regression test. - Exact failing turmoil seed now passes. - Full clippy, formatting, and diff checks pass. --- src/control_plane/consensus/raft/state.rs | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index 61420a22..ceb19c16 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -211,6 +211,13 @@ impl Raft { if *member == self.node_id || !live_set.contains(member) { continue; } + if self.peers.contains(member) { + // Bootstrap group creation can observe different ring snapshots + // on different nodes. Re-applying an existing voter is a "no-op" + // here, but heals followers whose initial voter set omitted it. + changed |= self.propose(RaftCommand::AddPeer(member.clone())).is_ok(); + continue; + } // Stage the ring member as a non-voting learner; it's promoted to a // voter once caught up. Never added straight to the quorum — an // un-participating ring member would otherwise freeze commits. @@ -3832,6 +3839,28 @@ mod tests { (raft, reader, members) } + #[test] + fn takeover_reasserts_bootstrap_voters_through_the_log() { + let (mut raft, reader, members) = ring_raft_with_stale(&[]); + let expected: HashSet = members + .iter() + .filter(|member| **member != node("node-1")) + .cloned() + .collect(); + + assert!(raft.reconcile(&reader, Some(members.0))); + + let asserted: HashSet = raft + .consensus + .uncommited_log_range() + .filter_map(|index| match &raft.consensus.log_entry(index)?.command { + RaftCommand::AddPeer(node_id) => Some(node_id.clone()), + RaftCommand::Noop | RaftCommand::RemovePeer(_) | RaftCommand::Metadata(_) => None, + }) + .collect(); + assert_eq!(asserted, expected); + } + /// Simulate `peer` confirming replication up to the leader's last index. fn ack_to_last(raft: &mut Raft, peer: &NodeId) { let term = raft.current_term(); From 8dc736c7a83f5f97549dabf78c080a9aaca6e83a Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 15:48:49 +0400 Subject: [PATCH 21/56] rename : add peer -> ensure peer --- src/control_plane/consensus/multi_raft.rs | 4 +-- src/control_plane/consensus/raft/command.rs | 2 +- src/control_plane/consensus/raft/state.rs | 36 ++++++++++++--------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index c9a91a5b..9a73cf11 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -1849,7 +1849,7 @@ mod tests { let log = store.storage.load_state(gid.0).log; assert!( !log.iter() - .any(|e| e.command == RaftCommand::AddPeer(node("n4"))), + .any(|e| e.command == RaftCommand::EnsurePeer(node("n4"))), "the learner must not be added straight to the quorum (no immediate AddPeer, log: {:?})", log.iter().map(|e| &e.command).collect::>() ); @@ -1926,7 +1926,7 @@ mod tests { ); assert!( !log.iter() - .any(|e| e.command == RaftCommand::AddPeer(node("n9"))), + .any(|e| e.command == RaftCommand::EnsurePeer(node("n9"))), "the eviction must not be paired with an AddPeer" ); } diff --git a/src/control_plane/consensus/raft/command.rs b/src/control_plane/consensus/raft/command.rs index 6d67e313..66bf7238 100644 --- a/src/control_plane/consensus/raft/command.rs +++ b/src/control_plane/consensus/raft/command.rs @@ -16,7 +16,7 @@ use crate::{impl_from_variant, impl_from_variant_via}; pub enum RaftCommand { Noop, Metadata(MetadataCommand), - AddPeer(NodeId), + EnsurePeer(NodeId), RemovePeer(NodeId), } diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index ceb19c16..615e2af2 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -215,7 +215,9 @@ impl Raft { // Bootstrap group creation can observe different ring snapshots // on different nodes. Re-applying an existing voter is a "no-op" // here, but heals followers whose initial voter set omitted it. - changed |= self.propose(RaftCommand::AddPeer(member.clone())).is_ok(); + changed |= self + .propose(RaftCommand::EnsurePeer(member.clone())) + .is_ok(); continue; } // Stage the ring member as a non-voting learner; it's promoted to a @@ -330,7 +332,7 @@ impl Raft { self.consensus.uncommited_log_range().any(|i| { matches!( self.consensus.log_entry(i).map(|e| &e.command), - Some(RaftCommand::AddPeer(_) | RaftCommand::RemovePeer(_)) + Some(RaftCommand::EnsurePeer(_) | RaftCommand::RemovePeer(_)) ) }) } @@ -1169,7 +1171,7 @@ impl Raft { return; } if self.consensus.is_learner_ready_for_promotion(node) { - let _ = self.propose(RaftCommand::AddPeer(node.clone())); + let _ = self.propose(RaftCommand::EnsurePeer(node.clone())); } } @@ -1532,7 +1534,7 @@ impl Raft { match entry.command { RaftCommand::Noop => {} RaftCommand::Metadata(cmd) => self.apply_metadata_entry(cmd, entry.index), - RaftCommand::AddPeer(node_id) => self.apply_add_peer(node_id), + RaftCommand::EnsurePeer(node_id) => self.apply_ensure_peer(node_id), RaftCommand::RemovePeer(node_id) => self.apply_remove_peer(node_id), } } @@ -1576,7 +1578,7 @@ impl Raft { /// Apply-only helper. Invoked from `apply_committed_entries()` when an /// `AddPeer` log entry commits. Never call directly — the peer set is part /// of the replicated state machine and must only mutate through the log. - fn apply_add_peer(&mut self, node_id: NodeId) { + fn apply_ensure_peer(&mut self, node_id: NodeId) { if node_id == self.node_id { return; } @@ -1905,7 +1907,7 @@ impl crate::test_traits::TAssertInvariant for Raft { // Invariant (partial): self is never in peers. The peer set is otherwise // mutated only via apply of committed AddPeer/RemovePeer entries — the - // discipline itself is enforced by keeping `apply_add_peer`/`apply_remove_peer` + // discipline itself is enforced by keeping `apply_ensure_peer`/`apply_remove_peer` // as the sole callers of `peers.insert`/`peers.remove` (callers checked at // compile time by their private visibility). assert!( @@ -3080,7 +3082,7 @@ mod tests { // Direct mutation is gone — the peer set is part of the replicated state. #[test] - fn add_peer_log_entry_inserts_into_peers_on_apply() { + fn ensure_peer_log_entry_inserts_into_peers_on_apply() { let mut raft = single_node_raft(); raft.handle_timeout(RaftTimeoutCallback::ElectionTimeout { shard_group_id: TEST_SHARD, @@ -3090,7 +3092,8 @@ mod tests { assert!(raft.is_leader()); assert_eq!(raft.peers_count(), 0); - raft.propose(RaftCommand::AddPeer(node("node-2"))).unwrap(); + raft.propose(RaftCommand::EnsurePeer(node("node-2"))) + .unwrap(); raft.simulate_flush(); assert!(raft.has_peer(&node("node-2"))); @@ -3098,7 +3101,7 @@ mod tests { } #[test] - fn add_peer_log_entry_skips_self() { + fn ensure_peer_log_entry_skips_self() { let mut raft = single_node_raft(); raft.handle_timeout(RaftTimeoutCallback::ElectionTimeout { shard_group_id: TEST_SHARD, @@ -3106,7 +3109,8 @@ mod tests { }); drain(&mut raft); - raft.propose(RaftCommand::AddPeer(node("node-1"))).unwrap(); + raft.propose(RaftCommand::EnsurePeer(node("node-1"))) + .unwrap(); raft.simulate_flush(); assert!(!raft.has_peer(&node("node-1"))); @@ -3114,7 +3118,7 @@ mod tests { } #[test] - fn add_peer_log_entry_leader_initializes_peer_state() { + fn ensure_peer_log_entry_leader_initializes_peer_state() { let mut raft = single_node_raft(); raft.handle_timeout(RaftTimeoutCallback::ElectionTimeout { shard_group_id: TEST_SHARD, @@ -3123,7 +3127,8 @@ mod tests { drain(&mut raft); assert!(raft.is_leader()); - raft.propose(RaftCommand::AddPeer(node("node-2"))).unwrap(); + raft.propose(RaftCommand::EnsurePeer(node("node-2"))) + .unwrap(); raft.simulate_flush(); drain(&mut raft); @@ -3854,7 +3859,7 @@ mod tests { .consensus .uncommited_log_range() .filter_map(|index| match &raft.consensus.log_entry(index)?.command { - RaftCommand::AddPeer(node_id) => Some(node_id.clone()), + RaftCommand::EnsurePeer(node_id) => Some(node_id.clone()), RaftCommand::Noop | RaftCommand::RemovePeer(_) | RaftCommand::Metadata(_) => None, }) .collect(); @@ -3926,7 +3931,8 @@ mod tests { // A membership entry is in flight: the window is complete, but the // one-config-change-at-a-time gate must park the eviction. - raft.propose(RaftCommand::AddPeer(node("node-7"))).unwrap(); + raft.propose(RaftCommand::EnsurePeer(node("node-7"))) + .unwrap(); assert!( raft.reconcile_stale_live_peers(&topology, &live).is_err(), "uncommitted AddPeer must park the eviction" @@ -3943,7 +3949,7 @@ mod tests { assert_eq!( proposals_after_become_leader(&raft), vec![ - RaftCommand::AddPeer(node("node-7")), + RaftCommand::EnsurePeer(node("node-7")), RaftCommand::RemovePeer(node("node-9")), ], ); From d6db30c14f396ecc71ad34baab834f181ae7ec0a Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 15:51:04 +0400 Subject: [PATCH 22/56] convert cold-read integration test to tokio --- src/data_plane/state.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/data_plane/state.rs b/src/data_plane/state.rs index c3f7e3c3..bf8facfd 100644 --- a/src/data_plane/state.rs +++ b/src/data_plane/state.rs @@ -3682,8 +3682,8 @@ mod tests { /// `WalRecord` format agreement with `checkpoint.rs`, and `record_count` /// round-tripping through the file. Bypasses the coordinator seal round-trip /// (driven directly) so it's deterministic. - #[test] - fn cold_fetch_serves_sealed_segment_from_disk() { + #[tokio::test] + async fn cold_fetch_serves_sealed_segment_from_disk() { use crate::connections::protocol::RangeProgressSignal; use crate::data_plane::cold_read::ColdReadPool; use crate::data_plane::messages::query::{DataPlaneQuery, Fetch}; @@ -3777,9 +3777,7 @@ mod tests { reply, }))); - let result = reply_rx - .blocking_recv() - .expect("cold-read pool dropped the reply"); + let result = reply_rx.await.expect("cold-read pool dropped the reply"); let FetchedRecords { entries, next_entry_id, From 5f38f9020ed255b61dc19257bda4068bb7131147 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 17:43:38 +0400 Subject: [PATCH 23/56] client principal --- src/security.rs | 71 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/src/security.rs b/src/security.rs index 0e26f076..bc58e7bf 100644 --- a/src/security.rs +++ b/src/security.rs @@ -224,14 +224,42 @@ impl NodeTransportSecurity { /// the admission-record key. This function only parses the certificate; callers /// must use it after rustls has authenticated the peer's certificate chain. pub(crate) fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Result { - const NODE_PRINCIPAL_URI_PREFIX: &str = "urn:eastguard:node:"; + certificate_principal( + certificate, + "urn:eastguard:node:", + "node", + "Node Certificate Principal", + ) +} + +/// Reads the client principal from exactly one +/// `urn:eastguard:client:` URI Subject Alternative Name. +/// +/// TLS authentication must succeed before callers use this parsed identity for +/// authorization. +pub(crate) fn client_certificate_principal(certificate: &CertificateDer<'_>) -> Result { + certificate_principal( + certificate, + "urn:eastguard:client:", + "client", + "Client Certificate Principal", + ) +} +fn certificate_principal( + certificate: &CertificateDer<'_>, + uri_prefix: &str, + certificate_kind: &str, + principal_name: &str, +) -> Result { let (_, certificate) = X509Certificate::from_der(certificate.as_ref()).context("invalid X.509 certificate")?; let subject_alt_name = certificate .subject_alternative_name() .context("invalid X.509 subject alternative name")? - .context("node certificate has no subject alternative name")?; + .with_context(|| { + format!("{certificate_kind} certificate has no subject alternative name") + })?; let mut principals = subject_alt_name @@ -239,16 +267,16 @@ pub(crate) fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Re .general_names .iter() .filter_map(|name| match name { - GeneralName::URI(uri) => uri.strip_prefix(NODE_PRINCIPAL_URI_PREFIX), + GeneralName::URI(uri) => uri.strip_prefix(uri_prefix), _ => None, }); let principal = principals .next() .filter(|principal| !principal.is_empty()) - .context("node certificate has no Node Certificate Principal")?; + .with_context(|| format!("{certificate_kind} certificate has no {principal_name}"))?; anyhow::ensure!( principals.next().is_none(), - "node certificate has multiple Node Certificate Principals" + "{certificate_kind} certificate has multiple {principal_name}s" ); Ok(principal.to_string()) } @@ -327,6 +355,39 @@ mod tests { ); } + #[test] + fn reads_client_principal_from_uri_subject_alternative_name() { + let certificate = + certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:client:producer-a"]); + + assert_eq!( + client_certificate_principal(&certificate).unwrap(), + "producer-a" + ); + } + + #[test] + fn requires_exactly_one_client_principal() { + let missing = certificate_with_uris(&["urn:example:unrelated"]); + let ambiguous = certificate_with_uris(&[ + "urn:eastguard:client:producer-a", + "urn:eastguard:client:producer-b", + ]); + + assert_eq!( + client_certificate_principal(&missing) + .unwrap_err() + .to_string(), + "client certificate has no Client Certificate Principal" + ); + assert_eq!( + client_certificate_principal(&ambiguous) + .unwrap_err() + .to_string(), + "client certificate has multiple Client Certificate Principals" + ); + } + #[test] fn server_verifier_requires_trust_and_node_principal() { let trusted = certificate_with_uris(&["urn:eastguard:node:broker-a"]); From 74329a7b0479a0c0e078707cbad126091110615e Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 18:01:28 +0400 Subject: [PATCH 24/56] client support for tls --- src/connections/controller.rs | 4 +- src/connections/reader.rs | 8 +- src/connections/writer.rs | 10 +- .../consensus/transport/inbound.rs | 6 +- src/control_plane/consensus/transport/mod.rs | 8 +- .../consensus/transport/outbound.rs | 12 +- src/data_plane/transport/mod.rs | 4 +- src/data_plane/transport/reader.rs | 9 +- src/data_plane/transport/writers.rs | 11 +- src/lib.rs | 12 +- src/net/mod.rs | 7 +- src/net/tcp.rs | 136 ++++++++++++++---- 12 files changed, 164 insertions(+), 63 deletions(-) diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 732cbbeb..cb426e87 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -28,7 +28,7 @@ use crate::data_plane::messages::command::{ use crate::data_plane::messages::query::{ DataPlaneQuery, Fetch, ListOffsets, ReadConsumerOffset, ReadConsumerOffsetResult, }; -use crate::net::TcpStream; +use crate::net::TransportTcpStream; use tokio::sync::mpsc; /// # Client ↔ Server request_id protocol @@ -615,7 +615,7 @@ fn keyspace_bound_matches_range(bound: &Option, range: &RangeMeta } pub async fn handle_client_stream( - stream: TcpStream, + stream: TransportTcpStream, node_id: NodeId, swim_sender: SwimSender, raft_sender: MutlRaftSender, diff --git a/src/connections/reader.rs b/src/connections/reader.rs index 24d4dd79..8ecc4a6b 100644 --- a/src/connections/reader.rs +++ b/src/connections/reader.rs @@ -16,19 +16,19 @@ use crate::connections::{LEN_PREFIX_SIZE, REQUEST_ID_SIZE}; use std::io::ErrorKind; -use crate::net::OwnedReadHalf; +use crate::net::TransportReadHalf; use bytes::{Buf, BytesMut}; use tokio::io::AsyncReadExt; pub struct ClientStreamReader { - pub(crate) stream: OwnedReadHalf, + pub(crate) stream: TransportReadHalf, buffer: BytesMut, } impl ClientStreamReader { - pub fn new(stream: OwnedReadHalf) -> Self { + pub fn new(stream: impl Into) -> Self { Self { - stream, + stream: stream.into(), buffer: BytesMut::with_capacity(1024), } } diff --git a/src/connections/writer.rs b/src/connections/writer.rs index d9ec3be7..1e28b339 100644 --- a/src/connections/writer.rs +++ b/src/connections/writer.rs @@ -2,17 +2,19 @@ use tokio::io::AsyncWriteExt; use crate::{ connections::{REQUEST_ID_SIZE, protocol::ClientResponse}, - net::OwnedWriteHalf, + net::TransportWriteHalf, }; use tokio::sync::mpsc; pub(crate) struct ClientRawWriter { - stream: OwnedWriteHalf, + stream: TransportWriteHalf, } impl ClientRawWriter { - pub fn new(write_half: OwnedWriteHalf) -> Self { - Self { stream: write_half } + pub fn new(write_half: impl Into) -> Self { + Self { + stream: write_half.into(), + } } pub async fn write( diff --git a/src/control_plane/consensus/transport/inbound.rs b/src/control_plane/consensus/transport/inbound.rs index e7862fb6..2fc82220 100644 --- a/src/control_plane/consensus/transport/inbound.rs +++ b/src/control_plane/consensus/transport/inbound.rs @@ -2,18 +2,18 @@ use crate::control_plane::NodeId; use crate::control_plane::consensus::actor::MutlRaftSender; use crate::control_plane::consensus::messages::InboundRaftRpc; use crate::control_plane::consensus::messages::WireRaftMessage; -use crate::net::NodeReadHalf; +use crate::net::TransportReadHalf; use crate::security::NodeTransportIdentity; use tokio::io::AsyncReadExt; pub(super) struct RaftRpcListener { - read_half: NodeReadHalf, + read_half: TransportReadHalf, transport_identity: NodeTransportIdentity, } impl RaftRpcListener { pub(super) fn new( - read_half: impl Into, + read_half: impl Into, transport_identity: NodeTransportIdentity, ) -> Self { Self { diff --git a/src/control_plane/consensus/transport/mod.rs b/src/control_plane/consensus/transport/mod.rs index 84ec681a..0fed5ff7 100644 --- a/src/control_plane/consensus/transport/mod.rs +++ b/src/control_plane/consensus/transport/mod.rs @@ -12,8 +12,8 @@ use crate::control_plane::consensus::actor::MutlRaftSender; use crate::control_plane::NodeId; use crate::control_plane::consensus::messages::RaftTransportCommand; use crate::control_plane::membership::actor::SwimSender; -use crate::net::NodeTcpStream; use crate::net::TcpListener; +use crate::net::TransportTcpStream; #[cfg(test)] use crate::security::NodeTransportIdentity; use crate::security::NodeTransportSecurity; @@ -40,7 +40,7 @@ impl RaftTransportActor { loop { tokio::select! { Ok((stream, _)) = listener.accept() => { - match NodeTcpStream::accept(stream, &security).await { + match TransportTcpStream::accept_node(stream, &security).await { Ok(stream) => dispatcher.accept(stream, &raft_tx).await, Err(error) => tracing::debug!("Raft TLS accept rejected: {error}"), } @@ -255,7 +255,7 @@ mod tests { let (stream, _) = listener.accept().await?; state - .accept(NodeTcpStream::TrustedDevelopment(stream), &raft_tx) + .accept(TransportTcpStream::TrustedDevelopment(stream), &raft_tx) .await; assert!( @@ -300,7 +300,7 @@ mod tests { // First connection from node-a let (stream, _) = listener.accept().await?; state - .accept(NodeTcpStream::TrustedDevelopment(stream), &raft_tx) + .accept(TransportTcpStream::TrustedDevelopment(stream), &raft_tx) .await; assert!(state.contains(&NodeId::new("node-a"))); diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index a313ae72..31989557 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -11,7 +11,7 @@ use crate::control_plane::consensus::messages::{OutboundRaftPacket, WireRaftMess use crate::control_plane::NodeId; use crate::control_plane::consensus::transport::RaftRpcListener; use crate::control_plane::membership::actor::SwimSender; -use crate::net::{NodeTcpStream, NodeWriteHalf}; +use crate::net::{TransportTcpStream, TransportWriteHalf}; use crate::security::NodeTransportSecurity; const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); @@ -29,7 +29,7 @@ const PENDING_DIAL_BUFFER_CAP: usize = 256; /// outbound connect), the tie is broken by NodeId ordering. pub(super) struct RaftRpcDispatcher { node_id: NodeId, - writers: HashMap, + writers: HashMap, /// Peers explicitly disconnected via DisconnectPeer. Outbound RPCs /// to these peers are silently dropped until a new connection is /// accepted (peer restart with new UUID won't hit this — different NodeId). @@ -48,7 +48,7 @@ pub(super) struct RaftRpcDispatcher { /// Result of a background dial attempt, delivered back to the transport loop. pub(super) struct DialOutcome { target: NodeId, - outcome: anyhow::Result<(RaftRpcListener, NodeWriteHalf)>, + outcome: anyhow::Result<(RaftRpcListener, TransportWriteHalf)>, } impl RaftRpcDispatcher { @@ -68,7 +68,7 @@ impl RaftRpcDispatcher { } } - pub(super) async fn accept(&mut self, stream: NodeTcpStream, raft_tx: &MutlRaftSender) { + pub(super) async fn accept(&mut self, stream: TransportTcpStream, raft_tx: &MutlRaftSender) { let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); let mut reader = RaftRpcListener::new(read_half, transport_identity); @@ -258,14 +258,14 @@ async fn dial( target_id: NodeId, swim_tx: SwimSender, security: NodeTransportSecurity, -) -> anyhow::Result<(RaftRpcListener, NodeWriteHalf)> { +) -> anyhow::Result<(RaftRpcListener, TransportWriteHalf)> { let Some(addr) = swim_tx.resolve_address(target_id.clone()).await? else { anyhow::bail!("[{}] Cannot resolve address for {:?}", node_id, target_id); }; let stream = tokio::time::timeout( std::time::Duration::from_secs(3), - NodeTcpStream::connect(addr.cluster_addr(), &security), + TransportTcpStream::connect_node(addr.cluster_addr(), &security), ) .await??; diff --git a/src/data_plane/transport/mod.rs b/src/data_plane/transport/mod.rs index 5cbc8da4..f5d89088 100644 --- a/src/data_plane/transport/mod.rs +++ b/src/data_plane/transport/mod.rs @@ -9,7 +9,7 @@ use crate::control_plane::membership::TopologyReader; use crate::control_plane::membership::actor::SwimSender; use crate::data_plane::actor::DataPlaneSender; -use crate::net::{NodeTcpStream, TcpListener}; +use crate::net::{TcpListener, TransportTcpStream}; use crate::security::NodeTransportSecurity; use command::DataTransportCommand; @@ -69,7 +69,7 @@ impl DataTransportActor { } Ok((stream, _)) = listener.accept() => { - let stream = match NodeTcpStream::accept(stream, &security).await { + let stream = match TransportTcpStream::accept_node(stream, &security).await { Ok(stream) => stream, Err(error) => { tracing::debug!("Data TLS accept rejected: {error}"); diff --git a/src/data_plane/transport/reader.rs b/src/data_plane/transport/reader.rs index 1dc7b839..6534666b 100644 --- a/src/data_plane/transport/reader.rs +++ b/src/data_plane/transport/reader.rs @@ -4,19 +4,22 @@ use tokio::sync::mpsc; use crate::control_plane::NodeId; use crate::data_plane::actor::DataPlaneSender; use crate::data_plane::messages::command::{DataPlaneCommand, ReceivePeerMessage}; -use crate::net::NodeReadHalf; +use crate::net::TransportReadHalf; use crate::security::NodeTransportIdentity; const NODE_ID_FRAME_MAX: usize = 1024; const DATA_FRAME_MAX: usize = 64 * 1024 * 1024; pub(super) struct DataReader { - read_half: NodeReadHalf, + read_half: TransportReadHalf, transport_identity: NodeTransportIdentity, } impl DataReader { - pub(super) fn new(read_half: NodeReadHalf, transport_identity: NodeTransportIdentity) -> Self { + pub(super) fn new( + read_half: TransportReadHalf, + transport_identity: NodeTransportIdentity, + ) -> Self { Self { read_half, transport_identity, diff --git a/src/data_plane/transport/writers.rs b/src/data_plane/transport/writers.rs index b3e63ff3..037d0067 100644 --- a/src/data_plane/transport/writers.rs +++ b/src/data_plane/transport/writers.rs @@ -11,7 +11,7 @@ use crate::data_plane::actor::DataPlaneSender; use crate::data_plane::messages::command::{ DataPlaneCommand, DataPlanePeerMessage, ReceivePeerMessage, }; -use crate::net::{NodeTcpStream, NodeWriteHalf}; +use crate::net::{TransportTcpStream, TransportWriteHalf}; use crate::security::NodeTransportSecurity; use super::reader::DataReader; @@ -20,7 +20,7 @@ const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); pub(super) struct TransportState { node_id: NodeId, - writers: HashMap, + writers: HashMap, dead_peers: HashSet, /// Tracks when the last connect attempt to a peer failed. Skips retry /// for CONNECT_BACKOFF (2s) to avoid blocking the select loop on repeated @@ -40,7 +40,10 @@ impl TransportState { } } - pub async fn accept(&mut self, stream: NodeTcpStream) -> anyhow::Result<(NodeId, DataReader)> { + pub async fn accept( + &mut self, + stream: TransportTcpStream, + ) -> anyhow::Result<(NodeId, DataReader)> { let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); let mut reader = DataReader::new(read_half, transport_identity); @@ -133,7 +136,7 @@ impl TransportState { let stream = tokio::time::timeout( std::time::Duration::from_secs(3), - NodeTcpStream::connect(node_addr.data_addr(), &self.security), + TransportTcpStream::connect_node(node_addr.data_addr(), &self.security), ) .await .context("connect timed out")? diff --git a/src/lib.rs b/src/lib.rs index 32ad86e6..43a961d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,7 +37,7 @@ use crate::data_plane::recovery; use crate::data_plane::transport::DataTransportActor; use crate::data_plane::transport::command::DataTransportCommand; use crate::impls::metadata_storage::MetadataStorage; -use crate::net::{TcpListener, UdpSocket}; +use crate::net::{TcpListener, TransportTcpStream, UdpSocket}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; use crate::security::NodeTransportSecurity; @@ -168,7 +168,7 @@ impl StartUp { // Client handler let _ = self - .receive_client_streams(node_id, swim_sender, raft_tx, data_plane_tx) + .receive_client_streams(node_id, swim_sender, raft_tx, data_plane_tx, security) .await; Ok(()) } @@ -179,6 +179,7 @@ impl StartUp { swim_sender: SwimSender, raft_tx: MutlRaftSender, data_plane_tx: DataPlaneSender, + security: NodeTransportSecurity, ) { let addr = self.env.bind_addr(); let listener = TcpListener::bind(&addr).await.unwrap(); @@ -189,6 +190,13 @@ impl StartUp { ); while let Ok((stream, _)) = listener.accept().await { + let stream = match TransportTcpStream::accept_client(stream, &security).await { + Ok(stream) => stream, + Err(error) => { + tracing::debug!("client authentication failed: {error}"); + continue; + } + }; let node_id = node_id.clone(); let swim_tx = swim_sender.clone(); let raft = raft_tx.clone(); diff --git a/src/net/mod.rs b/src/net/mod.rs index 6dbfba31..70dc9afc 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -3,10 +3,9 @@ mod tcp; pub(crate) mod udp; -pub use tcp::{ - NodeReadHalf, NodeTcpStream, NodeWriteHalf, OwnedReadHalf, OwnedWriteHalf, TcpListener, - TcpStream, -}; +#[cfg(test)] +pub use tcp::OwnedWriteHalf; +pub use tcp::{TcpListener, TcpStream, TransportReadHalf, TransportTcpStream, TransportWriteHalf}; pub use udp::UdpSocket; #[cfg(not(test))] diff --git a/src/net/tcp.rs b/src/net/tcp.rs index d1620b48..7a62c98d 100644 --- a/src/net/tcp.rs +++ b/src/net/tcp.rs @@ -9,7 +9,10 @@ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio_rustls::{TlsAcceptor, TlsConnector, TlsStream}; use super::inner; -use crate::security::{NodeTransportIdentity, NodeTransportSecurity, node_certificate_principal}; +use crate::security::{ + NodeTransportIdentity, NodeTransportSecurity, client_certificate_principal, + node_certificate_principal, +}; macro_rules! tcp_wrapper { ($name:ident) => { @@ -49,7 +52,16 @@ pub struct AuthenticatedTcpStream { impl AuthenticatedTcpStream { pub async fn accept(stream: TcpStream, config: Arc) -> Result { let stream = TlsAcceptor::from(config).accept(stream).await?; - Self::from_tls_stream(stream.into()) + + Self::from_tls_stream(stream.into(), node_certificate_principal) + } + + pub async fn accept_client( + stream: TcpStream, + config: Arc, + ) -> Result { + let stream = TlsAcceptor::from(config).accept(stream).await?; + Self::from_tls_stream(stream.into(), client_certificate_principal) } pub async fn connect( @@ -63,17 +75,20 @@ impl AuthenticatedTcpStream { let stream = TlsConnector::from(config) .connect(server_name, stream) .await?; - Self::from_tls_stream(stream.into()) + Self::from_tls_stream(stream.into(), node_certificate_principal) } - fn from_tls_stream(stream: TlsStream) -> Result { + fn from_tls_stream( + stream: TlsStream, + read_principal: fn(&rustls::pki_types::CertificateDer<'_>) -> Result, + ) -> Result { let certificate = stream .get_ref() .1 .peer_certificates() .and_then(|certificates| certificates.first()) .context("authenticated TLS peer supplied no certificate")?; - let peer_principal = node_certificate_principal(certificate)?; + let peer_principal = read_principal(certificate)?; Ok(Self { peer_principal, stream, @@ -94,37 +109,37 @@ impl AuthenticatedTcpStream { } } -/// Cluster TCP connection. Secure mode carries an authenticated certificate +/// TCP transport connection. Secure mode carries an authenticated certificate /// principal; trusted-development mode preserves the existing plaintext path. -pub enum NodeTcpStream { +pub enum TransportTcpStream { TrustedDevelopment(TcpStream), Secure(Box), } -pub enum NodeReadHalf { +pub enum TransportReadHalf { TrustedDevelopment(OwnedReadHalf), Secure(tokio::io::ReadHalf), } -pub enum NodeWriteHalf { +pub enum TransportWriteHalf { TrustedDevelopment(OwnedWriteHalf), Secure(tokio::io::WriteHalf), } -impl From for NodeReadHalf { +impl From for TransportReadHalf { fn from(value: OwnedReadHalf) -> Self { Self::TrustedDevelopment(value) } } -impl From for NodeWriteHalf { +impl From for TransportWriteHalf { fn from(value: OwnedWriteHalf) -> Self { Self::TrustedDevelopment(value) } } -impl NodeTcpStream { - pub async fn accept(stream: TcpStream, security: &NodeTransportSecurity) -> Result { +impl TransportTcpStream { + pub async fn accept_node(stream: TcpStream, security: &NodeTransportSecurity) -> Result { match security { NodeTransportSecurity::Secure { server, .. } => { AuthenticatedTcpStream::accept(stream, server.clone()) @@ -136,7 +151,22 @@ impl NodeTcpStream { } } - pub async fn connect( + pub async fn accept_client( + stream: TcpStream, + security: &NodeTransportSecurity, + ) -> Result { + match security { + NodeTransportSecurity::Secure { server, .. } => { + AuthenticatedTcpStream::accept_client(stream, server.clone()) + .await + .map(Box::new) + .map(Self::Secure) + } + NodeTransportSecurity::TrustedDevelopment => Ok(Self::TrustedDevelopment(stream)), + } + } + + pub async fn connect_node( addr: A, security: &NodeTransportSecurity, ) -> Result { @@ -163,24 +193,27 @@ impl NodeTcpStream { } } - pub fn into_split(self) -> (NodeReadHalf, NodeWriteHalf) { + pub fn into_split(self) -> (TransportReadHalf, TransportWriteHalf) { match self { Self::TrustedDevelopment(stream) => { let (read, write) = TcpStream::into_split(stream); ( - NodeReadHalf::TrustedDevelopment(read), - NodeWriteHalf::TrustedDevelopment(write), + TransportReadHalf::TrustedDevelopment(read), + TransportWriteHalf::TrustedDevelopment(write), ) } Self::Secure(stream) => { let (read, write) = AuthenticatedTcpStream::into_split(*stream); - (NodeReadHalf::Secure(read), NodeWriteHalf::Secure(write)) + ( + TransportReadHalf::Secure(read), + TransportWriteHalf::Secure(write), + ) } } } } -macro_rules! impl_node_io { +macro_rules! impl_transport_io { ($type:ty) => { impl AsyncRead for $type { fn poll_read( @@ -230,9 +263,9 @@ macro_rules! impl_node_io { }; } -impl_node_io!(NodeTcpStream); +impl_transport_io!(TransportTcpStream); -impl AsyncRead for NodeReadHalf { +impl AsyncRead for TransportReadHalf { fn poll_read( mut self: Pin<&mut Self>, cx: &mut Context<'_>, @@ -245,7 +278,7 @@ impl AsyncRead for NodeReadHalf { } } -impl AsyncWrite for NodeWriteHalf { +impl AsyncWrite for TransportWriteHalf { fn poll_write( mut self: Pin<&mut Self>, cx: &mut Context<'_>, @@ -393,6 +426,7 @@ mod tests { use turmoil::Builder; fn certificate( + principal_kind: &str, principal: &str, dns_name: Option<&str>, ) -> ( @@ -401,7 +435,7 @@ mod tests { ) { let mut params = CertificateParams::default(); params.subject_alt_names.push(SanType::URI( - Ia5String::try_from(format!("urn:eastguard:node:{principal}")).unwrap(), + Ia5String::try_from(format!("urn:eastguard:{principal_kind}:{principal}")).unwrap(), )); if let Some(dns_name) = dns_name { params @@ -416,8 +450,8 @@ mod tests { fn tls_configs() -> (Arc, Arc) { let (server_certificate, server_key) = - certificate("broker-server", Some("unused.eastguard")); - let (client_certificate, client_key) = certificate("broker-client", None); + certificate("node", "broker-server", Some("unused.eastguard")); + let (client_certificate, client_key) = certificate("node", "broker-client", None); let mut client_roots = RootCertStore::empty(); client_roots.add(client_certificate.clone()).unwrap(); @@ -475,4 +509,56 @@ mod tests { sim.run() } + + #[test] + fn mutual_tls_exposes_client_principal_under_turmoil() -> turmoil::Result { + let (server_certificate, server_key) = + certificate("node", "broker-server", Some("unused.eastguard")); + let (client_certificate, client_key) = certificate("client", "producer-a", None); + + let mut client_roots = RootCertStore::empty(); + client_roots.add(client_certificate.clone()).unwrap(); + let client_verifier = WebPkiClientVerifier::builder(Arc::new(client_roots)) + .build() + .unwrap(); + let server_config = Arc::new( + ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_client_cert_verifier(client_verifier) + .with_single_cert(vec![server_certificate.clone()], server_key) + .unwrap(), + ); + + let mut server_roots = RootCertStore::empty(); + server_roots.add(server_certificate).unwrap(); + let client_config = Arc::new( + ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_root_certificates(server_roots) + .with_client_auth_cert(vec![client_certificate], client_key) + .unwrap(), + ); + + let mut sim = Builder::new().build(); + sim.host("server", move || { + let server_config = server_config.clone(); + async move { + let listener = TcpListener::bind("0.0.0.0:9000").await?; + let (stream, _) = listener.accept().await?; + let stream = AuthenticatedTcpStream::accept_client(stream, server_config) + .await + .unwrap(); + assert_eq!(stream.peer_principal(), "producer-a"); + Ok(()) + } + }); + sim.client("client", async move { + let stream = + AuthenticatedTcpStream::connect((turmoil::lookup("server"), 9000), client_config) + .await + .unwrap(); + assert_eq!(stream.peer_principal(), "broker-server"); + Ok(()) + }); + + sim.run() + } } From 1c97a2922ade361da4865d04b0a16d48687df339 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Tue, 28 Jul 2026 20:41:59 +0400 Subject: [PATCH 25/56] feat: client controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TLS client principal ↓ TransportIdentity ↓ ClientController ↓ future ACL checks Changes: - Renamed NodeTransportIdentity to shared TransportIdentity. - Captures identity before splitting the stream. - Stores identity on ClientController. - Trusted-development tests explicitly use TrustedDevelopment. - No ACL behavior added yet. --- src/connections/controller.rs | 179 ++++++++++-------- .../consensus/transport/inbound.rs | 6 +- src/control_plane/consensus/transport/mod.rs | 16 +- src/data_plane/transport/reader.rs | 9 +- src/net/tcp.rs | 8 +- src/security.rs | 2 +- 6 files changed, 116 insertions(+), 104 deletions(-) diff --git a/src/connections/controller.rs b/src/connections/controller.rs index cb426e87..4a76e288 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -29,6 +29,7 @@ use crate::data_plane::messages::query::{ DataPlaneQuery, Fetch, ListOffsets, ReadConsumerOffset, ReadConsumerOffsetResult, }; use crate::net::TransportTcpStream; +use crate::security::TransportIdentity; use tokio::sync::mpsc; /// # Client ↔ Server request_id protocol @@ -47,6 +48,7 @@ use tokio::sync::mpsc; /// with responses arriving in any order. #[derive(Clone)] pub struct ClientController { + transport_identity: TransportIdentity, node_id: NodeId, swim_sender: SwimSender, raft_sender: MutlRaftSender, @@ -55,12 +57,14 @@ pub struct ClientController { impl ClientController { fn new( + transport_identity: TransportIdentity, node_id: NodeId, swim_sender: SwimSender, raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, ) -> Self { Self { + transport_identity, node_id, swim_sender, raft_sender, @@ -103,6 +107,10 @@ impl ClientController { } pub async fn dispatch(&self, request: ClientRequest) -> ClientResponse { + tracing::trace!( + transport_identity = ?self.transport_identity, + "dispatching client request" + ); match request { ClientRequest::ControlPlane(cp) => self.handle_control_plane(cp).await, ClientRequest::DataPlane(dp) => self.handle_data_plane(dp).await, @@ -621,9 +629,16 @@ pub async fn handle_client_stream( raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, ) { + let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); let (writer_tx, writer_rx) = mpsc::channel(128); - let handler = ClientController::new(node_id, swim_sender, raft_sender, data_plane_tx); + let handler = ClientController::new( + transport_identity, + node_id, + swim_sender, + raft_sender, + data_plane_tx, + ); tokio::spawn(run_client_writer( ClientRawWriter::new(write_half), writer_rx, @@ -715,6 +730,21 @@ mod tests { DataPlaneSender(tx) } + fn trusted_controller( + node_id: NodeId, + swim_sender: SwimSender, + raft_sender: MutlRaftSender, + data_plane_tx: DataPlaneSender, + ) -> ClientController { + ClientController::new( + TransportIdentity::TrustedDevelopment, + node_id, + swim_sender, + raft_sender, + data_plane_tx, + ) + } + fn produce_req() -> ClientRequest { ClientRequest::DataPlane(ClientDataPlaneRequest::Produce(ProduceRequest { topic_name: "t1".into(), @@ -750,10 +780,9 @@ mod tests { let _ = reply.send(None); } }); - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(produce_req()) - .await; + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(produce_req()) + .await; assert!( matches!( resp, @@ -782,10 +811,9 @@ mod tests { } _ => {} }); - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(produce_req()) - .await; + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(produce_req()) + .await; let ClientResponse::Err(ServerError::ShardNotLocal { hint_node }) = resp else { panic!("expected ShardNotLocal, got {resp:?}"); }; @@ -817,7 +845,7 @@ mod tests { let _ = reply.send(Some(topic_meta("leader"))); } }); - let resp = ClientController::new(me, swim, raft, dp_stub()) + let resp = trusted_controller(me, swim, raft, dp_stub()) .dispatch(produce_req()) .await; let ClientResponse::Err(ServerError::NotWriteLeader { @@ -847,7 +875,7 @@ mod tests { let _ = reply.send(Some(topic_meta("self"))); } }); - let resp = ClientController::new(me, swim, raft, dp_acking()) + let resp = trusted_controller(me, swim, raft, dp_acking()) .dispatch(produce_req()) .await; let ClientResponse::Ok(ClientSuccess::Produced(entry_id)) = resp else { @@ -875,19 +903,18 @@ mod tests { } _ => {} }); - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(ClientRequest::ControlPlane( - ControlPlaneRequest::CreateTopic { - name: "t1".into(), - storage_policy: StoragePolicy { - retention_ms: Some(3_600_000), - replication_factor: 1, - partition_strategy: PartitionStrategy::AutoSplit, - }, + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(ClientRequest::ControlPlane( + ControlPlaneRequest::CreateTopic { + name: "t1".into(), + storage_policy: StoragePolicy { + retention_ms: Some(3_600_000), + replication_factor: 1, + partition_strategy: PartitionStrategy::AutoSplit, }, - )) - .await; + }, + )) + .await; let ClientResponse::Err(ServerError::TopicMetadataRedirect { owner: redirect_owner, }) = resp @@ -910,7 +937,7 @@ mod tests { let _ = reply.send(Ok(())); } }); - let resp = ClientController::new(node_id("node-1"), swim, raft, dp_stub()) + let resp = trusted_controller(node_id("node-1"), swim, raft, dp_stub()) .dispatch(ClientRequest::ControlPlane( ControlPlaneRequest::CreateTopic { name: "t1".into(), @@ -935,19 +962,18 @@ mod tests { let _ = reply.send(None); } }); - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(ClientRequest::ControlPlane( - ControlPlaneRequest::CreateTopic { - name: "t1".into(), - storage_policy: StoragePolicy { - retention_ms: Some(3_600_000), - replication_factor: 1, - partition_strategy: PartitionStrategy::AutoSplit, - }, + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(ClientRequest::ControlPlane( + ControlPlaneRequest::CreateTopic { + name: "t1".into(), + storage_policy: StoragePolicy { + retention_ms: Some(3_600_000), + replication_factor: 1, + partition_strategy: PartitionStrategy::AutoSplit, }, - )) - .await; + }, + )) + .await; assert!( matches!(resp, ClientResponse::Err(ServerError::Internal(_))), "expected InternalError, got {resp:?}" @@ -966,7 +992,7 @@ mod tests { let _ = reply.send(Ok(())); } }); - let resp = ClientController::new(node_id("node-1"), swim, raft, dp_stub()) + let resp = trusted_controller(node_id("node-1"), swim, raft, dp_stub()) .dispatch(ClientRequest::ControlPlane( ControlPlaneRequest::DeleteTopic { name: "t1".into() }, )) @@ -984,12 +1010,11 @@ mod tests { let _ = reply.send(Box::new(["alpha".into(), "beta".into()])); } }); - let resp = - ClientController::new(node_id("self"), swim_sender_with(|_| {}), raft, dp_stub()) - .dispatch(ClientRequest::ControlPlane( - ControlPlaneRequest::ListHostedTopics, - )) - .await; + let resp = trusted_controller(node_id("self"), swim_sender_with(|_| {}), raft, dp_stub()) + .dispatch(ClientRequest::ControlPlane( + ControlPlaneRequest::ListHostedTopics, + )) + .await; let ClientResponse::Ok(ClientSuccess::TopicList { topics }) = resp else { panic!("expected TopicList, got {resp:?}"); }; @@ -1022,14 +1047,13 @@ mod tests { _ => {} }) }; - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(ClientRequest::ControlPlane( - ControlPlaneRequest::DescribeTopic { - name: "elsewhere".into(), - }, - )) - .await; + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(ClientRequest::ControlPlane( + ControlPlaneRequest::DescribeTopic { + name: "elsewhere".into(), + }, + )) + .await; let ClientResponse::Err(ServerError::TopicMetadataRedirect { owner: redirect_owner, }) = resp @@ -1060,7 +1084,7 @@ mod tests { let _ = reply.send(None); } }); - let resp = ClientController::new(me, swim, raft, dp_stub()) + let resp = trusted_controller(me, swim, raft, dp_stub()) .dispatch(ClientRequest::ControlPlane( ControlPlaneRequest::DescribeTopic { name: "missing".into(), @@ -1098,10 +1122,9 @@ mod tests { } }) }; - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(ClientRequest::Admin(AdminRequest::DescribeCluster)) - .await; + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(ClientRequest::Admin(AdminRequest::DescribeCluster)) + .await; let ClientResponse::Ok(ClientSuccess::ClusterInfo { nodes: info }) = resp else { panic!("expected ClusterInfo, got {resp:?}"); }; @@ -1128,12 +1151,11 @@ mod tests { } }) }; - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(ClientRequest::Admin(AdminRequest::GetShardInfo { - key: b"any".to_vec(), - })) - .await; + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(ClientRequest::Admin(AdminRequest::GetShardInfo { + key: b"any".to_vec(), + })) + .await; let ClientResponse::Ok(ClientSuccess::ShardInfo { detail: Some(d) }) = resp else { panic!("expected ShardInfo with detail, got {resp:?}"); }; @@ -1151,12 +1173,11 @@ mod tests { let _ = reply.send(None); } }); - let resp = - ClientController::new(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) - .dispatch(ClientRequest::Admin(AdminRequest::GetShardInfo { - key: b"x".to_vec(), - })) - .await; + let resp = trusted_controller(node_id("self"), swim, raft_sender_with(|_| {}), dp_stub()) + .dispatch(ClientRequest::Admin(AdminRequest::GetShardInfo { + key: b"x".to_vec(), + })) + .await; assert!( matches!( resp, @@ -1173,12 +1194,11 @@ mod tests { let _ = reply.send(Some(node_id("n1"))); } }); - let resp = - ClientController::new(node_id("self"), swim_sender_with(|_| {}), raft, dp_stub()) - .dispatch(ClientRequest::Admin(AdminRequest::GetShardLeader { - shard_group_id: ShardGroupId(42), - })) - .await; + let resp = trusted_controller(node_id("self"), swim_sender_with(|_| {}), raft, dp_stub()) + .dispatch(ClientRequest::Admin(AdminRequest::GetShardLeader { + shard_group_id: ShardGroupId(42), + })) + .await; let ClientResponse::Ok(ClientSuccess::ShardLeader { leader }) = resp else { panic!("expected ShardLeader, got {resp:?}"); }; @@ -1196,12 +1216,11 @@ mod tests { }])); } }); - let resp = - ClientController::new(node_id("self"), swim_sender_with(|_| {}), raft, dp_stub()) - .dispatch(ClientRequest::Admin( - AdminRequest::ListHostedTopicsWithStats, - )) - .await; + let resp = trusted_controller(node_id("self"), swim_sender_with(|_| {}), raft, dp_stub()) + .dispatch(ClientRequest::Admin( + AdminRequest::ListHostedTopicsWithStats, + )) + .await; let ClientResponse::Ok(ClientSuccess::TopicStats { topics }) = resp else { panic!("expected TopicStats, got {resp:?}"); }; diff --git a/src/control_plane/consensus/transport/inbound.rs b/src/control_plane/consensus/transport/inbound.rs index 2fc82220..f4ad23de 100644 --- a/src/control_plane/consensus/transport/inbound.rs +++ b/src/control_plane/consensus/transport/inbound.rs @@ -3,18 +3,18 @@ use crate::control_plane::consensus::actor::MutlRaftSender; use crate::control_plane::consensus::messages::InboundRaftRpc; use crate::control_plane::consensus::messages::WireRaftMessage; use crate::net::TransportReadHalf; -use crate::security::NodeTransportIdentity; +use crate::security::TransportIdentity; use tokio::io::AsyncReadExt; pub(super) struct RaftRpcListener { read_half: TransportReadHalf, - transport_identity: NodeTransportIdentity, + transport_identity: TransportIdentity, } impl RaftRpcListener { pub(super) fn new( read_half: impl Into, - transport_identity: NodeTransportIdentity, + transport_identity: TransportIdentity, ) -> Self { Self { read_half: read_half.into(), diff --git a/src/control_plane/consensus/transport/mod.rs b/src/control_plane/consensus/transport/mod.rs index 0fed5ff7..33d64812 100644 --- a/src/control_plane/consensus/transport/mod.rs +++ b/src/control_plane/consensus/transport/mod.rs @@ -14,9 +14,9 @@ use crate::control_plane::consensus::messages::RaftTransportCommand; use crate::control_plane::membership::actor::SwimSender; use crate::net::TcpListener; use crate::net::TransportTcpStream; -#[cfg(test)] -use crate::security::NodeTransportIdentity; use crate::security::NodeTransportSecurity; +#[cfg(test)] +use crate::security::TransportIdentity; const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); @@ -109,8 +109,7 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = - RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); + let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); let peer_id = reader.read_node_id().await.unwrap(); assert_eq!(peer_id, NodeId::new("node-abc")); @@ -139,8 +138,7 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = - RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); + let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); let msg = reader.read_message().await.unwrap(); assert_eq!(msg.shard_group_id, ShardGroupId(42)); @@ -190,8 +188,7 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = - RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); + let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); let peer = reader.read_node_id().await?; let (raft_tx, mut raft_rx) = MultiRaftActor::channel(8); @@ -307,8 +304,7 @@ mod tests { // Second connection from node-a (simulating simultaneous connect) let (stream2, _) = dummy_listener.accept().await?; let (read_half, _write_half) = stream2.into_split(); - let mut reader = - RaftRpcListener::new(read_half, NodeTransportIdentity::TrustedDevelopment); + let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); let peer_id = reader.read_node_id().await.unwrap(); assert_eq!(peer_id, NodeId::new("node-a")); diff --git a/src/data_plane/transport/reader.rs b/src/data_plane/transport/reader.rs index 6534666b..a21f0ab4 100644 --- a/src/data_plane/transport/reader.rs +++ b/src/data_plane/transport/reader.rs @@ -5,21 +5,18 @@ use crate::control_plane::NodeId; use crate::data_plane::actor::DataPlaneSender; use crate::data_plane::messages::command::{DataPlaneCommand, ReceivePeerMessage}; use crate::net::TransportReadHalf; -use crate::security::NodeTransportIdentity; +use crate::security::TransportIdentity; const NODE_ID_FRAME_MAX: usize = 1024; const DATA_FRAME_MAX: usize = 64 * 1024 * 1024; pub(super) struct DataReader { read_half: TransportReadHalf, - transport_identity: NodeTransportIdentity, + transport_identity: TransportIdentity, } impl DataReader { - pub(super) fn new( - read_half: TransportReadHalf, - transport_identity: NodeTransportIdentity, - ) -> Self { + pub(super) fn new(read_half: TransportReadHalf, transport_identity: TransportIdentity) -> Self { Self { read_half, transport_identity, diff --git a/src/net/tcp.rs b/src/net/tcp.rs index 7a62c98d..6e0ad407 100644 --- a/src/net/tcp.rs +++ b/src/net/tcp.rs @@ -10,7 +10,7 @@ use tokio_rustls::{TlsAcceptor, TlsConnector, TlsStream}; use super::inner; use crate::security::{ - NodeTransportIdentity, NodeTransportSecurity, client_certificate_principal, + NodeTransportSecurity, TransportIdentity, client_certificate_principal, node_certificate_principal, }; @@ -184,12 +184,12 @@ impl TransportTcpStream { } } - pub fn peer_identity(&self) -> NodeTransportIdentity { + pub fn peer_identity(&self) -> TransportIdentity { match self { Self::Secure(stream) => { - NodeTransportIdentity::CertificatePrincipal(stream.peer_principal().to_string()) + TransportIdentity::CertificatePrincipal(stream.peer_principal().to_string()) } - Self::TrustedDevelopment(_) => NodeTransportIdentity::TrustedDevelopment, + Self::TrustedDevelopment(_) => TransportIdentity::TrustedDevelopment, } } diff --git a/src/security.rs b/src/security.rs index bc58e7bf..af994173 100644 --- a/src/security.rs +++ b/src/security.rs @@ -115,7 +115,7 @@ pub(crate) enum NodeTransportSecurity { } #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum NodeTransportIdentity { +pub(crate) enum TransportIdentity { CertificatePrincipal(String), TrustedDevelopment, } From 6537e6611e7ebee542ecde87828e68f905931c0c Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 08:13:31 +0400 Subject: [PATCH 26/56] roadmap update --- docs/security/roadmap.md | 67 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index baf0b238..959574b2 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -141,7 +141,7 @@ permission checks; its text grants no authority by itself. | :--- | :--- | | `cluster` | Membership inspection, topology lookup, operator diagnostics | | `topic-admin/{topic}` | Create, delete, describe topic metadata | -| `topic-data/{topic}` | Produce, fetch, list offsets for topic | +| `topic-data/{topic-id}` | Produce, fetch, list offsets for topic | | `consumer-group/{topic}/{group}` | Consume messages, read/commit consumer offsets | | `producer-session/{topic}/{session}` | Renew producer session (permanently bound to creator principal) | | `security/cluster` | Read/write ACLs, manage admissions and revocations, inspect security audit | @@ -173,7 +173,11 @@ security/node/{node-certificate-principal} ``` - **Local Authorization:** Brokers evaluate ACLs against local cached security records. -- **Freshness & Expiry:** Cached records include a monotonic deadline (max 60s) and revision counter. Expired entries require re-validation from the owner metadata shard; if the owner is offline, authorization fails closed. + Data permissions use the stable topic ID, so a data replica can authorize a + request without hosting that topic's metadata shard. +- **Freshness & Expiry:** Cached records include a monotonic deadline (max 60s) + and revision counter. An expired entry cannot authorize a request. The chosen + distribution mechanism must obtain current state or fail closed. --- @@ -204,9 +208,66 @@ security/node/{node-certificate-principal} ## 6. Resource Limits & Security Audit ### Rate & Memory Bounds -- Every listener enforces strict limits on unauthenticated handshakes, concurrent connections, in-flight frames, memory allocations, and per-source request rates. +- Every listener enforces strict limits on unauthenticated handshakes, concurrent connections, in-flight frames, and memory allocations. +- Client request-rate limits are deferred. The design must first define whether + replicas of one application share a principal, which limits are node-wide, and + how limits behave as a Kubernetes workload scales. - The future secure UDP transport must define a payload budget that avoids IP fragmentation after authentication and encryption overhead. +### Client Request Boundary + +Clients normally route directly to the data replica named by their current topic +metadata. A redirect is only recovery from stale routing; brokers never proxy a +produce or fetch to another data node. + +``` +Client request + │ + ▼ +Authenticate certificate + │ + ▼ +Check local ACL cache + ├── Current grant + ├── Current denial ─► Return unauthorized + └── Missing / expired + │ + ▼ + Obtain current ACL state + (distribution design deferred) + │ + ┌─────┴─────┐ + ▼ ▼ + Grant Deny / unavailable ──► Fail closed + │ + ▼ +Does this node serve the requested data? + ├── No ──► Return data-node redirect + └── Yes ──► Execute locally +``` + +Authorization precedes redirects so an ungranted client cannot use stale-route +responses to discover data placement. Any remote authorization request may +update only the ACL cache; it must never carry or execute the client's data +operation. + +The ACL distribution mechanism is intentionally deferred. A later phase may use +lazy pull, proactive push, or a push-and-pull hybrid: + +| Model | Benefit | Failure to handle | +| :--- | :--- | :--- | +| Pull on cache miss or expiry | Simple; clients that leave create no update traffic | A data node must fail closed if the owner is unavailable | +| Push changed records | Fast local decisions | A disconnected data node can miss an update | +| Push plus periodic pull | Fast normally; repairs missed updates | More protocol and cache synchronization logic | + +The same decision gate covers request-rate limiting. A Kubernetes deployment may +give all replicas of one application a shared principal even though each replica +has a different leaf certificate. A fixed per-principal table size or request +budget is therefore premature: one shared principal can represent many clients, +while one principal per replica can grow with autoscaling. No client request-rate +limit is implemented until identity granularity, node-wide capacity bounds, and +cache distribution are chosen together. + ### Secure UDP Decision EastGuard retains UDP for SWIM because connection-oriented transport does not From ee5d0dbfee06de2dbbb29593700e22a3a94ecf7e Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 08:24:44 +0400 Subject: [PATCH 27/56] authorize at metadata level --- src/control_plane/consensus/actor.rs | 22 +++++++++++++ src/control_plane/consensus/messages/actor.rs | 20 +++++++++++- src/control_plane/consensus/multi_raft.rs | 31 +++++++++++++++++-- src/control_plane/consensus/raft/state.rs | 4 +++ .../consensus/raft/states/metadata_state.rs | 21 +++++++++++++ .../consensus/raft/states/security.rs | 30 ++++++++++++++++++ 6 files changed, 125 insertions(+), 3 deletions(-) diff --git a/src/control_plane/consensus/actor.rs b/src/control_plane/consensus/actor.rs index 7679972f..b0023ebc 100644 --- a/src/control_plane/consensus/actor.rs +++ b/src/control_plane/consensus/actor.rs @@ -278,6 +278,28 @@ impl MutlRaftSender { recv.await.unwrap_or_default() } + /// Checks ACL state only when this node hosts the selected metadata shard. + /// + /// `None` means the shard is not local or the actor stopped. `Some(false)` + /// is an authoritative default-deny decision from local replicated state. + pub(crate) async fn authorize_principal( + &self, + shard_group_id: ShardGroupId, + resource: String, + principal: String, + ) -> Option { + let (reply, recv) = tokio::sync::oneshot::channel(); + let _ = self + .send(AuthorizePrincipal { + shard_group_id, + resource, + principal, + reply, + }) + .await; + recv.await.ok().flatten() + } + pub(crate) async fn send( &self, cmd: impl Into, diff --git a/src/control_plane/consensus/messages/actor.rs b/src/control_plane/consensus/messages/actor.rs index d123a6de..c184b34c 100644 --- a/src/control_plane/consensus/messages/actor.rs +++ b/src/control_plane/consensus/messages/actor.rs @@ -52,6 +52,7 @@ pub enum MultiRaftActorCommand { topic_name: String, reply: oneshot::Sender>, }, + AuthorizePrincipal(AuthorizePrincipal), GetConsumerGroupAssignment(GetConsumerGroupAssignment), /// Data-plane request forwarded to the metadata coordinator for proposal. ProposeSegmentRoll(ProposeSegmentRoll), @@ -74,6 +75,13 @@ pub struct GetConsumerGroupAssignment { pub(crate) reply: oneshot::Sender>, } +pub struct AuthorizePrincipal { + pub(crate) shard_group_id: ShardGroupId, + pub(crate) resource: String, + pub(crate) principal: String, + pub(crate) reply: oneshot::Sender>, +} + impl From for MultiRaftActorCommand { fn from(cmd: RaftProtocolMessage) -> Self { MultiRaftActorCommand::ProtocolMessage(cmd) @@ -94,13 +102,22 @@ impl_from_variant_via!( RemoveGroup, ); -impl_from_variant!(MultiRaftActorCommand, GetConsumerGroupAssignment); +impl_from_variant!( + MultiRaftActorCommand, + AuthorizePrincipal, + GetConsumerGroupAssignment, +); pub(crate) struct DeferredConsumerGroupAssignment { pub(crate) reply: oneshot::Sender>, pub(crate) value: Option, } +pub(crate) struct DeferredAuthorization { + pub(crate) reply: oneshot::Sender>, + pub(crate) value: Option, +} + pub(crate) enum DeferredReply { GetLeader(oneshot::Sender>, Option), GetPeers(oneshot::Sender>, Box<[NodeId]>), @@ -111,5 +128,6 @@ pub(crate) enum DeferredReply { GetTopics(oneshot::Sender>, Box<[String]>), GetTopicStats(oneshot::Sender>, Box<[TopicStats]>), GetTopicMetadata(oneshot::Sender>, Box>), + AuthorizePrincipal(DeferredAuthorization), GetConsumerGroupAssignment(DeferredConsumerGroupAssignment), } diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index 9a73cf11..11ba6991 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -3,8 +3,9 @@ use crate::control_plane::consensus::boundary_recovery::{ BoundaryRecoveryAction, SegmentBoundaryRecovery, }; use crate::control_plane::consensus::messages::{ - DeferredConsumerGroupAssignment, DeferredReply, InboundRaftRpc, LogMutation, MetadataProposal, - MultiRaftActorCommand, ProposeSegmentRoll, RaftEvent, RaftProtocolMessage, RaftTimeoutCallback, + DeferredAuthorization, DeferredConsumerGroupAssignment, DeferredReply, InboundRaftRpc, + LogMutation, MetadataProposal, MultiRaftActorCommand, ProposeSegmentRoll, RaftEvent, + RaftProtocolMessage, RaftTimeoutCallback, }; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::consensus::raft::state::{Raft, TimerSeqs}; @@ -261,6 +262,18 @@ impl MultiRaft { self.deferred .push(DeferredReply::GetTopicMetadata(reply, Box::new(meta))); } + MultiRaftActorCommand::AuthorizePrincipal(query) => { + let value = self.authorize_principal( + query.shard_group_id, + &query.resource, + &query.principal, + ); + self.deferred + .push(DeferredReply::AuthorizePrincipal(DeferredAuthorization { + reply: query.reply, + value, + })); + } MultiRaftActorCommand::GetConsumerGroupAssignment(query) => { let value = self.get_consumer_group_assignment( &query.topic_name, @@ -311,6 +324,9 @@ impl MultiRaft { DeferredReply::GetTopicMetadata(sender, v) => { let _ = sender.send(*v); } + DeferredReply::AuthorizePrincipal(deferred) => { + let _ = deferred.reply.send(deferred.value); + } DeferredReply::GetConsumerGroupAssignment(deferred) => { let _ = deferred.reply.send(deferred.value); } @@ -478,6 +494,17 @@ impl MultiRaft { .find_map(|raft| raft.get_topic_by_name(name).cloned()) } + fn authorize_principal( + &self, + shard_group_id: ShardGroupId, + resource: &str, + principal: &str, + ) -> Option { + self.groups + .get(&shard_group_id) + .map(|raft| raft.authorizes(resource, principal)) + } + fn get_consumer_group_assignment( &self, topic_name: &str, diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index 615e2af2..5eba491d 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -161,6 +161,10 @@ impl Raft { self.metadata.get_topic_by_name(name) } + pub(crate) fn authorizes(&self, resource: &str, principal: &str) -> bool { + self.metadata.authorizes(resource, principal) + } + pub(crate) fn get_consumer_group_assignment( &self, topic_name: &str, diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index d59ebfe7..ba7a8109 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -89,6 +89,10 @@ impl MetadataState { range.segments.get(&key.segment_id) } + pub(crate) fn authorizes(&self, resource: &str, principal: &str) -> bool { + self.security.authorizes(resource, principal) + } + pub(crate) fn get_consumer_group_assignment( &self, topic_name: &str, @@ -686,6 +690,23 @@ mod tests { restored.assert_invariants(); } + #[test] + fn metadata_authorization_defaults_to_deny() { + let mut state = MetadataState::new(ShardGroupId(1)); + state.security.acls.insert( + "topic-data/42".to_string(), + AclRecord { + resource: "topic-data/42".to_string(), + revision: 1, + principals: vec!["orders-service".to_string()].into_boxed_slice(), + }, + ); + + assert!(state.authorizes("topic-data/42", "orders-service")); + assert!(!state.authorizes("topic-data/42", "unknown-service")); + assert!(!state.authorizes("topic-data/43", "orders-service")); + } + fn replica_set() -> Replicas { Replicas::new(vec![ NodeId::new("node-1"), diff --git a/src/control_plane/consensus/raft/states/security.rs b/src/control_plane/consensus/raft/states/security.rs index 5b8901ca..f6653162 100644 --- a/src/control_plane/consensus/raft/states/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -52,6 +52,18 @@ pub(crate) struct RevocationRecord { pub revoked_at: u64, } +impl SecurityState { + /// Returns whether an exact principal is listed on an exact ACL resource. + /// + /// Missing records and missing principals deny by default. Resource + /// hierarchy or wildcard matching is intentionally not inferred here. + pub(crate) fn authorizes(&self, resource: &str, principal: &str) -> bool { + self.acls + .get(resource) + .is_some_and(|acl| acl.principals.iter().any(|entry| entry == principal)) + } +} + #[cfg(any(test, debug_assertions))] impl crate::test_traits::TAssertInvariant for SecurityState { fn assert_invariants(&self) { @@ -113,4 +125,22 @@ mod tests { revoked_at: 100, }); } + + #[test] + fn acl_authorization_is_exact_and_defaults_to_deny() { + let mut security = SecurityState::default(); + security.acls.insert( + "topic-data/42".to_string(), + AclRecord { + resource: "topic-data/42".to_string(), + revision: 1, + principals: vec!["orders-service".to_string()].into_boxed_slice(), + }, + ); + + assert!(security.authorizes("topic-data/42", "orders-service")); + assert!(!security.authorizes("topic-data/42", "unknown-service")); + assert!(!security.authorizes("topic-data/43", "orders-service")); + assert!(!security.authorizes("topic-data", "orders-service")); + } } From e551cce8950279a8f72429d5764fc886c75bccf3 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 08:37:21 +0400 Subject: [PATCH 28/56] feat: authorize_data_access Certificate-authenticated clients now require topic-data/{topic-id} authorization before: - Produce - Fetch and fetch-by-ID - List offsets - Commit consumer offset - Fetch consumer offset - Data-locality/write-leader checks - Any data-plane dispatch --- src/client/mod.rs | 1 + src/connections/controller.rs | 98 ++++++++++++++++++++++++++++++- src/connections/protocol/error.rs | 3 + 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/src/client/mod.rs b/src/client/mod.rs index a545188d..1f73cb36 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -506,6 +506,7 @@ impl Client { ServerError::TopicNotFound => Redirect::NotFound, ServerError::SegmentNotLocal | ServerError::Internal(_) => Redirect::Reresolve, ServerError::AlreadyExists + | ServerError::Unauthorized | ServerError::StaleRange | ServerError::ProduceRejected(_) | ServerError::EntryIdOutOfRange diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 4a76e288..98a8b2b7 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -4,7 +4,8 @@ use crate::connections::{protocol::*, run_client_writer}; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::metadata::{ - OpenProducerSession, RangeMeta, SyncConsumerGroup, SyncConsumerGroupRequest, TopicState, + OpenProducerSession, RangeMeta, SyncConsumerGroup, SyncConsumerGroupRequest, TopicId, + TopicState, }; use crate::control_plane::{ NodeId, SwimNodeState, @@ -269,6 +270,26 @@ impl ClientController { .await } + async fn authorize_data_access(&self, topic_id: TopicId) -> Result<(), ServerError> { + let TransportIdentity::CertificatePrincipal(principal) = &self.transport_identity else { + return Ok(()); + }; + + let resource = format!("topic-data/{}", topic_id.0); + let ShardRouting::Local(group) = self.route(resource.as_bytes().to_vec()).await? else { + return Err(ServerError::Unauthorized); + }; + + match self + .raft_sender + .authorize_principal(group.id, resource, principal.clone()) + .await + { + Some(true) => Ok(()), + Some(false) | None => Err(ServerError::Unauthorized), + } + } + /// Structural redirect for a control-plane op that isn't local: to the member if /// one resolves, else a retriable error (no member's address known here yet). fn control_plane_redirect(&self, member: Option) -> ServerError { @@ -344,6 +365,8 @@ impl ClientController { &self, req: CommitConsumerOffsetRequest, ) -> Result { + self.authorize_data_access(req.key.topic_id).await?; + let (tx, recv) = tokio::sync::oneshot::channel(); self.data_plane_tx .send_async(CommitConsumerOffset { @@ -372,6 +395,8 @@ impl ClientController { &self, req: FetchConsumerOffsetRequest, ) -> Result { + self.authorize_data_access(req.key.topic_id).await?; + let (reply, recv) = tokio::sync::oneshot::channel(); self.data_plane_tx .send_async(ReadConsumerOffset { @@ -410,6 +435,7 @@ impl ClientController { } let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; + self.authorize_data_access(topic.id).await?; let producer_identity = req .producer_identity @@ -463,6 +489,7 @@ impl ClientController { /// state machine needs to answer without any further I/O. async fn fetch(&self, req: FetchRequest) -> Result { let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; + self.authorize_data_access(topic.id).await?; let range = topic.get_range(&req.range_id)?; @@ -495,6 +522,8 @@ impl ClientController { /// serve it. No proxying: a miss returns `SegmentNotLocal` and the client /// retries another replica. async fn fetch_by_id(&self, req: FetchByIdRequest) -> Result { + self.authorize_data_access(req.topic_id).await?; + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); let query = Fetch { topic_id: req.topic_id, @@ -518,6 +547,7 @@ impl ClientController { /// for the range's currently-active segment on this node. async fn list_offsets(&self, req: RangeOffsetRequest) -> Result { let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; + self.authorize_data_access(topic.id).await?; let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); @@ -745,6 +775,21 @@ mod tests { ) } + fn authenticated_controller( + principal: &str, + node_id: NodeId, + swim_sender: SwimSender, + raft_sender: MutlRaftSender, + ) -> ClientController { + ClientController::new( + TransportIdentity::CertificatePrincipal(principal.to_string()), + node_id, + swim_sender, + raft_sender, + dp_stub(), + ) + } + fn produce_req() -> ClientRequest { ClientRequest::DataPlane(ClientDataPlaneRequest::Produce(ProduceRequest { topic_name: "t1".into(), @@ -771,6 +816,57 @@ mod tests { ) } + #[tokio::test] + async fn certificate_client_requires_exact_topic_acl() { + let group = ShardGroup { + id: ShardGroupId(42), + replicas: Replicas::new(vec![node_id("self")]), + }; + let swim = swim_sender_with(move |cmd| { + if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { + let _ = reply.send(Some(group.clone())); + } + }); + let raft = raft_sender_with(|cmd| { + if let MultiRaftActorCommand::AuthorizePrincipal(query) = cmd { + assert_eq!(query.shard_group_id, ShardGroupId(42)); + assert_eq!(query.resource, "topic-data/7"); + let _ = query.reply.send(Some(query.principal == "orders-service")); + } + }); + let controller = authenticated_controller("orders-service", node_id("self"), swim, raft); + + assert_eq!(controller.authorize_data_access(TopicId(7)).await, Ok(())); + } + + #[tokio::test] + async fn certificate_client_fails_closed_when_acl_shard_is_remote() { + let group = ShardGroup { + id: ShardGroupId(42), + replicas: Replicas::new(vec![node_id("other")]), + }; + let swim = swim_sender_with(move |cmd| match cmd { + SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) => { + let _ = reply.send(Some(group.clone())); + } + SwimActorCommand::Query(QueryCommand::ResolveAddress { reply, .. }) => { + let _ = reply.send(None); + } + _ => {} + }); + let controller = authenticated_controller( + "orders-service", + node_id("self"), + swim, + raft_sender_with(|_| panic!("remote ACL shard must not be queried locally")), + ); + + assert_eq!( + controller.authorize_data_access(TopicId(7)).await, + Err(ServerError::Unauthorized) + ); + } + /// Ring can't map the key yet (topology not converged) → retriable /// `ShardNotLocal` with no hint, not `TopicNotFound`. #[tokio::test] diff --git a/src/connections/protocol/error.rs b/src/connections/protocol/error.rs index 11c225fc..64fa1739 100644 --- a/src/connections/protocol/error.rs +++ b/src/connections/protocol/error.rs @@ -29,6 +29,9 @@ pub enum ServerError { #[error("topic already exists")] AlreadyExists, + #[error("unauthorized")] + Unauthorized, + #[error("stale range")] StaleRange, From 6e00e16e8b4f70329ece05b11f46fac7ceb08309 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 08:49:37 +0400 Subject: [PATCH 29/56] feat: grant/revoke acl Behavior: - Operates on one exact resource/principal pair. - Duplicate grant and revoke are no-ops. - Revision advances only when authorization actually changes. - Revoking the final principal retains an empty ACL re --- .../consensus/raft/states/metadata_state.rs | 25 ++++++++++++++ .../consensus/raft/states/security.rs | 33 +++++++++++++++++++ src/control_plane/metadata/command.rs | 18 +++++++++- 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index ba7a8109..58f3640c 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -220,6 +220,8 @@ impl MetadataState { SyncConsumerGroup(cmd) => self.sync_consumer_group(cmd)?, OpenProducerSession(cmd) => self.open_producer_session(cmd)?, ExpireProducerSessions(cmd) => self.expire_producer_sessions(cmd)?, + GrantAcl(cmd) => self.security.grant(cmd.resource, cmd.principal), + RevokeAcl(cmd) => self.security.revoke(cmd.resource, &cmd.principal), } #[cfg(any(test, debug_assertions))] self.assert_invariants(); @@ -707,6 +709,29 @@ mod tests { assert!(!state.authorizes("topic-data/43", "orders-service")); } + #[test] + fn acl_grant_and_revoke_are_idempotent() { + let mut state = MetadataState::new(ShardGroupId(1)); + let grant = GrantAcl { + resource: "topic-data/42".to_string(), + principal: "orders-service".to_string(), + }; + let revoke = RevokeAcl { + resource: grant.resource.clone(), + principal: grant.principal.clone(), + }; + + state.apply(grant.clone().into()).unwrap(); + state.apply(grant.into()).unwrap(); + assert!(state.authorizes("topic-data/42", "orders-service")); + assert_eq!(state.security.acls["topic-data/42"].revision, 1); + + state.apply(revoke.clone().into()).unwrap(); + state.apply(revoke.into()).unwrap(); + assert!(!state.authorizes("topic-data/42", "orders-service")); + assert_eq!(state.security.acls["topic-data/42"].revision, 2); + } + fn replica_set() -> Replicas { Replicas::new(vec![ NodeId::new("node-1"), diff --git a/src/control_plane/consensus/raft/states/security.rs b/src/control_plane/consensus/raft/states/security.rs index f6653162..7b9900a3 100644 --- a/src/control_plane/consensus/raft/states/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -62,6 +62,39 @@ impl SecurityState { .get(resource) .is_some_and(|acl| acl.principals.iter().any(|entry| entry == principal)) } + + pub(super) fn grant(&mut self, resource: String, principal: String) { + let acl = self + .acls + .entry(resource.clone()) + .or_insert_with(|| AclRecord { + resource, + revision: 0, + principals: Box::new([]), + }); + if acl.principals.contains(&principal) { + return; + } + + let mut principals = std::mem::take(&mut acl.principals).into_vec(); + principals.push(principal); + acl.principals = principals.into_boxed_slice(); + acl.revision += 1; + } + + pub(super) fn revoke(&mut self, resource: String, principal: &str) { + let Some(acl) = self.acls.get_mut(&resource) else { + return; + }; + let Some(index) = acl.principals.iter().position(|entry| entry == principal) else { + return; + }; + + let mut principals = std::mem::take(&mut acl.principals).into_vec(); + principals.remove(index); + acl.principals = principals.into_boxed_slice(); + acl.revision += 1; + } } #[cfg(any(test, debug_assertions))] diff --git a/src/control_plane/metadata/command.rs b/src/control_plane/metadata/command.rs index de8395b4..b7a00ff4 100644 --- a/src/control_plane/metadata/command.rs +++ b/src/control_plane/metadata/command.rs @@ -106,6 +106,18 @@ pub struct ExpireProducerSessions { pub observed_at: u64, } +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub struct GrantAcl { + pub resource: String, + pub principal: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub struct RevokeAcl { + pub resource: String, + pub principal: String, +} + #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub struct SyncConsumerGroupRequest { pub topic_name: String, @@ -146,6 +158,8 @@ pub enum MetadataCommand { SyncConsumerGroup(SyncConsumerGroup), OpenProducerSession(OpenProducerSession), ExpireProducerSessions(ExpireProducerSessions), + GrantAcl(GrantAcl), + RevokeAcl(RevokeAcl), } impl_from_variant!( @@ -159,5 +173,7 @@ impl_from_variant!( DeleteSegments, SyncConsumerGroup, OpenProducerSession, - ExpireProducerSessions + ExpireProducerSessions, + GrantAcl, + RevokeAcl ); From 0ef80b4fbf8d7380d6c174090a88e22cdb0b5ddf Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 09:02:55 +0400 Subject: [PATCH 30/56] feat: AclResource Added typed variants for: - cluster - topic-admin/{topic-id} - topic-data/{topic-id} - consumer-group/{topic-id}/{group-id} - producer-session/{topic-id}/{producer-id} - security/cluster Also added strict string parsing via FromStr: - Invalid topic IDs are rejected. - Invalid producer UUIDs are rejected. - Empty group IDs are rejected. - Unknown and incomplete resource keys are rejected. - Group IDs containing / round-trip correctly. --- docs/security/roadmap.md | 10 +- src/connections/controller.rs | 10 +- src/control_plane/consensus/actor.rs | 4 +- src/control_plane/consensus/messages/actor.rs | 4 +- src/control_plane/consensus/multi_raft.rs | 5 +- src/control_plane/consensus/raft/state.rs | 6 +- .../consensus/raft/states/metadata_state.rs | 33 ++-- .../consensus/raft/states/security.rs | 26 ++-- src/control_plane/metadata/command.rs | 6 +- src/control_plane/metadata/mod.rs | 147 ++++++++++++++++++ 10 files changed, 203 insertions(+), 48 deletions(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 959574b2..59006f3e 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -140,14 +140,16 @@ permission checks; its text grants no authority by itself. | Resource Key Format | Granted Actions | | :--- | :--- | | `cluster` | Membership inspection, topology lookup, operator diagnostics | -| `topic-admin/{topic}` | Create, delete, describe topic metadata | +| `topic-admin/{topic-id}` | Create, delete, describe topic metadata | | `topic-data/{topic-id}` | Produce, fetch, list offsets for topic | -| `consumer-group/{topic}/{group}` | Consume messages, read/commit consumer offsets | -| `producer-session/{topic}/{session}` | Renew producer session (permanently bound to creator principal) | +| `consumer-group/{topic-id}/{group-id}` | Coordinate the group and read/commit its offsets | +| `producer-session/{topic-id}/{producer-id}` | Renew the producer session permanently bound to its creator | | `security/cluster` | Read/write ACLs, manage admissions and revocations, inspect security audit | Consumer-group access permits group coordination and offset read/commit. Reading -records separately requires `Fetch` on `topic-data/{topic}`. +records separately requires `Fetch` on `topic-data/{topic-id}`. Text resource +keys are used only at routing and administrative boundaries; replicated records +store the typed resource directly. ### Sharded Metadata Storage diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 98a8b2b7..bf8cb625 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -4,8 +4,8 @@ use crate::connections::{protocol::*, run_client_writer}; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::metadata::{ - OpenProducerSession, RangeMeta, SyncConsumerGroup, SyncConsumerGroupRequest, TopicId, - TopicState, + AclResource, OpenProducerSession, RangeMeta, SyncConsumerGroup, SyncConsumerGroupRequest, + TopicId, TopicState, }; use crate::control_plane::{ NodeId, SwimNodeState, @@ -275,8 +275,8 @@ impl ClientController { return Ok(()); }; - let resource = format!("topic-data/{}", topic_id.0); - let ShardRouting::Local(group) = self.route(resource.as_bytes().to_vec()).await? else { + let resource = AclResource::TopicData(topic_id); + let ShardRouting::Local(group) = self.route(resource.routing_key()).await? else { return Err(ServerError::Unauthorized); }; @@ -830,7 +830,7 @@ mod tests { let raft = raft_sender_with(|cmd| { if let MultiRaftActorCommand::AuthorizePrincipal(query) = cmd { assert_eq!(query.shard_group_id, ShardGroupId(42)); - assert_eq!(query.resource, "topic-data/7"); + assert_eq!(query.resource, AclResource::TopicData(TopicId(7))); let _ = query.reply.send(Some(query.principal == "orders-service")); } }); diff --git a/src/control_plane/consensus/actor.rs b/src/control_plane/consensus/actor.rs index b0023ebc..4915a457 100644 --- a/src/control_plane/consensus/actor.rs +++ b/src/control_plane/consensus/actor.rs @@ -12,7 +12,7 @@ use crate::control_plane::consensus::raft::storage::RaftStorage; use crate::control_plane::membership::actor::SwimSender; use crate::control_plane::membership::{ShardGroupId, SwimCommand, TopologyReader}; use crate::control_plane::metadata::{ - ConsumerGroupAssignment, MetadataCommand, TopicMeta, TopicStats, + AclResource, ConsumerGroupAssignment, MetadataCommand, TopicMeta, TopicStats, }; use crate::data_plane::transport::command::DataTransportCommand; use crate::schedulers::ticker_message::{SchedulerSender, TickerCommand}; @@ -285,7 +285,7 @@ impl MutlRaftSender { pub(crate) async fn authorize_principal( &self, shard_group_id: ShardGroupId, - resource: String, + resource: AclResource, principal: String, ) -> Option { let (reply, recv) = tokio::sync::oneshot::channel(); diff --git a/src/control_plane/consensus/messages/actor.rs b/src/control_plane/consensus/messages/actor.rs index c184b34c..58dc5652 100644 --- a/src/control_plane/consensus/messages/actor.rs +++ b/src/control_plane/consensus/messages/actor.rs @@ -4,7 +4,7 @@ use uuid::Uuid; use crate::control_plane::NodeId; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::membership::ShardGroupId; -use crate::control_plane::metadata::{ConsumerGroupAssignment, TopicMeta, TopicStats}; +use crate::control_plane::metadata::{AclResource, ConsumerGroupAssignment, TopicMeta, TopicStats}; use crate::data_plane::messages::command::{ DurableSegmentEndReported, SegmentCaughtUp, SegmentPlaced, }; @@ -77,7 +77,7 @@ pub struct GetConsumerGroupAssignment { pub struct AuthorizePrincipal { pub(crate) shard_group_id: ShardGroupId, - pub(crate) resource: String, + pub(crate) resource: AclResource, pub(crate) principal: String, pub(crate) reply: oneshot::Sender>, } diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index 11ba6991..302f606b 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -16,7 +16,8 @@ use crate::control_plane::membership::{ShardGroup, ShardGroupId, TopologyReader} use crate::control_plane::metadata::command::RollSegment; use crate::control_plane::metadata::event::MetadataEvent; use crate::control_plane::metadata::{ - ConsumerGroupAssignment, EntryId, SegmentRollIntent, TopicId, TopicMeta, TopicStats, + AclResource, ConsumerGroupAssignment, EntryId, SegmentRollIntent, TopicId, TopicMeta, + TopicStats, }; use crate::data_plane::SegmentKey; use crate::data_plane::messages::command::{ @@ -497,7 +498,7 @@ impl MultiRaft { fn authorize_principal( &self, shard_group_id: ShardGroupId, - resource: &str, + resource: &AclResource, principal: &str, ) -> Option { self.groups diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index 5eba491d..c777f7ad 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -16,8 +16,8 @@ use crate::control_plane::membership::{ShardGroupId, TopologyReader}; use crate::control_plane::metadata::command::{DeleteSegments, ExpireProducerSessions}; use crate::control_plane::metadata::event::MetadataEvent; use crate::control_plane::metadata::{ - ConsumerGroupAssignment, ConsumerMemberId, MetadataCommand, ReassignSegment, RollSegment, - SegmentRollIntent, TopicId, TopicMeta, TopicStats, + AclResource, ConsumerGroupAssignment, ConsumerMemberId, MetadataCommand, ReassignSegment, + RollSegment, SegmentRollIntent, TopicId, TopicMeta, TopicStats, }; use crate::control_plane::{NodeId, Replicas}; use crate::data_plane::SegmentKey; @@ -161,7 +161,7 @@ impl Raft { self.metadata.get_topic_by_name(name) } - pub(crate) fn authorizes(&self, resource: &str, principal: &str) -> bool { + pub(crate) fn authorizes(&self, resource: &AclResource, principal: &str) -> bool { self.metadata.authorizes(resource, principal) } diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index 58f3640c..ea086d6e 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -9,7 +9,9 @@ use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::ConsumerGroupAssignment; use crate::control_plane::metadata::topic::{TopicMeta, TopicState, TopicStats}; -use crate::control_plane::metadata::{EntryId, RangeId, SegmentId, TopicId, error::MetadataError}; +use crate::control_plane::metadata::{ + AclResource, EntryId, RangeId, SegmentId, TopicId, error::MetadataError, +}; use crate::data_plane::SegmentKey; #[cfg(any(test, debug_assertions))] use crate::test_traits::TAssertInvariant; @@ -89,7 +91,7 @@ impl MetadataState { range.segments.get(&key.segment_id) } - pub(crate) fn authorizes(&self, resource: &str, principal: &str) -> bool { + pub(crate) fn authorizes(&self, resource: &AclResource, principal: &str) -> bool { self.security.authorizes(resource, principal) } @@ -648,7 +650,7 @@ mod tests { process_public_key: vec![1, 2, 3].into_boxed_slice(), }; let acl = AclRecord { - resource: "security/cluster".to_string(), + resource: AclResource::TopicData(TopicId(42)), revision: 4, principals: vec!["operator".to_string()].into_boxed_slice(), }; @@ -680,7 +682,7 @@ mod tests { restored.security.admissions.get("broker-a"), Some(&admission) ); - assert_eq!(restored.security.acls.get("security/cluster"), Some(&acl)); + assert_eq!(restored.security.acls.get(&acl.resource), Some(&acl)); assert_eq!( restored.security.revocations.get(&( "cluster-ca".to_string(), @@ -695,25 +697,26 @@ mod tests { #[test] fn metadata_authorization_defaults_to_deny() { let mut state = MetadataState::new(ShardGroupId(1)); + let resource = AclResource::TopicData(TopicId(42)); state.security.acls.insert( - "topic-data/42".to_string(), + resource.clone(), AclRecord { - resource: "topic-data/42".to_string(), + resource: resource.clone(), revision: 1, principals: vec!["orders-service".to_string()].into_boxed_slice(), }, ); - assert!(state.authorizes("topic-data/42", "orders-service")); - assert!(!state.authorizes("topic-data/42", "unknown-service")); - assert!(!state.authorizes("topic-data/43", "orders-service")); + assert!(state.authorizes(&resource, "orders-service")); + assert!(!state.authorizes(&resource, "unknown-service")); + assert!(!state.authorizes(&AclResource::TopicData(TopicId(43)), "orders-service")); } #[test] fn acl_grant_and_revoke_are_idempotent() { let mut state = MetadataState::new(ShardGroupId(1)); let grant = GrantAcl { - resource: "topic-data/42".to_string(), + resource: AclResource::TopicData(TopicId(42)), principal: "orders-service".to_string(), }; let revoke = RevokeAcl { @@ -723,13 +726,13 @@ mod tests { state.apply(grant.clone().into()).unwrap(); state.apply(grant.into()).unwrap(); - assert!(state.authorizes("topic-data/42", "orders-service")); - assert_eq!(state.security.acls["topic-data/42"].revision, 1); + assert!(state.authorizes(&revoke.resource, "orders-service")); + assert_eq!(state.security.acls[&revoke.resource].revision, 1); state.apply(revoke.clone().into()).unwrap(); - state.apply(revoke.into()).unwrap(); - assert!(!state.authorizes("topic-data/42", "orders-service")); - assert_eq!(state.security.acls["topic-data/42"].revision, 2); + state.apply(revoke.clone().into()).unwrap(); + assert!(!state.authorizes(&revoke.resource, "orders-service")); + assert_eq!(state.security.acls[&revoke.resource].revision, 2); } fn replica_set() -> Replicas { diff --git a/src/control_plane/consensus/raft/states/security.rs b/src/control_plane/consensus/raft/states/security.rs index 7b9900a3..c42df26f 100644 --- a/src/control_plane/consensus/raft/states/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use borsh::{BorshDeserialize, BorshSerialize}; use crate::control_plane::NodeId; +use crate::control_plane::metadata::AclResource; /// Security records replicated by one metadata shard. /// @@ -11,7 +12,7 @@ use crate::control_plane::NodeId; #[derive(Debug, Clone, Default, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct SecurityState { pub(super) admissions: HashMap, - pub(super) acls: HashMap, + pub(super) acls: HashMap, pub(super) revocations: HashMap<(String, Box<[u8]>), RevocationRecord>, } @@ -35,7 +36,7 @@ pub(crate) struct AdmissionRecord { /// principals and missing records deny access. #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct AclRecord { - pub resource: String, + pub resource: AclResource, pub revision: u64, pub principals: Box<[String]>, } @@ -57,13 +58,13 @@ impl SecurityState { /// /// Missing records and missing principals deny by default. Resource /// hierarchy or wildcard matching is intentionally not inferred here. - pub(crate) fn authorizes(&self, resource: &str, principal: &str) -> bool { + pub(crate) fn authorizes(&self, resource: &AclResource, principal: &str) -> bool { self.acls .get(resource) .is_some_and(|acl| acl.principals.iter().any(|entry| entry == principal)) } - pub(super) fn grant(&mut self, resource: String, principal: String) { + pub(super) fn grant(&mut self, resource: AclResource, principal: String) { let acl = self .acls .entry(resource.clone()) @@ -82,7 +83,7 @@ impl SecurityState { acl.revision += 1; } - pub(super) fn revoke(&mut self, resource: String, principal: &str) { + pub(super) fn revoke(&mut self, resource: AclResource, principal: &str) { let Some(acl) = self.acls.get_mut(&resource) else { return; }; @@ -128,6 +129,7 @@ impl crate::test_traits::TAssertInvariant for SecurityState { #[cfg(test)] mod tests { use super::*; + use crate::control_plane::metadata::TopicId; fn round_trip(value: &T) where @@ -147,7 +149,7 @@ mod tests { process_public_key: vec![1, 2, 3].into_boxed_slice(), }); round_trip(&AclRecord { - resource: "security/cluster".to_string(), + resource: AclResource::TopicData(TopicId(42)), revision: 4, principals: vec!["operator".to_string()].into_boxed_slice(), }); @@ -162,18 +164,18 @@ mod tests { #[test] fn acl_authorization_is_exact_and_defaults_to_deny() { let mut security = SecurityState::default(); + let resource = AclResource::TopicData(TopicId(42)); security.acls.insert( - "topic-data/42".to_string(), + resource.clone(), AclRecord { - resource: "topic-data/42".to_string(), + resource: resource.clone(), revision: 1, principals: vec!["orders-service".to_string()].into_boxed_slice(), }, ); - assert!(security.authorizes("topic-data/42", "orders-service")); - assert!(!security.authorizes("topic-data/42", "unknown-service")); - assert!(!security.authorizes("topic-data/43", "orders-service")); - assert!(!security.authorizes("topic-data", "orders-service")); + assert!(security.authorizes(&resource, "orders-service")); + assert!(!security.authorizes(&resource, "unknown-service")); + assert!(!security.authorizes(&AclResource::TopicData(TopicId(43)), "orders-service")); } } diff --git a/src/control_plane/metadata/command.rs b/src/control_plane/metadata/command.rs index b7a00ff4..2d2ea2b0 100644 --- a/src/control_plane/metadata/command.rs +++ b/src/control_plane/metadata/command.rs @@ -7,7 +7,7 @@ use crate::{ connections::protocol::ConsumerGroupSyncAction, control_plane::{ Replicas, - metadata::{EntryId, RangeId, SegmentId, TopicId, strategy::StoragePolicy}, + metadata::{AclResource, EntryId, RangeId, SegmentId, TopicId, strategy::StoragePolicy}, }, data_plane::SegmentKey, impl_from_variant, @@ -108,13 +108,13 @@ pub struct ExpireProducerSessions { #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub struct GrantAcl { - pub resource: String, + pub resource: AclResource, pub principal: String, } #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub struct RevokeAcl { - pub resource: String, + pub resource: AclResource, pub principal: String, } diff --git a/src/control_plane/metadata/mod.rs b/src/control_plane/metadata/mod.rs index a3806137..723388cd 100644 --- a/src/control_plane/metadata/mod.rs +++ b/src/control_plane/metadata/mod.rs @@ -17,6 +17,7 @@ pub(crate) use topic::{TopicMeta, TopicState, TopicStats}; pub(crate) mod segment; use borsh::{BorshDeserialize as Deser, BorshSerialize as Ser}; +use uuid::Uuid; pub(crate) use command::*; pub(crate) use consumer_group::{ConsumerGroupAssignment, ConsumerGroupMeta, ConsumerMemberId}; @@ -30,6 +31,106 @@ pub struct TopicId(pub(crate) u64); impl_new_struct_wrapper!(TopicId, u64); +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Ser, Deser)] +pub enum AclResource { + Cluster, + TopicAdmin(TopicId), + TopicData(TopicId), + ConsumerGroup(ConsumerGroupResource), + ProducerSession(ProducerSessionResource), + SecurityCluster, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Ser, Deser)] +pub struct ConsumerGroupResource { + pub topic_id: TopicId, + pub group_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Ser, Deser)] +pub struct ProducerSessionResource { + pub topic_id: TopicId, + pub producer_id: Uuid, +} + +impl AclResource { + pub(crate) fn routing_key(&self) -> Vec { + self.to_string().into_bytes() + } +} + +impl std::fmt::Display for AclResource { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Cluster => formatter.write_str("cluster"), + Self::TopicAdmin(topic_id) => write!(formatter, "topic-admin/{}", topic_id.0), + Self::TopicData(topic_id) => write!(formatter, "topic-data/{}", topic_id.0), + Self::ConsumerGroup(resource) => write!( + formatter, + "consumer-group/{}/{}", + resource.topic_id.0, resource.group_id + ), + Self::ProducerSession(resource) => write!( + formatter, + "producer-session/{}/{}", + resource.topic_id.0, resource.producer_id + ), + Self::SecurityCluster => formatter.write_str("security/cluster"), + } + } +} + +impl std::str::FromStr for AclResource { + type Err = String; + + fn from_str(input: &str) -> Result { + let topic_id = |candidate: &str| { + candidate + .parse::() + .map(TopicId) + .map_err(|_| format!("invalid ACL topic ID: {candidate}")) + }; + + if input == "cluster" { + return Ok(Self::Cluster); + } + if input == "security/cluster" { + return Ok(Self::SecurityCluster); + } + if let Some(topic) = input.strip_prefix("topic-admin/") { + return topic_id(topic).map(Self::TopicAdmin); + } + if let Some(topic) = input.strip_prefix("topic-data/") { + return topic_id(topic).map(Self::TopicData); + } + if let Some(consumer_group) = input.strip_prefix("consumer-group/") { + let (topic, group_id) = consumer_group + .split_once('/') + .ok_or_else(|| "consumer-group ACL requires a group ID".to_string())?; + if group_id.is_empty() { + return Err("consumer-group ACL requires a non-empty group ID".to_string()); + } + return Ok(Self::ConsumerGroup(ConsumerGroupResource { + topic_id: topic_id(topic)?, + group_id: group_id.to_string(), + })); + } + if let Some(producer_session) = input.strip_prefix("producer-session/") { + let (topic, producer_id) = producer_session + .split_once('/') + .ok_or_else(|| "producer-session ACL requires a producer ID".to_string())?; + let producer_id = Uuid::parse_str(producer_id) + .map_err(|_| format!("invalid ACL producer ID: {producer_id}"))?; + return Ok(Self::ProducerSession(ProducerSessionResource { + topic_id: topic_id(topic)?, + producer_id, + })); + } + + Err(format!("unknown ACL resource: {input}")) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ser, Deser, PartialOrd, Ord)] pub struct RangeId(pub(crate) u64); @@ -157,3 +258,49 @@ impl std::ops::Sub for EntryId { } } } + +#[cfg(test)] +mod acl_resource_tests { + use super::*; + + #[test] + fn resources_round_trip_through_canonical_routing_keys() { + let producer_id = Uuid::parse_str("6ba7b810-9dad-11d1-80b4-00c04fd430c8").unwrap(); + let resources = [ + AclResource::Cluster, + AclResource::TopicAdmin(TopicId(42)), + AclResource::TopicData(TopicId(42)), + AclResource::ConsumerGroup(ConsumerGroupResource { + topic_id: TopicId(42), + group_id: "billing/readers".to_string(), + }), + AclResource::ProducerSession(ProducerSessionResource { + topic_id: TopicId(42), + producer_id, + }), + AclResource::SecurityCluster, + ]; + + for resource in resources { + let key = resource.to_string(); + assert_eq!(key.parse::(), Ok(resource.clone())); + assert_eq!(resource.routing_key(), key.as_bytes()); + } + } + + #[test] + fn malformed_resource_keys_are_rejected() { + for key in [ + "", + "topic-data/", + "topic-data/name", + "topic-data/42/extra", + "consumer-group/42", + "consumer-group/42/", + "producer-session/42/not-a-uuid", + "unknown/42", + ] { + assert!(key.parse::().is_err(), "{key} was accepted"); + } + } +} From 3dcc0bc133f46cfa514df3fd12fcea42db892f17 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 09:09:35 +0400 Subject: [PATCH 31/56] fix: correct ACL scope in client controller Corrected ACL scope: - Produce, fetch, and list-offsets use TopicData. - Consumer-offset reads and commits use the exact ConsumerGroup(topic ID, group ID) resource. - Authorization still occurs before any data-plane dispatch. - Trusted-development behavior remains unchanged. --- src/connections/controller.rs | 75 ++++++++++++++++--- .../consumer_offsets/state.rs | 8 +- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/src/connections/controller.rs b/src/connections/controller.rs index bf8cb625..9410e274 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -5,7 +5,7 @@ use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::metadata::{ AclResource, OpenProducerSession, RangeMeta, SyncConsumerGroup, SyncConsumerGroupRequest, - TopicId, TopicState, + TopicState, }; use crate::control_plane::{ NodeId, SwimNodeState, @@ -270,12 +270,11 @@ impl ClientController { .await } - async fn authorize_data_access(&self, topic_id: TopicId) -> Result<(), ServerError> { + async fn authorize_data_access(&self, resource: AclResource) -> Result<(), ServerError> { let TransportIdentity::CertificatePrincipal(principal) = &self.transport_identity else { return Ok(()); }; - let resource = AclResource::TopicData(topic_id); let ShardRouting::Local(group) = self.route(resource.routing_key()).await? else { return Err(ServerError::Unauthorized); }; @@ -365,7 +364,7 @@ impl ClientController { &self, req: CommitConsumerOffsetRequest, ) -> Result { - self.authorize_data_access(req.key.topic_id).await?; + self.authorize_data_access(req.key.acl()).await?; let (tx, recv) = tokio::sync::oneshot::channel(); self.data_plane_tx @@ -395,7 +394,7 @@ impl ClientController { &self, req: FetchConsumerOffsetRequest, ) -> Result { - self.authorize_data_access(req.key.topic_id).await?; + self.authorize_data_access(req.key.acl()).await?; let (reply, recv) = tokio::sync::oneshot::channel(); self.data_plane_tx @@ -435,7 +434,8 @@ impl ClientController { } let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; - self.authorize_data_access(topic.id).await?; + self.authorize_data_access(AclResource::TopicData(topic.id)) + .await?; let producer_identity = req .producer_identity @@ -489,7 +489,8 @@ impl ClientController { /// state machine needs to answer without any further I/O. async fn fetch(&self, req: FetchRequest) -> Result { let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; - self.authorize_data_access(topic.id).await?; + self.authorize_data_access(AclResource::TopicData(topic.id)) + .await?; let range = topic.get_range(&req.range_id)?; @@ -522,7 +523,8 @@ impl ClientController { /// serve it. No proxying: a miss returns `SegmentNotLocal` and the client /// retries another replica. async fn fetch_by_id(&self, req: FetchByIdRequest) -> Result { - self.authorize_data_access(req.topic_id).await?; + self.authorize_data_access(AclResource::TopicData(req.topic_id)) + .await?; let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); let query = Fetch { @@ -547,7 +549,8 @@ impl ClientController { /// for the range's currently-active segment on this node. async fn list_offsets(&self, req: RangeOffsetRequest) -> Result { let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; - self.authorize_data_access(topic.id).await?; + self.authorize_data_access(AclResource::TopicData(topic.id)) + .await?; let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); @@ -691,11 +694,13 @@ mod tests { use crate::control_plane::membership::{ QueryCommand, ShardGroup, ShardGroupId, ShardLeaderEntry, SwimActorCommand, }; - use crate::control_plane::metadata::TopicStats as MetadataTopicStats; + use crate::control_plane::metadata::consumer_group::GenerationId; use crate::control_plane::metadata::strategy::{PartitionStrategy, StoragePolicy}; + use crate::control_plane::metadata::{ConsumerGroupResource, TopicStats as MetadataTopicStats}; use crate::control_plane::metadata::{RangeId, TopicId, TopicMeta}; use crate::control_plane::{NodeAddress, NodeId, Replicas, SwimNode, SwimNodeState}; use crate::data_plane::actor::DataPlaneSender; + use crate::data_plane::auxiliary_states::consumer_offsets::state::ConsumerOffsetKey; use crate::data_plane::messages::DataPlaneMessage; use crate::data_plane::messages::command::{DataPlaneCommand, ProduceAck}; use std::net::SocketAddr; @@ -836,7 +841,51 @@ mod tests { }); let controller = authenticated_controller("orders-service", node_id("self"), swim, raft); - assert_eq!(controller.authorize_data_access(TopicId(7)).await, Ok(())); + assert_eq!( + controller + .authorize_data_access(AclResource::TopicData(TopicId(7))) + .await, + Ok(()) + ); + } + + #[tokio::test] + async fn consumer_offset_requires_its_group_acl() { + let group = ShardGroup { + id: ShardGroupId(42), + replicas: Replicas::new(vec![node_id("self")]), + }; + let swim = swim_sender_with(move |cmd| { + if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { + let _ = reply.send(Some(group.clone())); + } + }); + let raft = raft_sender_with(|cmd| { + if let MultiRaftActorCommand::AuthorizePrincipal(query) = cmd { + assert_eq!( + query.resource, + AclResource::ConsumerGroup(ConsumerGroupResource { + topic_id: TopicId(7), + group_id: "billing".to_string(), + }) + ); + let _ = query.reply.send(Some(false)); + } + }); + let controller = authenticated_controller("orders-service", node_id("self"), swim, raft); + + let result = controller + .handle_fetch_consumer_offset(FetchConsumerOffsetRequest { + key: ConsumerOffsetKey { + topic_id: TopicId(7), + range_id: RangeId(0), + group_id: "billing".to_string(), + }, + generation: GenerationId(1), + }) + .await; + + assert_eq!(result, Err(ServerError::Unauthorized)); } #[tokio::test] @@ -862,7 +911,9 @@ mod tests { ); assert_eq!( - controller.authorize_data_access(TopicId(7)).await, + controller + .authorize_data_access(AclResource::TopicData(TopicId(7))) + .await, Err(ServerError::Unauthorized) ); } diff --git a/src/data_plane/auxiliary_states/consumer_offsets/state.rs b/src/data_plane/auxiliary_states/consumer_offsets/state.rs index 5f019576..779648d5 100644 --- a/src/data_plane/auxiliary_states/consumer_offsets/state.rs +++ b/src/data_plane/auxiliary_states/consumer_offsets/state.rs @@ -5,7 +5,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use crate::client::RangeId; use crate::control_plane::metadata::consumer_group::GenerationId; -use crate::control_plane::metadata::{EntryId, TopicId}; +use crate::control_plane::metadata::{AclResource, ConsumerGroupResource, EntryId, TopicId}; use crate::data_plane::SegmentKey; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, BorshSerialize, BorshDeserialize)] @@ -19,6 +19,12 @@ impl ConsumerOffsetKey { pub(crate) fn placement_key(&self) -> (TopicId, RangeId) { (self.topic_id, self.range_id) } + pub(crate) fn acl(&self) -> AclResource { + AclResource::ConsumerGroup(ConsumerGroupResource { + topic_id: self.topic_id, + group_id: self.group_id.clone(), + }) + } } #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, BorshSerialize, BorshDeserialize)] From db33728b71fcc08ab899d00e4ea25480d773c250 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 09:20:41 +0400 Subject: [PATCH 32/56] rename + consumer group coordination --- src/client/consumer/group.rs | 16 ++-- src/client/mod.rs | 4 +- src/connections/controller.rs | 91 +++++++++++++++---- src/connections/protocol/control_plane.rs | 8 +- src/connections/protocol/mod.rs | 4 +- src/control_plane/consensus/raft/command.rs | 4 +- .../consensus/raft/states/metadata_state.rs | 12 +-- src/control_plane/metadata/command.rs | 24 ++--- src/control_plane/metadata/consumer_group.rs | 20 ++-- src/control_plane/metadata/topic.rs | 2 +- 10 files changed, 122 insertions(+), 63 deletions(-) diff --git a/src/client/consumer/group.rs b/src/client/consumer/group.rs index d0c55202..cc7305db 100644 --- a/src/client/consumer/group.rs +++ b/src/client/consumer/group.rs @@ -8,9 +8,9 @@ use dashmap::DashMap; use uuid::Uuid; use crate::client::{Client, ClientError, ClientSuccess}; -use crate::connections::protocol::{ClientResponse, ConsumerGroupSyncAction}; +use crate::connections::protocol::{ClientResponse, ConsumerGroupMemberAction}; use crate::control_plane::metadata::consumer_group::GenerationId; -use crate::control_plane::metadata::{EntryId, RangeId, SyncConsumerGroupRequest, TopicId}; +use crate::control_plane::metadata::{EntryId, RangeId, TopicId, UpdateConsumerGroupMemberRequest}; use crate::data_plane::auxiliary_states::consumer_offsets::state::{ ConsumerOffsetKey, ConsumerOffsetPosition, ConsumerOffsetUpdate, }; @@ -190,11 +190,11 @@ impl ConsumerGroup { } pub(crate) async fn request_assignment(&self) -> Result, ClientError> { - let request = SyncConsumerGroupRequest { + let request = UpdateConsumerGroupMemberRequest { topic_name: self.topic.to_string(), group_id: self.group_id.to_string(), member_id: self.consumer_id, - action: ConsumerGroupSyncAction::Heartbeat, + action: ConsumerGroupMemberAction::Heartbeat, }; let assignment = match self @@ -219,11 +219,11 @@ impl ConsumerGroup { if self.quit.load(AtomicOrdering::Acquire) { return Ok(()); } - let request = SyncConsumerGroupRequest { + let request = UpdateConsumerGroupMemberRequest { topic_name: self.topic.clone(), group_id: self.group_id.clone(), member_id: self.consumer_id, - action: ConsumerGroupSyncAction::Leave, + action: ConsumerGroupMemberAction::Leave, }; let served = self .client @@ -294,11 +294,11 @@ impl Drop for ConsumerGroup { return; } let client = self.client.clone(); - let req = SyncConsumerGroupRequest { + let req = UpdateConsumerGroupMemberRequest { topic_name: self.topic.clone(), group_id: self.group_id.clone(), member_id: self.consumer_id, - action: ConsumerGroupSyncAction::Leave, + action: ConsumerGroupMemberAction::Leave, }; if let Ok(handle) = tokio::runtime::Handle::try_current() { handle.spawn(async move { diff --git a/src/client/mod.rs b/src/client/mod.rs index 1f73cb36..a51c88e5 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -36,7 +36,7 @@ use crate::control_plane::NodeAddressInfo; use crate::control_plane::metadata::consumer_group::GenerationId; pub use crate::control_plane::metadata::strategy::{PartitionStrategy, StoragePolicy}; pub use crate::control_plane::metadata::{EntryId, RangeId}; -use crate::control_plane::metadata::{SyncConsumerGroupRequest, TopicId}; +use crate::control_plane::metadata::{TopicId, UpdateConsumerGroupMemberRequest}; use crate::data_plane::auxiliary_states::consumer_offsets::state::{ ConsumerOffsetKey, ConsumerOffsetPosition, }; @@ -53,7 +53,7 @@ use uuid::Uuid; use crate::connections::protocol::{ ClientDataPlaneRequest, ClientRequest, ClientResponse, CommitConsumerOffsetRequest, - ConsumerGroupAssignmentResponse, ConsumerGroupSyncAction, ControlPlaneRequest, + ConsumerGroupAssignmentResponse, ConsumerGroupMemberAction, ControlPlaneRequest, FetchConsumerOffsetRequest, OpenProducerSessionRequest, ProduceRequest, ProducerSessionOpened, RangeOffsetRequest, }; diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 9410e274..9ab7f9ef 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -4,8 +4,8 @@ use crate::connections::{protocol::*, run_client_writer}; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::metadata::{ - AclResource, OpenProducerSession, RangeMeta, SyncConsumerGroup, SyncConsumerGroupRequest, - TopicState, + AclResource, ConsumerGroupResource, OpenProducerSession, RangeMeta, TopicState, + UpdateConsumerGroupMember, UpdateConsumerGroupMemberRequest, }; use crate::control_plane::{ NodeId, SwimNodeState, @@ -130,7 +130,7 @@ impl ClientController { DeleteTopic { name } => self.delete_topic(name).await, ListHostedTopics => self.list_hosted_topics().await, DescribeTopic { name } => self.describe_topic(name).await, - SyncConsumerGroup(req) => self.sync_consumer_group(req).await, + SyncConsumerGroup(req) => self.update_consumer_group_member(req).await, OpenProducerSession(req) => self.open_producer_session(req).await, }; res.into() @@ -172,9 +172,9 @@ impl ClientController { )) } - async fn sync_consumer_group( + async fn update_consumer_group_member( &self, - req: SyncConsumerGroupRequest, + req: UpdateConsumerGroupMemberRequest, ) -> Result { let group = match self.route(req.topic_name.as_bytes().to_vec()).await? { ShardRouting::Local(group) => group, @@ -183,10 +183,20 @@ impl ClientController { } }; - self.propose_topic_write(group.id, SyncConsumerGroup::new(req.clone())) + let topic = self + .raft_sender + .get_topic_metadata(req.topic_name.clone()) + .await?; + self.authorize_acl_resource(AclResource::ConsumerGroup(ConsumerGroupResource { + topic_id: topic.id, + group_id: req.group_id.clone(), + })) + .await?; + + self.propose_topic_write(group.id, UpdateConsumerGroupMember::new(req.clone())) .await?; - if req.action == ConsumerGroupSyncAction::Leave { + if req.action == ConsumerGroupMemberAction::Leave { return Ok(ClientSuccess::ConsumerGroupLeft); } @@ -270,7 +280,7 @@ impl ClientController { .await } - async fn authorize_data_access(&self, resource: AclResource) -> Result<(), ServerError> { + async fn authorize_acl_resource(&self, resource: AclResource) -> Result<(), ServerError> { let TransportIdentity::CertificatePrincipal(principal) = &self.transport_identity else { return Ok(()); }; @@ -364,7 +374,7 @@ impl ClientController { &self, req: CommitConsumerOffsetRequest, ) -> Result { - self.authorize_data_access(req.key.acl()).await?; + self.authorize_acl_resource(req.key.acl()).await?; let (tx, recv) = tokio::sync::oneshot::channel(); self.data_plane_tx @@ -394,7 +404,7 @@ impl ClientController { &self, req: FetchConsumerOffsetRequest, ) -> Result { - self.authorize_data_access(req.key.acl()).await?; + self.authorize_acl_resource(req.key.acl()).await?; let (reply, recv) = tokio::sync::oneshot::channel(); self.data_plane_tx @@ -434,7 +444,7 @@ impl ClientController { } let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; - self.authorize_data_access(AclResource::TopicData(topic.id)) + self.authorize_acl_resource(AclResource::TopicData(topic.id)) .await?; let producer_identity = req @@ -489,7 +499,7 @@ impl ClientController { /// state machine needs to answer without any further I/O. async fn fetch(&self, req: FetchRequest) -> Result { let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; - self.authorize_data_access(AclResource::TopicData(topic.id)) + self.authorize_acl_resource(AclResource::TopicData(topic.id)) .await?; let range = topic.get_range(&req.range_id)?; @@ -523,7 +533,7 @@ impl ClientController { /// serve it. No proxying: a miss returns `SegmentNotLocal` and the client /// retries another replica. async fn fetch_by_id(&self, req: FetchByIdRequest) -> Result { - self.authorize_data_access(AclResource::TopicData(req.topic_id)) + self.authorize_acl_resource(AclResource::TopicData(req.topic_id)) .await?; let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); @@ -549,7 +559,7 @@ impl ClientController { /// for the range's currently-active segment on this node. async fn list_offsets(&self, req: RangeOffsetRequest) -> Result { let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; - self.authorize_data_access(AclResource::TopicData(topic.id)) + self.authorize_acl_resource(AclResource::TopicData(topic.id)) .await?; let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); @@ -704,6 +714,8 @@ mod tests { use crate::data_plane::messages::DataPlaneMessage; use crate::data_plane::messages::command::{DataPlaneCommand, ProduceAck}; use std::net::SocketAddr; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; fn addr(port: u16) -> SocketAddr { format!("127.0.0.1:{port}").parse().unwrap() @@ -843,7 +855,7 @@ mod tests { assert_eq!( controller - .authorize_data_access(AclResource::TopicData(TopicId(7))) + .authorize_acl_resource(AclResource::TopicData(TopicId(7))) .await, Ok(()) ); @@ -888,6 +900,53 @@ mod tests { assert_eq!(result, Err(ServerError::Unauthorized)); } + #[tokio::test] + async fn consumer_group_sync_is_authorized_before_proposal() { + let group = ShardGroup { + id: ShardGroupId(42), + replicas: Replicas::new(vec![node_id("self")]), + }; + let swim = swim_sender_with(move |cmd| { + if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { + let _ = reply.send(Some(group.clone())); + } + }); + let proposal_count = Arc::new(AtomicUsize::new(0)); + let observed_proposals = proposal_count.clone(); + let raft = raft_sender_with(move |cmd| match cmd { + MultiRaftActorCommand::GetTopicMetadata { reply, .. } => { + let _ = reply.send(Some(topic_meta("self"))); + } + MultiRaftActorCommand::AuthorizePrincipal(query) => { + assert_eq!( + query.resource, + AclResource::ConsumerGroup(ConsumerGroupResource { + topic_id: TopicId(1), + group_id: "billing".to_string(), + }) + ); + let _ = query.reply.send(Some(false)); + } + MultiRaftActorCommand::ClientProposal { .. } => { + observed_proposals.fetch_add(1, Ordering::Relaxed); + } + _ => {} + }); + let controller = authenticated_controller("orders-service", node_id("self"), swim, raft); + + let result = controller + .update_consumer_group_member(UpdateConsumerGroupMemberRequest { + topic_name: "t1".to_string(), + group_id: "billing".to_string(), + member_id: uuid::Uuid::new_v4(), + action: ConsumerGroupMemberAction::Heartbeat, + }) + .await; + + assert_eq!(result, Err(ServerError::Unauthorized)); + assert_eq!(proposal_count.load(Ordering::Relaxed), 0); + } + #[tokio::test] async fn certificate_client_fails_closed_when_acl_shard_is_remote() { let group = ShardGroup { @@ -912,7 +971,7 @@ mod tests { assert_eq!( controller - .authorize_data_access(AclResource::TopicData(TopicId(7))) + .authorize_acl_resource(AclResource::TopicData(TopicId(7))) .await, Err(ServerError::Unauthorized) ); diff --git a/src/connections/protocol/control_plane.rs b/src/connections/protocol/control_plane.rs index 9392a83a..820278d0 100644 --- a/src/connections/protocol/control_plane.rs +++ b/src/connections/protocol/control_plane.rs @@ -22,7 +22,7 @@ use std::collections::{HashMap, HashSet}; use crate::control_plane::metadata::{ EntryId, OpenProducerSession, RangeId, RangeMeta, RangeState, SegmentId, SegmentMeta, - SegmentMetaState, SyncConsumerGroupRequest, TopicId, TopicMeta, TopicState, + SegmentMetaState, TopicId, TopicMeta, TopicState, UpdateConsumerGroupMemberRequest, }; #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] @@ -38,13 +38,13 @@ pub enum ControlPlaneRequest { DescribeTopic { name: String, }, - SyncConsumerGroup(SyncConsumerGroupRequest), + SyncConsumerGroup(UpdateConsumerGroupMemberRequest), OpenProducerSession(OpenProducerSessionRequest), } impl_from_variant!( ControlPlaneRequest, - SyncConsumerGroup(SyncConsumerGroupRequest), + SyncConsumerGroup(UpdateConsumerGroupMemberRequest), OpenProducerSession(OpenProducerSessionRequest) ); @@ -70,7 +70,7 @@ impl OpenProducerSessionRequest { } #[derive(Debug, Clone, Copy, PartialEq, Eq, BorshSerialize, BorshDeserialize)] -pub enum ConsumerGroupSyncAction { +pub enum ConsumerGroupMemberAction { Heartbeat, Leave, } diff --git a/src/connections/protocol/mod.rs b/src/connections/protocol/mod.rs index 9e8b2c56..8b806413 100644 --- a/src/connections/protocol/mod.rs +++ b/src/connections/protocol/mod.rs @@ -30,7 +30,7 @@ pub use error::*; use borsh::{BorshDeserialize, BorshSerialize}; use crate::{ - control_plane::metadata::{EntryId, SyncConsumerGroupRequest}, + control_plane::metadata::{EntryId, UpdateConsumerGroupMemberRequest}, data_plane::{ auxiliary_states::consumer_offsets::state::ConsumerOffsetPosition, messages::query::RangeOffsets, @@ -126,4 +126,4 @@ impl_from_variant_via!( FetchConsumerOffsetRequest ); -impl_from_variant_via!(ClientRequest, ControlPlaneRequest, SyncConsumerGroupRequest); +impl_from_variant_via!(ClientRequest, ControlPlaneRequest, UpdateConsumerGroupMemberRequest); diff --git a/src/control_plane/consensus/raft/command.rs b/src/control_plane/consensus/raft/command.rs index 66bf7238..ebb2c1bd 100644 --- a/src/control_plane/consensus/raft/command.rs +++ b/src/control_plane/consensus/raft/command.rs @@ -4,7 +4,7 @@ use crate::control_plane::NodeId; use crate::control_plane::metadata::ReassignSegment; use crate::control_plane::metadata::command::{ CreateTopic, DeleteSegments, DeleteTopic, ExpireProducerSessions, MergeRange, MetadataCommand, - OpenProducerSession, RollSegment, SplitRange, SyncConsumerGroup, + OpenProducerSession, RollSegment, SplitRange, UpdateConsumerGroupMember, }; use crate::{impl_from_variant, impl_from_variant_via}; @@ -31,7 +31,7 @@ impl_from_variant_via!( DeleteTopic, ReassignSegment, DeleteSegments, - SyncConsumerGroup, + UpdateConsumerGroupMember, OpenProducerSession, ExpireProducerSessions ); diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index ea086d6e..e07feaaf 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -219,7 +219,7 @@ impl MetadataState { DeleteTopic(cmd) => self.delete_topic(cmd)?, ReassignSegment(cmd) => self.reassign_segment(cmd)?, DeleteSegments(cmd) => self.delete_segments(cmd)?, - SyncConsumerGroup(cmd) => self.sync_consumer_group(cmd)?, + UpdateConsumerGroupMember(cmd) => self.sync_consumer_group(cmd)?, OpenProducerSession(cmd) => self.open_producer_session(cmd)?, ExpireProducerSessions(cmd) => self.expire_producer_sessions(cmd)?, GrantAcl(cmd) => self.security.grant(cmd.resource, cmd.principal), @@ -509,7 +509,7 @@ impl MetadataState { Ok(()) } - fn sync_consumer_group(&mut self, cmd: SyncConsumerGroup) -> Result<(), MetadataError> { + fn sync_consumer_group(&mut self, cmd: UpdateConsumerGroupMember) -> Result<(), MetadataError> { let group_id = cmd.group_id.clone(); let topic = self @@ -608,7 +608,7 @@ impl crate::test_traits::TAssertInvariant for MetadataState { #[cfg(test)] mod tests { use super::*; - use crate::connections::protocol::ConsumerGroupSyncAction; + use crate::connections::protocol::ConsumerGroupMemberAction; use crate::control_plane::consensus::raft::states::security::{ AclRecord, AdmissionRecord, RevocationRecord, }; @@ -802,12 +802,12 @@ mod tests { let mut sm = MetadataState::new(ShardGroupId(1)); create_topic(&mut sm, "orders"); let member = uuid::Uuid::new_v4(); - let command = SyncConsumerGroup { - req: SyncConsumerGroupRequest { + let command = UpdateConsumerGroupMember { + req: UpdateConsumerGroupMemberRequest { topic_name: "orders".into(), group_id: "workers".into(), member_id: member, - action: ConsumerGroupSyncAction::Heartbeat, + action: ConsumerGroupMemberAction::Heartbeat, }, observed_at: 100, session_timeout_ms: 10_000, diff --git a/src/control_plane/metadata/command.rs b/src/control_plane/metadata/command.rs index 2d2ea2b0..3cc43248 100644 --- a/src/control_plane/metadata/command.rs +++ b/src/control_plane/metadata/command.rs @@ -4,7 +4,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use uuid::Uuid; use crate::{ - connections::protocol::ConsumerGroupSyncAction, + connections::protocol::ConsumerGroupMemberAction, control_plane::{ Replicas, metadata::{AclResource, EntryId, RangeId, SegmentId, TopicId, strategy::StoragePolicy}, @@ -84,8 +84,8 @@ pub struct DeleteSegments { } #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] -pub struct SyncConsumerGroup { - pub req: SyncConsumerGroupRequest, +pub struct UpdateConsumerGroupMember { + pub req: UpdateConsumerGroupMemberRequest, // TODO consider using logical clock pub observed_at: u64, pub session_timeout_ms: u64, @@ -119,26 +119,26 @@ pub struct RevokeAcl { } #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] -pub struct SyncConsumerGroupRequest { +pub struct UpdateConsumerGroupMemberRequest { pub topic_name: String, pub group_id: String, pub member_id: Uuid, - pub action: ConsumerGroupSyncAction, + pub action: ConsumerGroupMemberAction, } -impl Deref for SyncConsumerGroup { - type Target = SyncConsumerGroupRequest; +impl Deref for UpdateConsumerGroupMember { + type Target = UpdateConsumerGroupMemberRequest; fn deref(&self) -> &Self::Target { &self.req } } -impl SyncConsumerGroup { - pub(crate) fn new(req: SyncConsumerGroupRequest) -> Self { +impl UpdateConsumerGroupMember { + pub(crate) fn new(req: UpdateConsumerGroupMemberRequest) -> Self { const SESSION_TIMEOUT_MS: u64 = 10_000; let observed_at = crate::now_ms(); - SyncConsumerGroup { + UpdateConsumerGroupMember { req, observed_at, session_timeout_ms: SESSION_TIMEOUT_MS, @@ -155,7 +155,7 @@ pub enum MetadataCommand { DeleteTopic(DeleteTopic), ReassignSegment(ReassignSegment), DeleteSegments(DeleteSegments), - SyncConsumerGroup(SyncConsumerGroup), + UpdateConsumerGroupMember(UpdateConsumerGroupMember), OpenProducerSession(OpenProducerSession), ExpireProducerSessions(ExpireProducerSessions), GrantAcl(GrantAcl), @@ -171,7 +171,7 @@ impl_from_variant!( DeleteTopic, ReassignSegment, DeleteSegments, - SyncConsumerGroup, + UpdateConsumerGroupMember, OpenProducerSession, ExpireProducerSessions, GrantAcl, diff --git a/src/control_plane/metadata/consumer_group.rs b/src/control_plane/metadata/consumer_group.rs index d4b1b365..8780073d 100644 --- a/src/control_plane/metadata/consumer_group.rs +++ b/src/control_plane/metadata/consumer_group.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use borsh::{BorshDeserialize, BorshSerialize}; use uuid::Uuid; -use crate::{connections::protocol::ConsumerGroupSyncAction, impl_new_struct_wrapper}; +use crate::{connections::protocol::ConsumerGroupMemberAction, impl_new_struct_wrapper}; use super::RangeId; @@ -38,7 +38,7 @@ impl ConsumerGroupMeta { pub(crate) fn sync_member( &mut self, member_id: ConsumerMemberId, - action: ConsumerGroupSyncAction, + action: ConsumerGroupMemberAction, observed_at: u64, session_timeout_ms: u64, active_ranges: &[RangeId], @@ -48,10 +48,10 @@ impl ConsumerGroupMeta { .retain(|_, last_seen| observed_at.saturating_sub(*last_seen) <= session_timeout_ms); match action { - ConsumerGroupSyncAction::Heartbeat => { + ConsumerGroupMemberAction::Heartbeat => { self.members.insert(member_id, observed_at); } - ConsumerGroupSyncAction::Leave => { + ConsumerGroupMemberAction::Leave => { self.members.remove(&member_id); } } @@ -150,14 +150,14 @@ mod tests { let a = uuid::Uuid::new_v4(); let b = uuid::Uuid::new_v4(); - assert!(group.sync_member(a, ConsumerGroupSyncAction::Heartbeat, 1, 10, &ranges)); + assert!(group.sync_member(a, ConsumerGroupMemberAction::Heartbeat, 1, 10, &ranges)); assert_eq!(*group.generation, 1); assert_eq!(group.ranges_for(a).as_ref(), &ranges); - assert!(!group.sync_member(a, ConsumerGroupSyncAction::Heartbeat, 2, 10, &ranges)); + assert!(!group.sync_member(a, ConsumerGroupMemberAction::Heartbeat, 2, 10, &ranges)); assert_eq!(*group.generation, 1); - assert!(group.sync_member(b, ConsumerGroupSyncAction::Heartbeat, 3, 10, &ranges)); + assert!(group.sync_member(b, ConsumerGroupMemberAction::Heartbeat, 3, 10, &ranges)); assert_eq!(*group.generation, 2); group.assert_assignments(&ranges); } @@ -168,10 +168,10 @@ mod tests { let ranges = [RangeId(1), RangeId(2)]; let stale = uuid::Uuid::new_v4(); let live = uuid::Uuid::new_v4(); - group.sync_member(stale, ConsumerGroupSyncAction::Heartbeat, 1, 10, &ranges); - group.sync_member(live, ConsumerGroupSyncAction::Heartbeat, 2, 10, &ranges); + group.sync_member(stale, ConsumerGroupMemberAction::Heartbeat, 1, 10, &ranges); + group.sync_member(live, ConsumerGroupMemberAction::Heartbeat, 2, 10, &ranges); - assert!(group.sync_member(live, ConsumerGroupSyncAction::Heartbeat, 20, 10, &ranges)); + assert!(group.sync_member(live, ConsumerGroupMemberAction::Heartbeat, 20, 10, &ranges)); assert_eq!(*group.generation, 3); assert!(group.ranges_for(stale).is_empty()); assert_eq!(group.ranges_for(live).as_ref(), &ranges); diff --git a/src/control_plane/metadata/topic.rs b/src/control_plane/metadata/topic.rs index 7b30dce2..7cf76139 100644 --- a/src/control_plane/metadata/topic.rs +++ b/src/control_plane/metadata/topic.rs @@ -449,7 +449,7 @@ impl TopicMeta { Ok(merged_id) } - pub(crate) fn sync_consumer_group(&mut self, cmd: SyncConsumerGroup) -> bool { + pub(crate) fn sync_consumer_group(&mut self, cmd: UpdateConsumerGroupMember) -> bool { let consumer_group_meta = self .consumer_groups .entry(cmd.group_id.clone()) From f9b7376bfa3ab5bd06845ace9fc64bb3f88107a6 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 10:03:46 +0400 Subject: [PATCH 33/56] feat: produer-session ownership checkpoint Implemented: - TransportIdentity::CertificatePrincipal(Box) for immutable connection identity. - Separate persisted ProducerSessionOwner, with comments explaining why transport evidence must not become snapshot schema. - Producer sessions store their owner. - Different principals cannot renew or recover an active producer ID. - Topic-data authorization occurs before opening a session. - Ownership is rechecked after commit to cover concurrent-open races. - Session expiry still removes metadata and data-plane deduplication state. - Roadmap wording now says ownership lasts for the session lifetime. --- docs/security/roadmap.md | 2 +- src/connections/controller.rs | 41 +++++++--- src/connections/protocol/control_plane.rs | 10 ++- src/connections/protocol/error.rs | 1 + src/connections/protocol/mod.rs | 6 +- src/control_plane/consensus/actor.rs | 2 +- src/control_plane/consensus/messages/actor.rs | 2 +- .../consensus/raft/states/metadata_state.rs | 12 +-- src/control_plane/metadata/command.rs | 8 +- src/control_plane/metadata/error.rs | 2 + src/control_plane/metadata/mod.rs | 24 +++++- .../metadata/producer_sessions.rs | 80 +++++++++++++++---- src/net/tcp.rs | 2 +- src/security.rs | 4 +- 14 files changed, 144 insertions(+), 52 deletions(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 59006f3e..06100ebb 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -143,7 +143,7 @@ permission checks; its text grants no authority by itself. | `topic-admin/{topic-id}` | Create, delete, describe topic metadata | | `topic-data/{topic-id}` | Produce, fetch, list offsets for topic | | `consumer-group/{topic-id}/{group-id}` | Coordinate the group and read/commit its offsets | -| `producer-session/{topic-id}/{producer-id}` | Renew the producer session permanently bound to its creator | +| `producer-session/{topic-id}/{producer-id}` | Renew the session, bound to its creator for the session lifetime | | `security/cluster` | Read/write ACLs, manage admissions and revocations, inspect security audit | Consumer-group access permits group coordination and offset read/commit. Reading diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 9ab7f9ef..52333a1e 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -4,8 +4,8 @@ use crate::connections::{protocol::*, run_client_writer}; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::metadata::{ - AclResource, ConsumerGroupResource, OpenProducerSession, RangeMeta, TopicState, - UpdateConsumerGroupMember, UpdateConsumerGroupMemberRequest, + AclResource, ConsumerGroupResource, OpenProducerSession, ProducerSessionOwner, RangeMeta, + TopicState, UpdateConsumerGroupMember, UpdateConsumerGroupMemberRequest, }; use crate::control_plane::{ NodeId, SwimNodeState, @@ -140,7 +140,12 @@ impl ClientController { &self, req: OpenProducerSessionRequest, ) -> Result { - let command: OpenProducerSession = req.into_command(); + // Convert ephemeral authentication evidence into the durable owner + // carried by the Raft command. The persisted type intentionally does + // not depend on TLS or stream implementation details. + let owner = ProducerSessionOwner::from(&self.transport_identity); + + let command: OpenProducerSession = req.into_command(owner.clone()); let group = match self.route(command.topic_name.as_bytes().to_vec()).await? { ShardRouting::Local(group) => group, @@ -149,20 +154,28 @@ impl ClientController { } }; + let topic_meta = self + .raft_sender + .get_topic_metadata(command.topic_name.to_string()) + .await?; + self.authorize_acl_resource(AclResource::TopicData(topic_meta.id)) + .await?; + topic_meta + .producer_sessions + .get_for_owner(&command.producer_id, &owner)?; + self.propose_topic_write(group.id, command.clone()).await?; - let topic_meta = self + let committed_topic = self .raft_sender - .get_topic_metadata(command.topic_name) + .get_topic_metadata(command.topic_name.to_string()) .await?; - let session = topic_meta + let session = committed_topic .producer_sessions - .get(&command.producer_id) - .copied() - .ok_or_else(|| { - ServerError::Internal("committed producer session is unavailable".into()) - })?; + .get_for_owner(&command.producer_id, &owner)? + .cloned() + .ok_or(ServerError::Unauthorized)?; Ok(ClientSuccess::ProducerSessionOpened( ProducerSessionOpened { @@ -799,7 +812,7 @@ mod tests { raft_sender: MutlRaftSender, ) -> ClientController { ClientController::new( - TransportIdentity::CertificatePrincipal(principal.to_string()), + TransportIdentity::CertificatePrincipal(principal.into()), node_id, swim_sender, raft_sender, @@ -848,7 +861,9 @@ mod tests { if let MultiRaftActorCommand::AuthorizePrincipal(query) = cmd { assert_eq!(query.shard_group_id, ShardGroupId(42)); assert_eq!(query.resource, AclResource::TopicData(TopicId(7))); - let _ = query.reply.send(Some(query.principal == "orders-service")); + let _ = query + .reply + .send(Some(query.principal.as_ref() == "orders-service")); } }); let controller = authenticated_controller("orders-service", node_id("self"), swim, raft); diff --git a/src/connections/protocol/control_plane.rs b/src/connections/protocol/control_plane.rs index 820278d0..5936035c 100644 --- a/src/connections/protocol/control_plane.rs +++ b/src/connections/protocol/control_plane.rs @@ -21,8 +21,9 @@ use borsh::{BorshDeserialize, BorshSerialize}; use std::collections::{HashMap, HashSet}; use crate::control_plane::metadata::{ - EntryId, OpenProducerSession, RangeId, RangeMeta, RangeState, SegmentId, SegmentMeta, - SegmentMetaState, TopicId, TopicMeta, TopicState, UpdateConsumerGroupMemberRequest, + EntryId, OpenProducerSession, ProducerSessionOwner, RangeId, RangeMeta, RangeState, SegmentId, + SegmentMeta, SegmentMetaState, TopicId, TopicMeta, TopicState, + UpdateConsumerGroupMemberRequest, }; #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] @@ -56,13 +57,14 @@ pub struct OpenProducerSessionRequest { } impl OpenProducerSessionRequest { - pub fn into_command(self) -> OpenProducerSession { + pub fn into_command(self, owner: ProducerSessionOwner) -> OpenProducerSession { const SESSION_TIMEOUT_MS: u64 = 60_000; let observed_at = crate::now_ms(); OpenProducerSession { - topic_name: self.topic_name, + topic_name: self.topic_name.into_boxed_str(), producer_id: self.producer_id, session_nonce: self.session_nonce, + owner, observed_at, session_timeout_ms: SESSION_TIMEOUT_MS, } diff --git a/src/connections/protocol/error.rs b/src/connections/protocol/error.rs index 64fa1739..252fde36 100644 --- a/src/connections/protocol/error.rs +++ b/src/connections/protocol/error.rs @@ -84,6 +84,7 @@ impl From for ServerError TopicNotActive(_) | RangeNotFound | RangeNotActive => ServerError::StaleRange, SegmentNotFound | SegmentNotActive | SegmentNotSealed => ServerError::SegmentNotLocal, InvalidSplitPoint => ServerError::InvalidSplitPoint, + ProducerSessionOwnerMismatch => ServerError::Unauthorized, SplitNotAllowed(_) | RangesNotAdjacent => ServerError::Internal(err.to_string()), } } diff --git a/src/connections/protocol/mod.rs b/src/connections/protocol/mod.rs index 8b806413..6fcdb198 100644 --- a/src/connections/protocol/mod.rs +++ b/src/connections/protocol/mod.rs @@ -126,4 +126,8 @@ impl_from_variant_via!( FetchConsumerOffsetRequest ); -impl_from_variant_via!(ClientRequest, ControlPlaneRequest, UpdateConsumerGroupMemberRequest); +impl_from_variant_via!( + ClientRequest, + ControlPlaneRequest, + UpdateConsumerGroupMemberRequest +); diff --git a/src/control_plane/consensus/actor.rs b/src/control_plane/consensus/actor.rs index 4915a457..56484e18 100644 --- a/src/control_plane/consensus/actor.rs +++ b/src/control_plane/consensus/actor.rs @@ -286,7 +286,7 @@ impl MutlRaftSender { &self, shard_group_id: ShardGroupId, resource: AclResource, - principal: String, + principal: Box, ) -> Option { let (reply, recv) = tokio::sync::oneshot::channel(); let _ = self diff --git a/src/control_plane/consensus/messages/actor.rs b/src/control_plane/consensus/messages/actor.rs index 58dc5652..4cc66cb1 100644 --- a/src/control_plane/consensus/messages/actor.rs +++ b/src/control_plane/consensus/messages/actor.rs @@ -78,7 +78,7 @@ pub struct GetConsumerGroupAssignment { pub struct AuthorizePrincipal { pub(crate) shard_group_id: ShardGroupId, pub(crate) resource: AclResource, - pub(crate) principal: String, + pub(crate) principal: Box, pub(crate) reply: oneshot::Sender>, } diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index e07feaaf..b1091824 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -233,20 +233,14 @@ impl MetadataState { fn open_producer_session(&mut self, cmd: OpenProducerSession) -> Result<(), MetadataError> { let topic_id = self .topic_name_index - .get(&cmd.topic_name) + .get(cmd.topic_name.as_ref()) .copied() - .ok_or_else(|| MetadataError::TopicNameNotFound(cmd.topic_name.clone()))?; + .ok_or_else(|| MetadataError::TopicNameNotFound(cmd.topic_name.to_string()))?; let topic = self .topics .get_mut(&topic_id) .ok_or(MetadataError::TopicNotFound(topic_id))?; - topic.producer_sessions.open_producer_session( - cmd.producer_id, - cmd.session_nonce, - cmd.observed_at, - cmd.session_timeout_ms, - ); - Ok(()) + topic.producer_sessions.open_producer_session(cmd) } fn expire_producer_sessions( diff --git a/src/control_plane/metadata/command.rs b/src/control_plane/metadata/command.rs index 3cc43248..53fa5cba 100644 --- a/src/control_plane/metadata/command.rs +++ b/src/control_plane/metadata/command.rs @@ -7,7 +7,10 @@ use crate::{ connections::protocol::ConsumerGroupMemberAction, control_plane::{ Replicas, - metadata::{AclResource, EntryId, RangeId, SegmentId, TopicId, strategy::StoragePolicy}, + metadata::{ + AclResource, EntryId, ProducerSessionOwner, RangeId, SegmentId, TopicId, + strategy::StoragePolicy, + }, }, data_plane::SegmentKey, impl_from_variant, @@ -93,9 +96,10 @@ pub struct UpdateConsumerGroupMember { #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub struct OpenProducerSession { - pub topic_name: String, + pub topic_name: Box, pub producer_id: Uuid, pub session_nonce: Uuid, + pub owner: ProducerSessionOwner, pub observed_at: u64, pub session_timeout_ms: u64, } diff --git a/src/control_plane/metadata/error.rs b/src/control_plane/metadata/error.rs index f6e63147..92f7e561 100644 --- a/src/control_plane/metadata/error.rs +++ b/src/control_plane/metadata/error.rs @@ -26,4 +26,6 @@ pub enum MetadataError { RangesNotAdjacent, #[error("invalid split point")] InvalidSplitPoint, + #[error("produce session belongs to another principal")] + ProducerSessionOwnerMismatch, } diff --git a/src/control_plane/metadata/mod.rs b/src/control_plane/metadata/mod.rs index 723388cd..cb48ca58 100644 --- a/src/control_plane/metadata/mod.rs +++ b/src/control_plane/metadata/mod.rs @@ -24,7 +24,7 @@ pub(crate) use consumer_group::{ConsumerGroupAssignment, ConsumerGroupMeta, Cons pub(crate) use segment::*; -use crate::impl_new_struct_wrapper; +use crate::{impl_new_struct_wrapper, security::TransportIdentity}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Ser, Deser)] pub struct TopicId(pub(crate) u64); @@ -53,6 +53,28 @@ pub struct ProducerSessionResource { pub producer_id: Uuid, } +/// Durable owner of one producer session. +/// +/// This is deliberately separate from the connection's transport identity: +/// Raft snapshots retain ownership after the TLS connection disappears, and +/// transport refactors must not change the persisted metadata schema. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Ser, Deser)] +pub enum ProducerSessionOwner { + CertificatePrincipal(Box), + TrustedDevelopment, +} + +impl ProducerSessionOwner { + pub(crate) fn from(transport_identity: &TransportIdentity) -> Self { + match transport_identity { + TransportIdentity::CertificatePrincipal(principal) => { + ProducerSessionOwner::CertificatePrincipal(principal.clone()) + } + TransportIdentity::TrustedDevelopment => ProducerSessionOwner::TrustedDevelopment, + } + } +} + impl AclResource { pub(crate) fn routing_key(&self) -> Vec { self.to_string().into_bytes() diff --git a/src/control_plane/metadata/producer_sessions.rs b/src/control_plane/metadata/producer_sessions.rs index 866480fc..0f8fe63b 100644 --- a/src/control_plane/metadata/producer_sessions.rs +++ b/src/control_plane/metadata/producer_sessions.rs @@ -2,12 +2,16 @@ use std::collections::HashMap; use borsh::{BorshDeserialize, BorshSerialize}; +use crate::control_plane::metadata::ProducerSessionOwner; +use crate::control_plane::metadata::command::OpenProducerSession; +use crate::control_plane::metadata::error::MetadataError; use crate::impl_new_struct_wrapper; -#[derive(Debug, Clone, Copy, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct ProducerSessionMeta { pub(crate) incarnation: u32, pub(crate) expires_at: u64, + pub(crate) owner: ProducerSessionOwner, session_nonce: uuid::Uuid, } @@ -16,28 +20,41 @@ pub(crate) struct ProducerSessions(HashMap); impl_new_struct_wrapper!(ProducerSessions,HashMap); impl ProducerSessions { + pub(crate) fn get_for_owner( + &self, + producer_id: &uuid::Uuid, + owner: &ProducerSessionOwner, + ) -> Result, MetadataError> { + match self.get(producer_id) { + Some(session) if &session.owner != owner => { + Err(MetadataError::ProducerSessionOwnerMismatch) + } + session => Ok(session), + } + } + pub(crate) fn open_producer_session( &mut self, - producer_id: uuid::Uuid, - session_nonce: uuid::Uuid, - observed_at: u64, - session_timeout_ms: u64, - ) -> ProducerSessionMeta { - self.expire_producer_sessions(observed_at); + command: OpenProducerSession, + ) -> Result<(), MetadataError> { + self.expire_producer_sessions(command.observed_at); - let incarnation = match self.get(&producer_id) { - Some(session) if session.session_nonce == session_nonce => session.incarnation, + let incarnation = match self.get_for_owner(&command.producer_id, &command.owner)? { + Some(session) if session.session_nonce == command.session_nonce => session.incarnation, Some(session) => session.incarnation.saturating_add(1), None => 0, }; - let expires_at = observed_at.saturating_add(session_timeout_ms); + let expires_at = command + .observed_at + .saturating_add(command.session_timeout_ms); let session = ProducerSessionMeta { incarnation, expires_at, - session_nonce, + owner: command.owner, + session_nonce: command.session_nonce, }; - self.insert(producer_id, session); - session + self.insert(command.producer_id, session); + Ok(()) } pub(crate) fn has_expired_producer_sessions(&self, observed_at: u64) -> bool { @@ -55,14 +72,43 @@ fn producer_session_recovery_bumps_incarnation_and_expiry_removes_it() { let mut producer_sessions = ProducerSessions::default(); let producer_id = uuid::Uuid::new_v4(); let first_nonce = uuid::Uuid::new_v4(); + let owner = ProducerSessionOwner::CertificatePrincipal("orders-service".into()); + + let command = |session_nonce, session_owner, observed_at| OpenProducerSession { + topic_name: "orders".into(), + producer_id, + session_nonce, + owner: session_owner, + observed_at, + session_timeout_ms: 100, + }; - let session = producer_sessions.open_producer_session(producer_id, first_nonce, 10, 100); + producer_sessions + .open_producer_session(command(first_nonce, owner.clone(), 10)) + .unwrap(); + let session = &producer_sessions[&producer_id]; assert_eq!((session.incarnation, session.expires_at), (0, 110)); - let session2 = producer_sessions.open_producer_session(producer_id, first_nonce, 20, 100); + + producer_sessions + .open_producer_session(command(first_nonce, owner.clone(), 20)) + .unwrap(); + let session2 = &producer_sessions[&producer_id]; assert_eq!((session2.incarnation, session2.expires_at), (0, 120)); - let session3 = - producer_sessions.open_producer_session(producer_id, uuid::Uuid::new_v4(), 20, 100); + + producer_sessions + .open_producer_session(command(uuid::Uuid::new_v4(), owner.clone(), 20)) + .unwrap(); + let session3 = &producer_sessions[&producer_id]; assert_eq!((session3.incarnation, session3.expires_at), (1, 120)); + assert_eq!( + producer_sessions.open_producer_session(command( + uuid::Uuid::new_v4(), + ProducerSessionOwner::CertificatePrincipal("other-service".into()), + 20, + )), + Err(MetadataError::ProducerSessionOwnerMismatch) + ); + assert_eq!(producer_sessions[&producer_id].owner, owner); assert!(producer_sessions.has_expired_producer_sessions(121)); producer_sessions.expire_producer_sessions(121); diff --git a/src/net/tcp.rs b/src/net/tcp.rs index 6e0ad407..11b8869d 100644 --- a/src/net/tcp.rs +++ b/src/net/tcp.rs @@ -187,7 +187,7 @@ impl TransportTcpStream { pub fn peer_identity(&self) -> TransportIdentity { match self { Self::Secure(stream) => { - TransportIdentity::CertificatePrincipal(stream.peer_principal().to_string()) + TransportIdentity::CertificatePrincipal(stream.peer_principal().into()) } Self::TrustedDevelopment(_) => TransportIdentity::TrustedDevelopment, } diff --git a/src/security.rs b/src/security.rs index af994173..063d392c 100644 --- a/src/security.rs +++ b/src/security.rs @@ -116,7 +116,9 @@ pub(crate) enum NodeTransportSecurity { #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum TransportIdentity { - CertificatePrincipal(String), + /// Principal authenticated for this live TLS connection. This is transport + /// evidence, not a durable authorization or ownership record. + CertificatePrincipal(Box), TrustedDevelopment, } From b995c33a238a4354709dafd51e6b4010344bb500 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 11:04:05 +0400 Subject: [PATCH 34/56] feat: admin acl --- docs/security/roadmap.md | 4 +- src/connections/controller.rs | 115 +++++++++++++++++++++++++++++++--- 2 files changed, 109 insertions(+), 10 deletions(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 06100ebb..363f3701 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -139,8 +139,8 @@ permission checks; its text grants no authority by itself. | Resource Key Format | Granted Actions | | :--- | :--- | -| `cluster` | Membership inspection, topology lookup, operator diagnostics | -| `topic-admin/{topic-id}` | Create, delete, describe topic metadata | +| `cluster` | Create and list topics, membership inspection, topology lookup, operator diagnostics | +| `topic-admin/{topic-id}` | Delete and describe topic metadata | | `topic-data/{topic-id}` | Produce, fetch, list offsets for topic | | `consumer-group/{topic-id}/{group-id}` | Coordinate the group and read/commit its offsets | | `producer-session/{topic-id}/{producer-id}` | Renew the session, bound to its creator for the session lifetime | diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 52333a1e..4d479dfc 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -115,7 +115,7 @@ impl ClientController { match request { ClientRequest::ControlPlane(cp) => self.handle_control_plane(cp).await, ClientRequest::DataPlane(dp) => self.handle_data_plane(dp).await, - ClientRequest::Admin(admin) => self.handle_admin(admin).await, + ClientRequest::Admin(admin) => self.handle_admin(admin).await.into(), } } @@ -239,6 +239,8 @@ impl ClientController { } let topic = self.raft_sender.get_topic_metadata(topic_name).await?; + self.authorize_acl_resource(AclResource::TopicAdmin(topic.id)) + .await?; let addresses = self.swim_sender.list_all_node_addresses().await?; let detail = TopicDetail::from_meta(topic, &addresses); @@ -256,6 +258,9 @@ impl ClientController { return Err(self.control_plane_redirect(member)); } }; + // A new topic has no stable ID yet, so its creator needs the cluster-wide + // grant. Once created, topic-admin/{topic-id} governs its metadata. + self.authorize_acl_resource(AclResource::Cluster).await?; let created_at = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -279,6 +284,12 @@ impl ClientController { return Err(self.control_plane_redirect(member)); } }; + let topic = self + .raft_sender + .get_topic_metadata(topic_name.clone()) + .await?; + self.authorize_acl_resource(AclResource::TopicAdmin(topic.id)) + .await?; let cmd = DeleteTopic { name: topic_name }; self.propose_topic_write(group.id, cmd).await?; @@ -355,6 +366,8 @@ impl ClientController { } async fn list_hosted_topics(&self) -> Result { + self.authorize_acl_resource(AclResource::Cluster).await?; + let topics = self .raft_sender .get_topics() @@ -593,17 +606,17 @@ impl ClientController { Ok(ClientSuccess::RangeOffset(range_offset)) } - async fn handle_admin(&self, request: AdminRequest) -> ClientResponse { + async fn handle_admin(&self, request: AdminRequest) -> Result { use AdminRequest::*; - let res = match request { + self.authorize_acl_resource(AclResource::Cluster).await?; + + match request { DescribeCluster => self.describe_cluster().await, ListHostedTopicsWithStats => self.list_hosted_topics_with_stats().await, - GetShardInfo { key } => self.get_shard_info(key).await, GetShardLeader { shard_group_id } => self.handle_get_shard_leader(shard_group_id).await, - }; - res.into() + } } async fn describe_cluster(&self) -> Result { @@ -876,6 +889,88 @@ mod tests { ); } + #[tokio::test] + async fn topic_metadata_operations_require_topic_admin_acl() { + let group = test_shard_group(); + let swim = swim_sender_with(move |cmd| { + if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { + let _ = reply.send(Some(group.clone())); + } + }); + let proposal_count = Arc::new(AtomicUsize::new(0)); + let observed_proposals = proposal_count.clone(); + let raft = raft_sender_with(move |cmd| match cmd { + MultiRaftActorCommand::GetTopicMetadata { reply, .. } => { + let _ = reply.send(Some(topic_meta("node-1"))); + } + MultiRaftActorCommand::AuthorizePrincipal(query) => { + assert_eq!(query.resource, AclResource::TopicAdmin(TopicId(1))); + let _ = query.reply.send(Some(false)); + } + MultiRaftActorCommand::ClientProposal { .. } => { + observed_proposals.fetch_add(1, Ordering::Relaxed); + } + _ => {} + }); + let controller = authenticated_controller("orders-service", node_id("node-1"), swim, raft); + + assert_eq!( + controller.describe_topic("t1".into()).await, + Err(ServerError::Unauthorized) + ); + assert_eq!( + controller.delete_topic("t1".into()).await, + Err(ServerError::Unauthorized) + ); + assert_eq!(proposal_count.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn cluster_scoped_apis_require_cluster_acl() { + let group = test_shard_group(); + let swim = swim_sender_with(move |cmd| { + if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { + let _ = reply.send(Some(group.clone())); + } + }); + let proposal_count = Arc::new(AtomicUsize::new(0)); + let observed_proposals = proposal_count.clone(); + let raft = raft_sender_with(move |cmd| match cmd { + MultiRaftActorCommand::AuthorizePrincipal(query) => { + assert_eq!(query.resource, AclResource::Cluster); + let _ = query.reply.send(Some(false)); + } + MultiRaftActorCommand::ClientProposal { .. } => { + observed_proposals.fetch_add(1, Ordering::Relaxed); + } + _ => {} + }); + let controller = authenticated_controller("orders-service", node_id("node-1"), swim, raft); + + assert_eq!( + controller + .handle_create_topic( + "t1".into(), + StoragePolicy { + retention_ms: Some(3_600_000), + replication_factor: 1, + partition_strategy: PartitionStrategy::AutoSplit, + }, + ) + .await, + Err(ServerError::Unauthorized) + ); + assert_eq!( + controller.list_hosted_topics().await, + Err(ServerError::Unauthorized) + ); + assert!(matches!( + controller.handle_admin(AdminRequest::DescribeCluster).await, + Err(ServerError::Unauthorized) + )); + assert_eq!(proposal_count.load(Ordering::Relaxed), 0); + } + #[tokio::test] async fn consumer_offset_requires_its_group_acl() { let group = ShardGroup { @@ -1208,10 +1303,14 @@ mod tests { let _ = reply.send(Some(test_shard_group())); } }); - let raft = raft_sender_with(|cmd| { - if let MultiRaftActorCommand::ClientProposal { reply, .. } = cmd { + let raft = raft_sender_with(|cmd| match cmd { + MultiRaftActorCommand::GetTopicMetadata { reply, .. } => { + let _ = reply.send(Some(topic_meta("node-1"))); + } + MultiRaftActorCommand::ClientProposal { reply, .. } => { let _ = reply.send(Ok(())); } + _ => {} }); let resp = trusted_controller(node_id("node-1"), swim, raft, dp_stub()) .dispatch(ClientRequest::ControlPlane( From d3730d1d9edc12df263a60ab908f039b1a30eada Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 11:40:03 +0400 Subject: [PATCH 35/56] feat: client -> data node -> cache flow for acl verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ACL records and data replicas are usually on different shard groups. Client → data replica │ ├─ serves topic-data/42 └─ may not host ACL topic-data/42 Today, the data replica can only check ACL state if it hosts that ACL’s metadata shard. Otherwise it denies the request—even when the client is allowed. The intended fix is: Client → data replica → local ACL cache │ fresh → allow / deny miss → fetch ACL record from owner shard │ └─ cache ≤60s → allow / deny --- src/security/acl_cache.rs | 194 +++++++++++++++++++++ src/security/certificates.rs | 150 ++++++++++++++++ src/security/mod.rs | 9 + src/{security.rs => security/transport.rs} | 133 +------------- 4 files changed, 354 insertions(+), 132 deletions(-) create mode 100644 src/security/acl_cache.rs create mode 100644 src/security/certificates.rs create mode 100644 src/security/mod.rs rename src/{security.rs => security/transport.rs} (69%) diff --git a/src/security/acl_cache.rs b/src/security/acl_cache.rs new file mode 100644 index 00000000..a3673434 --- /dev/null +++ b/src/security/acl_cache.rs @@ -0,0 +1,194 @@ +use std::collections::HashMap; +use std::time::Duration; + +use tokio::time::Instant; + +use crate::control_plane::membership::ShardGroupId; +use crate::control_plane::metadata::AclResource; + +/// Maximum time an ACL entry may authorize without a fresh read from its +/// owning metadata shard. +pub(crate) const MAX_ACL_CACHE_TTL: Duration = Duration::from_secs(60); + +/// One ACL record copied from its owning metadata shard. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CachedAcl { + pub(crate) source_shard_id: ShardGroupId, + pub(crate) revision: u64, + pub(crate) principals: Box<[Box]>, +} + +/// Result of checking one principal against the local ACL cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CachedAuthorization { + Authorized, + Denied, + Miss, +} + +#[derive(Debug)] +struct AclCacheEntry { + acl: CachedAcl, + expires_at: Instant, +} + +/// Short-lived, revision-aware copies of ACL records. +/// +/// The cache has no authority of its own. A caller must supply the shard that +/// currently owns the ACL resource; an entry from a previous owner, an expired +/// entry, or no entry at all is a miss and must be refreshed or denied. +#[derive(Debug, Default)] +pub(crate) struct AclCache { + entries: HashMap, +} + +impl AclCache { + pub(crate) fn authorize( + &self, + resource: &AclResource, + source_shard_id: ShardGroupId, + principal: &str, + now: Instant, + ) -> CachedAuthorization { + let Some(entry) = self.entries.get(resource) else { + return CachedAuthorization::Miss; + }; + if entry.acl.source_shard_id != source_shard_id || entry.expires_at <= now { + return CachedAuthorization::Miss; + } + if entry + .acl + .principals + .iter() + .any(|candidate| candidate.as_ref() == principal) + { + CachedAuthorization::Authorized + } else { + CachedAuthorization::Denied + } + } + + /// Retains the newest known revision from a shard and caps its authority + /// window at [`MAX_ACL_CACHE_TTL`]. A record from a newly assigned owner + /// replaces the old owner's revision, because revisions are shard-local. + pub(crate) fn insert(&mut self, resource: AclResource, acl: CachedAcl, now: Instant) { + let replace = match self.entries.get(&resource) { + Some(entry) if entry.acl.source_shard_id == acl.source_shard_id => { + entry.acl.revision <= acl.revision + } + Some(_) | None => true, + }; + if replace { + self.entries.insert( + resource, + AclCacheEntry { + acl, + expires_at: now + MAX_ACL_CACHE_TTL, + }, + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control_plane::metadata::TopicId; + + fn resource() -> AclResource { + AclResource::TopicData(TopicId(7)) + } + + fn acl(source_shard_id: ShardGroupId, revision: u64, principals: &[&str]) -> CachedAcl { + CachedAcl { + source_shard_id, + revision, + principals: principals + .iter() + .map(|principal| (*principal).into()) + .collect(), + } + } + + #[test] + fn authorizes_and_denies_exact_principals_from_a_fresh_entry() { + let now = Instant::now(); + let mut cache = AclCache::default(); + cache.insert( + resource(), + acl(ShardGroupId(3), 4, &["orders-service"]), + now, + ); + + assert_eq!( + cache.authorize(&resource(), ShardGroupId(3), "orders-service", now), + CachedAuthorization::Authorized + ); + assert_eq!( + cache.authorize(&resource(), ShardGroupId(3), "billing-service", now), + CachedAuthorization::Denied + ); + } + + #[test] + fn expires_entries_after_the_bounded_ttl() { + let now = Instant::now(); + let mut cache = AclCache::default(); + cache.insert( + resource(), + acl(ShardGroupId(3), 4, &["orders-service"]), + now, + ); + + assert_eq!( + cache.authorize( + &resource(), + ShardGroupId(3), + "orders-service", + now + MAX_ACL_CACHE_TTL + Duration::from_millis(1), + ), + CachedAuthorization::Miss + ); + } + + #[test] + fn rejects_an_entry_from_a_previous_owner_shard() { + let now = Instant::now(); + let mut cache = AclCache::default(); + cache.insert( + resource(), + acl(ShardGroupId(3), 4, &["orders-service"]), + now, + ); + + assert_eq!( + cache.authorize(&resource(), ShardGroupId(4), "orders-service", now), + CachedAuthorization::Miss + ); + } + + #[test] + fn does_not_replace_a_newer_revision_from_the_same_shard() { + let now = Instant::now(); + let mut cache = AclCache::default(); + cache.insert( + resource(), + acl(ShardGroupId(3), 5, &["orders-service"]), + now, + ); + cache.insert( + resource(), + acl(ShardGroupId(3), 4, &["billing-service"]), + now, + ); + + assert_eq!( + cache.authorize(&resource(), ShardGroupId(3), "orders-service", now), + CachedAuthorization::Authorized + ); + assert_eq!( + cache.authorize(&resource(), ShardGroupId(3), "billing-service", now), + CachedAuthorization::Denied + ); + } +} diff --git a/src/security/certificates.rs b/src/security/certificates.rs new file mode 100644 index 00000000..cf5e9d22 --- /dev/null +++ b/src/security/certificates.rs @@ -0,0 +1,150 @@ +use anyhow::{Context, Result}; +use rustls::pki_types::CertificateDer; +use x509_parser::extensions::GeneralName; +use x509_parser::prelude::{FromDer, X509Certificate}; + +/// Reads the stable node principal from a leaf certificate's URI Subject +/// Alternative Name. +/// +/// The certificate must contain exactly one URI beginning with +/// `urn:eastguard:node:`. The text after that prefix is the principal used as +/// the admission-record key. This function only parses the certificate; callers +/// must use it after rustls has authenticated the peer's certificate chain. +pub(crate) fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Result { + certificate_principal( + certificate, + "urn:eastguard:node:", + "node", + "Node Certificate Principal", + ) +} + +/// Reads the client principal from exactly one +/// `urn:eastguard:client:` URI Subject Alternative Name. +/// +/// TLS authentication must succeed before callers use this parsed identity for +/// authorization. +pub(crate) fn client_certificate_principal(certificate: &CertificateDer<'_>) -> Result { + certificate_principal( + certificate, + "urn:eastguard:client:", + "client", + "Client Certificate Principal", + ) +} + +fn certificate_principal( + certificate: &CertificateDer<'_>, + uri_prefix: &str, + certificate_kind: &str, + principal_name: &str, +) -> Result { + let (_, certificate) = + X509Certificate::from_der(certificate.as_ref()).context("invalid X.509 certificate")?; + let subject_alt_name = certificate + .subject_alternative_name() + .context("invalid X.509 subject alternative name")? + .with_context(|| { + format!("{certificate_kind} certificate has no subject alternative name") + })?; + + let mut principals = + subject_alt_name + .value + .general_names + .iter() + .filter_map(|name| match name { + GeneralName::URI(uri) => uri.strip_prefix(uri_prefix), + _ => None, + }); + let principal = principals + .next() + .filter(|principal| !principal.is_empty()) + .with_context(|| format!("{certificate_kind} certificate has no {principal_name}"))?; + anyhow::ensure!( + principals.next().is_none(), + "{certificate_kind} certificate has multiple {principal_name}s" + ); + Ok(principal.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rcgen::string::Ia5String; + use rcgen::{CertificateParams, KeyPair, SanType}; + + fn certificate_with_uris(uris: &[&str]) -> CertificateDer<'static> { + let mut params = CertificateParams::default(); + params.subject_alt_names = uris + .iter() + .map(|uri| SanType::URI(Ia5String::try_from(*uri).unwrap())) + .collect(); + let key = KeyPair::generate().unwrap(); + params.self_signed(&key).unwrap().der().clone() + } + + #[test] + fn reads_node_principal_from_uri_subject_alternative_name() { + let certificate = + certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:node:broker-a"]); + + assert_eq!( + node_certificate_principal(&certificate).unwrap(), + "broker-a" + ); + } + + #[test] + fn requires_exactly_one_node_principal() { + let missing = certificate_with_uris(&["urn:example:unrelated"]); + let ambiguous = + certificate_with_uris(&["urn:eastguard:node:broker-a", "urn:eastguard:node:broker-b"]); + + assert_eq!( + node_certificate_principal(&missing) + .unwrap_err() + .to_string(), + "node certificate has no Node Certificate Principal" + ); + assert_eq!( + node_certificate_principal(&ambiguous) + .unwrap_err() + .to_string(), + "node certificate has multiple Node Certificate Principals" + ); + } + + #[test] + fn reads_client_principal_from_uri_subject_alternative_name() { + let certificate = + certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:client:producer-a"]); + + assert_eq!( + client_certificate_principal(&certificate).unwrap(), + "producer-a" + ); + } + + #[test] + fn requires_exactly_one_client_principal() { + let missing = certificate_with_uris(&["urn:example:unrelated"]); + let ambiguous = certificate_with_uris(&[ + "urn:eastguard:client:producer-a", + "urn:eastguard:client:producer-b", + ]); + + assert_eq!( + client_certificate_principal(&missing) + .unwrap_err() + .to_string(), + "client certificate has no Client Certificate Principal" + ); + assert_eq!( + client_certificate_principal(&ambiguous) + .unwrap_err() + .to_string(), + "client certificate has multiple Client Certificate Principals" + ); + } +} diff --git a/src/security/mod.rs b/src/security/mod.rs new file mode 100644 index 00000000..11aaa36a --- /dev/null +++ b/src/security/mod.rs @@ -0,0 +1,9 @@ +#![allow(dead_code)] + +pub(crate) mod acl_cache; + +mod certificates; +mod transport; + +pub(crate) use certificates::{client_certificate_principal, node_certificate_principal}; +pub(crate) use transport::{NodeTransportSecurity, TransportIdentity}; diff --git a/src/security.rs b/src/security/transport.rs similarity index 69% rename from src/security.rs rename to src/security/transport.rs index 063d392c..ff2a57d1 100644 --- a/src/security.rs +++ b/src/security/transport.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use std::fmt; use std::fs::File; use std::io::BufReader; @@ -16,9 +15,8 @@ use rustls::{ CertificateError, ClientConfig, DigitallySignedStruct, Error as RustlsError, OtherError, RootCertStore, ServerConfig, SignatureScheme, }; -use x509_parser::extensions::GeneralName; -use x509_parser::prelude::{FromDer, X509Certificate}; +use super::certificates::node_certificate_principal; use crate::config::{Environment, SecurityMode}; /// Verifies certificates presented to EastGuard's outbound node connections. @@ -218,71 +216,6 @@ impl NodeTransportSecurity { } } -/// Reads the stable node principal from a leaf certificate's URI Subject -/// Alternative Name. -/// -/// The certificate must contain exactly one URI beginning with -/// `urn:eastguard:node:`. The text after that prefix is the principal used as -/// the admission-record key. This function only parses the certificate; callers -/// must use it after rustls has authenticated the peer's certificate chain. -pub(crate) fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Result { - certificate_principal( - certificate, - "urn:eastguard:node:", - "node", - "Node Certificate Principal", - ) -} - -/// Reads the client principal from exactly one -/// `urn:eastguard:client:` URI Subject Alternative Name. -/// -/// TLS authentication must succeed before callers use this parsed identity for -/// authorization. -pub(crate) fn client_certificate_principal(certificate: &CertificateDer<'_>) -> Result { - certificate_principal( - certificate, - "urn:eastguard:client:", - "client", - "Client Certificate Principal", - ) -} - -fn certificate_principal( - certificate: &CertificateDer<'_>, - uri_prefix: &str, - certificate_kind: &str, - principal_name: &str, -) -> Result { - let (_, certificate) = - X509Certificate::from_der(certificate.as_ref()).context("invalid X.509 certificate")?; - let subject_alt_name = certificate - .subject_alternative_name() - .context("invalid X.509 subject alternative name")? - .with_context(|| { - format!("{certificate_kind} certificate has no subject alternative name") - })?; - - let mut principals = - subject_alt_name - .value - .general_names - .iter() - .filter_map(|name| match name { - GeneralName::URI(uri) => uri.strip_prefix(uri_prefix), - _ => None, - }); - let principal = principals - .next() - .filter(|principal| !principal.is_empty()) - .with_context(|| format!("{certificate_kind} certificate has no {principal_name}"))?; - anyhow::ensure!( - principals.next().is_none(), - "{certificate_kind} certificate has multiple {principal_name}s" - ); - Ok(principal.to_string()) -} - #[cfg(test)] mod tests { use super::*; @@ -326,70 +259,6 @@ mod tests { )); } - #[test] - fn reads_node_principal_from_uri_subject_alternative_name() { - let certificate = - certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:node:broker-a"]); - - assert_eq!( - node_certificate_principal(&certificate).unwrap(), - "broker-a" - ); - } - - #[test] - fn requires_exactly_one_node_principal() { - let missing = certificate_with_uris(&["urn:example:unrelated"]); - let ambiguous = - certificate_with_uris(&["urn:eastguard:node:broker-a", "urn:eastguard:node:broker-b"]); - - assert_eq!( - node_certificate_principal(&missing) - .unwrap_err() - .to_string(), - "node certificate has no Node Certificate Principal" - ); - assert_eq!( - node_certificate_principal(&ambiguous) - .unwrap_err() - .to_string(), - "node certificate has multiple Node Certificate Principals" - ); - } - - #[test] - fn reads_client_principal_from_uri_subject_alternative_name() { - let certificate = - certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:client:producer-a"]); - - assert_eq!( - client_certificate_principal(&certificate).unwrap(), - "producer-a" - ); - } - - #[test] - fn requires_exactly_one_client_principal() { - let missing = certificate_with_uris(&["urn:example:unrelated"]); - let ambiguous = certificate_with_uris(&[ - "urn:eastguard:client:producer-a", - "urn:eastguard:client:producer-b", - ]); - - assert_eq!( - client_certificate_principal(&missing) - .unwrap_err() - .to_string(), - "client certificate has no Client Certificate Principal" - ); - assert_eq!( - client_certificate_principal(&ambiguous) - .unwrap_err() - .to_string(), - "client certificate has multiple Client Certificate Principals" - ); - } - #[test] fn server_verifier_requires_trust_and_node_principal() { let trusted = certificate_with_uris(&["urn:eastguard:node:broker-a"]); From 63d91902a3a529bbe49fe4155f53b4f3a641f63b Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 12:28:32 +0400 Subject: [PATCH 36/56] refactor: acl cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ClientController └─ cache.authorize_or_refresh(resource, shard, principal, lazy_fetch) ├─ fresh cached grant → Ok(()) ├─ fresh cached denial → Err(Unauthorized) └─ cache miss ├─ snapshot fetched → store → recheck → Ok / Unauthorized └─ no snapshot → Err(Unauthorized) - Removed CachedAuthorization; cache internals no longer leak Authorized / Denied / Miss. - SharedAclCache now returns Result<(), ServerError> and owns the fail-closed policy. - ClientController no longer contains cache-miss, snapshot-to-cache, or recheck logic—it simply awaits the cache result. - The snapshot fetch is lazy: on a fresh cache entry, no Raft query is made and the resource is not cloned for the query. - CachedAcl and conversion from a replicated ACL record are private implementation details. - Tests now assert success/failure results instead of cache-state variants. --- docs/security/roadmap.md | 41 ++-- src/connections/controller.rs | 128 ++++++++++-- src/control_plane/consensus/actor.rs | 16 +- src/control_plane/consensus/messages/actor.rs | 47 +++-- src/control_plane/consensus/multi_raft.rs | 90 ++++---- src/control_plane/consensus/raft/state.rs | 5 +- .../consensus/raft/states/metadata_state.rs | 28 ++- .../consensus/raft/states/security.rs | 46 ++++- src/lib.rs | 11 +- src/security/acl_cache.rs | 193 ++++++++++++++---- 10 files changed, 438 insertions(+), 167 deletions(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 363f3701..7efaf1ec 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -235,26 +235,37 @@ Check local ACL cache └── Missing / expired │ ▼ - Obtain current ACL state - (distribution design deferred) - │ - ┌─────┴─────┐ - ▼ ▼ - Grant Deny / unavailable ──► Fail closed - │ - ▼ -Does this node serve the requested data? + Is the ACL shard local? + ├── No ──► Return unauthorized + └── Yes + │ + ▼ + Read committed ACL state + │ + ▼ + Refresh cache + │ + ┌─────┴─────┐ + ▼ ▼ + Grant Deny / unavailable ──► Fail closed + │ + ▼ + Does this node serve the requested data? ├── No ──► Return data-node redirect └── Yes ──► Execute locally ``` Authorization precedes redirects so an ungranted client cannot use stale-route -responses to discover data placement. Any remote authorization request may -update only the ACL cache; it must never carry or execute the client's data -operation. - -The ACL distribution mechanism is intentionally deferred. A later phase may use -lazy pull, proactive push, or a push-and-pull hybrid: +responses to discover data placement. The current implementation refreshes a +missing or expired cache entry only when the broker hosts the ACL's metadata +shard. It reads that shard's committed ACL record, including an empty record for +default denial, and stores it for no more than 60 seconds. When another broker +owns the ACL shard, the request fails closed; it does not forward or proxy the +client data operation. + +Fetching an ACL record from a remote owner is still deferred. That later path may +use lazy pull, proactive push, or a push-and-pull hybrid. It may update only the +ACL cache; it must never carry or execute the client's data operation: | Model | Benefit | Failure to handle | | :--- | :--- | :--- | diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 4d479dfc..cc8ac528 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -31,6 +31,7 @@ use crate::data_plane::messages::query::{ }; use crate::net::TransportTcpStream; use crate::security::TransportIdentity; +use crate::security::acl_cache::SharedAclCache; use tokio::sync::mpsc; /// # Client ↔ Server request_id protocol @@ -54,6 +55,7 @@ pub struct ClientController { swim_sender: SwimSender, raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, + acl_cache: SharedAclCache, } impl ClientController { @@ -63,6 +65,7 @@ impl ClientController { swim_sender: SwimSender, raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, + acl_cache: SharedAclCache, ) -> Self { Self { transport_identity, @@ -70,6 +73,7 @@ impl ClientController { swim_sender, raft_sender, data_plane_tx, + acl_cache, } } @@ -313,14 +317,12 @@ impl ClientController { return Err(ServerError::Unauthorized); }; - match self - .raft_sender - .authorize_principal(group.id, resource, principal.clone()) + self.acl_cache + .authorize_or_refresh(&resource, group.id, principal, || { + self.raft_sender + .get_acl_snapshot(group.id, resource.clone()) + }) .await - { - Some(true) => Ok(()), - Some(false) | None => Err(ServerError::Unauthorized), - } } /// Structural redirect for a control-plane op that isn't local: to the member if @@ -697,6 +699,7 @@ pub async fn handle_client_stream( swim_sender: SwimSender, raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, + acl_cache: SharedAclCache, ) { let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); @@ -707,6 +710,7 @@ pub async fn handle_client_stream( swim_sender, raft_sender, data_plane_tx, + acl_cache, ); tokio::spawn(run_client_writer( ClientRawWriter::new(write_half), @@ -726,6 +730,7 @@ mod tests { }; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::MultiRaftActorCommand; + use crate::control_plane::consensus::raft::states::security::AclRecord; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ QueryCommand, ShardGroup, ShardGroupId, ShardLeaderEntry, SwimActorCommand, @@ -815,6 +820,7 @@ mod tests { swim_sender, raft_sender, data_plane_tx, + SharedAclCache::default(), ) } @@ -823,6 +829,22 @@ mod tests { node_id: NodeId, swim_sender: SwimSender, raft_sender: MutlRaftSender, + ) -> ClientController { + authenticated_controller_with_cache( + principal, + node_id, + swim_sender, + raft_sender, + SharedAclCache::default(), + ) + } + + fn authenticated_controller_with_cache( + principal: &str, + node_id: NodeId, + swim_sender: SwimSender, + raft_sender: MutlRaftSender, + acl_cache: SharedAclCache, ) -> ClientController { ClientController::new( TransportIdentity::CertificatePrincipal(principal.into()), @@ -830,9 +852,21 @@ mod tests { swim_sender, raft_sender, dp_stub(), + acl_cache, ) } + fn acl_snapshot(resource: AclResource, principals: &[&str]) -> AclRecord { + AclRecord { + resource, + revision: 1, + principals: principals + .iter() + .map(|principal| (*principal).to_owned()) + .collect(), + } + } + fn produce_req() -> ClientRequest { ClientRequest::DataPlane(ClientDataPlaneRequest::Produce(ProduceRequest { topic_name: "t1".into(), @@ -871,12 +905,12 @@ mod tests { } }); let raft = raft_sender_with(|cmd| { - if let MultiRaftActorCommand::AuthorizePrincipal(query) = cmd { + if let MultiRaftActorCommand::GetAclSnapshot(query) = cmd { assert_eq!(query.shard_group_id, ShardGroupId(42)); assert_eq!(query.resource, AclResource::TopicData(TopicId(7))); let _ = query .reply - .send(Some(query.principal.as_ref() == "orders-service")); + .send(Some(acl_snapshot(query.resource, &["orders-service"]))); } }); let controller = authenticated_controller("orders-service", node_id("self"), swim, raft); @@ -889,6 +923,66 @@ mod tests { ); } + #[tokio::test] + async fn fresh_acl_cache_authorizes_without_a_raft_query() { + let group = test_shard_group(); + let swim = swim_sender_with(move |cmd| { + if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { + let _ = reply.send(Some(group.clone())); + } + }); + let cache = SharedAclCache::default(); + let resource = AclResource::TopicData(TopicId(7)); + assert_eq!( + cache + .authorize_or_refresh(&resource, ShardGroupId(42), "orders-service", || async { + Some(acl_snapshot(resource.clone(), &["orders-service"])) + }) + .await, + Ok(()) + ); + let controller = authenticated_controller_with_cache( + "orders-service", + node_id("node-1"), + swim, + raft_sender_with(|_| panic!("fresh ACL cache must not query Raft")), + cache, + ); + + assert_eq!(controller.authorize_acl_resource(resource).await, Ok(())); + } + + #[tokio::test] + async fn acl_cache_miss_reads_the_local_snapshot_once() { + let group = test_shard_group(); + let swim = swim_sender_with(move |cmd| { + if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { + let _ = reply.send(Some(group.clone())); + } + }); + let query_count = Arc::new(AtomicUsize::new(0)); + let observed_query_count = query_count.clone(); + let raft = raft_sender_with(move |cmd| { + if let MultiRaftActorCommand::GetAclSnapshot(query) = cmd { + observed_query_count.fetch_add(1, Ordering::Relaxed); + let _ = query + .reply + .send(Some(acl_snapshot(query.resource, &["orders-service"]))); + } + }); + let controller = authenticated_controller("orders-service", node_id("node-1"), swim, raft); + + for _ in 0..2 { + assert_eq!( + controller + .authorize_acl_resource(AclResource::TopicData(TopicId(7))) + .await, + Ok(()) + ); + } + assert_eq!(query_count.load(Ordering::Relaxed), 1); + } + #[tokio::test] async fn topic_metadata_operations_require_topic_admin_acl() { let group = test_shard_group(); @@ -903,9 +997,9 @@ mod tests { MultiRaftActorCommand::GetTopicMetadata { reply, .. } => { let _ = reply.send(Some(topic_meta("node-1"))); } - MultiRaftActorCommand::AuthorizePrincipal(query) => { + MultiRaftActorCommand::GetAclSnapshot(query) => { assert_eq!(query.resource, AclResource::TopicAdmin(TopicId(1))); - let _ = query.reply.send(Some(false)); + let _ = query.reply.send(Some(acl_snapshot(query.resource, &[]))); } MultiRaftActorCommand::ClientProposal { .. } => { observed_proposals.fetch_add(1, Ordering::Relaxed); @@ -936,9 +1030,9 @@ mod tests { let proposal_count = Arc::new(AtomicUsize::new(0)); let observed_proposals = proposal_count.clone(); let raft = raft_sender_with(move |cmd| match cmd { - MultiRaftActorCommand::AuthorizePrincipal(query) => { + MultiRaftActorCommand::GetAclSnapshot(query) => { assert_eq!(query.resource, AclResource::Cluster); - let _ = query.reply.send(Some(false)); + let _ = query.reply.send(Some(acl_snapshot(query.resource, &[]))); } MultiRaftActorCommand::ClientProposal { .. } => { observed_proposals.fetch_add(1, Ordering::Relaxed); @@ -983,7 +1077,7 @@ mod tests { } }); let raft = raft_sender_with(|cmd| { - if let MultiRaftActorCommand::AuthorizePrincipal(query) = cmd { + if let MultiRaftActorCommand::GetAclSnapshot(query) = cmd { assert_eq!( query.resource, AclResource::ConsumerGroup(ConsumerGroupResource { @@ -991,7 +1085,7 @@ mod tests { group_id: "billing".to_string(), }) ); - let _ = query.reply.send(Some(false)); + let _ = query.reply.send(Some(acl_snapshot(query.resource, &[]))); } }); let controller = authenticated_controller("orders-service", node_id("self"), swim, raft); @@ -1027,7 +1121,7 @@ mod tests { MultiRaftActorCommand::GetTopicMetadata { reply, .. } => { let _ = reply.send(Some(topic_meta("self"))); } - MultiRaftActorCommand::AuthorizePrincipal(query) => { + MultiRaftActorCommand::GetAclSnapshot(query) => { assert_eq!( query.resource, AclResource::ConsumerGroup(ConsumerGroupResource { @@ -1035,7 +1129,7 @@ mod tests { group_id: "billing".to_string(), }) ); - let _ = query.reply.send(Some(false)); + let _ = query.reply.send(Some(acl_snapshot(query.resource, &[]))); } MultiRaftActorCommand::ClientProposal { .. } => { observed_proposals.fetch_add(1, Ordering::Relaxed); diff --git a/src/control_plane/consensus/actor.rs b/src/control_plane/consensus/actor.rs index 56484e18..51b039b2 100644 --- a/src/control_plane/consensus/actor.rs +++ b/src/control_plane/consensus/actor.rs @@ -8,6 +8,7 @@ use crate::control_plane::NodeId; use crate::control_plane::consensus::messages::*; use crate::control_plane::consensus::multi_raft::MultiRaft; use crate::control_plane::consensus::raft::errors::ProposalError; +use crate::control_plane::consensus::raft::states::security::AclRecord; use crate::control_plane::consensus::raft::storage::RaftStorage; use crate::control_plane::membership::actor::SwimSender; use crate::control_plane::membership::{ShardGroupId, SwimCommand, TopologyReader}; @@ -278,22 +279,19 @@ impl MutlRaftSender { recv.await.unwrap_or_default() } - /// Checks ACL state only when this node hosts the selected metadata shard. - /// - /// `None` means the shard is not local or the actor stopped. `Some(false)` - /// is an authoritative default-deny decision from local replicated state. - pub(crate) async fn authorize_principal( + /// Reads the committed ACL snapshot only when this node hosts the selected + /// metadata shard. A missing ACL is an empty, cacheable denial; `None` + /// means the shard is no longer local or the actor stopped. + pub(crate) async fn get_acl_snapshot( &self, shard_group_id: ShardGroupId, resource: AclResource, - principal: Box, - ) -> Option { + ) -> Option { let (reply, recv) = tokio::sync::oneshot::channel(); let _ = self - .send(AuthorizePrincipal { + .send(GetAclSnapshot { shard_group_id, resource, - principal, reply, }) .await; diff --git a/src/control_plane/consensus/messages/actor.rs b/src/control_plane/consensus/messages/actor.rs index 4cc66cb1..a7f1f0f3 100644 --- a/src/control_plane/consensus/messages/actor.rs +++ b/src/control_plane/consensus/messages/actor.rs @@ -3,6 +3,7 @@ use uuid::Uuid; use crate::control_plane::NodeId; use crate::control_plane::consensus::raft::errors::ProposalError; +use crate::control_plane::consensus::raft::states::security::AclRecord; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::{AclResource, ConsumerGroupAssignment, TopicMeta, TopicStats}; use crate::data_plane::messages::command::{ @@ -52,7 +53,7 @@ pub enum MultiRaftActorCommand { topic_name: String, reply: oneshot::Sender>, }, - AuthorizePrincipal(AuthorizePrincipal), + GetAclSnapshot(GetAclSnapshot), GetConsumerGroupAssignment(GetConsumerGroupAssignment), /// Data-plane request forwarded to the metadata coordinator for proposal. ProposeSegmentRoll(ProposeSegmentRoll), @@ -75,11 +76,14 @@ pub struct GetConsumerGroupAssignment { pub(crate) reply: oneshot::Sender>, } -pub struct AuthorizePrincipal { +/// Returns this node's committed ACL state for a shard it hosts. +/// +/// The caller has already routed the resource to this shard. A missing ACL is +/// returned as an empty record so it can be cached as a bounded denial. +pub struct GetAclSnapshot { pub(crate) shard_group_id: ShardGroupId, pub(crate) resource: AclResource, - pub(crate) principal: Box, - pub(crate) reply: oneshot::Sender>, + pub(crate) reply: oneshot::Sender>, } impl From for MultiRaftActorCommand { @@ -104,30 +108,29 @@ impl_from_variant_via!( impl_from_variant!( MultiRaftActorCommand, - AuthorizePrincipal, + GetAclSnapshot, GetConsumerGroupAssignment, ); -pub(crate) struct DeferredConsumerGroupAssignment { - pub(crate) reply: oneshot::Sender>, - pub(crate) value: Option, +/// A synchronous actor result held until the end-of-batch reply flush. +pub(crate) struct DeferredResponse { + pub(crate) reply: oneshot::Sender, + pub(crate) value: T, } -pub(crate) struct DeferredAuthorization { - pub(crate) reply: oneshot::Sender>, - pub(crate) value: Option, +impl DeferredResponse { + pub(crate) fn send(self) { + let _ = self.reply.send(self.value); + } } pub(crate) enum DeferredReply { - GetLeader(oneshot::Sender>, Option), - GetPeers(oneshot::Sender>, Box<[NodeId]>), - Propose( - oneshot::Sender>, - Result<(), ProposalError>, - ), - GetTopics(oneshot::Sender>, Box<[String]>), - GetTopicStats(oneshot::Sender>, Box<[TopicStats]>), - GetTopicMetadata(oneshot::Sender>, Box>), - AuthorizePrincipal(DeferredAuthorization), - GetConsumerGroupAssignment(DeferredConsumerGroupAssignment), + GetLeader(DeferredResponse>), + GetPeers(DeferredResponse>), + Propose(DeferredResponse>), + GetTopics(DeferredResponse>), + GetTopicStats(DeferredResponse>), + GetTopicMetadata(DeferredResponse>), + GetAclSnapshot(DeferredResponse>), + GetConsumerGroupAssignment(DeferredResponse>), } diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index 302f606b..cf5aa19a 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -3,13 +3,13 @@ use crate::control_plane::consensus::boundary_recovery::{ BoundaryRecoveryAction, SegmentBoundaryRecovery, }; use crate::control_plane::consensus::messages::{ - DeferredAuthorization, DeferredConsumerGroupAssignment, DeferredReply, InboundRaftRpc, - LogMutation, MetadataProposal, MultiRaftActorCommand, ProposeSegmentRoll, RaftEvent, - RaftProtocolMessage, RaftTimeoutCallback, + DeferredReply, DeferredResponse, InboundRaftRpc, LogMutation, MetadataProposal, + MultiRaftActorCommand, ProposeSegmentRoll, RaftEvent, RaftProtocolMessage, RaftTimeoutCallback, }; use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::consensus::raft::state::{Raft, TimerSeqs}; use crate::control_plane::consensus::raft::states::consensus::LeaderlessSegments; +use crate::control_plane::consensus::raft::states::security::AclRecord; use crate::control_plane::consensus::raft::storage::RaftStorage; use crate::control_plane::consensus::raft::{compute_replacement_replica_set, now_ms}; use crate::control_plane::membership::{ShardGroup, ShardGroupId, TopologyReader}; @@ -239,11 +239,19 @@ impl MultiRaft { } MultiRaftActorCommand::GetLeader { group_id, reply } => { let result = self.get_leader(group_id); - self.deferred.push(DeferredReply::GetLeader(reply, result)); + self.deferred + .push(DeferredReply::GetLeader(DeferredResponse { + reply, + value: result, + })); } MultiRaftActorCommand::GetPeers { group_id, reply } => { let result = self.get_peers(group_id); - self.deferred.push(DeferredReply::GetPeers(reply, result)); + self.deferred + .push(DeferredReply::GetPeers(DeferredResponse { + reply, + value: result, + })); } MultiRaftActorCommand::ClientProposal { propose, reply } => { self.propose(propose, reply); @@ -251,26 +259,32 @@ impl MultiRaft { MultiRaftActorCommand::GetTopics { reply } => { let topics = self.get_topics(); - self.deferred.push(DeferredReply::GetTopics(reply, topics)); + self.deferred + .push(DeferredReply::GetTopics(DeferredResponse { + reply, + value: topics, + })); } MultiRaftActorCommand::GetTopicStats { reply } => { let stats = self.get_topic_stats(); self.deferred - .push(DeferredReply::GetTopicStats(reply, stats)); + .push(DeferredReply::GetTopicStats(DeferredResponse { + reply, + value: stats, + })); } MultiRaftActorCommand::GetTopicMetadata { topic_name, reply } => { let meta = self.get_topic_metadata(&topic_name); self.deferred - .push(DeferredReply::GetTopicMetadata(reply, Box::new(meta))); + .push(DeferredReply::GetTopicMetadata(DeferredResponse { + reply, + value: meta, + })); } - MultiRaftActorCommand::AuthorizePrincipal(query) => { - let value = self.authorize_principal( - query.shard_group_id, - &query.resource, - &query.principal, - ); + MultiRaftActorCommand::GetAclSnapshot(query) => { + let value = self.acl_snapshot(query.shard_group_id, &query.resource); self.deferred - .push(DeferredReply::AuthorizePrincipal(DeferredAuthorization { + .push(DeferredReply::GetAclSnapshot(DeferredResponse { reply: query.reply, value, })); @@ -283,7 +297,7 @@ impl MultiRaft { ); self.deferred .push(DeferredReply::GetConsumerGroupAssignment( - DeferredConsumerGroupAssignment { + DeferredResponse { reply: query.reply, value, }, @@ -307,30 +321,14 @@ impl MultiRaft { pub(crate) fn fire_deferred(&mut self) { for reply in self.deferred.drain(..) { match reply { - DeferredReply::GetLeader(sender, v) => { - let _ = sender.send(v); - } - DeferredReply::GetPeers(sender, v) => { - let _ = sender.send(v); - } - DeferredReply::Propose(sender, v) => { - let _ = sender.send(v); - } - DeferredReply::GetTopics(sender, v) => { - let _ = sender.send(v); - } - DeferredReply::GetTopicStats(sender, v) => { - let _ = sender.send(v); - } - DeferredReply::GetTopicMetadata(sender, v) => { - let _ = sender.send(*v); - } - DeferredReply::AuthorizePrincipal(deferred) => { - let _ = deferred.reply.send(deferred.value); - } - DeferredReply::GetConsumerGroupAssignment(deferred) => { - let _ = deferred.reply.send(deferred.value); - } + DeferredReply::GetLeader(deferred) => deferred.send(), + DeferredReply::GetPeers(deferred) => deferred.send(), + DeferredReply::Propose(deferred) => deferred.send(), + DeferredReply::GetTopics(deferred) => deferred.send(), + DeferredReply::GetTopicStats(deferred) => deferred.send(), + DeferredReply::GetTopicMetadata(deferred) => deferred.send(), + DeferredReply::GetAclSnapshot(deferred) => deferred.send(), + DeferredReply::GetConsumerGroupAssignment(deferred) => deferred.send(), } } } @@ -495,15 +493,14 @@ impl MultiRaft { .find_map(|raft| raft.get_topic_by_name(name).cloned()) } - fn authorize_principal( + fn acl_snapshot( &self, shard_group_id: ShardGroupId, resource: &AclResource, - principal: &str, - ) -> Option { + ) -> Option { self.groups .get(&shard_group_id) - .map(|raft| raft.authorizes(resource, principal)) + .map(|raft| raft.acl_snapshot(resource)) } fn get_consumer_group_assignment( @@ -775,7 +772,10 @@ impl MultiRaft { self.pending_proposes.insert((gid, index), reply); } Err(e) => { - self.deferred.push(DeferredReply::Propose(reply, Err(e))); + self.deferred.push(DeferredReply::Propose(DeferredResponse { + reply, + value: Err(e), + })); } } } diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index c777f7ad..47885f3f 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -8,6 +8,7 @@ use crate::control_plane::consensus::raft::states::consensus::{ ConsensusState, PeerState, Role, SNAPSHOT_CHUNK_BYTES, SnapshotInstallOutcome, }; use crate::control_plane::consensus::raft::states::metadata_state::MetadataState; +use crate::control_plane::consensus::raft::states::security::AclRecord; use crate::control_plane::consensus::raft::storage::{ RaftPersistentState, RaftSnapshot, SnapshotData, }; @@ -161,8 +162,8 @@ impl Raft { self.metadata.get_topic_by_name(name) } - pub(crate) fn authorizes(&self, resource: &AclResource, principal: &str) -> bool { - self.metadata.authorizes(resource, principal) + pub(crate) fn acl_snapshot(&self, resource: &AclResource) -> AclRecord { + self.metadata.acl_snapshot(resource) } pub(crate) fn get_consumer_group_assignment( diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index b1091824..98bcaa99 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -1,4 +1,4 @@ -use crate::control_plane::consensus::raft::states::security::SecurityState; +use crate::control_plane::consensus::raft::states::security::{AclRecord, SecurityState}; use crate::control_plane::metadata::SegmentMeta; use crate::control_plane::metadata::command::*; use crate::control_plane::metadata::event::*; @@ -91,8 +91,8 @@ impl MetadataState { range.segments.get(&key.segment_id) } - pub(crate) fn authorizes(&self, resource: &AclResource, principal: &str) -> bool { - self.security.authorizes(resource, principal) + pub(crate) fn acl_snapshot(&self, resource: &AclResource) -> AclRecord { + self.security.acl_snapshot(resource) } pub(crate) fn get_consumer_group_assignment( @@ -689,7 +689,7 @@ mod tests { } #[test] - fn metadata_authorization_defaults_to_deny() { + fn metadata_acl_snapshot_returns_the_exact_record_or_an_empty_denial() { let mut state = MetadataState::new(ShardGroupId(1)); let resource = AclResource::TopicData(TopicId(42)); state.security.acls.insert( @@ -701,9 +701,16 @@ mod tests { }, ); - assert!(state.authorizes(&resource, "orders-service")); - assert!(!state.authorizes(&resource, "unknown-service")); - assert!(!state.authorizes(&AclResource::TopicData(TopicId(43)), "orders-service")); + assert_eq!( + state.acl_snapshot(&resource).principals, + vec!["orders-service".to_string()].into_boxed_slice() + ); + assert_eq!( + state + .acl_snapshot(&AclResource::TopicData(TopicId(43))) + .revision, + 0 + ); } #[test] @@ -720,12 +727,15 @@ mod tests { state.apply(grant.clone().into()).unwrap(); state.apply(grant.into()).unwrap(); - assert!(state.authorizes(&revoke.resource, "orders-service")); + assert_eq!( + state.acl_snapshot(&revoke.resource).principals, + vec!["orders-service".to_string()].into_boxed_slice() + ); assert_eq!(state.security.acls[&revoke.resource].revision, 1); state.apply(revoke.clone().into()).unwrap(); state.apply(revoke.clone().into()).unwrap(); - assert!(!state.authorizes(&revoke.resource, "orders-service")); + assert!(state.acl_snapshot(&revoke.resource).principals.is_empty()); assert_eq!(state.security.acls[&revoke.resource].revision, 2); } diff --git a/src/control_plane/consensus/raft/states/security.rs b/src/control_plane/consensus/raft/states/security.rs index c42df26f..50fdfb23 100644 --- a/src/control_plane/consensus/raft/states/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -54,14 +54,19 @@ pub(crate) struct RevocationRecord { } impl SecurityState { - /// Returns whether an exact principal is listed on an exact ACL resource. - /// - /// Missing records and missing principals deny by default. Resource - /// hierarchy or wildcard matching is intentionally not inferred here. - pub(crate) fn authorizes(&self, resource: &AclResource, principal: &str) -> bool { + /// Returns the current ACL record, or an empty revision-zero record when + /// the resource has never been granted to any principal. Both forms deny + /// by default; representing absence explicitly lets callers cache that + /// denial and avoid repeatedly querying the owning shard. + pub(crate) fn acl_snapshot(&self, resource: &AclResource) -> AclRecord { self.acls .get(resource) - .is_some_and(|acl| acl.principals.iter().any(|entry| entry == principal)) + .cloned() + .unwrap_or_else(|| AclRecord { + resource: resource.clone(), + revision: 0, + principals: Box::new([]), + }) } pub(super) fn grant(&mut self, resource: AclResource, principal: String) { @@ -162,7 +167,7 @@ mod tests { } #[test] - fn acl_authorization_is_exact_and_defaults_to_deny() { + fn acl_snapshot_returns_the_exact_record_or_an_empty_denial() { let mut security = SecurityState::default(); let resource = AclResource::TopicData(TopicId(42)); security.acls.insert( @@ -174,8 +179,29 @@ mod tests { }, ); - assert!(security.authorizes(&resource, "orders-service")); - assert!(!security.authorizes(&resource, "unknown-service")); - assert!(!security.authorizes(&AclResource::TopicData(TopicId(43)), "orders-service")); + assert_eq!( + security.acl_snapshot(&resource).principals, + vec!["orders-service".to_string()].into_boxed_slice() + ); + assert_eq!( + security + .acl_snapshot(&AclResource::TopicData(TopicId(43))) + .revision, + 0 + ); + } + + #[test] + fn missing_acl_snapshot_is_an_empty_revision_zero_record() { + let resource = AclResource::TopicData(TopicId(42)); + + assert_eq!( + SecurityState::default().acl_snapshot(&resource), + AclRecord { + resource, + revision: 0, + principals: Box::new([]), + } + ); } } diff --git a/src/lib.rs b/src/lib.rs index 43a961d3..46197c79 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,6 +41,7 @@ use crate::net::{TcpListener, TransportTcpStream, UdpSocket}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; use crate::security::NodeTransportSecurity; +use crate::security::acl_cache::SharedAclCache; use crate::{ config::ENV, control_plane::membership::{actor::SwimActor, transport::SwimTransportActor}, @@ -181,6 +182,7 @@ impl StartUp { data_plane_tx: DataPlaneSender, security: NodeTransportSecurity, ) { + let acl_cache = SharedAclCache::default(); let addr = self.env.bind_addr(); let listener = TcpListener::bind(&addr).await.unwrap(); tracing::info!( @@ -202,7 +204,14 @@ impl StartUp { let raft = raft_tx.clone(); let dp = data_plane_tx.clone(); - tokio::spawn(handle_client_stream(stream, node_id, swim_tx, raft, dp)); + tokio::spawn(handle_client_stream( + stream, + node_id, + swim_tx, + raft, + dp, + acl_cache.clone(), + )); } } } diff --git a/src/security/acl_cache.rs b/src/security/acl_cache.rs index a3673434..67788e9a 100644 --- a/src/security/acl_cache.rs +++ b/src/security/acl_cache.rs @@ -1,8 +1,12 @@ use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; +use tokio::sync::RwLock; use tokio::time::Instant; +use crate::connections::protocol::ServerError; +use crate::control_plane::consensus::raft::states::security::AclRecord; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::AclResource; @@ -12,18 +16,24 @@ pub(crate) const MAX_ACL_CACHE_TTL: Duration = Duration::from_secs(60); /// One ACL record copied from its owning metadata shard. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct CachedAcl { +struct CachedAcl { pub(crate) source_shard_id: ShardGroupId, pub(crate) revision: u64, pub(crate) principals: Box<[Box]>, } -/// Result of checking one principal against the local ACL cache. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum CachedAuthorization { - Authorized, - Denied, - Miss, +impl CachedAcl { + fn from_snapshot(source_shard_id: ShardGroupId, snapshot: AclRecord) -> Self { + Self { + source_shard_id, + revision: snapshot.revision, + principals: snapshot + .principals + .into_iter() + .map(String::into_boxed_str) + .collect(), + } + } } #[derive(Debug)] @@ -38,40 +48,36 @@ struct AclCacheEntry { /// currently owns the ACL resource; an entry from a previous owner, an expired /// entry, or no entry at all is a miss and must be refreshed or denied. #[derive(Debug, Default)] -pub(crate) struct AclCache { +struct AclCache { entries: HashMap, } impl AclCache { - pub(crate) fn authorize( + /// Returns `None` when there is no current cache entry for this resource. + fn authorization( &self, resource: &AclResource, source_shard_id: ShardGroupId, principal: &str, now: Instant, - ) -> CachedAuthorization { - let Some(entry) = self.entries.get(resource) else { - return CachedAuthorization::Miss; - }; + ) -> Option { + let entry = self.entries.get(resource)?; if entry.acl.source_shard_id != source_shard_id || entry.expires_at <= now { - return CachedAuthorization::Miss; - } - if entry - .acl - .principals - .iter() - .any(|candidate| candidate.as_ref() == principal) - { - CachedAuthorization::Authorized - } else { - CachedAuthorization::Denied + return None; } + Some( + entry + .acl + .principals + .iter() + .any(|candidate| candidate.as_ref() == principal), + ) } /// Retains the newest known revision from a shard and caps its authority /// window at [`MAX_ACL_CACHE_TTL`]. A record from a newly assigned owner /// replaces the old owner's revision, because revisions are shard-local. - pub(crate) fn insert(&mut self, resource: AclResource, acl: CachedAcl, now: Instant) { + fn insert(&mut self, resource: AclResource, acl: CachedAcl, now: Instant) { let replace = match self.entries.get(&resource) { Some(entry) if entry.acl.source_shard_id == acl.source_shard_id => { entry.acl.revision <= acl.revision @@ -90,6 +96,71 @@ impl AclCache { } } +/// Shareable cache boundary for concurrent client request handlers. +/// +/// The lock protects only an in-memory map lookup or replacement. +/// No network or Raft operation is performed while it is held. +/// Use of `RwLock` over lock-free data structure is jusitified because +/// the dominant cost factor won't be write lock but raft snapshot query +#[derive(Debug, Clone, Default)] +pub(crate) struct SharedAclCache(Arc>); + +impl SharedAclCache { + /// Authorizes from a fresh cache entry, or lazily refreshes the entry and + /// checks it again. The refresh future is created only after a cache miss. + /// A missing snapshot fails closed. + pub(crate) async fn authorize_or_refresh( + &self, + resource: &AclResource, + source_shard_id: ShardGroupId, + principal: &str, + refresh: F, + ) -> Result<(), ServerError> + where + F: AsyncFnOnce() -> Option, + { + if let Some(authorized) = self + .cached_authorization(resource, source_shard_id, principal) + .await + { + return if authorized { + Ok(()) + } else { + Err(ServerError::Unauthorized) + }; + } + + // Cache Miss Case + let Some(snapshot) = refresh().await else { + return Err(ServerError::Unauthorized); + }; + self.0.write().await.insert( + resource.clone(), + CachedAcl::from_snapshot(source_shard_id, snapshot), + Instant::now(), + ); + match self + .cached_authorization(resource, source_shard_id, principal) + .await + { + Some(true) => Ok(()), + Some(false) | None => Err(ServerError::Unauthorized), + } + } + + async fn cached_authorization( + &self, + resource: &AclResource, + source_shard_id: ShardGroupId, + principal: &str, + ) -> Option { + self.0 + .read() + .await + .authorization(resource, source_shard_id, principal, Instant::now()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -110,6 +181,17 @@ mod tests { } } + fn snapshot(resource: AclResource, revision: u64, principals: &[&str]) -> AclRecord { + AclRecord { + resource, + revision, + principals: principals + .iter() + .map(|principal| (*principal).to_owned()) + .collect(), + } + } + #[test] fn authorizes_and_denies_exact_principals_from_a_fresh_entry() { let now = Instant::now(); @@ -121,12 +203,12 @@ mod tests { ); assert_eq!( - cache.authorize(&resource(), ShardGroupId(3), "orders-service", now), - CachedAuthorization::Authorized + cache.authorization(&resource(), ShardGroupId(3), "orders-service", now), + Some(true) ); assert_eq!( - cache.authorize(&resource(), ShardGroupId(3), "billing-service", now), - CachedAuthorization::Denied + cache.authorization(&resource(), ShardGroupId(3), "billing-service", now), + Some(false) ); } @@ -141,13 +223,13 @@ mod tests { ); assert_eq!( - cache.authorize( + cache.authorization( &resource(), ShardGroupId(3), "orders-service", now + MAX_ACL_CACHE_TTL + Duration::from_millis(1), ), - CachedAuthorization::Miss + None ); } @@ -162,8 +244,8 @@ mod tests { ); assert_eq!( - cache.authorize(&resource(), ShardGroupId(4), "orders-service", now), - CachedAuthorization::Miss + cache.authorization(&resource(), ShardGroupId(4), "orders-service", now), + None ); } @@ -183,12 +265,49 @@ mod tests { ); assert_eq!( - cache.authorize(&resource(), ShardGroupId(3), "orders-service", now), - CachedAuthorization::Authorized + cache.authorization(&resource(), ShardGroupId(3), "orders-service", now), + Some(true) ); assert_eq!( - cache.authorize(&resource(), ShardGroupId(3), "billing-service", now), - CachedAuthorization::Denied + cache.authorization(&resource(), ShardGroupId(3), "billing-service", now), + Some(false) + ); + } + + #[tokio::test] + async fn refreshes_a_miss_then_reuses_the_fresh_entry() { + let cache = SharedAclCache::default(); + let resource = resource(); + + assert_eq!( + cache + .authorize_or_refresh(&resource, ShardGroupId(3), "orders-service", || async { + Some(snapshot(resource.clone(), 1, &["orders-service"])) + }) + .await, + Ok(()) + ); + assert_eq!( + cache + .authorize_or_refresh(&resource, ShardGroupId(3), "orders-service", || async { + panic!("fresh entry must not refresh") + }) + .await, + Ok(()) + ); + } + + #[tokio::test] + async fn missing_snapshot_leaves_a_miss_for_fail_closed_handling() { + let cache = SharedAclCache::default(); + + assert_eq!( + cache + .authorize_or_refresh(&resource(), ShardGroupId(3), "orders-service", || async { + None + },) + .await, + Err(ServerError::Unauthorized) ); } } From 8a02bb63bee6a881506dc98d73ae7c88ff934a1f Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 16:15:41 +0400 Subject: [PATCH 37/56] feat(security): refresh ACLs from shard hosts - Read ACLs locally or from the current remote shard host without proxying client data. - Bound and coalesce remote reads; deny on failure or saturation. - Carry the initial Raft RPC or ACL request in the opening frame. --- .claude/rules/raft-transport.md | 49 ++- docs/security/roadmap.md | 72 ++-- src/connections/controller.rs | 189 +++++++--- .../consensus/messages/command.rs | 2 +- src/control_plane/consensus/multi_raft.rs | 10 +- .../consensus/transport/acl/message.rs | 46 +++ .../consensus/transport/acl/mod.rs | 336 ++++++++++++++++++ .../consensus/transport/inbound.rs | 55 +-- src/control_plane/consensus/transport/mod.rs | 202 +++++++++-- .../consensus/transport/outbound.rs | 118 ++++-- .../consensus/transport/protocol.rs | 49 +++ src/control_plane/membership/actor.rs | 25 +- src/lib.rs | 15 +- 13 files changed, 984 insertions(+), 184 deletions(-) create mode 100644 src/control_plane/consensus/transport/acl/message.rs create mode 100644 src/control_plane/consensus/transport/acl/mod.rs create mode 100644 src/control_plane/consensus/transport/protocol.rs diff --git a/.claude/rules/raft-transport.md b/.claude/rules/raft-transport.md index 5498b237..30031394 100644 --- a/.claude/rules/raft-transport.md +++ b/.claude/rules/raft-transport.md @@ -1,27 +1,41 @@ # Raft Transport (Invariants) -`RaftTransportActor` — async TCP transport for Raft RPCs. Manages persistent bidirectional connections between nodes. Each connection split into a reader task and a writer half held in the per-node `writers` map. +`RaftTransportActor` — async TCP transport for Raft RPCs. It manages persistent +bidirectional Raft connections between nodes. Each connection splits into a +reader task and a writer half held in the per-node `writers` map. The same +authenticated cluster listener also serves a one-shot ACL snapshot read used to +refresh a broker's local authorization cache. Separate from SWIM's UDP transport. Raft uses TCP for reliable, ordered delivery. ## Architecture (brief) ``` -RaftTransportActor -├── listener (raft_port, TCP) -├── writers: HashMap (one write half per peer) -└── reader tasks (one per accepted/established connection) +cluster listener (TCP) + │ + ├── initial Raft message ──► persistent reader + one writer per peer + │ + └── initial ACL request ──► read committed ACL → reply → close ``` ## Wire Protocol -Length-prefixed bincode frames: -1. **Handshake** (first frame on every connection): `[len: u32][NodeId: bincode]` -2. **Messages**: `[len: u32][WireRaftMessage: bincode]`, where `WireRaftMessage` carries `shard_group_id` so transport can dispatch to the correct Raft group. +Length-prefixed Borsh frames: +1. **Initial message** (first frame on every connection): either a Raft message + or an ACL snapshot request. + - The initial Raft message carries its sender, which establishes the peer + identity, then later frames are raw `WireRaftMessage` values until close. + Each message carries `shard_group_id` so transport can dispatch it to the + correct Raft group. + - The initial ACL snapshot request carries its requesting node, shard, and + resource. Its response is one `AclSnapshotResponse`, then the connection + closes. ## Invariants -1. **Connection identity is established by handshake before any RPCs.** First frame on every connection is the initiator's `NodeId`. The acceptor uses this to key the writer slot and to detect the simultaneous-connect race. Without the handshake, the acceptor cannot route inbound messages to a peer-identified slot. +1. **Connection identity is established by the initial Raft message.** Its +sender keys the writer slot and detects the simultaneous-connect race. Without +it, the acceptor cannot route later frames to a peer-identified slot. 2. **At most one writer per peer.** `writers` is keyed by `NodeId`. Coexisting writers would split messages to the same peer across two TCP connections; per-connection ordering would let later messages overtake earlier ones in unpredictable patterns, causing the leader to chase its own retries. @@ -29,10 +43,21 @@ Length-prefixed bincode frames: 4. **Address resolution is always live.** Every connect attempt queries SWIM for the peer's current address; the transport keeps no local address cache. A stale local cache would connect to the wrong host after a peer moves or restarts on a different address. -5. **Frame sizes are bounded.** Handshake frames capped at 1KB; message frames at 4MB. Without bounds, a malicious or buggy peer can exhaust memory by sending a giant length prefix before any payload. +5. **Frame sizes are bounded.** Every initial, Raft, and ACL response frame is +capped at 4MB. Without bounds, a malicious or buggy peer can exhaust memory by +sending a giant length prefix before any payload. -6. **Transport validates envelope identity but never interprets the RPC.** The -connection peer must match the envelope `sender`; a mismatch closes that +6. **Transport validates message identity but never interprets the RPC.** The +connection peer must match the message `sender`; a mismatch closes that connection. The transport routes by `shard_group_id` and passes the authenticated peer onward, but the RPC remains opaque. Voter, learner, leader, term, and log checks belong to the target Raft state machine. + +## ACL Snapshot Rule + +An ACL snapshot request is not a Raft RPC and never enters a Raft state machine. +It asks the local multi-Raft actor for the selected shard's committed ACL record, +returns that record on the same authenticated connection, then closes the +connection. It carries no client data request and cannot proxy one. A cluster +connection that begins with a Raft message carries only raw Raft frames after +that message; an invalid frame closes the connection. diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 7efaf1ec..4a0e0402 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -23,7 +23,7 @@ The production boundary defends against external network attackers attempting ea | Listener | Port | Protocol | Peer Authentication | Purpose | | :--- | :--- | :--- | :--- | :--- | | **Client** | TCP 2921 | TLS 1.3 | Mutual X.509 | Metadata queries, administration, produce, fetch | -| **Raft** | TCP 2922 | TLS 1.3 | Mutual X.509 | Metadata shard consensus log replication | +| **Raft** | TCP 2922 | TLS 1.3 | Mutual X.509 | Metadata shard consensus log replication and one-shot ACL-cache refreshes between brokers | | **Data** | TCP 2923 | TLS 1.3 | Mutual X.509 | Segment replication, repair, and coordination | | **SWIM** | UDP 2922 | Secure datagrams (deferred) | Mutual X.509 | Membership gossip and failure detection | @@ -178,8 +178,8 @@ security/node/{node-certificate-principal} Data permissions use the stable topic ID, so a data replica can authorize a request without hosting that topic's metadata shard. - **Freshness & Expiry:** Cached records include a monotonic deadline (max 60s) - and revision counter. An expired entry cannot authorize a request. The chosen - distribution mechanism must obtain current state or fail closed. + and revision counter. An expired entry cannot authorize a request. The broker + reads the current owner directly when needed, or fails closed. --- @@ -235,12 +235,9 @@ Check local ACL cache └── Missing / expired │ ▼ - Is the ACL shard local? - ├── No ──► Return unauthorized - └── Yes - │ - ▼ - Read committed ACL state + Read committed ACL state + ├── Local shard ─────────► Local metadata read + └── Remote shard ────────► One authenticated node request │ ▼ Refresh cache @@ -256,22 +253,47 @@ Check local ACL cache ``` Authorization precedes redirects so an ungranted client cannot use stale-route -responses to discover data placement. The current implementation refreshes a -missing or expired cache entry only when the broker hosts the ACL's metadata -shard. It reads that shard's committed ACL record, including an empty record for -default denial, and stores it for no more than 60 seconds. When another broker -owns the ACL shard, the request fails closed; it does not forward or proxy the -client data operation. - -Fetching an ACL record from a remote owner is still deferred. That later path may -use lazy pull, proactive push, or a push-and-pull hybrid. It may update only the -ACL cache; it must never carry or execute the client's data operation: - -| Model | Benefit | Failure to handle | -| :--- | :--- | :--- | -| Pull on cache miss or expiry | Simple; clients that leave create no update traffic | A data node must fail closed if the owner is unavailable | -| Push changed records | Fast local decisions | A disconnected data node can miss an update | -| Push plus periodic pull | Fast normally; repairs missed updates | More protocol and cache synchronization logic | +responses to discover data placement. A missing or expired entry triggers a +lazy pull from the ACL shard's current host. A local host reads its committed +state directly; a remote host receives one short authenticated node request and +returns only the ACL record. The request never carries client data and never +turns the broker into a data proxy. A missing record is a bounded cached denial; +an unavailable owner also denies the request. + +``` +Data broker ACL shard host + │ │ + │── authenticated ACL-only read ──────►│ + │ │── read committed record + │◄──────── record or unavailable ──────│ + │ + └── refresh local cache (at most 60s) ──► allow or deny client request +``` + +This pull-on-miss path is deliberately small: cache expiry is 60 seconds, so a +second connection pool would add persistent state for infrequent reads. A future +push or hybrid distribution scheme remains optional; it must preserve the same +fail-closed behavior and update only the ACL cache. + +One broker-local refresh worker owns these remote reads. It bounds active and +queued work, and combines simultaneous requests for the same ACL record into +one read. When that worker is full, stopped, or too slow, callers deny rather +than opening more connections: + +``` +many cache misses + │ + ▼ + ACL refresh worker + ├── same shard + resource ──► one read, reply to all waiters + ├── different records ──────► bounded active work and queue + └── full / unavailable ─────► deny +``` + +The worker combines only identical records. Several different records owned by +the same remote shard host can still create several short connections. If that +becomes material, the next step is one request containing several resources for +the same owner and shard—not delaying mailbox reads or adding a connection pool. The same decision gate covers request-rate limiting. A Kubernetes deployment may give all replicas of one application a shared principal even though each replica diff --git a/src/connections/controller.rs b/src/connections/controller.rs index cc8ac528..7768f959 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -3,6 +3,7 @@ use crate::connections::writer::ClientRawWriter; use crate::connections::{protocol::*, run_client_writer}; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::errors::ProposalError; +use crate::control_plane::consensus::transport::AclSnapshotSender; use crate::control_plane::metadata::{ AclResource, ConsumerGroupResource, OpenProducerSession, ProducerSessionOwner, RangeMeta, TopicState, UpdateConsumerGroupMember, UpdateConsumerGroupMemberRequest, @@ -11,7 +12,7 @@ use crate::control_plane::{ NodeId, SwimNodeState, consensus::actor::MutlRaftSender, membership::{ - ShardGroupId, + ShardGroup, ShardGroupId, actor::{ShardRouting, SwimSender}, }, metadata::{ @@ -56,6 +57,7 @@ pub struct ClientController { raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, acl_cache: SharedAclCache, + acl_snapshot_sender: AclSnapshotSender, } impl ClientController { @@ -66,6 +68,7 @@ impl ClientController { raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, acl_cache: SharedAclCache, + acl_snapshot_sender: AclSnapshotSender, ) -> Self { Self { transport_identity, @@ -74,6 +77,7 @@ impl ClientController { raft_sender, data_plane_tx, acl_cache, + acl_snapshot_sender, } } @@ -151,12 +155,9 @@ impl ClientController { let command: OpenProducerSession = req.into_command(owner.clone()); - let group = match self.route(command.topic_name.as_bytes().to_vec()).await? { - ShardRouting::Local(group) => group, - ShardRouting::Redirect(member) => { - return Err(self.control_plane_redirect(member)); - } - }; + let group = self + .route_local(command.topic_name.as_bytes().to_vec()) + .await?; let topic_meta = self .raft_sender @@ -193,12 +194,7 @@ impl ClientController { &self, req: UpdateConsumerGroupMemberRequest, ) -> Result { - let group = match self.route(req.topic_name.as_bytes().to_vec()).await? { - ShardRouting::Local(group) => group, - ShardRouting::Redirect(member) => { - return Err(self.control_plane_redirect(member)); - } - }; + let group = self.route_local(req.topic_name.as_bytes().to_vec()).await?; let topic = self .raft_sender @@ -238,9 +234,7 @@ impl ClientController { /// otherwise it redirects to a member so the consumer retries against the right /// node — no server-side proxying. async fn describe_topic(&self, topic_name: String) -> Result { - if let ShardRouting::Redirect(member) = self.route(topic_name.as_bytes().to_vec()).await? { - return Err(self.control_plane_redirect(member)); - } + self.route_local(topic_name.as_bytes().to_vec()).await?; let topic = self.raft_sender.get_topic_metadata(topic_name).await?; self.authorize_acl_resource(AclResource::TopicAdmin(topic.id)) @@ -256,12 +250,7 @@ impl ClientController { name: String, storage_policy: StoragePolicy, ) -> Result { - let group = match self.route(name.as_bytes().to_vec()).await? { - ShardRouting::Local(group) => group, - ShardRouting::Redirect(member) => { - return Err(self.control_plane_redirect(member)); - } - }; + let group = self.route_local(name.as_bytes().to_vec()).await?; // A new topic has no stable ID yet, so its creator needs the cluster-wide // grant. Once created, topic-admin/{topic-id} governs its metadata. self.authorize_acl_resource(AclResource::Cluster).await?; @@ -282,12 +271,7 @@ impl ClientController { } async fn delete_topic(&self, topic_name: String) -> Result { - let group = match self.route(topic_name.as_bytes().to_vec()).await? { - ShardRouting::Local(group) => group, - ShardRouting::Redirect(member) => { - return Err(self.control_plane_redirect(member)); - } - }; + let group = self.route_local(topic_name.as_bytes().to_vec()).await?; let topic = self .raft_sender .get_topic_metadata(topic_name.clone()) @@ -308,21 +292,48 @@ impl ClientController { .await } + /// Resolves a local metadata shard or returns a redirect for a normal + /// client request. ACL authorization needs the richer route directly. + async fn route_local(&self, key: Vec) -> Result { + match self.route(key).await? { + ShardRouting::Local(group) => Ok(group), + ShardRouting::Redirect(remote) => { + Err(self.control_plane_redirect(remote.and_then(|remote| remote.member))) + } + } + } + async fn authorize_acl_resource(&self, resource: AclResource) -> Result<(), ServerError> { let TransportIdentity::CertificatePrincipal(principal) = &self.transport_identity else { return Ok(()); }; - let ShardRouting::Local(group) = self.route(resource.routing_key()).await? else { - return Err(ServerError::Unauthorized); - }; - - self.acl_cache - .authorize_or_refresh(&resource, group.id, principal, || { - self.raft_sender - .get_acl_snapshot(group.id, resource.clone()) - }) - .await + match self.route(resource.routing_key()).await? { + ShardRouting::Local(group) => { + self.acl_cache + .authorize_or_refresh(&resource, group.id, principal, || { + self.raft_sender + .get_acl_snapshot(group.id, resource.clone()) + }) + .await + } + ShardRouting::Redirect(Some(remote)) => { + let Some(owner) = remote.member else { + return Err(ServerError::Unauthorized); + }; + self.acl_cache + .authorize_or_refresh(&resource, remote.group_id, principal, || { + self.acl_snapshot_sender.fetch( + self.node_id.clone(), + owner, + remote.group_id, + resource.clone(), + ) + }) + .await + } + ShardRouting::Redirect(None) => Err(ServerError::Unauthorized), + } } /// Structural redirect for a control-plane op that isn't local: to the member if @@ -465,10 +476,12 @@ impl ClientController { let received_at_ms = crate::now_ms(); // Not local (ring unconverged or this node isn't a member) → retriable // redirect; the hint is best-effort, absent until SWIM converges. - if let ShardRouting::Redirect(hint_node) = + if let ShardRouting::Redirect(remote) = self.route(req.topic_name.as_bytes().to_vec()).await? { - return Err(ServerError::ShardNotLocal { hint_node }); + return Err(ServerError::ShardNotLocal { + hint_node: remote.and_then(|remote| remote.member), + }); } let topic = self.raft_sender.get_topic_metadata(req.topic_name).await?; @@ -700,6 +713,7 @@ pub async fn handle_client_stream( raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, acl_cache: SharedAclCache, + acl_snapshot_sender: AclSnapshotSender, ) { let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); @@ -711,6 +725,7 @@ pub async fn handle_client_stream( raft_sender, data_plane_tx, acl_cache, + acl_snapshot_sender, ); tokio::spawn(run_client_writer( ClientRawWriter::new(write_half), @@ -731,6 +746,7 @@ mod tests { use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::MultiRaftActorCommand; use crate::control_plane::consensus::raft::states::security::AclRecord; + use crate::control_plane::consensus::transport::{AclSnapshotActor, RaftTransportActor}; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ QueryCommand, ShardGroup, ShardGroupId, ShardLeaderEntry, SwimActorCommand, @@ -744,9 +760,14 @@ mod tests { use crate::data_plane::auxiliary_states::consumer_offsets::state::ConsumerOffsetKey; use crate::data_plane::messages::DataPlaneMessage; use crate::data_plane::messages::command::{DataPlaneCommand, ProduceAck}; + use crate::net::TcpListener; + use crate::security::NodeTransportSecurity; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + use tokio::sync::Notify; + use turmoil::Builder; fn addr(port: u16) -> SocketAddr { format!("127.0.0.1:{port}").parse().unwrap() @@ -814,6 +835,8 @@ mod tests { raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, ) -> ClientController { + let acl_snapshot_sender = + AclSnapshotActor::spawn(NodeTransportSecurity::TrustedDevelopment); ClientController::new( TransportIdentity::TrustedDevelopment, node_id, @@ -821,6 +844,7 @@ mod tests { raft_sender, data_plane_tx, SharedAclCache::default(), + acl_snapshot_sender, ) } @@ -846,6 +870,8 @@ mod tests { raft_sender: MutlRaftSender, acl_cache: SharedAclCache, ) -> ClientController { + let acl_snapshot_sender = + AclSnapshotActor::spawn(NodeTransportSecurity::TrustedDevelopment); ClientController::new( TransportIdentity::CertificatePrincipal(principal.into()), node_id, @@ -853,6 +879,7 @@ mod tests { raft_sender, dp_stub(), acl_cache, + acl_snapshot_sender, ) } @@ -1181,6 +1208,88 @@ mod tests { ); } + #[test] + fn certificate_client_refreshes_acl_from_remote_shard_host() -> turmoil::Result { + let resource = AclResource::TopicData(TopicId(7)); + let snapshot = acl_snapshot(resource.clone(), &["orders-service"]); + let response_received = Arc::new(Notify::new()); + let mut sim = Builder::new() + .simulation_duration(Duration::from_secs(5)) + .build(); + + let owner_snapshot = snapshot.clone(); + let owner_resource = resource.clone(); + let owner_response_received = response_received.clone(); + sim.host("owner", move || { + let expected_snapshot = owner_snapshot.clone(); + let expected_resource = owner_resource.clone(); + let owner_completion = owner_response_received.clone(); + async move { + let listener = TcpListener::bind("0.0.0.0:9000").await?; + let (raft_tx, mut raft_rx) = MultiRaftActor::channel(8); + let (_transport_tx, transport_rx) = mpsc::channel(1); + let (swim_tx, _swim_rx) = SwimActor::channel(1); + tokio::spawn(RaftTransportActor::run( + node_id("owner"), + listener, + raft_tx, + transport_rx, + swim_tx, + NodeTransportSecurity::TrustedDevelopment, + )); + + let Some(MultiRaftActorCommand::GetAclSnapshot(query)) = raft_rx.recv().await + else { + panic!("expected remote ACL snapshot query"); + }; + assert_eq!(query.shard_group_id, ShardGroupId(42)); + assert_eq!(query.resource, expected_resource); + let _ = query.reply.send(Some(expected_snapshot)); + owner_completion.notified().await; + Ok(()) + } + }); + + let requester_resource = resource.clone(); + sim.host("requester", move || { + let requested_resource = requester_resource.clone(); + let requester_completion = response_received.clone(); + async move { + let owner_addr = turmoil::lookup("owner"); + let group = ShardGroup { + id: ShardGroupId(42), + replicas: Replicas::new(vec![node_id("owner")]), + }; + let owner = NodeAddress::test( + SocketAddr::new(owner_addr, 9000), + SocketAddr::new(owner_addr, 9001), + ); + let swim = swim_sender_with(move |cmd| match cmd { + SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) => { + let _ = reply.send(Some(group.clone())); + } + SwimActorCommand::Query(QueryCommand::ResolveAddress { reply, .. }) => { + let _ = reply.send(Some(owner)); + } + _ => {} + }); + let controller = authenticated_controller( + "orders-service", + node_id("requester"), + swim, + raft_sender_with(|_| panic!("remote ACL refresh must not query local Raft")), + ); + + let result = controller.authorize_acl_resource(requested_resource).await; + requester_completion.notify_one(); + assert_eq!(result, Ok(())); + Ok(()) + } + }); + + sim.run() + } + /// Ring can't map the key yet (topology not converged) → retriable /// `ShardNotLocal` with no hint, not `TopicNotFound`. #[tokio::test] diff --git a/src/control_plane/consensus/messages/command.rs b/src/control_plane/consensus/messages/command.rs index dc0c2f02..d68aa117 100644 --- a/src/control_plane/consensus/messages/command.rs +++ b/src/control_plane/consensus/messages/command.rs @@ -8,7 +8,7 @@ use crate::{impl_from_variant, impl_new_struct_wrapper}; pub struct InboundRaftRpc { pub shard_group_id: ShardGroupId, - pub from: NodeId, + pub peer_id: NodeId, pub rpc: RaftRpc, } diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index cf5aa19a..86643883 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -422,10 +422,10 @@ impl MultiRaft { tracing::info!("[{}] Removed Raft group {:?}", self.node_id, group_id); } - #[tracing::instrument(level = "trace", skip_all, fields(group = cmd.shard_group_id.0, from = %cmd.from))] + #[tracing::instrument(level = "trace", skip_all, fields(group = cmd.shard_group_id.0, from = %cmd.peer_id))] fn handle_rpc(&mut self, cmd: InboundRaftRpc) { if let Some(raft) = self.groups.get_mut(&cmd.shard_group_id) { - raft.handle_rpc(cmd.from, cmd.rpc); + raft.handle_rpc(cmd.peer_id, cmd.rpc); self.dirty.insert(cmd.shard_group_id); } } @@ -1376,7 +1376,7 @@ mod tests { // n2 is acting as leader at term 1. Send two entries to n1 (follower). store.handle_consensus(InboundRaftRpc { shard_group_id: TEST_GROUP_ID, - from: n2.clone(), + peer_id: n2.clone(), rpc: RaftRpc::AppendEntries(AppendEntries { term: 1, leader_id: n2.clone(), @@ -1411,7 +1411,7 @@ mod tests { // Raft truncates from index 1 and replaces with the new entry. store.handle_consensus(InboundRaftRpc { shard_group_id: TEST_GROUP_ID, - from: n2.clone(), + peer_id: n2.clone(), rpc: RaftRpc::AppendEntries(AppendEntries { term: 2, leader_id: n2.clone(), @@ -2150,7 +2150,7 @@ mod tests { // recovery roll, and the leader-gated ring-check won't fire to prune it. store.handle_consensus(InboundRaftRpc { shard_group_id: TEST_GROUP_ID, - from: peer.clone(), + peer_id: peer.clone(), rpc: RaftRpc::AppendEntries(AppendEntries { term: 99, leader_id: peer, diff --git a/src/control_plane/consensus/transport/acl/message.rs b/src/control_plane/consensus/transport/acl/message.rs new file mode 100644 index 00000000..a270ede2 --- /dev/null +++ b/src/control_plane/consensus/transport/acl/message.rs @@ -0,0 +1,46 @@ +use tokio::sync::oneshot; + +use crate::control_plane::{ + NodeAddressInfo, NodeId, consensus::raft::states::security::AclRecord, + membership::ShardGroupId, metadata::AclResource, +}; + +pub(super) struct AclSnapshotFetchRequest { + pub(super) fetch: AclSnapshotFetch, + pub(super) reply: oneshot::Sender>, +} + +#[derive(Clone, Debug)] +pub(super) struct AclSnapshotFetch { + pub(super) node_id: NodeId, + pub(super) key: AclSnapshotKey, + pub(super) owner: NodeAddressInfo, +} +impl AclSnapshotFetch { + pub(crate) fn new( + node_id: NodeId, + shard_group_id: ShardGroupId, + resource: AclResource, + owner: NodeAddressInfo, + ) -> Self { + AclSnapshotFetch { + node_id, + key: AclSnapshotKey { + shard_group_id, + resource, + }, + owner, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) struct AclSnapshotKey { + pub(super) shard_group_id: ShardGroupId, + pub(super) resource: AclResource, +} + +pub(super) struct AclSnapshotCompleted { + pub(super) key: AclSnapshotKey, + pub(super) snapshot: Option, +} diff --git a/src/control_plane/consensus/transport/acl/mod.rs b/src/control_plane/consensus/transport/acl/mod.rs new file mode 100644 index 00000000..40d617cd --- /dev/null +++ b/src/control_plane/consensus/transport/acl/mod.rs @@ -0,0 +1,336 @@ +mod message; + +use anyhow::{Context, Result}; +use std::collections::{HashMap, HashSet, VecDeque}; +use tokio::io::AsyncWriteExt; +use tokio::sync::{mpsc, oneshot}; + +use crate::control_plane::NodeAddressInfo; +use crate::control_plane::NodeId; +use crate::control_plane::consensus::raft::states::security::AclRecord; +use crate::control_plane::membership::ShardGroupId; +use crate::control_plane::metadata::AclResource; +use crate::net::TransportTcpStream; +use crate::security::NodeTransportSecurity; +use message::*; + +use super::inbound::ClusterMessageReader; +use super::protocol::{AclSnapshotRequest, InitialClusterMessage, encode_frame}; + +const ACL_SNAPSHOT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); +const MAX_IN_FLIGHT_FETCHES: usize = 16; +const MAX_QUEUED_FETCHES: usize = 128; +const MAX_WAITERS_PER_FETCH: usize = 256; +const ACL_SNAPSHOT_MAILBOX_CAPACITY: usize = 256; + +/// Bounded async boundary for remote ACL cache refreshes. +/// +/// A cache miss never opens a socket from a client-request task directly. This +/// actor limits concurrent dials and makes all callers waiting for the same +/// record share one read and one response. +pub(crate) struct AclSnapshotActor; + +impl AclSnapshotActor { + pub(crate) fn spawn(security: NodeTransportSecurity) -> AclSnapshotSender { + let (sender, mailbox) = mpsc::channel(ACL_SNAPSHOT_MAILBOX_CAPACITY); + tokio::spawn(Self::run(security, mailbox)); + AclSnapshotSender(sender) + } + + async fn run( + security: NodeTransportSecurity, + mut mailbox: mpsc::Receiver, + ) { + let (completed_tx, mut completed_rx) = + mpsc::channel::(MAX_IN_FLIGHT_FETCHES); + let mut state = AclSnapshotState::new(security); + let mut requests = Vec::with_capacity(64); + + loop { + tokio::select! { + count = mailbox.recv_many(&mut requests, 64) => { + if count == 0 { + break; + } + for request in requests.drain(..) { + state.request_snapshot(request); + } + } + Some(completed) = completed_rx.recv() => { + state.complete(completed); + } + } + + for fetch in state.take_pending() { + let completed_tx = completed_tx.clone(); + let sec = state.security.clone(); + tokio::spawn(async move { + let key = fetch.key.clone(); + let snapshot = AclSnapshotActor::fetch_snapshot(sec, fetch).await; + let _ = completed_tx + .send(AclSnapshotCompleted { key, snapshot }) + .await; + }); + } + } + } + + async fn fetch_snapshot( + security: NodeTransportSecurity, + fetch: AclSnapshotFetch, + ) -> Option { + let owner_id = fetch.owner.node_id.clone(); + match tokio::time::timeout( + ACL_SNAPSHOT_TIMEOUT, + Self::fetch_snapshot_inner(security, fetch), + ) + .await + { + Ok(Ok(snapshot)) => snapshot, + Ok(Err(error)) => { + tracing::debug!(owner = %owner_id, "ACL snapshot fetch failed: {error}"); + None + } + Err(_) => { + tracing::debug!(owner = %owner_id, "ACL snapshot fetch timed out"); + None + } + } + } + + async fn fetch_snapshot_inner( + security: NodeTransportSecurity, + fetch: AclSnapshotFetch, + ) -> Result> { + let stream = + TransportTcpStream::connect_node(fetch.owner.cluster_addr(), &security).await?; + let transport_identity = stream.peer_identity(); + let (read_half, mut write_half) = stream.into_split(); + write_half + .write_all(&encode_frame(&InitialClusterMessage::AclSnapshot( + AclSnapshotRequest { + requester_node_id: fetch.node_id, + shard_group_id: fetch.key.shard_group_id, + resource: fetch.key.resource, + }, + ))?) + .await + .context("write initial ACL snapshot request")?; + + let mut reader = ClusterMessageReader::new(read_half, transport_identity); + Ok(reader.read_acl_snapshot_response().await?.snapshot) + } +} + +/// Scheduling state for bounded, coalesced snapshot reads. +struct AclSnapshotState { + security: NodeTransportSecurity, + active: HashSet, + queued: VecDeque, + waiters: HashMap>>>, + pending_events: Vec, +} + +impl AclSnapshotState { + fn new(security: NodeTransportSecurity) -> Self { + Self { + security, + active: HashSet::with_capacity(MAX_IN_FLIGHT_FETCHES), + queued: VecDeque::with_capacity(MAX_QUEUED_FETCHES), + waiters: HashMap::with_capacity(MAX_WAITERS_PER_FETCH), + pending_events: Vec::new(), + } + } + + fn request_snapshot(&mut self, request: AclSnapshotFetchRequest) { + let AclSnapshotFetchRequest { fetch, reply } = request; + + if let Some(waiting) = self.waiters.get_mut(&fetch.key) { + if waiting.len() == MAX_WAITERS_PER_FETCH { + tracing::debug!(?fetch.key, "ACL snapshot refresh has too many waiting callers"); + let _ = reply.send(None); + return; + } + waiting.push(reply); + return; + } + + if self.active.len() < MAX_IN_FLIGHT_FETCHES { + self.active.insert(fetch.key.clone()); + self.waiters.insert(fetch.key.clone(), vec![reply]); + self.pending_events.push(fetch); + return; + } + if self.queued.len() < MAX_QUEUED_FETCHES { + self.waiters.insert(fetch.key.clone(), vec![reply]); + self.queued.push_back(fetch); + return; + } + tracing::debug!(?fetch.key, "ACL snapshot refresh queue is full"); + let _ = reply.send(None); + } + + fn complete(&mut self, completed: AclSnapshotCompleted) { + debug_assert!(self.active.remove(&completed.key)); + if let Some(waiting) = self.waiters.remove(&completed.key) { + for reply in waiting { + let _ = reply.send(completed.snapshot.clone()); + } + } else { + tracing::debug!("ACL snapshot completed without waiting callers"); + } + if let Some(next) = self.queued.pop_front() { + self.active.insert(next.key.clone()); + self.pending_events.push(next); + } + } + + fn take_pending(&mut self) -> Vec { + std::mem::take(&mut self.pending_events) + } +} + +/// Sends bounded ACL snapshot refresh requests to [`AclSnapshotActor`]. +#[derive(Clone)] +pub(crate) struct AclSnapshotSender(mpsc::Sender); + +impl AclSnapshotSender { + /// Reads one committed ACL record from a shard host. + /// + /// A saturated or stopped refresh actor fails closed. The actor logs the + /// reason and coalesces concurrent reads for the same shard resource. + pub(crate) async fn fetch( + &self, + node_id: NodeId, + owner: NodeAddressInfo, + shard_group_id: ShardGroupId, + resource: AclResource, + ) -> Option { + let (reply, recv) = oneshot::channel(); + match self.0.try_send(AclSnapshotFetchRequest { + fetch: AclSnapshotFetch::new(node_id, shard_group_id, resource, owner), + reply, + }) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(_)) => { + tracing::debug!("ACL snapshot refresh actor mailbox is full"); + return None; + } + Err(mpsc::error::TrySendError::Closed(_)) => { + tracing::debug!("ACL snapshot refresh actor is stopped"); + return None; + } + } + match tokio::time::timeout(ACL_SNAPSHOT_TIMEOUT, recv).await { + Ok(Ok(snapshot)) => snapshot, + Ok(Err(_)) => { + tracing::debug!("ACL snapshot refresh actor stopped before replying"); + None + } + Err(_) => { + tracing::debug!("ACL snapshot refresh did not complete before its deadline"); + None + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control_plane::NodeAddress; + use crate::control_plane::metadata::TopicId; + use std::net::SocketAddr; + use tokio::sync::oneshot; + + fn fetch(topic_id: u64) -> AclSnapshotFetch { + AclSnapshotFetch { + node_id: NodeId::new("requester"), + key: AclSnapshotKey { + shard_group_id: ShardGroupId(42), + resource: AclResource::TopicData(TopicId(topic_id)), + }, + owner: NodeAddressInfo::new( + NodeId::new("owner"), + NodeAddress::test( + "127.0.0.1:9000".parse::().unwrap(), + "127.0.0.1:9001".parse::().unwrap(), + ), + ), + } + } + + fn request( + topic_id: u64, + ) -> ( + AclSnapshotFetchRequest, + oneshot::Receiver>, + ) { + let (reply, receiver) = oneshot::channel(); + ( + AclSnapshotFetchRequest { + fetch: fetch(topic_id), + reply, + }, + receiver, + ) + } + + #[test] + fn joins_identical_fetches_without_starting_another_dial() { + let mut state = AclSnapshotState::new(NodeTransportSecurity::TrustedDevelopment); + let (first_request, mut first_reply) = request(7); + let (second_request, mut second_reply) = request(7); + + state.request_snapshot(first_request); + state.request_snapshot(second_request); + + assert_eq!(state.take_pending().len(), 1); + state.complete(AclSnapshotCompleted { + key: fetch(7).key, + snapshot: None, + }); + assert_eq!(first_reply.try_recv(), Ok(None)); + assert_eq!(second_reply.try_recv(), Ok(None)); + } + + #[test] + fn completion_starts_the_next_queued_fetch() { + let mut state = AclSnapshotState::new(NodeTransportSecurity::TrustedDevelopment); + let first = fetch(0); + let (first_request, _first_reply) = request(0); + state.request_snapshot(first_request); + for topic_id in 1..MAX_IN_FLIGHT_FETCHES as u64 { + let (request, _reply) = request(topic_id); + state.request_snapshot(request); + } + let (queued_request, _queued_reply) = request(99); + state.request_snapshot(queued_request); + assert_eq!(state.take_pending().len(), MAX_IN_FLIGHT_FETCHES); + + state.complete(AclSnapshotCompleted { + key: first.key, + snapshot: None, + }); + + let started = state.take_pending(); + assert_eq!(started.len(), 1); + let [started] = started.as_slice() else { + panic!("expected one fetch event"); + }; + assert_eq!(started.key, fetch(99).key); + } + + #[test] + fn rejects_a_new_fetch_when_the_queue_is_full() { + let mut state = AclSnapshotState::new(NodeTransportSecurity::TrustedDevelopment); + for topic_id in 0..MAX_IN_FLIGHT_FETCHES as u64 + MAX_QUEUED_FETCHES as u64 { + let (request, _reply) = request(topic_id); + state.request_snapshot(request); + } + + let (request, mut reply) = request(999); + state.request_snapshot(request); + assert_eq!(reply.try_recv(), Ok(None)); + } +} diff --git a/src/control_plane/consensus/transport/inbound.rs b/src/control_plane/consensus/transport/inbound.rs index f4ad23de..a781aa9d 100644 --- a/src/control_plane/consensus/transport/inbound.rs +++ b/src/control_plane/consensus/transport/inbound.rs @@ -4,14 +4,17 @@ use crate::control_plane::consensus::messages::InboundRaftRpc; use crate::control_plane::consensus::messages::WireRaftMessage; use crate::net::TransportReadHalf; use crate::security::TransportIdentity; +use borsh::BorshDeserialize; use tokio::io::AsyncReadExt; -pub(super) struct RaftRpcListener { +use super::protocol::{AclSnapshotResponse, InitialClusterMessage}; + +pub(super) struct ClusterMessageReader { read_half: TransportReadHalf, transport_identity: TransportIdentity, } -impl RaftRpcListener { +impl ClusterMessageReader { pub(super) fn new( read_half: impl Into, transport_identity: TransportIdentity, @@ -22,25 +25,35 @@ impl RaftRpcListener { } } - pub(super) async fn read_node_id(&mut self) -> anyhow::Result { - let len = self.read_half.read_u32().await? as usize; - anyhow::ensure!(len <= 1024, "NodeId frame too large: {len} bytes"); - let mut buf = vec![0u8; len]; - self.read_half.read_exact(&mut buf).await?; - let id = borsh::from_slice::(&buf)?; - Ok(id) + pub(super) async fn read_initial_message(&mut self) -> anyhow::Result { + self.read_frame(4 * 1024 * 1024, "initial cluster message") + .await + } + + pub(super) async fn read_raft_message(&mut self) -> anyhow::Result { + self.read_frame(4 * 1024 * 1024, "Raft message").await + } + + pub(super) async fn read_acl_snapshot_response( + &mut self, + ) -> anyhow::Result { + self.read_frame(4 * 1024 * 1024, "ACL snapshot response") + .await } - pub(super) async fn read_message(&mut self) -> anyhow::Result { + async fn read_frame( + &mut self, + maximum_size: usize, + frame_name: &str, + ) -> anyhow::Result { let len = self.read_half.read_u32().await? as usize; anyhow::ensure!( - len <= 4 * 1024 * 1024, - "Raft message frame too large: {len} bytes" + len <= maximum_size, + "{frame_name} frame too large: {len} bytes" ); let mut buf = vec![0u8; len]; self.read_half.read_exact(&mut buf).await?; - let msg = borsh::from_slice::(&buf)?; - Ok(msg) + Ok(borsh::from_slice(&buf)?) } #[tracing::instrument( @@ -50,21 +63,21 @@ impl RaftRpcListener { )] pub(super) async fn run(mut self, tx: MutlRaftSender, peer: NodeId) { loop { - match self.read_message().await { - Ok(msg) => { - if msg.sender != peer { + match self.read_raft_message().await { + Ok(message) => { + if message.sender != peer { tracing::warn!( transport_peer = %peer, - claimed_sender = %msg.sender, + claimed_sender = %message.sender, "rejected Raft message whose sender differs from the connection peer", ); break; } let _ = tx .send(InboundRaftRpc { - shard_group_id: msg.shard_group_id, - from: peer.clone(), - rpc: msg.rpc, + shard_group_id: message.shard_group_id, + peer_id: peer.clone(), + rpc: message.rpc, }) .await; } diff --git a/src/control_plane/consensus/transport/mod.rs b/src/control_plane/consensus/transport/mod.rs index 33d64812..cd4d411b 100644 --- a/src/control_plane/consensus/transport/mod.rs +++ b/src/control_plane/consensus/transport/mod.rs @@ -1,9 +1,12 @@ #![allow(dead_code)] +mod acl; +pub(crate) use acl::{AclSnapshotActor, AclSnapshotSender}; mod inbound; use inbound::*; mod outbound; use outbound::*; +mod protocol; use tokio::sync::mpsc; @@ -74,16 +77,21 @@ impl RaftTransportActor { #[cfg(test)] mod tests { + use super::protocol::InitialClusterMessage; use super::*; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::{ MultiRaftActorCommand, RaftProtocolMessage, RaftRpc, RequestVote, WireRaftMessage, }; + use crate::control_plane::consensus::raft::states::security::AclRecord; use crate::control_plane::membership::ShardGroupId; + use crate::control_plane::metadata::{AclResource, TopicId}; + use crate::control_plane::{NodeAddress, NodeAddressInfo}; use crate::net::OwnedWriteHalf; use crate::net::TcpStream; use std::time::Duration; use tokio::io::AsyncWriteExt; + use tokio::sync::Notify; use turmoil::Builder; /// Write a length-prefixed borsh-encoded value to a raw write half. @@ -99,8 +107,21 @@ mod tests { Ok(()) } + fn request_vote_message(shard_group_id: u64, sender: &str) -> WireRaftMessage { + WireRaftMessage { + shard_group_id: ShardGroupId(shard_group_id), + sender: NodeId::new(sender), + rpc: RaftRpc::RequestVote(RequestVote { + term: 1, + candidate_id: NodeId::new(sender), + last_log_index: 0, + last_log_term: 0, + }), + } + } + #[test] - fn handshake_write_then_read_node_id() -> turmoil::Result { + fn initial_raft_message_identifies_the_peer() -> turmoil::Result { let mut sim = Builder::new() .simulation_duration(Duration::from_secs(5)) .build(); @@ -109,10 +130,14 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); + let mut reader = + ClusterMessageReader::new(read_half, TransportIdentity::TrustedDevelopment); - let peer_id = reader.read_node_id().await.unwrap(); - assert_eq!(peer_id, NodeId::new("node-abc")); + let InitialClusterMessage::Raft(message) = reader.read_initial_message().await.unwrap() + else { + panic!("expected initial Raft message"); + }; + assert_eq!(message.sender, NodeId::new("node-abc")); Ok(()) }); @@ -121,7 +146,11 @@ mod tests { let stream = TcpStream::connect((addr, 9000)).await?; let (_, mut write_half) = stream.into_split(); - write_frame(&mut write_half, &NodeId::new("node-abc")).await?; + write_frame( + &mut write_half, + &InitialClusterMessage::Raft(request_vote_message(42, "node-abc")), + ) + .await?; Ok(()) }); @@ -138,9 +167,10 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); + let mut reader = + ClusterMessageReader::new(read_half, TransportIdentity::TrustedDevelopment); - let msg = reader.read_message().await.unwrap(); + let msg = reader.read_raft_message().await.unwrap(); assert_eq!(msg.shard_group_id, ShardGroupId(42)); assert_eq!(msg.sender, NodeId::new("sender-1")); match msg.rpc { @@ -188,8 +218,9 @@ mod tests { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; let (read_half, _) = stream.into_split(); - let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); - let peer = reader.read_node_id().await?; + let reader = + ClusterMessageReader::new(read_half, TransportIdentity::TrustedDevelopment); + let peer = NodeId::new("node-a"); let (raft_tx, mut raft_rx) = MultiRaftActor::channel(8); reader.run(raft_tx, peer.clone()).await; @@ -200,7 +231,7 @@ mod tests { else { panic!("expected one inbound Raft RPC") }; - assert_eq!(cmd.from, peer); + assert_eq!(cmd.peer_id, peer); assert_eq!(cmd.shard_group_id, ShardGroupId(1)); assert!(raft_rx.try_recv().is_err()); Ok(()) @@ -210,24 +241,8 @@ mod tests { let addr = turmoil::lookup("server"); let stream = TcpStream::connect((addr, 9000)).await?; let (_, mut writer) = stream.into_split(); - write_frame(&mut writer, &NodeId::new("node-a")).await?; - - for (group, sender) in [(1, "node-a"), (2, "node-b")] { - write_frame( - &mut writer, - &WireRaftMessage { - shard_group_id: ShardGroupId(group), - sender: NodeId::new(sender), - rpc: RaftRpc::RequestVote(RequestVote { - term: 1, - candidate_id: NodeId::new(sender), - last_log_index: 0, - last_log_term: 0, - }), - }, - ) - .await?; - } + write_frame(&mut writer, &request_vote_message(1, "node-a")).await?; + write_frame(&mut writer, &request_vote_message(2, "node-b")).await?; Ok(()) }); @@ -235,13 +250,13 @@ mod tests { } #[test] - fn accepted_connection_registers_writer_after_handshake() -> turmoil::Result { + fn accepted_connection_registers_writer_after_initial_raft_message() -> turmoil::Result { let mut sim = Builder::new() .simulation_duration(Duration::from_secs(5)) .build(); sim.host("acceptor", || async { - let (raft_tx, _raft_rx) = MultiRaftActor::channel(16); + let (raft_tx, mut raft_rx) = MultiRaftActor::channel(16); let listener = TcpListener::bind("0.0.0.0:9000").await?; let (dial_tx, _dial_rx) = tokio::sync::mpsc::channel(8); let mut state = RaftRpcDispatcher::new( @@ -257,8 +272,15 @@ mod tests { assert!( state.contains(&NodeId::new("node-a")), - "writer should be registered after handshake" + "writer should be registered after the initial Raft message" ); + let Some(MultiRaftActorCommand::ProtocolMessage(RaftProtocolMessage::InboundRaftRpc( + rpc, + ))) = raft_rx.recv().await + else { + panic!("expected the initial Raft RPC") + }; + assert_eq!(rpc.peer_id, NodeId::new("node-a")); Ok(()) }); @@ -266,7 +288,11 @@ mod tests { let addr = turmoil::lookup("acceptor"); let stream = TcpStream::connect((addr, 9000)).await?; let (_, mut write_half) = stream.into_split(); - write_frame(&mut write_half, &NodeId::new("node-a")).await?; + write_frame( + &mut write_half, + &InitialClusterMessage::Raft(request_vote_message(1, "node-a")), + ) + .await?; Ok(()) }); @@ -304,8 +330,14 @@ mod tests { // Second connection from node-a (simulating simultaneous connect) let (stream2, _) = dummy_listener.accept().await?; let (read_half, _write_half) = stream2.into_split(); - let mut reader = RaftRpcListener::new(read_half, TransportIdentity::TrustedDevelopment); - let peer_id = reader.read_node_id().await.unwrap(); + let mut reader = + ClusterMessageReader::new(read_half, TransportIdentity::TrustedDevelopment); + let InitialClusterMessage::Raft(initial_raft_message) = + reader.read_initial_message().await.unwrap() + else { + panic!("expected initial Raft message"); + }; + let peer_id = initial_raft_message.sender; assert_eq!(peer_id, NodeId::new("node-a")); // Conflict: node-a < node-b → incoming wins, replace @@ -323,15 +355,113 @@ mod tests { let stream1 = TcpStream::connect((addr, 9000)).await?; let (_, mut write_half) = stream1.into_split(); - write_frame(&mut write_half, &NodeId::new("node-a")).await?; + write_frame( + &mut write_half, + &InitialClusterMessage::Raft(request_vote_message(1, "node-a")), + ) + .await?; let stream2 = TcpStream::connect((addr, 9001)).await?; let (_, mut write_half2) = stream2.into_split(); - write_frame(&mut write_half2, &NodeId::new("node-a")).await?; + write_frame( + &mut write_half2, + &InitialClusterMessage::Raft(request_vote_message(1, "node-a")), + ) + .await?; Ok(()) }); sim.run() } + + #[test] + fn acl_snapshot_actor_coalesces_concurrent_refreshes() -> turmoil::Result { + let resource = AclResource::TopicData(TopicId(7)); + let snapshot = AclRecord { + resource: resource.clone(), + revision: 3, + principals: vec!["orders-service".to_owned()].into(), + }; + let owner = NodeAddressInfo::new( + NodeId::new("owner"), + NodeAddress::test( + "127.0.0.1:9000".parse().unwrap(), + "127.0.0.1:9001".parse().unwrap(), + ), + ); + let response_received = std::sync::Arc::new(Notify::new()); + let mut sim = Builder::new() + .simulation_duration(Duration::from_secs(5)) + .build(); + + let server_snapshot = snapshot.clone(); + let server_resource = resource.clone(); + let server_response_received = response_received.clone(); + sim.host("owner", move || { + let expected_snapshot = server_snapshot.clone(); + let expected_resource = server_resource.clone(); + let owner_completion = server_response_received.clone(); + async move { + let listener = TcpListener::bind("0.0.0.0:9000").await?; + let (raft_tx, mut raft_rx) = MultiRaftActor::channel(8); + let (dial_tx, _dial_rx) = tokio::sync::mpsc::channel(8); + let mut dispatcher = RaftRpcDispatcher::new( + NodeId::new("owner"), + dial_tx, + NodeTransportSecurity::TrustedDevelopment, + ); + + let (stream, _) = listener.accept().await?; + dispatcher + .accept(TransportTcpStream::TrustedDevelopment(stream), &raft_tx) + .await; + + let Some(MultiRaftActorCommand::GetAclSnapshot(query)) = raft_rx.recv().await + else { + panic!("expected ACL snapshot query"); + }; + assert_eq!(query.shard_group_id, ShardGroupId(42)); + assert_eq!(query.resource, expected_resource); + let _ = query.reply.send(Some(expected_snapshot)); + owner_completion.notified().await; + Ok(()) + } + }); + + let client_resource = resource.clone(); + let client_snapshot = snapshot.clone(); + let client_owner = owner.clone(); + sim.host("requester", move || { + let requested_resource = client_resource.clone(); + let expected_snapshot = client_snapshot.clone(); + let remote_owner = client_owner.clone(); + let requester_completion = response_received.clone(); + async move { + let client = AclSnapshotActor::spawn(NodeTransportSecurity::TrustedDevelopment); + let first_owner = remote_owner.clone(); + let first_resource = requested_resource.clone(); + let (first, second) = tokio::join!( + client.fetch( + NodeId::new("requester"), + first_owner, + ShardGroupId(42), + first_resource, + ), + client.fetch( + NodeId::new("requester"), + remote_owner, + ShardGroupId(42), + requested_resource, + ), + ); + assert_eq!(first, Some(expected_snapshot.clone())); + assert_eq!(second, Some(expected_snapshot)); + requester_completion.notify_one(); + Ok(()) + } + }); + + sim.run() + } } diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index 31989557..6415fa19 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -6,14 +6,20 @@ use tokio::time::Instant; use crate::control_plane::consensus::actor::MutlRaftSender; -use crate::control_plane::consensus::messages::{OutboundRaftPacket, WireRaftMessage}; +use crate::control_plane::consensus::messages::{ + InboundRaftRpc, OutboundRaftPacket, WireRaftMessage, +}; use crate::control_plane::NodeId; -use crate::control_plane::consensus::transport::RaftRpcListener; +use crate::control_plane::consensus::transport::ClusterMessageReader; use crate::control_plane::membership::actor::SwimSender; use crate::net::{TransportTcpStream, TransportWriteHalf}; use crate::security::NodeTransportSecurity; +use super::protocol::{ + AclSnapshotRequest, AclSnapshotResponse, InitialClusterMessage, encode_frame, +}; + const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); /// Upper bound on messages buffered per peer while its dial is in flight; /// overflow is dropped (raft retries by timer). @@ -24,9 +30,9 @@ const PENDING_DIAL_BUFFER_CAP: usize = 256; /// On simultaneous connect, the connection initiated by the **lower `NodeId`** /// wins; the other is dropped. /// -/// Handshake: after connecting, the initiator sends its `NodeId`. The acceptor -/// reads it, and if a connection to that peer already exists (from our own -/// outbound connect), the tie is broken by NodeId ordering. +/// A connection begins with its first Raft message, which identifies the peer. +/// The acceptor uses that sender to key the writer slot and resolve simultaneous +/// connection conflicts. pub(super) struct RaftRpcDispatcher { node_id: NodeId, writers: HashMap, @@ -48,7 +54,7 @@ pub(super) struct RaftRpcDispatcher { /// Result of a background dial attempt, delivered back to the transport loop. pub(super) struct DialOutcome { target: NodeId, - outcome: anyhow::Result<(RaftRpcListener, TransportWriteHalf)>, + outcome: anyhow::Result<(ClusterMessageReader, TransportWriteHalf)>, } impl RaftRpcDispatcher { @@ -71,24 +77,44 @@ impl RaftRpcDispatcher { pub(super) async fn accept(&mut self, stream: TransportTcpStream, raft_tx: &MutlRaftSender) { let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); - let mut reader = RaftRpcListener::new(read_half, transport_identity); + let mut reader = ClusterMessageReader::new(read_half, transport_identity); - let Ok(peer_id) = reader.read_node_id().await else { - tracing::error!("Failed to read peer NodeId during accept"); + let Ok(initial_message) = reader.read_initial_message().await else { + tracing::debug!("cluster connection closed before its initial message"); return; }; - if self.writers.contains_key(&peer_id) && peer_id > self.node_id { - tracing::debug!( - peer = %peer_id, - "simultaneous connect: dropping accepted connection, \ - our outbound dial wins the tie-break (lower NodeId)", - ); - return; + match initial_message { + InitialClusterMessage::AclSnapshot(request) => { + let peer_id = request.requester_node_id.clone(); + tokio::spawn(serve_acl_snapshot_request( + peer_id, + request, + raft_tx.clone(), + write_half, + )); + } + InitialClusterMessage::Raft(initial_raft_message) => { + let initial_rpc = InboundRaftRpc { + shard_group_id: initial_raft_message.shard_group_id, + peer_id: initial_raft_message.sender, + rpc: initial_raft_message.rpc, + }; + if self.writers.contains_key(&initial_rpc.peer_id) + && initial_rpc.peer_id > self.node_id + { + // simultaneous connect: dropping accepted connection + return; + } + self.writers.insert(initial_rpc.peer_id.clone(), write_half); + let raft_tx = raft_tx.clone(); + tokio::spawn(async move { + let peer_id = initial_rpc.peer_id.clone(); + let _ = raft_tx.send(initial_rpc).await; + reader.run(raft_tx, peer_id).await; + }); + } } - - self.writers.insert(peer_id.clone(), write_half); - tokio::spawn(reader.run(raft_tx.clone(), peer_id)); } pub(super) async fn send(&mut self, packets: Vec, swim_tx: &SwimSender) { @@ -121,7 +147,7 @@ impl RaftRpcDispatcher { async fn send_to_target( &mut self, target_id: NodeId, - msgs: Vec, + mut msgs: Vec, swim_tx: &SwimSender, ) { if let Some(&failed_at) = self.connect_backoffs.get(&target_id) { @@ -146,13 +172,15 @@ impl RaftRpcDispatcher { } return; } + let initial_raft_message = msgs.remove(0); self.pending_dials.insert(target_id.clone(), msgs); let dial_task = dial( - self.node_id.clone(), target_id.clone(), swim_tx.clone(), self.security.clone(), + initial_raft_message, ); + let dial_tx = self.dial_tx.clone(); tokio::spawn(async move { let outcome = dial_task.await; @@ -230,10 +258,8 @@ impl RaftRpcDispatcher { })?; let mut buf = Vec::new(); for msg in msgs { - let bytes = borsh::to_vec(msg)?; - let len = bytes.len() as u32; - buf.extend_from_slice(&len.to_be_bytes()); - buf.extend_from_slice(&bytes); + let frame = encode_frame(msg).map_err(std::io::Error::other)?; + buf.extend_from_slice(&frame); } let result = writer.write_all(&buf).await; if result.is_err() { @@ -248,19 +274,19 @@ impl RaftRpcDispatcher { } } -/// Resolve, connect (3secs cap), and handshake — on a spawned task, so a hung -/// connect can never block the transport select loop. The loop +/// Resolve, connect (3secs cap), and send the opening Raft message on a spawned +/// task, so a hung connect can never block the transport select loop. The loop /// installs the writer and flushes buffered messages in `on_dial_result`. -// ! never inline this. Actor Model should onkly do work whose duration it controls. +// ! never inline this. Actor Model should only do work whose duration it controls. // ! Anything whose latency the outside actor controls must not be awaited in the handler. async fn dial( - node_id: NodeId, target_id: NodeId, swim_tx: SwimSender, security: NodeTransportSecurity, -) -> anyhow::Result<(RaftRpcListener, TransportWriteHalf)> { + initial_raft_message: WireRaftMessage, +) -> anyhow::Result<(ClusterMessageReader, TransportWriteHalf)> { let Some(addr) = swim_tx.resolve_address(target_id.clone()).await? else { - anyhow::bail!("[{}] Cannot resolve address for {:?}", node_id, target_id); + anyhow::bail!("cannot resolve address for {target_id}"); }; let stream = tokio::time::timeout( @@ -271,13 +297,31 @@ async fn dial( let transport_identity = stream.peer_identity(); let (read_half, mut write_half) = stream.into_split(); - let bytes = borsh::to_vec(&node_id) - .map_err(|e| anyhow::anyhow!("[{}] Handshake encode failed: {e}", node_id))?; - let len = bytes.len() as u32; - write_half.write_all(&len.to_be_bytes()).await?; - write_half.write_all(&bytes).await?; + write_half + .write_all(&encode_frame(&InitialClusterMessage::Raft( + initial_raft_message, + ))?) + .await?; Ok(( - RaftRpcListener::new(read_half, transport_identity), + ClusterMessageReader::new(read_half, transport_identity), write_half, )) } + +async fn serve_acl_snapshot_request( + peer_id: NodeId, + request: AclSnapshotRequest, + raft_tx: MutlRaftSender, + mut writer: TransportWriteHalf, +) { + let snapshot = raft_tx + .get_acl_snapshot(request.shard_group_id, request.resource) + .await; + let Ok(frame) = encode_frame(&AclSnapshotResponse { snapshot }) else { + tracing::debug!(peer = %peer_id, "failed to encode ACL snapshot response"); + return; + }; + if let Err(error) = writer.write_all(&frame).await { + tracing::debug!(peer = %peer_id, "failed to send ACL snapshot response: {error}"); + } +} diff --git a/src/control_plane/consensus/transport/protocol.rs b/src/control_plane/consensus/transport/protocol.rs new file mode 100644 index 00000000..d61eda06 --- /dev/null +++ b/src/control_plane/consensus/transport/protocol.rs @@ -0,0 +1,49 @@ +use anyhow::Result; +use borsh::{BorshDeserialize, BorshSerialize}; + +use crate::control_plane::NodeId; +use crate::control_plane::consensus::messages::WireRaftMessage; +use crate::control_plane::consensus::raft::states::security::AclRecord; +use crate::control_plane::membership::ShardGroupId; +use crate::control_plane::metadata::AclResource; +use crate::impl_from_variant; + +/// The first frame on a cluster TCP connection. +/// +/// A Raft connection begins with its first Raft message, which already names +/// its sender. An ACL snapshot connection begins with its requester and read. +/// Later Raft frames are raw; an ACL connection returns one response and closes. +#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] +pub(super) enum InitialClusterMessage { + Raft(WireRaftMessage), + AclSnapshot(AclSnapshotRequest), +} + +/// One read of a committed ACL record from a shard host. +#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] +pub(super) struct AclSnapshotRequest { + pub(super) requester_node_id: NodeId, + pub(super) shard_group_id: ShardGroupId, + pub(super) resource: AclResource, +} + +/// The response to one ACL snapshot request on its dedicated connection. +#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] +pub(super) struct AclSnapshotResponse { + pub(super) snapshot: Option, +} + +impl_from_variant!( + InitialClusterMessage, + Raft(WireRaftMessage), + AclSnapshot(AclSnapshotRequest), +); + +pub(super) fn encode_frame(value: &impl BorshSerialize) -> Result> { + let bytes = borsh::to_vec(value)?; + let len = u32::try_from(bytes.len())?; + let mut frame = Vec::with_capacity(std::mem::size_of::() + bytes.len()); + frame.extend_from_slice(&len.to_be_bytes()); + frame.extend_from_slice(&bytes); + Ok(frame) +} diff --git a/src/control_plane/membership/actor.rs b/src/control_plane/membership/actor.rs index ff3fda0b..6e4fe9c2 100644 --- a/src/control_plane/membership/actor.rs +++ b/src/control_plane/membership/actor.rs @@ -144,9 +144,18 @@ pub(crate) enum ShardRouting { /// This node hosts the key's shard group — proceed locally. Carries the group /// for callers that need its members/id (control-plane writes). Local(ShardGroup), - /// Not local; redirect the client to a member. `None` until the ring/addresses - /// converge here — still retriable. - Redirect(Option), + /// A client should retry elsewhere. `None` means the ring cannot map the + /// key yet; a remote shard can still lack an address while SWIM converges. + Redirect(Option), +} + +/// A shard this node does not host, with an optional reachable member. +/// +/// The shard ID lets internal callers validate cached records against the +/// current owner even when the client-facing redirect has no usable address. +pub(crate) struct RemoteShard { + pub(crate) group_id: ShardGroupId, + pub(crate) member: Option, } #[derive(Clone, Debug)] @@ -229,8 +238,9 @@ impl SwimSender { Ok(None) } - /// Route a key relative to `node_id`: `Local` if it hosts the key's shard group, - /// else a `Redirect` to a resolvable member (no hint until the ring converges). + /// Route a key relative to `node_id`: `Local` if it hosts the key's shard + /// group, otherwise a redirect. A resolved remote shard retains its ID for + /// internal ACL refreshes; an unresolved ring has no remote shard yet. pub(crate) async fn resolve_shard_routing( &self, key: Vec, @@ -243,7 +253,10 @@ impl SwimSender { return Ok(ShardRouting::Local(group)); } let member = self.resolve_any(&group.replicas).await?; - Ok(ShardRouting::Redirect(member)) + Ok(ShardRouting::Redirect(Some(RemoteShard { + group_id: group.id, + member, + }))) } pub(crate) async fn list_all_node_addresses( diff --git a/src/lib.rs b/src/lib.rs index 46197c79..80402c22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ use crate::control_plane::consensus::messages::{ MultiRaftActorCommand, RaftTimer, RaftTransportCommand, }; use crate::control_plane::consensus::transport::RaftTransportActor; +use crate::control_plane::consensus::transport::{AclSnapshotActor, AclSnapshotSender}; use crate::control_plane::membership::OutboundPacket; use crate::control_plane::membership::actor::SwimSender; use crate::control_plane::membership::topology_channel; @@ -167,9 +168,18 @@ impl StartUp { self.env.raft_snapshot_entry_threshold, ); + let acl_snapshot_sender = AclSnapshotActor::spawn(security.clone()); + // Client handler let _ = self - .receive_client_streams(node_id, swim_sender, raft_tx, data_plane_tx, security) + .receive_client_streams( + node_id, + swim_sender, + raft_tx, + data_plane_tx, + security, + acl_snapshot_sender, + ) .await; Ok(()) } @@ -181,6 +191,7 @@ impl StartUp { raft_tx: MutlRaftSender, data_plane_tx: DataPlaneSender, security: NodeTransportSecurity, + acl_snapshot_sender: AclSnapshotSender, ) { let acl_cache = SharedAclCache::default(); let addr = self.env.bind_addr(); @@ -203,6 +214,7 @@ impl StartUp { let swim_tx = swim_sender.clone(); let raft = raft_tx.clone(); let dp = data_plane_tx.clone(); + let acl_sender = acl_snapshot_sender.clone(); tokio::spawn(handle_client_stream( stream, @@ -211,6 +223,7 @@ impl StartUp { raft, dp, acl_cache.clone(), + acl_sender, )); } } From 8da30c25542b84d6a8647bd77007d4602600f208 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 16:44:12 +0400 Subject: [PATCH 38/56] feat: ProcessSigningKey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ed25519 process signing key using rustls’s existing ring backend. - Fresh 32-byte admission challenges. - Serializable admission proofs. - Tests covering valid proof, replay with another challenge, wrong principal, changed identity/epoch, and old process key. --- Cargo.lock | 1 + Cargo.toml | 1 + src/security/admission_proof.rs | 174 ++++++++++++++++++++++++++++++++ src/security/mod.rs | 1 + 4 files changed, 177 insertions(+) create mode 100644 src/security/admission_proof.rs diff --git a/Cargo.lock b/Cargo.lock index 676ebce7..6bacc64d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -403,6 +403,7 @@ dependencies = [ "murmur3", "rand 0.10.1", "rcgen", + "ring", "rocksdb", "rustls", "rustls-pemfile", diff --git a/Cargo.toml b/Cargo.toml index 1b29c8c3..8654f8cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ rustls = { version = "0.23.42", default-features = false, features = ["ring", "s rustls-pemfile = "2.2.0" tokio-rustls = { version = "0.26.4", default-features = false, features = ["ring"] } x509-parser = "0.18.1" +ring = "0.17.14" [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" diff --git a/src/security/admission_proof.rs b/src/security/admission_proof.rs new file mode 100644 index 00000000..710856a7 --- /dev/null +++ b/src/security/admission_proof.rs @@ -0,0 +1,174 @@ +use anyhow::{Context, Result}; +use borsh::{BorshDeserialize, BorshSerialize}; +use ring::{ + rand::{SecureRandom, SystemRandom}, + signature::{ED25519, Ed25519KeyPair, KeyPair, UnparsedPublicKey}, +}; + +use crate::control_plane::NodeId; + +const ADMISSION_PROOF_DOMAIN: &str = "eastguard-node-admission-v1"; +const ADMISSION_CHALLENGE_BYTES: usize = 32; +const ED25519_SIGNATURE_BYTES: usize = 64; + +/// Fresh value supplied by the accepting broker for one connection. +/// +/// A proof from an earlier connection cannot be replayed because it was signed +/// for a different challenge. +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub(crate) struct AdmissionChallenge([u8; ADMISSION_CHALLENGE_BYTES]); + +impl AdmissionChallenge { + pub(crate) fn generate() -> Result { + let mut bytes = [0; ADMISSION_CHALLENGE_BYTES]; + SystemRandom::new() + .fill(&mut bytes) + .map_err(|_| anyhow::anyhow!("failed to generate admission challenge"))?; + Ok(Self(bytes)) + } +} + +/// Private signing key generated for one broker process. +/// +/// Only its public key is committed in the admission record. The private key +/// remains in this process and proves that a connection belongs to the exact +/// process approved for the current admission epoch. +pub(crate) struct ProcessSigningKey(Ed25519KeyPair); + +impl ProcessSigningKey { + pub(crate) fn generate() -> Result { + let random = SystemRandom::new(); + let encoded = Ed25519KeyPair::generate_pkcs8(&random) + .map_err(|_| anyhow::anyhow!("failed to generate process signing key"))?; + let key = Ed25519KeyPair::from_pkcs8(encoded.as_ref()) + .map_err(|_| anyhow::anyhow!("failed to load generated process signing key"))?; + Ok(Self(key)) + } + + pub(crate) fn public_key(&self) -> Box<[u8]> { + self.0.public_key().as_ref().into() + } + + pub(crate) fn sign( + &self, + node_certificate_principal: &str, + node_id: &NodeId, + epoch: u64, + challenge: &AdmissionChallenge, + ) -> Result { + let message = + admission_proof_message(node_certificate_principal, node_id, epoch, challenge)?; + let signature = self.0.sign(&message); + let signature = signature + .as_ref() + .try_into() + .expect("Ed25519 signatures are always 64 bytes"); + Ok(AdmissionProof { + node_id: node_id.clone(), + epoch, + signature, + }) + } +} + +/// Claim sent by a broker process after it receives a fresh challenge. +/// +/// The receiver still checks `node_id`, `epoch`, and the public key against the +/// current admission record. The signature makes that comparison meaningful: +/// a process holding only the reusable node certificate cannot forge the proof. +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub(crate) struct AdmissionProof { + pub(crate) node_id: NodeId, + pub(crate) epoch: u64, + signature: [u8; ED25519_SIGNATURE_BYTES], +} + +impl AdmissionProof { + pub(crate) fn verify( + &self, + process_public_key: &[u8], + node_certificate_principal: &str, + challenge: &AdmissionChallenge, + ) -> bool { + let Ok(message) = admission_proof_message( + node_certificate_principal, + &self.node_id, + self.epoch, + challenge, + ) else { + return false; + }; + UnparsedPublicKey::new(&ED25519, process_public_key) + .verify(&message, &self.signature) + .is_ok() + } +} + +fn admission_proof_message( + node_certificate_principal: &str, + node_id: &NodeId, + epoch: u64, + challenge: &AdmissionChallenge, +) -> Result> { + borsh::to_vec(&( + ADMISSION_PROOF_DOMAIN, + node_certificate_principal, + node_id, + epoch, + challenge, + )) + .context("failed to encode admission proof") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn current_process_proves_its_admitted_identity() { + let key = ProcessSigningKey::generate().unwrap(); + let node_id = NodeId::new("broker-a::process-2"); + let challenge = AdmissionChallenge::generate().unwrap(); + let proof = key.sign("broker-a", &node_id, 8, &challenge).unwrap(); + + assert!(proof.verify(&key.public_key(), "broker-a", &challenge)); + } + + #[test] + fn proof_is_bound_to_the_certificate_principal_and_challenge() { + let key = ProcessSigningKey::generate().unwrap(); + let node_id = NodeId::new("broker-a::process-2"); + let challenge = AdmissionChallenge::generate().unwrap(); + let other_challenge = AdmissionChallenge::generate().unwrap(); + let proof = key.sign("broker-a", &node_id, 8, &challenge).unwrap(); + + assert!(!proof.verify(&key.public_key(), "broker-b", &challenge)); + assert!(!proof.verify(&key.public_key(), "broker-a", &other_challenge)); + } + + #[test] + fn old_process_key_cannot_prove_the_current_admission() { + let current_key = ProcessSigningKey::generate().unwrap(); + let old_key = ProcessSigningKey::generate().unwrap(); + let node_id = NodeId::new("broker-a::process-2"); + let challenge = AdmissionChallenge::generate().unwrap(); + let proof = old_key.sign("broker-a", &node_id, 8, &challenge).unwrap(); + + assert!(!proof.verify(¤t_key.public_key(), "broker-a", &challenge)); + } + + #[test] + fn changing_the_claimed_identity_invalidates_the_signature() { + let key = ProcessSigningKey::generate().unwrap(); + let node_id = NodeId::new("broker-a::process-2"); + let challenge = AdmissionChallenge::generate().unwrap(); + let mut proof = key.sign("broker-a", &node_id, 8, &challenge).unwrap(); + + proof.node_id = NodeId::new("broker-a::process-3"); + assert!(!proof.verify(&key.public_key(), "broker-a", &challenge)); + + proof.node_id = node_id; + proof.epoch = 9; + assert!(!proof.verify(&key.public_key(), "broker-a", &challenge)); + } +} diff --git a/src/security/mod.rs b/src/security/mod.rs index 11aaa36a..cbb4e134 100644 --- a/src/security/mod.rs +++ b/src/security/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod acl_cache; +mod admission_proof; mod certificates; mod transport; From 25fd3d50d344ef871fabb9011600f2dbd622dbf6 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Wed, 29 Jul 2026 17:20:23 +0400 Subject: [PATCH 39/56] feat: Admission cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admission cache │ ├── fresh ───────────────► return record/denial │ └── missing or expired │ ├── local shard ──► MultiRaft committed read └── remote shard ─► limited admission lookup │ └── reply once, then close Implemented: - Local committed admission-record queries. - Limited bootstrap endpoint, accepted before process admission but restricted to one admission read. - Local/remote lookup actor with: - 16 active lookups - 128 queued lookups - 256 waiters per record - concurrent identical requests combined - Revision-aware 60-second cache. - Lookup failures are not cached. - Older revisions cannot renew expired authority. - Records returned for another certificate principal are rejected. --- src/control_plane/consensus/actor.rs | 22 +- src/control_plane/consensus/messages/actor.rs | 13 +- src/control_plane/consensus/multi_raft.rs | 24 +- src/control_plane/consensus/raft/state.rs | 6 +- .../consensus/raft/states/metadata_state.rs | 8 +- .../consensus/raft/states/security.rs | 23 ++ .../consensus/transport/admission/message.rs | 59 ++++ .../consensus/transport/admission/mod.rs | 188 +++++++++++ .../consensus/transport/admission/state.rs | 302 ++++++++++++++++++ .../consensus/transport/inbound.rs | 9 +- src/control_plane/consensus/transport/mod.rs | 95 +++++- .../consensus/transport/outbound.rs | 49 ++- .../consensus/transport/protocol.rs | 26 +- 13 files changed, 811 insertions(+), 13 deletions(-) create mode 100644 src/control_plane/consensus/transport/admission/message.rs create mode 100644 src/control_plane/consensus/transport/admission/mod.rs create mode 100644 src/control_plane/consensus/transport/admission/state.rs diff --git a/src/control_plane/consensus/actor.rs b/src/control_plane/consensus/actor.rs index 51b039b2..0d4d14e9 100644 --- a/src/control_plane/consensus/actor.rs +++ b/src/control_plane/consensus/actor.rs @@ -8,7 +8,7 @@ use crate::control_plane::NodeId; use crate::control_plane::consensus::messages::*; use crate::control_plane::consensus::multi_raft::MultiRaft; use crate::control_plane::consensus::raft::errors::ProposalError; -use crate::control_plane::consensus::raft::states::security::AclRecord; +use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::consensus::raft::storage::RaftStorage; use crate::control_plane::membership::actor::SwimSender; use crate::control_plane::membership::{ShardGroupId, SwimCommand, TopologyReader}; @@ -298,6 +298,26 @@ impl MutlRaftSender { recv.await.ok().flatten() } + /// Reads the current admission record only when this node hosts the + /// selected metadata shard. Absence is an authoritative `None`; routing or + /// actor failures remain observable errors and must not be cached. + pub(crate) async fn get_admission( + &self, + shard_group_id: ShardGroupId, + node_certificate_principal: Box, + ) -> Result, ServerError> { + let (reply, recv) = tokio::sync::oneshot::channel(); + self.send(GetAdmission { + shard_group_id, + node_certificate_principal, + reply, + }) + .await + .map_err(|error| ServerError::Internal(error.to_string()))?; + recv.await + .map_err(|error| ServerError::Internal(error.to_string()))? + } + pub(crate) async fn send( &self, cmd: impl Into, diff --git a/src/control_plane/consensus/messages/actor.rs b/src/control_plane/consensus/messages/actor.rs index a7f1f0f3..946605f5 100644 --- a/src/control_plane/consensus/messages/actor.rs +++ b/src/control_plane/consensus/messages/actor.rs @@ -1,9 +1,10 @@ use tokio::sync::oneshot; use uuid::Uuid; +use crate::connections::protocol::ServerError; use crate::control_plane::NodeId; use crate::control_plane::consensus::raft::errors::ProposalError; -use crate::control_plane::consensus::raft::states::security::AclRecord; +use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::{AclResource, ConsumerGroupAssignment, TopicMeta, TopicStats}; use crate::data_plane::messages::command::{ @@ -54,6 +55,7 @@ pub enum MultiRaftActorCommand { reply: oneshot::Sender>, }, GetAclSnapshot(GetAclSnapshot), + GetAdmission(GetAdmission), GetConsumerGroupAssignment(GetConsumerGroupAssignment), /// Data-plane request forwarded to the metadata coordinator for proposal. ProposeSegmentRoll(ProposeSegmentRoll), @@ -86,6 +88,13 @@ pub struct GetAclSnapshot { pub(crate) reply: oneshot::Sender>, } +/// Returns one admission record from a metadata shard hosted by this node. +pub struct GetAdmission { + pub(crate) shard_group_id: ShardGroupId, + pub(crate) node_certificate_principal: Box, + pub(crate) reply: oneshot::Sender, ServerError>>, +} + impl From for MultiRaftActorCommand { fn from(cmd: RaftProtocolMessage) -> Self { MultiRaftActorCommand::ProtocolMessage(cmd) @@ -109,6 +118,7 @@ impl_from_variant_via!( impl_from_variant!( MultiRaftActorCommand, GetAclSnapshot, + GetAdmission, GetConsumerGroupAssignment, ); @@ -132,5 +142,6 @@ pub(crate) enum DeferredReply { GetTopicStats(DeferredResponse>), GetTopicMetadata(DeferredResponse>), GetAclSnapshot(DeferredResponse>), + GetAdmission(DeferredResponse, ServerError>>), GetConsumerGroupAssignment(DeferredResponse>), } diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index 86643883..029a5e8f 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -1,3 +1,4 @@ +use crate::client::ServerError; use crate::control_plane::NodeId; use crate::control_plane::consensus::boundary_recovery::{ BoundaryRecoveryAction, SegmentBoundaryRecovery, @@ -9,7 +10,7 @@ use crate::control_plane::consensus::messages::{ use crate::control_plane::consensus::raft::errors::ProposalError; use crate::control_plane::consensus::raft::state::{Raft, TimerSeqs}; use crate::control_plane::consensus::raft::states::consensus::LeaderlessSegments; -use crate::control_plane::consensus::raft::states::security::AclRecord; +use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::consensus::raft::storage::RaftStorage; use crate::control_plane::consensus::raft::{compute_replacement_replica_set, now_ms}; use crate::control_plane::membership::{ShardGroup, ShardGroupId, TopologyReader}; @@ -289,6 +290,14 @@ impl MultiRaft { value, })); } + MultiRaftActorCommand::GetAdmission(query) => { + let value = self.admission(query.shard_group_id, &query.node_certificate_principal); + self.deferred + .push(DeferredReply::GetAdmission(DeferredResponse { + reply: query.reply, + value, + })); + } MultiRaftActorCommand::GetConsumerGroupAssignment(query) => { let value = self.get_consumer_group_assignment( &query.topic_name, @@ -328,6 +337,7 @@ impl MultiRaft { DeferredReply::GetTopicStats(deferred) => deferred.send(), DeferredReply::GetTopicMetadata(deferred) => deferred.send(), DeferredReply::GetAclSnapshot(deferred) => deferred.send(), + DeferredReply::GetAdmission(deferred) => deferred.send(), DeferredReply::GetConsumerGroupAssignment(deferred) => deferred.send(), } } @@ -503,6 +513,18 @@ impl MultiRaft { .map(|raft| raft.acl_snapshot(resource)) } + fn admission( + &self, + shard_group_id: ShardGroupId, + node_certificate_principal: &str, + ) -> Result, ServerError> { + let raft = self + .groups + .get(&shard_group_id) + .ok_or(ServerError::ShardNotLocal { hint_node: None })?; + Ok(raft.admission(node_certificate_principal)) + } + fn get_consumer_group_assignment( &self, topic_name: &str, diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index 47885f3f..51bc06bb 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -8,7 +8,7 @@ use crate::control_plane::consensus::raft::states::consensus::{ ConsensusState, PeerState, Role, SNAPSHOT_CHUNK_BYTES, SnapshotInstallOutcome, }; use crate::control_plane::consensus::raft::states::metadata_state::MetadataState; -use crate::control_plane::consensus::raft::states::security::AclRecord; +use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::consensus::raft::storage::{ RaftPersistentState, RaftSnapshot, SnapshotData, }; @@ -166,6 +166,10 @@ impl Raft { self.metadata.acl_snapshot(resource) } + pub(crate) fn admission(&self, node_certificate_principal: &str) -> Option { + self.metadata.admission(node_certificate_principal) + } + pub(crate) fn get_consumer_group_assignment( &self, topic_name: &str, diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index 98bcaa99..eaf2e97d 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -1,4 +1,6 @@ -use crate::control_plane::consensus::raft::states::security::{AclRecord, SecurityState}; +use crate::control_plane::consensus::raft::states::security::{ + AclRecord, AdmissionRecord, SecurityState, +}; use crate::control_plane::metadata::SegmentMeta; use crate::control_plane::metadata::command::*; use crate::control_plane::metadata::event::*; @@ -95,6 +97,10 @@ impl MetadataState { self.security.acl_snapshot(resource) } + pub(crate) fn admission(&self, node_certificate_principal: &str) -> Option { + self.security.admission(node_certificate_principal) + } + pub(crate) fn get_consumer_group_assignment( &self, topic_name: &str, diff --git a/src/control_plane/consensus/raft/states/security.rs b/src/control_plane/consensus/raft/states/security.rs index 50fdfb23..ec81cd90 100644 --- a/src/control_plane/consensus/raft/states/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -54,6 +54,10 @@ pub(crate) struct RevocationRecord { } impl SecurityState { + pub(crate) fn admission(&self, node_certificate_principal: &str) -> Option { + self.admissions.get(node_certificate_principal).cloned() + } + /// Returns the current ACL record, or an empty revision-zero record when /// the resource has never been granted to any principal. Both forms deny /// by default; representing absence explicitly lets callers cache that @@ -166,6 +170,25 @@ mod tests { }); } + #[test] + fn admission_lookup_uses_the_certificate_principal() { + let mut security = SecurityState::default(); + let admission = AdmissionRecord { + node_certificate_principal: "broker-a".to_string(), + revision: 3, + epoch: 2, + node_id: NodeId::new("broker-a::process-2"), + process_public_key: vec![1, 2, 3].into_boxed_slice(), + }; + security.admissions.insert( + admission.node_certificate_principal.clone(), + admission.clone(), + ); + + assert_eq!(security.admission("broker-a"), Some(admission)); + assert_eq!(security.admission("broker-b"), None); + } + #[test] fn acl_snapshot_returns_the_exact_record_or_an_empty_denial() { let mut security = SecurityState::default(); diff --git a/src/control_plane/consensus/transport/admission/message.rs b/src/control_plane/consensus/transport/admission/message.rs new file mode 100644 index 00000000..a79ac574 --- /dev/null +++ b/src/control_plane/consensus/transport/admission/message.rs @@ -0,0 +1,59 @@ +use tokio::sync::oneshot; + +use crate::control_plane::NodeAddressInfo; +use crate::control_plane::consensus::raft::states::security::AdmissionRecord; +use crate::control_plane::membership::ShardGroupId; +use crate::impl_from_variant; + +pub(super) type AdmissionLookupResult = Result, AdmissionLookupUnavailable>; +pub(super) type AdmissionLookupReply = oneshot::Sender; + +pub(super) struct LookupAdmission { + pub(super) fetch: AdmissionFetch, + pub(super) reply: AdmissionLookupReply, +} + +#[derive(Clone, Debug)] +pub(super) enum AdmissionFetch { + Local(LocalAdmissionFetch), + Remote(RemoteAdmissionFetch), +} + +#[derive(Clone, Debug)] +pub(super) struct LocalAdmissionFetch(pub(super) AdmissionLookupKey); + +#[derive(Clone, Debug)] +pub(super) struct RemoteAdmissionFetch { + pub(super) key: AdmissionLookupKey, + pub(super) owner: NodeAddressInfo, +} + +impl_from_variant!( + AdmissionFetch, + Local(LocalAdmissionFetch), + Remote(RemoteAdmissionFetch), +); + +impl AdmissionFetch { + pub(super) fn key(&self) -> &AdmissionLookupKey { + match self { + Self::Local(fetch) => &fetch.0, + Self::Remote(fetch) => &fetch.key, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) struct AdmissionLookupKey { + pub(super) shard_group_id: ShardGroupId, + pub(super) node_certificate_principal: Box, +} + +pub(super) struct AdmissionLookupCompleted { + pub(super) key: AdmissionLookupKey, + pub(super) result: Result, AdmissionLookupUnavailable>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("admission lookup unavailable")] +pub(crate) struct AdmissionLookupUnavailable; diff --git a/src/control_plane/consensus/transport/admission/mod.rs b/src/control_plane/consensus/transport/admission/mod.rs new file mode 100644 index 00000000..d5a6132d --- /dev/null +++ b/src/control_plane/consensus/transport/admission/mod.rs @@ -0,0 +1,188 @@ +mod message; +mod state; +use state::*; + +use std::time::Duration; + +use anyhow::{Context, Result as AnyResult}; +use tokio::io::AsyncWriteExt; +use tokio::sync::{mpsc, oneshot}; + +use crate::control_plane::consensus::actor::MutlRaftSender; +use crate::control_plane::consensus::raft::states::security::AdmissionRecord; +use crate::control_plane::membership::actor::ShardRouting; +use crate::net::TransportTcpStream; +use crate::security::NodeTransportSecurity; + +use message::*; + +use super::inbound::ClusterMessageReader; +use super::protocol::{AdmissionLookupRequest, InitialClusterMessage, encode_frame}; + +const ADMISSION_FETCH_TIMEOUT: Duration = Duration::from_secs(3); +const ADMISSION_REQUEST_TIMEOUT: Duration = Duration::from_secs(4); + +const ADMISSION_LOOKUP_MAILBOX_CAPACITY: usize = 256; + +/// Bounded admission-record reader used while cluster connections are admitted. +/// +/// One actor owns the short-lived cache and combines simultaneous lookups for +/// the same principal. Local records come from this node's multi-Raft actor; +/// remote records use the limited admission-only connection. +pub(crate) struct AdmissionLookupActor; + +impl AdmissionLookupActor { + pub(crate) fn spawn( + raft_tx: MutlRaftSender, + security: NodeTransportSecurity, + ) -> AdmissionLookupSender { + let (sender, mailbox) = mpsc::channel(ADMISSION_LOOKUP_MAILBOX_CAPACITY); + tokio::spawn(Self::run(raft_tx, security, mailbox)); + AdmissionLookupSender(sender) + } + + async fn run( + raft_tx: MutlRaftSender, + security: NodeTransportSecurity, + mut mailbox: mpsc::Receiver, + ) { + let (completed_tx, mut completed_rx) = + mpsc::channel::(MAX_IN_FLIGHT_LOOKUPS); + let started_at = tokio::time::Instant::now(); + let mut state = AdmissionLookupState::default(); + let mut requests = Vec::with_capacity(64); + + loop { + tokio::select! { + count = mailbox.recv_many(&mut requests, 64) => { + if count == 0 { + break; + } + let now = started_at.elapsed(); + for request in requests.drain(..) { + state.lookup(request, now); + } + } + Some(completed) = completed_rx.recv() => { + state.complete(completed, started_at.elapsed()); + } + } + + for fetch in state.take_pending() { + tokio::spawn(Self::fetch( + raft_tx.clone(), + security.clone(), + fetch, + completed_tx.clone(), + )); + } + } + } + + async fn fetch( + raft_tx: MutlRaftSender, + security: NodeTransportSecurity, + fetch: AdmissionFetch, + completed_tx: mpsc::Sender, + ) { + let key = fetch.key().clone(); + let res = match tokio::time::timeout( + ADMISSION_FETCH_TIMEOUT, + Self::fetch_inner(raft_tx, security, fetch), + ) + .await + { + Ok(Ok(admission)) => Ok(admission), + Ok(Err(error)) => { + tracing::debug!(?key, "admission lookup failed: {error}"); + Err(AdmissionLookupUnavailable) + } + Err(_) => { + tracing::debug!(?key, "admission lookup timed out"); + Err(AdmissionLookupUnavailable) + } + }; + + let _ = completed_tx + .send(AdmissionLookupCompleted { key, result: res }) + .await; + } + + async fn fetch_inner( + raft_tx: MutlRaftSender, + security: NodeTransportSecurity, + fetch: AdmissionFetch, + ) -> AnyResult> { + match fetch { + AdmissionFetch::Local(LocalAdmissionFetch(key)) => Ok(raft_tx + .get_admission(key.shard_group_id, key.node_certificate_principal) + .await?), + AdmissionFetch::Remote(fetch) => { + let stream = + TransportTcpStream::connect_node(fetch.owner.cluster_addr(), &security).await?; + let transport_identity = stream.peer_identity(); + let (read_half, mut write_half) = stream.into_split(); + write_half + .write_all(&encode_frame(&InitialClusterMessage::AdmissionLookup( + AdmissionLookupRequest { + shard_group_id: fetch.key.shard_group_id, + node_certificate_principal: fetch.key.node_certificate_principal, + }, + ))?) + .await + .context("write initial admission lookup request")?; + + let mut reader = ClusterMessageReader::new(read_half, transport_identity); + Ok(reader.read_admission_lookup_response().await?.admission) + } + } + } +} +/// Sends admission-record requests to [`AdmissionLookupActor`]. +#[derive(Clone)] +pub(crate) struct AdmissionLookupSender(mpsc::Sender); + +impl AdmissionLookupSender { + pub(crate) async fn lookup( + &self, + routing: ShardRouting, + node_certificate_principal: Box, + ) -> Result, AdmissionLookupUnavailable> { + let fetch = match routing { + ShardRouting::Local(group) => LocalAdmissionFetch(AdmissionLookupKey { + shard_group_id: group.id, + node_certificate_principal, + }) + .into(), + ShardRouting::Redirect(Some(remote)) => { + let Some(owner) = remote.member else { + return Err(AdmissionLookupUnavailable); + }; + RemoteAdmissionFetch { + key: AdmissionLookupKey { + shard_group_id: remote.group_id, + node_certificate_principal, + }, + owner, + } + .into() + } + ShardRouting::Redirect(None) => return Err(AdmissionLookupUnavailable), + }; + let (reply, recv) = oneshot::channel(); + self.0 + .try_send(LookupAdmission { fetch, reply }) + .map_err(|e| { + tracing::debug!("admission lookup actor unavailable {}", e); + AdmissionLookupUnavailable + })?; + + match tokio::time::timeout(ADMISSION_REQUEST_TIMEOUT, recv).await { + Ok(Ok(result)) => result, + err => { + tracing::debug!("admission lookup actor unavailable {:?}", err); + Err(AdmissionLookupUnavailable) + } + } + } +} diff --git a/src/control_plane/consensus/transport/admission/state.rs b/src/control_plane/consensus/transport/admission/state.rs new file mode 100644 index 00000000..ed9d62ca --- /dev/null +++ b/src/control_plane/consensus/transport/admission/state.rs @@ -0,0 +1,302 @@ +use std::{ + collections::{HashMap, HashSet, VecDeque}, + time::Duration, +}; + +use crate::control_plane::consensus::{ + raft::states::security::AdmissionRecord, + transport::admission::{ + AdmissionFetch, AdmissionLookupCompleted, AdmissionLookupKey, AdmissionLookupReply, + LookupAdmission, message::AdmissionLookupUnavailable, + }, +}; + +const MAX_ADMISSION_CACHE_TTL: Duration = Duration::from_secs(60); +pub(super) const MAX_IN_FLIGHT_LOOKUPS: usize = 16; +const MAX_QUEUED_LOOKUPS: usize = 128; +const MAX_WAITERS_PER_LOOKUP: usize = 256; + +/// Pure scheduling and cache state for admission lookups. +#[derive(Default)] +pub(super) struct AdmissionLookupState { + cache: AdmissionCache, + active: HashSet, + queued: VecDeque, + waiters: HashMap>, + pending_events: Vec, +} + +impl AdmissionLookupState { + pub(super) fn lookup(&mut self, request: LookupAdmission, now: Duration) { + let LookupAdmission { fetch, reply } = request; + let key = fetch.key().clone(); + + if let Some(admission) = self.cache.fresh_admission(&key, now) { + let _ = reply.send(Ok(admission.clone())); + return; + } + if let Some(waiting) = self.waiters.get_mut(&key) { + if waiting.len() == MAX_WAITERS_PER_LOOKUP { + tracing::debug!(?key, "admission lookup has too many waiting callers"); + let _ = reply.send(Err(AdmissionLookupUnavailable)); + return; + } + waiting.push(reply); + return; + } + if self.active.len() < MAX_IN_FLIGHT_LOOKUPS { + self.active.insert(key.clone()); + self.waiters.insert(key, vec![reply]); + self.pending_events.push(fetch); + return; + } + if self.queued.len() < MAX_QUEUED_LOOKUPS { + self.waiters.insert(key, vec![reply]); + self.queued.push_back(fetch); + return; + } + tracing::debug!(?key, "admission lookup queue is full"); + let _ = reply.send(Err(AdmissionLookupUnavailable)); + } + + pub(super) fn complete(&mut self, completed: AdmissionLookupCompleted, now: Duration) { + debug_assert!(self.active.remove(&completed.key)); + if let Ok(admission) = &completed.result { + self.cache.insert(&completed.key, admission.clone(), now); + } + + let response = match completed.result { + Ok(_) => self + .cache + .fresh_admission(&completed.key, now) + .cloned() + .ok_or(AdmissionLookupUnavailable), + Err(error) => Err(error), + }; + if let Some(waiting) = self.waiters.remove(&completed.key) { + for reply in waiting { + let _ = reply.send(response.clone()); + } + } else { + tracing::debug!("admission lookup completed without waiting callers"); + } + if let Some(next) = self.queued.pop_front() { + self.active.insert(next.key().clone()); + self.pending_events.push(next); + } + } + + pub(super) fn take_pending(&mut self) -> Vec { + std::mem::take(&mut self.pending_events) + } +} + +#[derive(Debug)] +struct AdmissionCacheEntry { + source_shard_id: crate::control_plane::membership::ShardGroupId, + admission: Option, + expires_at: Duration, +} + +#[derive(Debug, Default)] +struct AdmissionCache { + entries: HashMap, AdmissionCacheEntry>, +} + +impl AdmissionCache { + fn fresh_admission( + &self, + key: &AdmissionLookupKey, + now: Duration, + ) -> Option<&Option> { + let entry = self.entries.get(key.node_certificate_principal.as_ref())?; + if entry.source_shard_id != key.shard_group_id || entry.expires_at <= now { + return None; + } + Some(&entry.admission) + } + + /// Keeps the highest revision observed from a shard. An older response + /// cannot extend the authority window of a newer, possibly expired record. + fn insert( + &mut self, + key: &AdmissionLookupKey, + admission: Option, + now: Duration, + ) { + if admission.as_ref().is_some_and(|record| { + record.node_certificate_principal != key.node_certificate_principal.as_ref() + }) { + return; + } + let replace = match self.entries.get(key.node_certificate_principal.as_ref()) { + Some(entry) if entry.source_shard_id == key.shard_group_id => { + match (&entry.admission, &admission) { + (Some(current), Some(incoming)) => current.revision <= incoming.revision, + (Some(_), None) => false, + (None, Some(_)) | (None, None) => true, + } + } + Some(_) | None => true, + }; + if replace { + self.entries.insert( + key.node_certificate_principal.clone(), + AdmissionCacheEntry { + source_shard_id: key.shard_group_id, + admission, + expires_at: now + MAX_ADMISSION_CACHE_TTL, + }, + ); + } + } +} + +#[cfg(test)] +pub mod tests { + use tokio::sync::oneshot; + + use super::*; + use crate::control_plane::{ + NodeId, + consensus::transport::admission::{AdmissionLookupResult, LocalAdmissionFetch}, + membership::ShardGroupId, + }; + + fn admission(revision: u64) -> AdmissionRecord { + AdmissionRecord { + node_certificate_principal: "broker-a".to_string(), + revision, + epoch: revision, + node_id: NodeId::new(format!("broker-a::process-{revision}")), + process_public_key: vec![revision as u8].into_boxed_slice(), + } + } + + fn local_fetch(principal: &str) -> AdmissionFetch { + LocalAdmissionFetch(AdmissionLookupKey { + shard_group_id: ShardGroupId(42), + node_certificate_principal: principal.into(), + }) + .into() + } + fn request(principal: &str) -> (LookupAdmission, oneshot::Receiver) { + let (reply, receiver) = oneshot::channel(); + ( + LookupAdmission { + fetch: local_fetch(principal), + reply, + }, + receiver, + ) + } + + #[test] + fn fresh_cache_entry_avoids_another_lookup() { + let mut state = AdmissionLookupState::default(); + let key = local_fetch("broker-a").key().clone(); + state.cache.insert(&key, Some(admission(3)), Duration::ZERO); + let (lookup_request, mut reply) = request("broker-a"); + + state.lookup(lookup_request, Duration::from_secs(1)); + + assert!(state.take_pending().is_empty()); + assert_eq!(reply.try_recv(), Ok(Ok(Some(admission(3))))); + } + + #[test] + fn older_revision_cannot_refresh_an_expired_admission() { + let mut state = AdmissionLookupState::default(); + let key = local_fetch("broker-a").key().clone(); + state.cache.insert(&key, Some(admission(3)), Duration::ZERO); + let expired = MAX_ADMISSION_CACHE_TTL + Duration::from_millis(1); + let (lookup_request, mut reply) = request("broker-a"); + state.lookup(lookup_request, expired); + state.take_pending(); + + state.complete( + AdmissionLookupCompleted { + key, + result: Ok(Some(admission(2))), + }, + expired, + ); + + assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); + } + + #[test] + fn combines_simultaneous_lookups_for_the_same_admission() { + let mut state = AdmissionLookupState::default(); + let (first, mut first_reply) = request("broker-a"); + let (second, mut second_reply) = request("broker-a"); + + state.lookup(first, Duration::ZERO); + state.lookup(second, Duration::ZERO); + + let pending = state.take_pending(); + assert_eq!(pending.len(), 1); + state.complete( + AdmissionLookupCompleted { + key: pending[0].key().clone(), + result: Ok(Some(admission(3))), + }, + Duration::ZERO, + ); + assert_eq!(first_reply.try_recv(), Ok(Ok(Some(admission(3))))); + assert_eq!(second_reply.try_recv(), Ok(Ok(Some(admission(3))))); + } + + #[test] + fn unavailable_lookup_is_not_cached() { + let mut state = AdmissionLookupState::default(); + let (lookup_request, mut reply) = request("broker-a"); + state.lookup(lookup_request, Duration::ZERO); + let pending = state.take_pending(); + state.complete( + AdmissionLookupCompleted { + key: pending[0].key().clone(), + result: Err(AdmissionLookupUnavailable), + }, + Duration::ZERO, + ); + assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); + + let (retry, _reply) = request("broker-a"); + state.lookup(retry, Duration::ZERO); + assert_eq!(state.take_pending().len(), 1); + } + + #[test] + fn response_for_another_principal_is_not_cached() { + let mut state = AdmissionLookupState::default(); + let (lookup_request, mut reply) = request("broker-b"); + state.lookup(lookup_request, Duration::ZERO); + let pending = state.take_pending(); + + state.complete( + AdmissionLookupCompleted { + key: pending[0].key().clone(), + result: Ok(Some(admission(3))), + }, + Duration::ZERO, + ); + + assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); + } + + #[test] + fn rejects_a_lookup_when_active_work_and_queue_are_full() { + let mut state = AdmissionLookupState::default(); + for index in 0..MAX_IN_FLIGHT_LOOKUPS + MAX_QUEUED_LOOKUPS { + let principal = format!("broker-{index}"); + let (lookup_request, _reply) = request(&principal); + state.lookup(lookup_request, Duration::ZERO); + } + + let (rejected, mut reply) = request("broker-overflow"); + state.lookup(rejected, Duration::ZERO); + + assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); + } +} diff --git a/src/control_plane/consensus/transport/inbound.rs b/src/control_plane/consensus/transport/inbound.rs index a781aa9d..455b3b05 100644 --- a/src/control_plane/consensus/transport/inbound.rs +++ b/src/control_plane/consensus/transport/inbound.rs @@ -7,7 +7,7 @@ use crate::security::TransportIdentity; use borsh::BorshDeserialize; use tokio::io::AsyncReadExt; -use super::protocol::{AclSnapshotResponse, InitialClusterMessage}; +use super::protocol::{AclSnapshotResponse, AdmissionLookupResponse, InitialClusterMessage}; pub(super) struct ClusterMessageReader { read_half: TransportReadHalf, @@ -41,6 +41,13 @@ impl ClusterMessageReader { .await } + pub(super) async fn read_admission_lookup_response( + &mut self, + ) -> anyhow::Result { + self.read_frame(4 * 1024 * 1024, "admission lookup response") + .await + } + async fn read_frame( &mut self, maximum_size: usize, diff --git a/src/control_plane/consensus/transport/mod.rs b/src/control_plane/consensus/transport/mod.rs index cd4d411b..2a8c9a7d 100644 --- a/src/control_plane/consensus/transport/mod.rs +++ b/src/control_plane/consensus/transport/mod.rs @@ -2,6 +2,7 @@ mod acl; pub(crate) use acl::{AclSnapshotActor, AclSnapshotSender}; +mod admission; mod inbound; use inbound::*; mod outbound; @@ -83,8 +84,9 @@ mod tests { use crate::control_plane::consensus::messages::{ MultiRaftActorCommand, RaftProtocolMessage, RaftRpc, RequestVote, WireRaftMessage, }; - use crate::control_plane::consensus::raft::states::security::AclRecord; + use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::membership::ShardGroupId; + use crate::control_plane::membership::actor::{RemoteShard, ShardRouting}; use crate::control_plane::metadata::{AclResource, TopicId}; use crate::control_plane::{NodeAddress, NodeAddressInfo}; use crate::net::OwnedWriteHalf; @@ -464,4 +466,95 @@ mod tests { sim.run() } + + #[test] + fn admission_lookup_actor_coalesces_remote_reads() -> turmoil::Result { + let admission = AdmissionRecord { + node_certificate_principal: "broker-a".to_string(), + revision: 3, + epoch: 8, + node_id: NodeId::new("broker-a::process-2"), + process_public_key: vec![1, 2, 3].into_boxed_slice(), + }; + let owner = NodeAddressInfo::new( + NodeId::new("owner"), + NodeAddress::test( + "127.0.0.1:9000".parse().unwrap(), + "127.0.0.1:9001".parse().unwrap(), + ), + ); + let response_received = std::sync::Arc::new(Notify::new()); + let mut sim = Builder::new() + .simulation_duration(Duration::from_secs(5)) + .build(); + + let server_admission = admission.clone(); + let server_response_received = response_received.clone(); + sim.host("owner", move || { + let expected_admission = server_admission.clone(); + let owner_completion = server_response_received.clone(); + async move { + let listener = TcpListener::bind("0.0.0.0:9000").await?; + let (raft_tx, mut raft_rx) = MultiRaftActor::channel(8); + let (dial_tx, _dial_rx) = tokio::sync::mpsc::channel(8); + let mut dispatcher = RaftRpcDispatcher::new( + NodeId::new("owner"), + dial_tx, + NodeTransportSecurity::TrustedDevelopment, + ); + + let (stream, _) = listener.accept().await?; + dispatcher + .accept(TransportTcpStream::TrustedDevelopment(stream), &raft_tx) + .await; + + let Some(MultiRaftActorCommand::GetAdmission(query)) = raft_rx.recv().await else { + panic!("expected admission query"); + }; + assert_eq!(query.shard_group_id, ShardGroupId(42)); + assert_eq!(query.node_certificate_principal.as_ref(), "broker-a"); + let _ = query.reply.send(Ok(Some(expected_admission))); + owner_completion.notified().await; + Ok(()) + } + }); + + let client_admission = admission.clone(); + let client_owner = owner.clone(); + sim.host("requester", move || { + let expected_admission = client_admission.clone(); + let remote_owner = client_owner.clone(); + let requester_completion = response_received.clone(); + async move { + let (raft_tx, _raft_rx) = MultiRaftActor::channel(8); + let lookup = admission::AdmissionLookupActor::spawn( + raft_tx, + NodeTransportSecurity::TrustedDevelopment, + ); + let first_owner = remote_owner.clone(); + let (first, second) = tokio::join!( + lookup.lookup( + ShardRouting::Redirect(Some(RemoteShard { + group_id: ShardGroupId(42), + member: Some(first_owner), + })), + "broker-a".into(), + ), + lookup.lookup( + ShardRouting::Redirect(Some(RemoteShard { + group_id: ShardGroupId(42), + member: Some(remote_owner), + })), + "broker-a".into(), + ), + ); + assert_eq!(first, Ok(Some(expected_admission.clone()))); + assert_eq!(second, Ok(Some(expected_admission))); + requester_completion.notify_one(); + Ok(()) + } + }); + + sim.run() + } } diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index 6415fa19..5b3717b4 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -14,10 +14,11 @@ use crate::control_plane::NodeId; use crate::control_plane::consensus::transport::ClusterMessageReader; use crate::control_plane::membership::actor::SwimSender; use crate::net::{TransportTcpStream, TransportWriteHalf}; -use crate::security::NodeTransportSecurity; +use crate::security::{NodeTransportSecurity, TransportIdentity}; use super::protocol::{ - AclSnapshotRequest, AclSnapshotResponse, InitialClusterMessage, encode_frame, + AclSnapshotRequest, AclSnapshotResponse, AdmissionLookupRequest, AdmissionLookupResponse, + InitialClusterMessage, encode_frame, }; const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); @@ -77,7 +78,7 @@ impl RaftRpcDispatcher { pub(super) async fn accept(&mut self, stream: TransportTcpStream, raft_tx: &MutlRaftSender) { let transport_identity = stream.peer_identity(); let (read_half, write_half) = stream.into_split(); - let mut reader = ClusterMessageReader::new(read_half, transport_identity); + let mut reader = ClusterMessageReader::new(read_half, transport_identity.clone()); let Ok(initial_message) = reader.read_initial_message().await else { tracing::debug!("cluster connection closed before its initial message"); @@ -85,6 +86,14 @@ impl RaftRpcDispatcher { }; match initial_message { + InitialClusterMessage::AdmissionLookup(request) => { + tokio::spawn(serve_admission_lookup_request( + transport_identity, + request, + raft_tx.clone(), + write_half, + )); + } InitialClusterMessage::AclSnapshot(request) => { let peer_id = request.requester_node_id.clone(); tokio::spawn(serve_acl_snapshot_request( @@ -308,6 +317,40 @@ async fn dial( )) } +async fn serve_admission_lookup_request( + transport_identity: TransportIdentity, + request: AdmissionLookupRequest, + raft_tx: MutlRaftSender, + mut writer: TransportWriteHalf, +) { + let admission = match raft_tx + .get_admission(request.shard_group_id, request.node_certificate_principal) + .await + { + Ok(admission) => admission, + Err(error) => { + tracing::debug!( + ?transport_identity, + "admission lookup failed before response: {error}" + ); + return; + } + }; + let Ok(frame) = encode_frame(&AdmissionLookupResponse { admission }) else { + tracing::debug!( + ?transport_identity, + "failed to encode admission lookup response" + ); + return; + }; + if let Err(error) = writer.write_all(&frame).await { + tracing::debug!( + ?transport_identity, + "failed to send admission lookup response: {error}" + ); + } +} + async fn serve_acl_snapshot_request( peer_id: NodeId, request: AclSnapshotRequest, diff --git a/src/control_plane/consensus/transport/protocol.rs b/src/control_plane/consensus/transport/protocol.rs index d61eda06..b4acaa82 100644 --- a/src/control_plane/consensus/transport/protocol.rs +++ b/src/control_plane/consensus/transport/protocol.rs @@ -3,7 +3,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use crate::control_plane::NodeId; use crate::control_plane::consensus::messages::WireRaftMessage; -use crate::control_plane::consensus::raft::states::security::AclRecord; +use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::AclResource; use crate::impl_from_variant; @@ -11,12 +11,13 @@ use crate::impl_from_variant; /// The first frame on a cluster TCP connection. /// /// A Raft connection begins with its first Raft message, which already names -/// its sender. An ACL snapshot connection begins with its requester and read. -/// Later Raft frames are raw; an ACL connection returns one response and closes. +/// its sender. ACL and admission lookup connections contain one read request. +/// Later Raft frames are raw; lookup connections return one response and close. #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] pub(super) enum InitialClusterMessage { Raft(WireRaftMessage), AclSnapshot(AclSnapshotRequest), + AdmissionLookup(AdmissionLookupRequest), } /// One read of a committed ACL record from a shard host. @@ -33,10 +34,29 @@ pub(super) struct AclSnapshotResponse { pub(super) snapshot: Option, } +/// Limited bootstrap read of one admission record from its metadata shard. +/// +/// In secure mode TLS authenticates the caller's node certificate, but this +/// request intentionally does not require process admission: admission is the +/// record the caller is trying to resolve. It cannot carry Raft, ACL, or client +/// data. +#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] +pub(super) struct AdmissionLookupRequest { + pub(super) shard_group_id: ShardGroupId, + pub(super) node_certificate_principal: Box, +} + +/// Response to one limited admission lookup, after which the connection closes. +#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] +pub(super) struct AdmissionLookupResponse { + pub(super) admission: Option, +} + impl_from_variant!( InitialClusterMessage, Raft(WireRaftMessage), AclSnapshot(AclSnapshotRequest), + AdmissionLookup(AdmissionLookupRequest), ); pub(super) fn encode_frame(value: &impl BorshSerialize) -> Result> { From a098bfa32f1ef1997ba094ce606642129bd39f0d Mon Sep 17 00:00:00 2001 From: Migorithm Date: Thu, 30 Jul 2026 21:13:41 +0400 Subject: [PATCH 40/56] update raft-transport rule --- .claude/rules/raft-transport.md | 82 ++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 26 deletions(-) diff --git a/.claude/rules/raft-transport.md b/.claude/rules/raft-transport.md index 30031394..d9ec18e0 100644 --- a/.claude/rules/raft-transport.md +++ b/.claude/rules/raft-transport.md @@ -2,9 +2,9 @@ `RaftTransportActor` — async TCP transport for Raft RPCs. It manages persistent bidirectional Raft connections between nodes. Each connection splits into a -reader task and a writer half held in the per-node `writers` map. The same -authenticated cluster listener also serves a one-shot ACL snapshot read used to -refresh a broker's local authorization cache. +reader task and a writer half held in the per-node `writers` map. +The same authenticated cluster listener also serves a one-shot ACL snapshot read used to +refresh a broker's local authorization cache and a limited admission-record read used to authenticate a connecting process. Separate from SWIM's UDP transport. Raft uses TCP for reliable, ordered delivery. @@ -13,29 +13,45 @@ Separate from SWIM's UDP transport. Raft uses TCP for reliable, ordered delivery ``` cluster listener (TCP) │ - ├── initial Raft message ──► persistent reader + one writer per peer + ├── limited admission read ──► read one record → reply → close │ - └── initial ACL request ──► read committed ACL → reply → close + └── admitted request + │ + ├── both sides verify a TLS-session-bound process proof + │ + ├── Raft ──► persistent reader + one writer per peer + └── ACL ───► read committed ACL → reply → close ``` ## Wire Protocol Length-prefixed Borsh frames: -1. **Initial message** (first frame on every connection): either a Raft message - or an ACL snapshot request. - - The initial Raft message carries its sender, which establishes the peer - identity, then later frames are raw `WireRaftMessage` values until close. - Each message carries `shard_group_id` so transport can dispatch it to the - correct Raft group. - - The initial ACL snapshot request carries its requesting node, shard, and - resource. Its response is one `AclSnapshotResponse`, then the connection - closes. + +1. **Secure initial message:** either one limited `AdmissionLookupRequest` or + `AdmittedClusterMessage`, which contains the dialer's process proof plus a + Raft message or ACL snapshot request. +2. **Mutual admission:** the acceptor verifies the dialer, then replies with its + own `AdmissionProof`. Both proofs sign the same TLS exporter value and are + checked against the peer's current admission record. The exporter lets both + ends derive identical connection-specific bytes without sending those bytes; + another TLS connection derives a different value. +3. **After mutual admission:** + - The first Raft message carries its sender. Later frames are raw + `WireRaftMessage` values until close. Each carries `shard_group_id`. + - An ACL snapshot request carries its requesting node, shard, and resource. + Its response is one `AclSnapshotResponse`, then the connection closes. +4. **Trusted-development initial message:** no cryptographic admission exchange; + the first frame remains either a Raft message or ACL snapshot request. ## Invariants -1. **Connection identity is established by the initial Raft message.** Its -sender keys the writer slot and detects the simultaneous-connect race. Without -it, the acceptor cannot route later frames to a peer-identified slot. +1. **Secure connection identity comes from mTLS plus mutual process admission.** + TLS supplies each stable Node Certificate Principal. A signature over the TLS + exporter value proves possession of the current process key and cannot be + replayed on another TLS session. Mutual proof is required because the Raft + connection carries traffic in both directions. The Raft sender or ACL requester + must equal its admitted `NodeId`. The first Raft sender then keys the writer slot + and detects the simultaneous-connect race. 2. **At most one writer per peer.** `writers` is keyed by `NodeId`. Coexisting writers would split messages to the same peer across two TCP connections; per-connection ordering would let later messages overtake earlier ones in unpredictable patterns, causing the leader to chase its own retries. @@ -43,21 +59,35 @@ it, the acceptor cannot route later frames to a peer-identified slot. 4. **Address resolution is always live.** Every connect attempt queries SWIM for the peer's current address; the transport keeps no local address cache. A stale local cache would connect to the wrong host after a peer moves or restarts on a different address. -5. **Frame sizes are bounded.** Every initial, Raft, and ACL response frame is -capped at 4MB. Without bounds, a malicious or buggy peer can exhaust memory by -sending a giant length prefix before any payload. +5. **Handshake work is bounded.** The listener acquires a permit before +spawning a handshake task, applies a total handshake deadline, and uses a +bounded queue to return verified Raft streams to the dispatcher. A slow TLS, +admission lookup, or proof exchange never blocks the transport select loop. + +6. **Frame sizes are bounded.** Initial, admitted, Raft, proof, and response +frames are capped before allocating their payload. -6. **Transport validates message identity but never interprets the RPC.** The +7. **Transport validates message identity but never interprets the RPC.** The connection peer must match the message `sender`; a mismatch closes that connection. The transport routes by `shard_group_id` and passes the authenticated peer onward, but the RPC remains opaque. Voter, learner, leader, term, and log checks belong to the target Raft state machine. +## Limited Admission Lookup Rule + +Admission records are sharded, so the acceptor may need another broker to read +the record required for its proof check. Requiring process admission for that +read would recurse. `AdmissionLookupRequest` is therefore accepted after mTLS +but before process admission. It can read one named admission record from one +shard, returns one `AdmissionLookupResponse`, and closes. It cannot carry Raft, +ACL, client, or admission-write traffic. + ## ACL Snapshot Rule An ACL snapshot request is not a Raft RPC and never enters a Raft state machine. -It asks the local multi-Raft actor for the selected shard's committed ACL record, -returns that record on the same authenticated connection, then closes the -connection. It carries no client data request and cannot proxy one. A cluster -connection that begins with a Raft message carries only raw Raft frames after -that message; an invalid frame closes the connection. +In secure mode it is served only after the requester completes process +admission. It asks the local multi-Raft actor for the selected shard's committed +ACL record, returns that record on the same connection, then closes. It carries +no client data request and cannot proxy one. A connection admitted for Raft +carries only raw Raft frames after its first message; an invalid frame closes +the connection. From 38efa2c0eba9bcd1d9a79c30598a4a644d381101 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Thu, 30 Jul 2026 21:25:01 +0400 Subject: [PATCH 41/56] roadmap updated --- docs/security/roadmap.md | 340 +++++++++++++++++++++------------------ 1 file changed, 180 insertions(+), 160 deletions(-) diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index 4a0e0402..b714c6a2 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -8,22 +8,23 @@ ## 1. Overview & Threat Model -EastGuard operates in two distinct security modes: - -- **Secure Mode (Default):** Mutual TLS 1.3 protects TCP listeners. Secure SWIM UDP is intentionally deferred until EastGuard can adopt a suitable datagram security implementation. Until then, secure mode fails startup rather than exposing plaintext SWIM. Unauthenticated or unauthorized traffic is immediately rejected. Plaintext connections and protocol downgrades are forbidden; invalid configuration prevents startup. -- **Trusted Development Mode:** Plaintext protocols enabled strictly via explicit opt-in configuration for isolated test environments. +| Mode | Boundary | +| :--- | :--- | +| **Secure (default)** | TLS 1.3 protects TCP. Plaintext, downgrade, invalid configuration, and unauthenticated or unauthorized traffic are rejected. Startup fails while secure SWIM UDP is unavailable. | +| **Trusted development** | Explicit opt-in permits plaintext only in isolated test environments. | ### Threat Model -The production boundary defends against external network attackers attempting eavesdropping, packet modification, replay, UDP spoofing, connection injection, or TLS downgrade. -> **Scope Note:** Compromised brokers or authorized clients are outside this threat model. Their impact is constrained by short-lived credentials, scoped ACLs, and strict principal binding. Byzantine consensus is out of scope. +The boundary covers eavesdropping, modification, replay, UDP spoofing, +connection injection, and downgrade. Compromised brokers, authorized clients, +and Byzantine consensus are out of scope. ### Listener Architecture | Listener | Port | Protocol | Peer Authentication | Purpose | | :--- | :--- | :--- | :--- | :--- | | **Client** | TCP 2921 | TLS 1.3 | Mutual X.509 | Metadata queries, administration, produce, fetch | -| **Raft** | TCP 2922 | TLS 1.3 | Mutual X.509 | Metadata shard consensus log replication and one-shot ACL-cache refreshes between brokers | +| **Raft** | TCP 2922 | TLS 1.3 | Mutual X.509 + process admission | Metadata shard consensus, one-shot ACL-cache refreshes, and limited admission-record reads between brokers | | **Data** | TCP 2923 | TLS 1.3 | Mutual X.509 | Segment replication, repair, and coordination | | **SWIM** | UDP 2922 | Secure datagrams (deferred) | Mutual X.509 | Membership gossip and failure detection | @@ -34,26 +35,29 @@ The production boundary defends against external network attackers attempting ea Security checks are split between the transport layer and application state machines to keep state machines free of security I/O: ``` - [ Authenticated Transport Layer ] +[ Authenticated Transport Layer ] TCP: TLS 1.3 UDP: secure datagrams (deferred) - Authenticate peer X.509 certificates + - Prove that a node connection belongs to the currently admitted process - Enforce framing, datagram MTU, and resource limits - Bind connection envelope sender to verified identity │ │ (Drop connection on transport failure) ▼ - [ Application Layer State Machines ] +[ Application Layer State Machines ] - Authorize requested operation against cached ACLs / placement - Execute SWIM / Raft / Data-Plane state transitions │ │ (Drop denied envelope only; connection stays open) ``` -- **Transport Responsibility:** Performs cryptographic handshakes, validates certificates, tracks cache expiry, and binds envelope senders to verified identities. Transport errors close the connection. -- **Application Responsibility:** Synchronous state machines (SWIM, Raft, Topology, Data Plane) process only pre-validated envelopes. An authorization failure drops the specific denied envelope without tearing down the underlying connection. -- **Placement-Based Data-Plane Authorization:** Data-plane nodes authorize incoming replication and repair requests against local committed placement state, ignoring sender-asserted replica lists. -- **Redirects:** A redirect is only an address hint. The destination authenticates the peer and repeats the authorization check. +| Boundary | Rule | +| :--- | :--- | +| Transport | Authenticates, bounds frames and caches, and binds senders to verified identities. Failure closes the connection. | +| State machine | Authorizes pre-validated envelopes. Denial drops one envelope, not the connection. | +| Data placement | Replication and repair use local committed placement, never a sender-asserted replica list. | +| Redirect | Carries only an address hint. The destination repeats authentication and authorization. | --- @@ -65,11 +69,11 @@ To allow safe node restarts and hardware replacement without exposing the cluste | Term | Scope | Lifetime / Ordering | Function | | :--- | :--- | :--- | :--- | -| **Node Certificate Principal** | X.509 Certificate | Long-lived / Reused | Operator-assigned node principal read from the authenticated certificate and used as the stable admission-record key. | -| **Admission Epoch** | Node Certificate Principal | Monotonically increasing `u64` | Assigned by metadata Raft upon restart; higher epoch **fences** older instances. | -| **NodeId** | Running Process | Single process lifetime | Unique ID generated on startup; used by SWIM, topology ring, Raft, and data placement. | +| **Node Certificate Principal** | X.509 Certificate | Long-lived / Reused | Stable admission-record key read from the certificate. | +| **Admission Epoch** | Node Certificate Principal | Increasing `u64` | Metadata Raft assigns it; a higher value fences older processes. | +| **NodeId** | Running Process | Single process lifetime | Startup-generated ID used by SWIM, topology, Raft, and placement. | | **Process Key** | Running Process | Single process lifetime | Proves that the connection belongs to the process admitted for this epoch. | -| **SWIM Incarnation** | Running Process | Monotonically increasing counter | Incremented by the *same* process instance to refute false `Suspect`/`Dead` gossip. | +| **SWIM Incarnation** | Running Process | Increasing counter | The same process increments it to refute stale liveness gossip. | The Node Certificate Principal is the value after `urn:eastguard:node:` in one URI Subject Alternative Name. A node certificate must contain exactly one such @@ -91,13 +95,13 @@ Node identity conflicts and stale gossip are resolved in this order: ### Partition Recovery vs. Node Restart -An **authorized operator** is a person or trusted automation using a client -certificate whose principal has the `security/cluster` grant. In an orchestrated -deployment, automation may approve restarts and scaling, but metadata Raft still -commits every admission. +An **authorized operator** is a person or automation whose client-certificate +principal has the `security/cluster` grant. -- **Healed Partition (Same Process):** Retains its existing `NodeId` and `Admission Epoch`. Increments its `SWIM Incarnation` counter to refute `Suspect` or `Dead` rumors spread during the partition. -- **Node Restart / Replacement:** The process generates a new `NodeId` and process key. An authorized operator approves that exact process, and metadata Raft commits the new `Admission Epoch`, `NodeId`, and process public key. The higher epoch fences older processes immediately where observed and everywhere else within 60 seconds. +| Event | Identity and recovery | +| :--- | :--- | +| Healed partition; same process | Keep the `NodeId` and epoch. Increase the SWIM incarnation to refute stale `Suspect` or `Dead` gossip. | +| Restart or replacement | Create a new `NodeId` and process key. The operator approves them; metadata Raft commits a higher epoch. Cache expiry fences the old process everywhere within 60 seconds. | ### Admission Gate & SWIM Separation @@ -122,18 +126,97 @@ SWIM liveness gossip is decoupled from cluster admission authority to prevent ne [ Raft Reconciliation ] ──► Commit AddPeer / RemovePeer ``` -1. **Process Proof:** A node session proves possession of the process private key bound to its admitted epoch. The process key signs the admitted identity and a session-specific value produced by the authenticated transport, so the proof cannot be replayed in another session. The reusable node certificate alone cannot create or claim a newer epoch. -2. **Admission Gate:** The SWIM actor checks incoming packets against a local admission cache backed by metadata Raft before passing facts to the SWIM state machine. -3. **Gossip Rule:** The secure datagram session authenticates the immediate sender. Every relayed membership fact is separately accepted only when its subject `NodeId` and `Admission Epoch` match an active admission record. -4. **Cache Policy:** Admission records are cached locally with a maximum TTL of 60 seconds. If a cache entry expires while the owning metadata shard is unreachable, the gate fails closed. +- Metadata Raft decides admission; SWIM reports only liveness. +- The transport authenticates the immediate sender. A relayed fact is accepted + only when its subject `NodeId` and epoch match an active admission. +- The local admission cache expires within 60 seconds and fails closed when its + owning shard is unavailable. + +### Why a TLS Session Proof Is Necessary + +A node certificate identifies a reusable broker principal, not one process +start. A UUID prevents accidental identity collisions, but it is public cluster +data. An old process that still has the reusable certificate can observe and +claim the current `NodeId` and `Admission Epoch`. + +``` +NodeId ──► which process the record names +Epoch ──► which admission is newer +Process key ──► proof that the speaker owns that admission +TLS value ──► proof is valid only on this connection +``` + +The admission record therefore stores a public key for one process start. It is +approved and committed before the connection. The broker sends a signature, not +a replacement key. Both sides prove their keys because Raft traffic is +bidirectional. + +``` +Connecting broker Accepting broker + │ │ + │◄────────────── mTLS ───────────────────►│ + │ derive the same fresh TLS session value │ + │── process proof + Raft / ACL ──────────►│ + │ │ verify current admission + │◄──────────── process proof ─────────────│ + │ verify current admission │ + │◄──────── admitted connection ──────────►│ +``` + +A TLS exporter derives application-specific bytes from a completed handshake: + +``` +connection 1: broker A derives X broker B derives X +connection 2: broker A derives Y broker B derives Y + X != Y +network: sends signatures over X or Y, never X or Y itself +``` + +“Shared” means both ends of one connection derive the same value. A signature +over `X` fails on a connection using `Y`; this removes the need for another +challenge. Missing or stale admission, identity mismatch, or bad signature +closes the connection before Raft or ACL dispatch. + +### Why Admission Lookup Is Separate + +The admission record may live on another broker. A normal cluster connection +would recurse: + +``` +Need record ──► open admitted connection ──► need record ──► loop +``` + +A narrow pre-admission path breaks the loop: + +``` +Accepting broker Admission shard host + │ │ + 1. Open ─────────────── mTLS ──────────────────►│ + │ │ + 2. Ask ─────── one admission-record key ───────►│ + │ │ read committed state + 3. Return ◄────────── record or no record ──────│ + │ │ + └──────────────── connection closes ───────┘ +``` + +- **Authentication:** Secure mode requires mTLS. It authenticates the reusable + node certificate, not the running process. +- **One purpose:** The connection reads one admission record. It cannot carry + Raft messages, ACL reads, client requests, or admission writes. +- **Bounded work:** One broker-local worker limits concurrent and queued reads + and combines simultaneous requests for the same record. +- **Cache result:** A record or confirmed missing record is cached for at most + 60 seconds. A missing record denies admission. +- **Do not cache failure:** Timeout, routing failure, or an unavailable shard + denies the current connection but is retried by a later lookup. --- ## 4. Authorization & Sharded Security Records -EastGuard enforces exact, default-deny access control lists (ACLs) without wildcards or inheritance. -A principal is the client ID read from an authenticated certificate and used for -permission checks; its text grants no authority by itself. +ACLs are exact and default deny, with no wildcards or inheritance. A principal +comes from the client certificate; its text grants no authority by itself. ### ACL Resource Catalog @@ -146,40 +229,31 @@ permission checks; its text grants no authority by itself. | `producer-session/{topic-id}/{producer-id}` | Renew the session, bound to its creator for the session lifetime | | `security/cluster` | Read/write ACLs, manage admissions and revocations, inspect security audit | -Consumer-group access permits group coordination and offset read/commit. Reading -records separately requires `Fetch` on `topic-data/{topic-id}`. Text resource -keys are used only at routing and administrative boundaries; replicated records -store the typed resource directly. +Consumer-group access covers coordination and offsets. Fetching records also +requires `topic-data/{topic-id}`. Text keys exist only at routing and +administrative boundaries; replicated state stores typed resources. ### Sharded Metadata Storage -A security record is one durable admission, ACL, or revocation entry. Its record -path selects one metadata shard; its revision lets brokers detect stale cached -copies. - -Security records (`security/node/{node-certificate-principal}`, `security/acl/{resource}`, `security/revocation/{issuer}/{serial}`) do not rely on a centralized security controller. Instead, they hash to standard metadata shards and replicate via Raft: +Admission, ACL, and revocation paths hash to standard metadata shards: ``` - Client/Node Request ──► Any Broker ──► Hash Record Path ──► Hosts Shard? ─┬─► Yes ──► Commit via Raft - └─► No ──► Return Owner Redirect +Request ──► hash record path ──► shard host? ─┬─► yes: commit through Raft + └─► no: return owner redirect ``` -- **Stable Admission Key:** A restart changes the process `NodeId` and key, but - not the Node Certificate Principal. The same record and metadata shard therefore - replace the old admitted process atomically: - ``` security/node/{node-certificate-principal} │ └── Admission Epoch + NodeId + Process Public Key ``` -- **Local Authorization:** Brokers evaluate ACLs against local cached security records. - Data permissions use the stable topic ID, so a data replica can authorize a - request without hosting that topic's metadata shard. -- **Freshness & Expiry:** Cached records include a monotonic deadline (max 60s) - and revision counter. An expired entry cannot authorize a request. The broker - reads the current owner directly when needed, or fails closed. +| Property | Rule | +| :--- | :--- | +| No controller | Every record is owned and replicated by its ordinary metadata shard. | +| Stable admission key | Restart changes the `NodeId` and process key, not the certificate principal, so one record atomically replaces the old process. | +| Local authorization | Cached ACLs and stable topic IDs let a data replica authorize without hosting topic metadata. | +| Freshness | Cache entries carry source shard, revision, and a monotonic deadline no later than 60 seconds. Expiry triggers refresh or denial. | --- @@ -187,34 +261,34 @@ security/node/{node-certificate-principal} ### Bootstrap & Node Joining -1. Operator initializes a cluster trust root and issues the first node certificate. -2. The first node generates its `NodeId` and process key. Initial metadata state - stores that admission, the first operator principal, and its - `security/cluster` grant. -3. A later joining node generates a new `NodeId` and process key. -4. An authorized operator approves that exact `NodeId` and process public key. The reusable node certificate alone cannot authorize replacement. -5. The joining node connects to a **limited admission endpoint** using its X.509 certificate. -6. The endpoint uses the authenticated Node Certificate Principal to route to its - admission record. The owning metadata shard atomically replaces the prior - process with the next `Admission Epoch`, `NodeId`, and process public key. -7. The joining node proves possession of the process private key before entering SWIM gossip and Raft membership reconciliation. +| Moment | Operator and cluster action | +| :--- | :--- | +| First broker | Create the trust root and node certificate. Initial metadata stores the first process admission, operator principal, and `security/cluster` grant. | +| Every later start | The process creates a new `NodeId` and process key. An authorized operator approves both. The owning shard increments the admission epoch and replaces the old process atomically. | +| Cluster connection | Each side follows the admission gate in Section 3. Raft or ACL traffic starts only after mutual process proof succeeds. | + +The reusable node certificate alone cannot replace an admitted process. SWIM +also remains blocked until the secure datagram admission gate in S3 exists. ### Online Credential Rotation -- **Zero-Downtime CA Rotation:** Brokers support dual trust chain loading. New root CAs can be added and leaf certificates reloaded online without restarting brokers or changing `Admission Epoch` / `NodeId`. -- **Revocation & Expiry:** Certificate revocations commit to metadata Raft records. Active authenticated sessions are terminated within the cache enforcement window. Expired certificates are rejected with clock-skew tolerance. -- **Recovery:** Runbooks cover lost authorized-operator access, lost issuing keys, expiry, accidental revocation, trust-root replacement, and cold-cluster restart. +| Operation | Required behavior | +| :--- | :--- | +| CA or leaf rotation | Load old and new trust chains together; reload leaves without restart or process-identity change. | +| Revocation or expiry | Commit revocations through metadata Raft; close active sessions within cache expiry; allow bounded clock skew. | +| Recovery | Cover lost operator access or issuing keys, accidental revocation, trust-root replacement, and cold restart. | --- ## 6. Resource Limits & Security Audit ### Rate & Memory Bounds -- Every listener enforces strict limits on unauthenticated handshakes, concurrent connections, in-flight frames, and memory allocations. -- Client request-rate limits are deferred. The design must first define whether - replicas of one application share a principal, which limits are node-wide, and - how limits behave as a Kubernetes workload scales. -- The future secure UDP transport must define a payload budget that avoids IP fragmentation after authentication and encryption overhead. + +| Boundary | Limit | +| :--- | :--- | +| Listener | Bound unauthenticated handshakes, connections, in-flight frames, and allocations. | +| Client requests | Rate limiting is deferred until principal sharing and node-wide limits are defined for autoscaling workloads. | +| Future secure UDP | Keep protected payloads below the IP-fragmentation threshold. | ### Client Request Boundary @@ -252,28 +326,15 @@ Check local ACL cache └── Yes ──► Execute locally ``` -Authorization precedes redirects so an ungranted client cannot use stale-route -responses to discover data placement. A missing or expired entry triggers a -lazy pull from the ACL shard's current host. A local host reads its committed -state directly; a remote host receives one short authenticated node request and -returns only the ACL record. The request never carries client data and never -turns the broker into a data proxy. A missing record is a bounded cached denial; -an unavailable owner also denies the request. - -``` -Data broker ACL shard host - │ │ - │── authenticated ACL-only read ──────►│ - │ │── read committed record - │◄──────── record or unavailable ──────│ - │ - └── refresh local cache (at most 60s) ──► allow or deny client request -``` +| Decision | Rule | +| :--- | :--- | +| Authorize before redirect | An ungranted client cannot discover placement through stale-route responses. | +| Refresh on cache miss | Read local committed state or make one authenticated ACL-only request to the shard host. Never proxy client data. | +| Fail closed | Cache a missing record as a bounded denial. An unavailable owner denies without caching the failure. | -This pull-on-miss path is deliberately small: cache expiry is 60 seconds, so a -second connection pool would add persistent state for infrequent reads. A future -push or hybrid distribution scheme remains optional; it must preserve the same -fail-closed behavior and update only the ACL cache. +Pull-on-miss avoids a second connection pool for reads needed at most once per +cache window. A future push or hybrid design may update only the same cache and +must remain fail closed. One broker-local refresh worker owns these remote reads. It bounds active and queued work, and combines simultaneous requests for the same ACL record into @@ -290,18 +351,19 @@ many cache misses └── full / unavailable ─────► deny ``` -The worker combines only identical records. Several different records owned by -the same remote shard host can still create several short connections. If that -becomes material, the next step is one request containing several resources for -the same owner and shard—not delaying mailbox reads or adding a connection pool. +Only identical records combine. If distinct records to one shard become costly, +batch those resource keys in one request; do not delay mailbox reads or add a +connection pool. + +Client rate limiting remains deferred: + +| Principal model | Problem with a fixed per-principal limit | +| :--- | :--- | +| Shared by application replicas | One limit represents an autoscaling workload. | +| Unique per replica | Principal state grows with replica count. | -The same decision gate covers request-rate limiting. A Kubernetes deployment may -give all replicas of one application a shared principal even though each replica -has a different leaf certificate. A fixed per-principal table size or request -budget is therefore premature: one shared principal can represent many clients, -while one principal per replica can grow with autoscaling. No client request-rate -limit is implemented until identity granularity, node-wide capacity bounds, and -cache distribution are chosen together. +Identity granularity, node-wide capacity, and cache distribution must be chosen +together before adding a limit. ### Secure UDP Decision @@ -320,18 +382,14 @@ constant socket count packet loss remains per node visible to SWIM ``` -TCP with bounded connection reuse was rejected because large clusters would -trade an unbounded connection mesh for continuous handshakes, connection churn, -kernel connection tracking, and head-of-line blocking. QUIC datagrams preserve -UDP delivery semantics, but their per-peer connection state and implementation -complexity are not justified for SWIM's sparse traffic. Current DTLS options do -not meet the combined requirements for maturity, permissive licensing, Rust -integration, and deterministic simulation. +| Alternative | Why it is not selected now | +| :--- | :--- | +| TCP | Either keeps a connection mesh or causes handshake churn, kernel tracking, and head-of-line blocking. | +| QUIC datagrams | Preserve loss, but add per-peer connection state and complexity for sparse probes. | +| Current DTLS libraries | Do not yet combine maturity, permissive licensing, Rust integration, and deterministic simulation. | -Secure SWIM therefore remains deferred. Trusted development mode may use -plaintext UDP in isolated environments. Secure mode must fail startup until a -secure datagram transport is selected and implemented; it must never fall back -to plaintext UDP. +Secure SWIM remains deferred. Trusted development may use plaintext UDP in +isolation; secure mode fails startup and never falls back to it. ### Acceptance Criteria for a Future Secure UDP Transport @@ -357,50 +415,12 @@ The selected transport must: ## 7. Delivery Plan (S0–S6) -``` -S0 ──► S1 ──► S2 ──► S3 ──► S4 ──► S5 ──► S6 -config records TCP SWIM clients operations production - mTLS deferred + ACLs + audit gate -``` - -| Phase | Target Scope | Key Deliverable | Exit Criteria | -| :--- | :--- | :--- | :--- | -| **S0** | Configuration | Security modes and certificate loader | Secure mode opens no plaintext listeners and fails startup while any required secure listener is unavailable | -| **S1** | Metadata Storage | Security record schema, sharded Raft state | Security records survive snapshot & recovery | -| **S2** | Cluster Transport | TLS 1.3 on TCP 2922/2923, Raft sender and role authorization | Authenticated and authorized cluster TCP traffic | -| **S3** | Membership (deferred) | Secure datagrams on UDP 2922 and SWIM admission gate | A transport meeting the secure UDP acceptance criteria provides authenticated gossip and partition-safe admission fencing | -| **S4** | Client API | Client mTLS on TCP 2921, principal binding, ACLs | Default-deny enforcement on all client APIs | -| **S5** | Operations | Certificate rotation, revocation, expiry, recovery, audit logging | Online credential operations and recovery runbooks | -| **S6** | Production Gate | Adversarial testing, fuzzing, partition stress | Passes all production readiness checks | - -S6 must verify node and client impersonation, stale-process replay, unauthorized -operations, protocol downgrade, rotation under live traffic, expired and revoked -credentials, cold-cluster restart, handshake and datagram fuzzing, resource -bounds, secret-free diagnostics, and reproducible secure-SWIM behavior under -turmoil with pinned randomness and node identities. S6 cannot pass while S3 is -deferred. - ---- - -## 8. Invariants & Security Rules - -### System Invariants - -1. **Single Connection Identity:** Every established client connection has exactly one authenticated principal; every node connection has exactly one `(Node Certificate Principal, Admission Epoch, NodeId, Process Public Key)`. -2. **Unique Active Node Admission:** Metadata state maintains at most one active `(Admission Epoch, NodeId, Process Public Key)` per Node Certificate Principal. -3. **Immutable Producer Session Principal:** Every producer session is immutably bound to the principal that created it. -4. **Explicit Cache Bounding:** Every cached security entry specifies its source metadata shard, revision, and expiry measured with a monotonic clock (≤ 60 seconds). -5. **Bounded Audit Footprint:** Audit queues and aggregate rate counters remain within configured capacity. - -### Operational Rules - -1. Secure mode never downgrades to plaintext or unauthenticated protocols. -2. Transport layers authenticate identity; application state machines authorize actions. -3. Transport identity mismatches close the connection immediately before payload dispatch. -4. Application authorization denials drop only the denied envelope, preserving the connection for valid traffic. -5. A higher admission epoch fences every older process immediately where observed and no later than admission-cache expiry elsewhere. -6. SWIM controls liveness; metadata Raft controls admission. -7. Missing or expired security records cause authorization and admission checks to fail closed. -8. Resource permits are acquired before allocating memory or spawning async tasks. -9. Audit logging backpressure must never block network protocol processing or consensus. -10. Only an authorized operator may approve a higher admission epoch. +| Phase | Complete when | +| :--- | :--- | +| **S0 — Configuration** | Secure configuration loads certificates, opens no plaintext listener, and fails startup when a required secure listener is unavailable. | +| **S1 — Records** | Sharded admission, ACL, and revocation records survive snapshot and recovery. | +| **S2 — Cluster TCP** | TLS 1.3 and session-bound process proof admit Raft, ACL, and data traffic only from the current process. | +| **S3 — SWIM (deferred)** | A transport meeting Section 6's UDP criteria provides authenticated gossip and partition-safe admission fencing. | +| **S4 — Clients** | Client mTLS and ACLs enforce default deny on every API. | +| **S5 — Operations** | Rotation, revocation, expiry, recovery, and non-blocking audit work online. | +| **S6 — Production** | Impersonation, replay, downgrade, fuzzing, resource bounds, recovery, and reproducible secure-SWIM tests pass. S3 must be complete. | From 046675aeffef6f088ef672d82ad4c8daabc177d2 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Fri, 31 Jul 2026 08:56:49 +0400 Subject: [PATCH 42/56] admission query, completion, param --- src/connections/controller.rs | 13 ++++---- .../consensus/transport/admission/message.rs | 33 +++---------------- 2 files changed, 10 insertions(+), 36 deletions(-) diff --git a/src/connections/controller.rs b/src/connections/controller.rs index 7768f959..c76ca970 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -746,7 +746,9 @@ mod tests { use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::MultiRaftActorCommand; use crate::control_plane::consensus::raft::states::security::AclRecord; - use crate::control_plane::consensus::transport::{AclSnapshotActor, RaftTransportActor}; + use crate::control_plane::consensus::transport::{ + AclSnapshotActor, ClusterSecurity, RaftTransportActor, + }; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ QueryCommand, ShardGroup, ShardGroupId, ShardLeaderEntry, SwimActorCommand, @@ -761,7 +763,6 @@ mod tests { use crate::data_plane::messages::DataPlaneMessage; use crate::data_plane::messages::command::{DataPlaneCommand, ProduceAck}; use crate::net::TcpListener; - use crate::security::NodeTransportSecurity; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -835,8 +836,7 @@ mod tests { raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, ) -> ClientController { - let acl_snapshot_sender = - AclSnapshotActor::spawn(NodeTransportSecurity::TrustedDevelopment); + let acl_snapshot_sender = AclSnapshotActor::spawn(ClusterSecurity::TrustedDevelopment); ClientController::new( TransportIdentity::TrustedDevelopment, node_id, @@ -870,8 +870,7 @@ mod tests { raft_sender: MutlRaftSender, acl_cache: SharedAclCache, ) -> ClientController { - let acl_snapshot_sender = - AclSnapshotActor::spawn(NodeTransportSecurity::TrustedDevelopment); + let acl_snapshot_sender = AclSnapshotActor::spawn(ClusterSecurity::TrustedDevelopment); ClientController::new( TransportIdentity::CertificatePrincipal(principal.into()), node_id, @@ -1235,7 +1234,7 @@ mod tests { raft_tx, transport_rx, swim_tx, - NodeTransportSecurity::TrustedDevelopment, + ClusterSecurity::TrustedDevelopment, )); let Some(MultiRaftActorCommand::GetAclSnapshot(query)) = raft_rx.recv().await diff --git a/src/control_plane/consensus/transport/admission/message.rs b/src/control_plane/consensus/transport/admission/message.rs index a79ac574..241b9b21 100644 --- a/src/control_plane/consensus/transport/admission/message.rs +++ b/src/control_plane/consensus/transport/admission/message.rs @@ -3,44 +3,19 @@ use tokio::sync::oneshot; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::states::security::AdmissionRecord; use crate::control_plane::membership::ShardGroupId; -use crate::impl_from_variant; pub(super) type AdmissionLookupResult = Result, AdmissionLookupUnavailable>; pub(super) type AdmissionLookupReply = oneshot::Sender; -pub(super) struct LookupAdmission { - pub(super) fetch: AdmissionFetch, +pub(super) struct AdmissionQuery { + pub(super) fetch: AdmissionTarget, pub(super) reply: AdmissionLookupReply, } #[derive(Clone, Debug)] -pub(super) enum AdmissionFetch { - Local(LocalAdmissionFetch), - Remote(RemoteAdmissionFetch), -} - -#[derive(Clone, Debug)] -pub(super) struct LocalAdmissionFetch(pub(super) AdmissionLookupKey); - -#[derive(Clone, Debug)] -pub(super) struct RemoteAdmissionFetch { +pub(super) struct AdmissionTarget { pub(super) key: AdmissionLookupKey, - pub(super) owner: NodeAddressInfo, -} - -impl_from_variant!( - AdmissionFetch, - Local(LocalAdmissionFetch), - Remote(RemoteAdmissionFetch), -); - -impl AdmissionFetch { - pub(super) fn key(&self) -> &AdmissionLookupKey { - match self { - Self::Local(fetch) => &fetch.0, - Self::Remote(fetch) => &fetch.key, - } - } + pub(super) remote_owner: Option, } #[derive(Clone, Debug, PartialEq, Eq, Hash)] From 7456be0ea593a86fdba5aa90368f4f192c2e58c9 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Fri, 31 Jul 2026 09:29:36 +0400 Subject: [PATCH 43/56] refactor: consolidate AdmissionLookupKey and AdmissionLookupRequest into AdmissionRecordKey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shard routing │ ▼ AdmissionRecordKey ├── cache/coalescing key └── remote wire lookup ProcessAdmissionRequest │ verify process proof ▼ AcceptedRaftConnection Changes: - Consolidated AdmissionLookupKey and AdmissionLookupRequest into AdmissionRecordKey. - The same key now flows directly from routing onto the wire. - Renamed: - InitialClusterMessage::Admitted → ProcessAdmission - AdmittedClusterMessage → ProcessAdmissionRequest - Updated the Raft transport rule. - Removed all old-name references. - Borsh variant positions and field order remain unchanged, so the wire layout is unchanged. --- .claude/rules/raft-transport.md | 16 ++--- .../consensus/transport/admission/message.rs | 13 ++-- .../consensus/transport/admission/state.rs | 59 ++++++++++--------- src/it/raft/election.rs | 5 +- src/it/raft/leader_event.rs | 5 +- src/it/raft/membership_change.rs | 5 +- src/it/raft/snapshot.rs | 5 +- src/security/mod.rs | 5 +- 8 files changed, 54 insertions(+), 59 deletions(-) diff --git a/.claude/rules/raft-transport.md b/.claude/rules/raft-transport.md index d9ec18e0..15773591 100644 --- a/.claude/rules/raft-transport.md +++ b/.claude/rules/raft-transport.md @@ -15,7 +15,7 @@ cluster listener (TCP) │ ├── limited admission read ──► read one record → reply → close │ - └── admitted request + └── request with process proof │ ├── both sides verify a TLS-session-bound process proof │ @@ -27,8 +27,8 @@ cluster listener (TCP) Length-prefixed Borsh frames: -1. **Secure initial message:** either one limited `AdmissionLookupRequest` or - `AdmittedClusterMessage`, which contains the dialer's process proof plus a +1. **Secure initial message:** either `AdmissionLookup(AdmissionRecordKey)` or + `ProcessAdmissionRequest`, which contains the dialer's process proof plus a Raft message or ACL snapshot request. 2. **Mutual admission:** the acceptor verifies the dialer, then replies with its own `AdmissionProof`. Both proofs sign the same TLS exporter value and are @@ -64,7 +64,7 @@ spawning a handshake task, applies a total handshake deadline, and uses a bounded queue to return verified Raft streams to the dispatcher. A slow TLS, admission lookup, or proof exchange never blocks the transport select loop. -6. **Frame sizes are bounded.** Initial, admitted, Raft, proof, and response +6. **Frame sizes are bounded.** Initial, process-admission, Raft, proof, and response frames are capped before allocating their payload. 7. **Transport validates message identity but never interprets the RPC.** The @@ -77,10 +77,10 @@ checks belong to the target Raft state machine. Admission records are sharded, so the acceptor may need another broker to read the record required for its proof check. Requiring process admission for that -read would recurse. `AdmissionLookupRequest` is therefore accepted after mTLS -but before process admission. It can read one named admission record from one -shard, returns one `AdmissionLookupResponse`, and closes. It cannot carry Raft, -ACL, client, or admission-write traffic. +read would recurse. `AdmissionLookup(AdmissionRecordKey)` is therefore accepted +after mTLS but before process admission. It can read one named admission record +from one shard, returns one `AdmissionLookupResponse`, and closes. It cannot +carry Raft, ACL, client, or admission-write traffic. ## ACL Snapshot Rule diff --git a/src/control_plane/consensus/transport/admission/message.rs b/src/control_plane/consensus/transport/admission/message.rs index 241b9b21..10ec79c1 100644 --- a/src/control_plane/consensus/transport/admission/message.rs +++ b/src/control_plane/consensus/transport/admission/message.rs @@ -2,7 +2,8 @@ use tokio::sync::oneshot; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::states::security::AdmissionRecord; -use crate::control_plane::membership::ShardGroupId; + +use super::super::protocol::AdmissionRecordKey; pub(super) type AdmissionLookupResult = Result, AdmissionLookupUnavailable>; pub(super) type AdmissionLookupReply = oneshot::Sender; @@ -14,18 +15,12 @@ pub(super) struct AdmissionQuery { #[derive(Clone, Debug)] pub(super) struct AdmissionTarget { - pub(super) key: AdmissionLookupKey, + pub(super) key: AdmissionRecordKey, pub(super) remote_owner: Option, } -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub(super) struct AdmissionLookupKey { - pub(super) shard_group_id: ShardGroupId, - pub(super) node_certificate_principal: Box, -} - pub(super) struct AdmissionLookupCompleted { - pub(super) key: AdmissionLookupKey, + pub(super) key: AdmissionRecordKey, pub(super) result: Result, AdmissionLookupUnavailable>, } diff --git a/src/control_plane/consensus/transport/admission/state.rs b/src/control_plane/consensus/transport/admission/state.rs index ed9d62ca..9de1040a 100644 --- a/src/control_plane/consensus/transport/admission/state.rs +++ b/src/control_plane/consensus/transport/admission/state.rs @@ -6,9 +6,10 @@ use std::{ use crate::control_plane::consensus::{ raft::states::security::AdmissionRecord, transport::admission::{ - AdmissionFetch, AdmissionLookupCompleted, AdmissionLookupKey, AdmissionLookupReply, - LookupAdmission, message::AdmissionLookupUnavailable, + AdmissionLookupCompleted, AdmissionLookupReply, AdmissionQuery, AdmissionTarget, + message::AdmissionLookupUnavailable, }, + transport::protocol::AdmissionRecordKey, }; const MAX_ADMISSION_CACHE_TTL: Duration = Duration::from_secs(60); @@ -20,16 +21,16 @@ const MAX_WAITERS_PER_LOOKUP: usize = 256; #[derive(Default)] pub(super) struct AdmissionLookupState { cache: AdmissionCache, - active: HashSet, - queued: VecDeque, - waiters: HashMap>, - pending_events: Vec, + active: HashSet, + queued: VecDeque, + waiters: HashMap>, + pending_events: Vec, } impl AdmissionLookupState { - pub(super) fn lookup(&mut self, request: LookupAdmission, now: Duration) { - let LookupAdmission { fetch, reply } = request; - let key = fetch.key().clone(); + pub(super) fn lookup(&mut self, request: AdmissionQuery, now: Duration) { + let AdmissionQuery { fetch, reply } = request; + let key = fetch.key.clone(); if let Some(admission) = self.cache.fresh_admission(&key, now) { let _ = reply.send(Ok(admission.clone())); @@ -81,12 +82,12 @@ impl AdmissionLookupState { tracing::debug!("admission lookup completed without waiting callers"); } if let Some(next) = self.queued.pop_front() { - self.active.insert(next.key().clone()); + self.active.insert(next.key.clone()); self.pending_events.push(next); } } - pub(super) fn take_pending(&mut self) -> Vec { + pub(super) fn take_pending(&mut self) -> Vec { std::mem::take(&mut self.pending_events) } } @@ -106,7 +107,7 @@ struct AdmissionCache { impl AdmissionCache { fn fresh_admission( &self, - key: &AdmissionLookupKey, + key: &AdmissionRecordKey, now: Duration, ) -> Option<&Option> { let entry = self.entries.get(key.node_certificate_principal.as_ref())?; @@ -120,7 +121,7 @@ impl AdmissionCache { /// cannot extend the authority window of a newer, possibly expired record. fn insert( &mut self, - key: &AdmissionLookupKey, + key: &AdmissionRecordKey, admission: Option, now: Duration, ) { @@ -158,9 +159,7 @@ pub mod tests { use super::*; use crate::control_plane::{ - NodeId, - consensus::transport::admission::{AdmissionLookupResult, LocalAdmissionFetch}, - membership::ShardGroupId, + NodeId, consensus::transport::admission::AdmissionLookupResult, membership::ShardGroupId, }; fn admission(revision: u64) -> AdmissionRecord { @@ -173,17 +172,19 @@ pub mod tests { } } - fn local_fetch(principal: &str) -> AdmissionFetch { - LocalAdmissionFetch(AdmissionLookupKey { - shard_group_id: ShardGroupId(42), - node_certificate_principal: principal.into(), - }) - .into() + fn local_fetch(principal: &str) -> AdmissionTarget { + AdmissionTarget { + key: AdmissionRecordKey { + shard_group_id: ShardGroupId(42), + node_certificate_principal: principal.into(), + }, + remote_owner: None, + } } - fn request(principal: &str) -> (LookupAdmission, oneshot::Receiver) { + fn request(principal: &str) -> (AdmissionQuery, oneshot::Receiver) { let (reply, receiver) = oneshot::channel(); ( - LookupAdmission { + AdmissionQuery { fetch: local_fetch(principal), reply, }, @@ -194,7 +195,7 @@ pub mod tests { #[test] fn fresh_cache_entry_avoids_another_lookup() { let mut state = AdmissionLookupState::default(); - let key = local_fetch("broker-a").key().clone(); + let key = local_fetch("broker-a").key.clone(); state.cache.insert(&key, Some(admission(3)), Duration::ZERO); let (lookup_request, mut reply) = request("broker-a"); @@ -207,7 +208,7 @@ pub mod tests { #[test] fn older_revision_cannot_refresh_an_expired_admission() { let mut state = AdmissionLookupState::default(); - let key = local_fetch("broker-a").key().clone(); + let key = local_fetch("broker-a").key.clone(); state.cache.insert(&key, Some(admission(3)), Duration::ZERO); let expired = MAX_ADMISSION_CACHE_TTL + Duration::from_millis(1); let (lookup_request, mut reply) = request("broker-a"); @@ -238,7 +239,7 @@ pub mod tests { assert_eq!(pending.len(), 1); state.complete( AdmissionLookupCompleted { - key: pending[0].key().clone(), + key: pending[0].key.clone(), result: Ok(Some(admission(3))), }, Duration::ZERO, @@ -255,7 +256,7 @@ pub mod tests { let pending = state.take_pending(); state.complete( AdmissionLookupCompleted { - key: pending[0].key().clone(), + key: pending[0].key.clone(), result: Err(AdmissionLookupUnavailable), }, Duration::ZERO, @@ -276,7 +277,7 @@ pub mod tests { state.complete( AdmissionLookupCompleted { - key: pending[0].key().clone(), + key: pending[0].key.clone(), result: Ok(Some(admission(3))), }, Duration::ZERO, diff --git a/src/it/raft/election.rs b/src/it/raft/election.rs index 2424350c..deba834e 100644 --- a/src/it/raft/election.rs +++ b/src/it/raft/election.rs @@ -9,7 +9,7 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::{EnsureGroup, MultiRaftActorCommand, RaftTimer}; -use crate::control_plane::consensus::transport::RaftTransportActor; +use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ShardGroup, ShardGroupId}; use crate::control_plane::{NodeId, Replicas}; @@ -19,7 +19,6 @@ use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; use crate::schedulers::ticker_message::{SchedulerSender, TickerCommand}; -use crate::security::NodeTransportSecurity; use super::{CLUSTER_PORT, QUERY_PORT, mock_swim_handler}; @@ -110,7 +109,7 @@ async fn run_raft_node( raft_tx.clone(), transport_rx, swim_tx.clone(), - NodeTransportSecurity::TrustedDevelopment, + ClusterSecurity::TrustedDevelopment, )); let db = MetadataStorage::open(std::env::temp_dir().join(uuid::Uuid::new_v4().to_string())); let election_jitter_seed = { diff --git a/src/it/raft/leader_event.rs b/src/it/raft/leader_event.rs index 97d6261b..dfbc2f86 100644 --- a/src/it/raft/leader_event.rs +++ b/src/it/raft/leader_event.rs @@ -9,7 +9,7 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::*; -use crate::control_plane::consensus::transport::RaftTransportActor; +use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ QueryCommand, ShardGroup, ShardGroupId, SwimActorCommand, SwimCommand, @@ -19,7 +19,6 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; -use crate::security::NodeTransportSecurity; use super::CLUSTER_PORT; @@ -121,7 +120,7 @@ fn leader_election_emits_leader_change_event() -> turmoil::Result { raft_tx.clone(), transport_rx, swim_tx.clone(), - NodeTransportSecurity::TrustedDevelopment, + ClusterSecurity::TrustedDevelopment, )); let db = MetadataStorage::open( std::env::temp_dir().join(uuid::Uuid::new_v4().to_string()), diff --git a/src/it/raft/membership_change.rs b/src/it/raft/membership_change.rs index e24b9196..41b91437 100644 --- a/src/it/raft/membership_change.rs +++ b/src/it/raft/membership_change.rs @@ -11,7 +11,7 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::{MultiRaftActor, MutlRaftSender}; use crate::control_plane::consensus::messages::*; -use crate::control_plane::consensus::transport::RaftTransportActor; +use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ShardGroup, ShardGroupId, Topology}; use crate::control_plane::{NodeId, Replicas, SwimNodeState}; @@ -20,7 +20,6 @@ use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; use crate::schedulers::ticker_message::{SchedulerSender, TickerCommand}; -use crate::security::NodeTransportSecurity; use super::{CLUSTER_PORT, QUERY_PORT, mock_swim_handler}; @@ -75,7 +74,7 @@ async fn start_raft_node( raft_tx.clone(), transport_rx, swim_tx.clone(), - NodeTransportSecurity::TrustedDevelopment, + ClusterSecurity::TrustedDevelopment, )); let db = MetadataStorage::open(std::env::temp_dir().join(uuid::Uuid::new_v4().to_string())); let election_jitter_seed = { diff --git a/src/it/raft/snapshot.rs b/src/it/raft/snapshot.rs index aaa49a9c..aa356d6f 100644 --- a/src/it/raft/snapshot.rs +++ b/src/it/raft/snapshot.rs @@ -9,7 +9,7 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::EnsureGroup; -use crate::control_plane::consensus::transport::RaftTransportActor; +use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ShardGroup, ShardGroupId}; use crate::control_plane::metadata::CreateTopic; @@ -19,7 +19,6 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; -use crate::security::NodeTransportSecurity; use super::{CLUSTER_PORT, mock_swim_handler}; @@ -56,7 +55,7 @@ async fn run_node(name: &'static str, ordinal: u16) -> Result<(), Box Date: Fri, 31 Jul 2026 10:23:49 +0400 Subject: [PATCH 44/56] clean up for security broker --- .claude/rules/raft-transport.md | 13 +- docs/security/roadmap.md | 41 ++- .../consensus/transport/acl/message.rs | 46 --- .../consensus/transport/acl/mod.rs | 336 ------------------ .../consensus/transport/admission/message.rs | 29 -- .../consensus/transport/admission/mod.rs | 188 ---------- .../consensus/transport/admission/state.rs | 303 ---------------- src/security/acl_cache.rs | 313 ---------------- 8 files changed, 32 insertions(+), 1237 deletions(-) delete mode 100644 src/control_plane/consensus/transport/acl/message.rs delete mode 100644 src/control_plane/consensus/transport/acl/mod.rs delete mode 100644 src/control_plane/consensus/transport/admission/message.rs delete mode 100644 src/control_plane/consensus/transport/admission/mod.rs delete mode 100644 src/control_plane/consensus/transport/admission/state.rs delete mode 100644 src/security/acl_cache.rs diff --git a/.claude/rules/raft-transport.md b/.claude/rules/raft-transport.md index 15773591..2b4e642f 100644 --- a/.claude/rules/raft-transport.md +++ b/.claude/rules/raft-transport.md @@ -13,14 +13,13 @@ Separate from SWIM's UDP transport. Raft uses TCP for reliable, ordered delivery ``` cluster listener (TCP) │ - ├── limited admission read ──► read one record → reply → close + ├── limited admission read ──► security actor → Raft → reply → close │ └── request with process proof │ ├── both sides verify a TLS-session-bound process proof - │ ├── Raft ──► persistent reader + one writer per peer - └── ACL ───► read committed ACL → reply → close + └── ACL ───► security actor → Raft → reply → close ``` ## Wire Protocol @@ -87,7 +86,7 @@ carry Raft, ACL, client, or admission-write traffic. An ACL snapshot request is not a Raft RPC and never enters a Raft state machine. In secure mode it is served only after the requester completes process admission. It asks the local multi-Raft actor for the selected shard's committed -ACL record, returns that record on the same connection, then closes. It carries -no client data request and cannot proxy one. A connection admitted for Raft -carries only raw Raft frames after its first message; an invalid frame closes -the connection. +ACL record through the broker security actor, returns that record on the same +connection, then closes. It carries no client data request and cannot proxy one. +A connection admitted for Raft carries only raw Raft frames after its first +message; an invalid frame closes the connection. diff --git a/docs/security/roadmap.md b/docs/security/roadmap.md index b714c6a2..7bb7a7c9 100644 --- a/docs/security/roadmap.md +++ b/docs/security/roadmap.md @@ -32,7 +32,8 @@ and Byzantine consensus are out of scope. ## 2. Layered Architecture -Security checks are split between the transport layer and application state machines to keep state machines free of security I/O: +Security checks are split across transport, one broker security actor, and the +application state machines. This keeps security I/O out of state machines: ``` [ Authenticated Transport Layer ] @@ -45,20 +46,31 @@ Security checks are split between the transport layer and application state mach │ │ (Drop connection on transport failure) ▼ +[ Broker Security Actor ] + - Cache admission and ACL records + - Route record reads to local Raft or one remote broker + - Authorize the authenticated principal + │ + │ (Deny one request on authorization failure) + ▼ [ Application Layer State Machines ] - - Authorize requested operation against cached ACLs / placement - Execute SWIM / Raft / Data-Plane state transitions - │ - │ (Drop denied envelope only; connection stays open) ``` | Boundary | Rule | | :--- | :--- | -| Transport | Authenticates, bounds frames and caches, and binds senders to verified identities. Failure closes the connection. | -| State machine | Authorizes pre-validated envelopes. Denial drops one envelope, not the connection. | +| Transport | Authenticates, bounds frames and handshakes, and binds senders to verified identities. Failure closes the connection. | +| Security actor | Owns caches, record reads, and authorization. Denial drops one request, not the connection. | +| State machine | Applies only authenticated and authorized operations. | | Data placement | Replication and repair use local committed placement, never a sender-asserted replica list. | | Redirect | Carries only an address hint. The destination repeats authentication and authorization. | +The actor owns only shared, non-durable state: both caches, record routing, and +identical-read combining. Certificates, connection lifetimes, frame parsing, +and durable records stay outside. Slow remote reads run in the background. +Admission and ACL refreshes share one active-read limit; saturation fails +closed instead of creating another queue or actor. + --- ## 3. Node Identity & Admission Model @@ -177,7 +189,7 @@ over `X` fails on a connection using `Y`; this removes the need for another challenge. Missing or stale admission, identity mismatch, or bad signature closes the connection before Raft or ACL dispatch. -### Why Admission Lookup Is Separate +### Why Admission Lookup Has a Narrow Wire Path The admission record may live on another broker. A normal cluster connection would recurse: @@ -204,8 +216,8 @@ Accepting broker Admission shard host node certificate, not the running process. - **One purpose:** The connection reads one admission record. It cannot carry Raft messages, ACL reads, client requests, or admission writes. -- **Bounded work:** One broker-local worker limits concurrent and queued reads - and combines simultaneous requests for the same record. +- **Bounded work:** The broker security actor combines simultaneous reads for + the same record. Admission and ACL reads share one active-read limit. - **Cache result:** A record or confirmed missing record is cached for at most 60 seconds. A missing record denies admission. - **Do not cache failure:** Timeout, routing failure, or an unavailable shard @@ -336,18 +348,17 @@ Pull-on-miss avoids a second connection pool for reads needed at most once per cache window. A future push or hybrid design may update only the same cache and must remain fail closed. -One broker-local refresh worker owns these remote reads. It bounds active and -queued work, and combines simultaneous requests for the same ACL record into -one read. When that worker is full, stopped, or too slow, callers deny rather -than opening more connections: +The broker security actor owns these remote reads and combines simultaneous +requests for the same ACL record. When its shared read limit is full, stopped, +or too slow, callers deny rather than opening more connections: ``` many cache misses │ ▼ - ACL refresh worker + broker security actor ├── same shard + resource ──► one read, reply to all waiters - ├── different records ──────► bounded active work and queue + ├── different records ──────► bounded background reads └── full / unavailable ─────► deny ``` diff --git a/src/control_plane/consensus/transport/acl/message.rs b/src/control_plane/consensus/transport/acl/message.rs deleted file mode 100644 index a270ede2..00000000 --- a/src/control_plane/consensus/transport/acl/message.rs +++ /dev/null @@ -1,46 +0,0 @@ -use tokio::sync::oneshot; - -use crate::control_plane::{ - NodeAddressInfo, NodeId, consensus::raft::states::security::AclRecord, - membership::ShardGroupId, metadata::AclResource, -}; - -pub(super) struct AclSnapshotFetchRequest { - pub(super) fetch: AclSnapshotFetch, - pub(super) reply: oneshot::Sender>, -} - -#[derive(Clone, Debug)] -pub(super) struct AclSnapshotFetch { - pub(super) node_id: NodeId, - pub(super) key: AclSnapshotKey, - pub(super) owner: NodeAddressInfo, -} -impl AclSnapshotFetch { - pub(crate) fn new( - node_id: NodeId, - shard_group_id: ShardGroupId, - resource: AclResource, - owner: NodeAddressInfo, - ) -> Self { - AclSnapshotFetch { - node_id, - key: AclSnapshotKey { - shard_group_id, - resource, - }, - owner, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub(super) struct AclSnapshotKey { - pub(super) shard_group_id: ShardGroupId, - pub(super) resource: AclResource, -} - -pub(super) struct AclSnapshotCompleted { - pub(super) key: AclSnapshotKey, - pub(super) snapshot: Option, -} diff --git a/src/control_plane/consensus/transport/acl/mod.rs b/src/control_plane/consensus/transport/acl/mod.rs deleted file mode 100644 index 40d617cd..00000000 --- a/src/control_plane/consensus/transport/acl/mod.rs +++ /dev/null @@ -1,336 +0,0 @@ -mod message; - -use anyhow::{Context, Result}; -use std::collections::{HashMap, HashSet, VecDeque}; -use tokio::io::AsyncWriteExt; -use tokio::sync::{mpsc, oneshot}; - -use crate::control_plane::NodeAddressInfo; -use crate::control_plane::NodeId; -use crate::control_plane::consensus::raft::states::security::AclRecord; -use crate::control_plane::membership::ShardGroupId; -use crate::control_plane::metadata::AclResource; -use crate::net::TransportTcpStream; -use crate::security::NodeTransportSecurity; -use message::*; - -use super::inbound::ClusterMessageReader; -use super::protocol::{AclSnapshotRequest, InitialClusterMessage, encode_frame}; - -const ACL_SNAPSHOT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); -const MAX_IN_FLIGHT_FETCHES: usize = 16; -const MAX_QUEUED_FETCHES: usize = 128; -const MAX_WAITERS_PER_FETCH: usize = 256; -const ACL_SNAPSHOT_MAILBOX_CAPACITY: usize = 256; - -/// Bounded async boundary for remote ACL cache refreshes. -/// -/// A cache miss never opens a socket from a client-request task directly. This -/// actor limits concurrent dials and makes all callers waiting for the same -/// record share one read and one response. -pub(crate) struct AclSnapshotActor; - -impl AclSnapshotActor { - pub(crate) fn spawn(security: NodeTransportSecurity) -> AclSnapshotSender { - let (sender, mailbox) = mpsc::channel(ACL_SNAPSHOT_MAILBOX_CAPACITY); - tokio::spawn(Self::run(security, mailbox)); - AclSnapshotSender(sender) - } - - async fn run( - security: NodeTransportSecurity, - mut mailbox: mpsc::Receiver, - ) { - let (completed_tx, mut completed_rx) = - mpsc::channel::(MAX_IN_FLIGHT_FETCHES); - let mut state = AclSnapshotState::new(security); - let mut requests = Vec::with_capacity(64); - - loop { - tokio::select! { - count = mailbox.recv_many(&mut requests, 64) => { - if count == 0 { - break; - } - for request in requests.drain(..) { - state.request_snapshot(request); - } - } - Some(completed) = completed_rx.recv() => { - state.complete(completed); - } - } - - for fetch in state.take_pending() { - let completed_tx = completed_tx.clone(); - let sec = state.security.clone(); - tokio::spawn(async move { - let key = fetch.key.clone(); - let snapshot = AclSnapshotActor::fetch_snapshot(sec, fetch).await; - let _ = completed_tx - .send(AclSnapshotCompleted { key, snapshot }) - .await; - }); - } - } - } - - async fn fetch_snapshot( - security: NodeTransportSecurity, - fetch: AclSnapshotFetch, - ) -> Option { - let owner_id = fetch.owner.node_id.clone(); - match tokio::time::timeout( - ACL_SNAPSHOT_TIMEOUT, - Self::fetch_snapshot_inner(security, fetch), - ) - .await - { - Ok(Ok(snapshot)) => snapshot, - Ok(Err(error)) => { - tracing::debug!(owner = %owner_id, "ACL snapshot fetch failed: {error}"); - None - } - Err(_) => { - tracing::debug!(owner = %owner_id, "ACL snapshot fetch timed out"); - None - } - } - } - - async fn fetch_snapshot_inner( - security: NodeTransportSecurity, - fetch: AclSnapshotFetch, - ) -> Result> { - let stream = - TransportTcpStream::connect_node(fetch.owner.cluster_addr(), &security).await?; - let transport_identity = stream.peer_identity(); - let (read_half, mut write_half) = stream.into_split(); - write_half - .write_all(&encode_frame(&InitialClusterMessage::AclSnapshot( - AclSnapshotRequest { - requester_node_id: fetch.node_id, - shard_group_id: fetch.key.shard_group_id, - resource: fetch.key.resource, - }, - ))?) - .await - .context("write initial ACL snapshot request")?; - - let mut reader = ClusterMessageReader::new(read_half, transport_identity); - Ok(reader.read_acl_snapshot_response().await?.snapshot) - } -} - -/// Scheduling state for bounded, coalesced snapshot reads. -struct AclSnapshotState { - security: NodeTransportSecurity, - active: HashSet, - queued: VecDeque, - waiters: HashMap>>>, - pending_events: Vec, -} - -impl AclSnapshotState { - fn new(security: NodeTransportSecurity) -> Self { - Self { - security, - active: HashSet::with_capacity(MAX_IN_FLIGHT_FETCHES), - queued: VecDeque::with_capacity(MAX_QUEUED_FETCHES), - waiters: HashMap::with_capacity(MAX_WAITERS_PER_FETCH), - pending_events: Vec::new(), - } - } - - fn request_snapshot(&mut self, request: AclSnapshotFetchRequest) { - let AclSnapshotFetchRequest { fetch, reply } = request; - - if let Some(waiting) = self.waiters.get_mut(&fetch.key) { - if waiting.len() == MAX_WAITERS_PER_FETCH { - tracing::debug!(?fetch.key, "ACL snapshot refresh has too many waiting callers"); - let _ = reply.send(None); - return; - } - waiting.push(reply); - return; - } - - if self.active.len() < MAX_IN_FLIGHT_FETCHES { - self.active.insert(fetch.key.clone()); - self.waiters.insert(fetch.key.clone(), vec![reply]); - self.pending_events.push(fetch); - return; - } - if self.queued.len() < MAX_QUEUED_FETCHES { - self.waiters.insert(fetch.key.clone(), vec![reply]); - self.queued.push_back(fetch); - return; - } - tracing::debug!(?fetch.key, "ACL snapshot refresh queue is full"); - let _ = reply.send(None); - } - - fn complete(&mut self, completed: AclSnapshotCompleted) { - debug_assert!(self.active.remove(&completed.key)); - if let Some(waiting) = self.waiters.remove(&completed.key) { - for reply in waiting { - let _ = reply.send(completed.snapshot.clone()); - } - } else { - tracing::debug!("ACL snapshot completed without waiting callers"); - } - if let Some(next) = self.queued.pop_front() { - self.active.insert(next.key.clone()); - self.pending_events.push(next); - } - } - - fn take_pending(&mut self) -> Vec { - std::mem::take(&mut self.pending_events) - } -} - -/// Sends bounded ACL snapshot refresh requests to [`AclSnapshotActor`]. -#[derive(Clone)] -pub(crate) struct AclSnapshotSender(mpsc::Sender); - -impl AclSnapshotSender { - /// Reads one committed ACL record from a shard host. - /// - /// A saturated or stopped refresh actor fails closed. The actor logs the - /// reason and coalesces concurrent reads for the same shard resource. - pub(crate) async fn fetch( - &self, - node_id: NodeId, - owner: NodeAddressInfo, - shard_group_id: ShardGroupId, - resource: AclResource, - ) -> Option { - let (reply, recv) = oneshot::channel(); - match self.0.try_send(AclSnapshotFetchRequest { - fetch: AclSnapshotFetch::new(node_id, shard_group_id, resource, owner), - reply, - }) { - Ok(()) => {} - Err(mpsc::error::TrySendError::Full(_)) => { - tracing::debug!("ACL snapshot refresh actor mailbox is full"); - return None; - } - Err(mpsc::error::TrySendError::Closed(_)) => { - tracing::debug!("ACL snapshot refresh actor is stopped"); - return None; - } - } - match tokio::time::timeout(ACL_SNAPSHOT_TIMEOUT, recv).await { - Ok(Ok(snapshot)) => snapshot, - Ok(Err(_)) => { - tracing::debug!("ACL snapshot refresh actor stopped before replying"); - None - } - Err(_) => { - tracing::debug!("ACL snapshot refresh did not complete before its deadline"); - None - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control_plane::NodeAddress; - use crate::control_plane::metadata::TopicId; - use std::net::SocketAddr; - use tokio::sync::oneshot; - - fn fetch(topic_id: u64) -> AclSnapshotFetch { - AclSnapshotFetch { - node_id: NodeId::new("requester"), - key: AclSnapshotKey { - shard_group_id: ShardGroupId(42), - resource: AclResource::TopicData(TopicId(topic_id)), - }, - owner: NodeAddressInfo::new( - NodeId::new("owner"), - NodeAddress::test( - "127.0.0.1:9000".parse::().unwrap(), - "127.0.0.1:9001".parse::().unwrap(), - ), - ), - } - } - - fn request( - topic_id: u64, - ) -> ( - AclSnapshotFetchRequest, - oneshot::Receiver>, - ) { - let (reply, receiver) = oneshot::channel(); - ( - AclSnapshotFetchRequest { - fetch: fetch(topic_id), - reply, - }, - receiver, - ) - } - - #[test] - fn joins_identical_fetches_without_starting_another_dial() { - let mut state = AclSnapshotState::new(NodeTransportSecurity::TrustedDevelopment); - let (first_request, mut first_reply) = request(7); - let (second_request, mut second_reply) = request(7); - - state.request_snapshot(first_request); - state.request_snapshot(second_request); - - assert_eq!(state.take_pending().len(), 1); - state.complete(AclSnapshotCompleted { - key: fetch(7).key, - snapshot: None, - }); - assert_eq!(first_reply.try_recv(), Ok(None)); - assert_eq!(second_reply.try_recv(), Ok(None)); - } - - #[test] - fn completion_starts_the_next_queued_fetch() { - let mut state = AclSnapshotState::new(NodeTransportSecurity::TrustedDevelopment); - let first = fetch(0); - let (first_request, _first_reply) = request(0); - state.request_snapshot(first_request); - for topic_id in 1..MAX_IN_FLIGHT_FETCHES as u64 { - let (request, _reply) = request(topic_id); - state.request_snapshot(request); - } - let (queued_request, _queued_reply) = request(99); - state.request_snapshot(queued_request); - assert_eq!(state.take_pending().len(), MAX_IN_FLIGHT_FETCHES); - - state.complete(AclSnapshotCompleted { - key: first.key, - snapshot: None, - }); - - let started = state.take_pending(); - assert_eq!(started.len(), 1); - let [started] = started.as_slice() else { - panic!("expected one fetch event"); - }; - assert_eq!(started.key, fetch(99).key); - } - - #[test] - fn rejects_a_new_fetch_when_the_queue_is_full() { - let mut state = AclSnapshotState::new(NodeTransportSecurity::TrustedDevelopment); - for topic_id in 0..MAX_IN_FLIGHT_FETCHES as u64 + MAX_QUEUED_FETCHES as u64 { - let (request, _reply) = request(topic_id); - state.request_snapshot(request); - } - - let (request, mut reply) = request(999); - state.request_snapshot(request); - assert_eq!(reply.try_recv(), Ok(None)); - } -} diff --git a/src/control_plane/consensus/transport/admission/message.rs b/src/control_plane/consensus/transport/admission/message.rs deleted file mode 100644 index 10ec79c1..00000000 --- a/src/control_plane/consensus/transport/admission/message.rs +++ /dev/null @@ -1,29 +0,0 @@ -use tokio::sync::oneshot; - -use crate::control_plane::NodeAddressInfo; -use crate::control_plane::consensus::raft::states::security::AdmissionRecord; - -use super::super::protocol::AdmissionRecordKey; - -pub(super) type AdmissionLookupResult = Result, AdmissionLookupUnavailable>; -pub(super) type AdmissionLookupReply = oneshot::Sender; - -pub(super) struct AdmissionQuery { - pub(super) fetch: AdmissionTarget, - pub(super) reply: AdmissionLookupReply, -} - -#[derive(Clone, Debug)] -pub(super) struct AdmissionTarget { - pub(super) key: AdmissionRecordKey, - pub(super) remote_owner: Option, -} - -pub(super) struct AdmissionLookupCompleted { - pub(super) key: AdmissionRecordKey, - pub(super) result: Result, AdmissionLookupUnavailable>, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[error("admission lookup unavailable")] -pub(crate) struct AdmissionLookupUnavailable; diff --git a/src/control_plane/consensus/transport/admission/mod.rs b/src/control_plane/consensus/transport/admission/mod.rs deleted file mode 100644 index d5a6132d..00000000 --- a/src/control_plane/consensus/transport/admission/mod.rs +++ /dev/null @@ -1,188 +0,0 @@ -mod message; -mod state; -use state::*; - -use std::time::Duration; - -use anyhow::{Context, Result as AnyResult}; -use tokio::io::AsyncWriteExt; -use tokio::sync::{mpsc, oneshot}; - -use crate::control_plane::consensus::actor::MutlRaftSender; -use crate::control_plane::consensus::raft::states::security::AdmissionRecord; -use crate::control_plane::membership::actor::ShardRouting; -use crate::net::TransportTcpStream; -use crate::security::NodeTransportSecurity; - -use message::*; - -use super::inbound::ClusterMessageReader; -use super::protocol::{AdmissionLookupRequest, InitialClusterMessage, encode_frame}; - -const ADMISSION_FETCH_TIMEOUT: Duration = Duration::from_secs(3); -const ADMISSION_REQUEST_TIMEOUT: Duration = Duration::from_secs(4); - -const ADMISSION_LOOKUP_MAILBOX_CAPACITY: usize = 256; - -/// Bounded admission-record reader used while cluster connections are admitted. -/// -/// One actor owns the short-lived cache and combines simultaneous lookups for -/// the same principal. Local records come from this node's multi-Raft actor; -/// remote records use the limited admission-only connection. -pub(crate) struct AdmissionLookupActor; - -impl AdmissionLookupActor { - pub(crate) fn spawn( - raft_tx: MutlRaftSender, - security: NodeTransportSecurity, - ) -> AdmissionLookupSender { - let (sender, mailbox) = mpsc::channel(ADMISSION_LOOKUP_MAILBOX_CAPACITY); - tokio::spawn(Self::run(raft_tx, security, mailbox)); - AdmissionLookupSender(sender) - } - - async fn run( - raft_tx: MutlRaftSender, - security: NodeTransportSecurity, - mut mailbox: mpsc::Receiver, - ) { - let (completed_tx, mut completed_rx) = - mpsc::channel::(MAX_IN_FLIGHT_LOOKUPS); - let started_at = tokio::time::Instant::now(); - let mut state = AdmissionLookupState::default(); - let mut requests = Vec::with_capacity(64); - - loop { - tokio::select! { - count = mailbox.recv_many(&mut requests, 64) => { - if count == 0 { - break; - } - let now = started_at.elapsed(); - for request in requests.drain(..) { - state.lookup(request, now); - } - } - Some(completed) = completed_rx.recv() => { - state.complete(completed, started_at.elapsed()); - } - } - - for fetch in state.take_pending() { - tokio::spawn(Self::fetch( - raft_tx.clone(), - security.clone(), - fetch, - completed_tx.clone(), - )); - } - } - } - - async fn fetch( - raft_tx: MutlRaftSender, - security: NodeTransportSecurity, - fetch: AdmissionFetch, - completed_tx: mpsc::Sender, - ) { - let key = fetch.key().clone(); - let res = match tokio::time::timeout( - ADMISSION_FETCH_TIMEOUT, - Self::fetch_inner(raft_tx, security, fetch), - ) - .await - { - Ok(Ok(admission)) => Ok(admission), - Ok(Err(error)) => { - tracing::debug!(?key, "admission lookup failed: {error}"); - Err(AdmissionLookupUnavailable) - } - Err(_) => { - tracing::debug!(?key, "admission lookup timed out"); - Err(AdmissionLookupUnavailable) - } - }; - - let _ = completed_tx - .send(AdmissionLookupCompleted { key, result: res }) - .await; - } - - async fn fetch_inner( - raft_tx: MutlRaftSender, - security: NodeTransportSecurity, - fetch: AdmissionFetch, - ) -> AnyResult> { - match fetch { - AdmissionFetch::Local(LocalAdmissionFetch(key)) => Ok(raft_tx - .get_admission(key.shard_group_id, key.node_certificate_principal) - .await?), - AdmissionFetch::Remote(fetch) => { - let stream = - TransportTcpStream::connect_node(fetch.owner.cluster_addr(), &security).await?; - let transport_identity = stream.peer_identity(); - let (read_half, mut write_half) = stream.into_split(); - write_half - .write_all(&encode_frame(&InitialClusterMessage::AdmissionLookup( - AdmissionLookupRequest { - shard_group_id: fetch.key.shard_group_id, - node_certificate_principal: fetch.key.node_certificate_principal, - }, - ))?) - .await - .context("write initial admission lookup request")?; - - let mut reader = ClusterMessageReader::new(read_half, transport_identity); - Ok(reader.read_admission_lookup_response().await?.admission) - } - } - } -} -/// Sends admission-record requests to [`AdmissionLookupActor`]. -#[derive(Clone)] -pub(crate) struct AdmissionLookupSender(mpsc::Sender); - -impl AdmissionLookupSender { - pub(crate) async fn lookup( - &self, - routing: ShardRouting, - node_certificate_principal: Box, - ) -> Result, AdmissionLookupUnavailable> { - let fetch = match routing { - ShardRouting::Local(group) => LocalAdmissionFetch(AdmissionLookupKey { - shard_group_id: group.id, - node_certificate_principal, - }) - .into(), - ShardRouting::Redirect(Some(remote)) => { - let Some(owner) = remote.member else { - return Err(AdmissionLookupUnavailable); - }; - RemoteAdmissionFetch { - key: AdmissionLookupKey { - shard_group_id: remote.group_id, - node_certificate_principal, - }, - owner, - } - .into() - } - ShardRouting::Redirect(None) => return Err(AdmissionLookupUnavailable), - }; - let (reply, recv) = oneshot::channel(); - self.0 - .try_send(LookupAdmission { fetch, reply }) - .map_err(|e| { - tracing::debug!("admission lookup actor unavailable {}", e); - AdmissionLookupUnavailable - })?; - - match tokio::time::timeout(ADMISSION_REQUEST_TIMEOUT, recv).await { - Ok(Ok(result)) => result, - err => { - tracing::debug!("admission lookup actor unavailable {:?}", err); - Err(AdmissionLookupUnavailable) - } - } - } -} diff --git a/src/control_plane/consensus/transport/admission/state.rs b/src/control_plane/consensus/transport/admission/state.rs deleted file mode 100644 index 9de1040a..00000000 --- a/src/control_plane/consensus/transport/admission/state.rs +++ /dev/null @@ -1,303 +0,0 @@ -use std::{ - collections::{HashMap, HashSet, VecDeque}, - time::Duration, -}; - -use crate::control_plane::consensus::{ - raft::states::security::AdmissionRecord, - transport::admission::{ - AdmissionLookupCompleted, AdmissionLookupReply, AdmissionQuery, AdmissionTarget, - message::AdmissionLookupUnavailable, - }, - transport::protocol::AdmissionRecordKey, -}; - -const MAX_ADMISSION_CACHE_TTL: Duration = Duration::from_secs(60); -pub(super) const MAX_IN_FLIGHT_LOOKUPS: usize = 16; -const MAX_QUEUED_LOOKUPS: usize = 128; -const MAX_WAITERS_PER_LOOKUP: usize = 256; - -/// Pure scheduling and cache state for admission lookups. -#[derive(Default)] -pub(super) struct AdmissionLookupState { - cache: AdmissionCache, - active: HashSet, - queued: VecDeque, - waiters: HashMap>, - pending_events: Vec, -} - -impl AdmissionLookupState { - pub(super) fn lookup(&mut self, request: AdmissionQuery, now: Duration) { - let AdmissionQuery { fetch, reply } = request; - let key = fetch.key.clone(); - - if let Some(admission) = self.cache.fresh_admission(&key, now) { - let _ = reply.send(Ok(admission.clone())); - return; - } - if let Some(waiting) = self.waiters.get_mut(&key) { - if waiting.len() == MAX_WAITERS_PER_LOOKUP { - tracing::debug!(?key, "admission lookup has too many waiting callers"); - let _ = reply.send(Err(AdmissionLookupUnavailable)); - return; - } - waiting.push(reply); - return; - } - if self.active.len() < MAX_IN_FLIGHT_LOOKUPS { - self.active.insert(key.clone()); - self.waiters.insert(key, vec![reply]); - self.pending_events.push(fetch); - return; - } - if self.queued.len() < MAX_QUEUED_LOOKUPS { - self.waiters.insert(key, vec![reply]); - self.queued.push_back(fetch); - return; - } - tracing::debug!(?key, "admission lookup queue is full"); - let _ = reply.send(Err(AdmissionLookupUnavailable)); - } - - pub(super) fn complete(&mut self, completed: AdmissionLookupCompleted, now: Duration) { - debug_assert!(self.active.remove(&completed.key)); - if let Ok(admission) = &completed.result { - self.cache.insert(&completed.key, admission.clone(), now); - } - - let response = match completed.result { - Ok(_) => self - .cache - .fresh_admission(&completed.key, now) - .cloned() - .ok_or(AdmissionLookupUnavailable), - Err(error) => Err(error), - }; - if let Some(waiting) = self.waiters.remove(&completed.key) { - for reply in waiting { - let _ = reply.send(response.clone()); - } - } else { - tracing::debug!("admission lookup completed without waiting callers"); - } - if let Some(next) = self.queued.pop_front() { - self.active.insert(next.key.clone()); - self.pending_events.push(next); - } - } - - pub(super) fn take_pending(&mut self) -> Vec { - std::mem::take(&mut self.pending_events) - } -} - -#[derive(Debug)] -struct AdmissionCacheEntry { - source_shard_id: crate::control_plane::membership::ShardGroupId, - admission: Option, - expires_at: Duration, -} - -#[derive(Debug, Default)] -struct AdmissionCache { - entries: HashMap, AdmissionCacheEntry>, -} - -impl AdmissionCache { - fn fresh_admission( - &self, - key: &AdmissionRecordKey, - now: Duration, - ) -> Option<&Option> { - let entry = self.entries.get(key.node_certificate_principal.as_ref())?; - if entry.source_shard_id != key.shard_group_id || entry.expires_at <= now { - return None; - } - Some(&entry.admission) - } - - /// Keeps the highest revision observed from a shard. An older response - /// cannot extend the authority window of a newer, possibly expired record. - fn insert( - &mut self, - key: &AdmissionRecordKey, - admission: Option, - now: Duration, - ) { - if admission.as_ref().is_some_and(|record| { - record.node_certificate_principal != key.node_certificate_principal.as_ref() - }) { - return; - } - let replace = match self.entries.get(key.node_certificate_principal.as_ref()) { - Some(entry) if entry.source_shard_id == key.shard_group_id => { - match (&entry.admission, &admission) { - (Some(current), Some(incoming)) => current.revision <= incoming.revision, - (Some(_), None) => false, - (None, Some(_)) | (None, None) => true, - } - } - Some(_) | None => true, - }; - if replace { - self.entries.insert( - key.node_certificate_principal.clone(), - AdmissionCacheEntry { - source_shard_id: key.shard_group_id, - admission, - expires_at: now + MAX_ADMISSION_CACHE_TTL, - }, - ); - } - } -} - -#[cfg(test)] -pub mod tests { - use tokio::sync::oneshot; - - use super::*; - use crate::control_plane::{ - NodeId, consensus::transport::admission::AdmissionLookupResult, membership::ShardGroupId, - }; - - fn admission(revision: u64) -> AdmissionRecord { - AdmissionRecord { - node_certificate_principal: "broker-a".to_string(), - revision, - epoch: revision, - node_id: NodeId::new(format!("broker-a::process-{revision}")), - process_public_key: vec![revision as u8].into_boxed_slice(), - } - } - - fn local_fetch(principal: &str) -> AdmissionTarget { - AdmissionTarget { - key: AdmissionRecordKey { - shard_group_id: ShardGroupId(42), - node_certificate_principal: principal.into(), - }, - remote_owner: None, - } - } - fn request(principal: &str) -> (AdmissionQuery, oneshot::Receiver) { - let (reply, receiver) = oneshot::channel(); - ( - AdmissionQuery { - fetch: local_fetch(principal), - reply, - }, - receiver, - ) - } - - #[test] - fn fresh_cache_entry_avoids_another_lookup() { - let mut state = AdmissionLookupState::default(); - let key = local_fetch("broker-a").key.clone(); - state.cache.insert(&key, Some(admission(3)), Duration::ZERO); - let (lookup_request, mut reply) = request("broker-a"); - - state.lookup(lookup_request, Duration::from_secs(1)); - - assert!(state.take_pending().is_empty()); - assert_eq!(reply.try_recv(), Ok(Ok(Some(admission(3))))); - } - - #[test] - fn older_revision_cannot_refresh_an_expired_admission() { - let mut state = AdmissionLookupState::default(); - let key = local_fetch("broker-a").key.clone(); - state.cache.insert(&key, Some(admission(3)), Duration::ZERO); - let expired = MAX_ADMISSION_CACHE_TTL + Duration::from_millis(1); - let (lookup_request, mut reply) = request("broker-a"); - state.lookup(lookup_request, expired); - state.take_pending(); - - state.complete( - AdmissionLookupCompleted { - key, - result: Ok(Some(admission(2))), - }, - expired, - ); - - assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); - } - - #[test] - fn combines_simultaneous_lookups_for_the_same_admission() { - let mut state = AdmissionLookupState::default(); - let (first, mut first_reply) = request("broker-a"); - let (second, mut second_reply) = request("broker-a"); - - state.lookup(first, Duration::ZERO); - state.lookup(second, Duration::ZERO); - - let pending = state.take_pending(); - assert_eq!(pending.len(), 1); - state.complete( - AdmissionLookupCompleted { - key: pending[0].key.clone(), - result: Ok(Some(admission(3))), - }, - Duration::ZERO, - ); - assert_eq!(first_reply.try_recv(), Ok(Ok(Some(admission(3))))); - assert_eq!(second_reply.try_recv(), Ok(Ok(Some(admission(3))))); - } - - #[test] - fn unavailable_lookup_is_not_cached() { - let mut state = AdmissionLookupState::default(); - let (lookup_request, mut reply) = request("broker-a"); - state.lookup(lookup_request, Duration::ZERO); - let pending = state.take_pending(); - state.complete( - AdmissionLookupCompleted { - key: pending[0].key.clone(), - result: Err(AdmissionLookupUnavailable), - }, - Duration::ZERO, - ); - assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); - - let (retry, _reply) = request("broker-a"); - state.lookup(retry, Duration::ZERO); - assert_eq!(state.take_pending().len(), 1); - } - - #[test] - fn response_for_another_principal_is_not_cached() { - let mut state = AdmissionLookupState::default(); - let (lookup_request, mut reply) = request("broker-b"); - state.lookup(lookup_request, Duration::ZERO); - let pending = state.take_pending(); - - state.complete( - AdmissionLookupCompleted { - key: pending[0].key.clone(), - result: Ok(Some(admission(3))), - }, - Duration::ZERO, - ); - - assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); - } - - #[test] - fn rejects_a_lookup_when_active_work_and_queue_are_full() { - let mut state = AdmissionLookupState::default(); - for index in 0..MAX_IN_FLIGHT_LOOKUPS + MAX_QUEUED_LOOKUPS { - let principal = format!("broker-{index}"); - let (lookup_request, _reply) = request(&principal); - state.lookup(lookup_request, Duration::ZERO); - } - - let (rejected, mut reply) = request("broker-overflow"); - state.lookup(rejected, Duration::ZERO); - - assert_eq!(reply.try_recv(), Ok(Err(AdmissionLookupUnavailable))); - } -} diff --git a/src/security/acl_cache.rs b/src/security/acl_cache.rs deleted file mode 100644 index 67788e9a..00000000 --- a/src/security/acl_cache.rs +++ /dev/null @@ -1,313 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use tokio::sync::RwLock; -use tokio::time::Instant; - -use crate::connections::protocol::ServerError; -use crate::control_plane::consensus::raft::states::security::AclRecord; -use crate::control_plane::membership::ShardGroupId; -use crate::control_plane::metadata::AclResource; - -/// Maximum time an ACL entry may authorize without a fresh read from its -/// owning metadata shard. -pub(crate) const MAX_ACL_CACHE_TTL: Duration = Duration::from_secs(60); - -/// One ACL record copied from its owning metadata shard. -#[derive(Debug, Clone, PartialEq, Eq)] -struct CachedAcl { - pub(crate) source_shard_id: ShardGroupId, - pub(crate) revision: u64, - pub(crate) principals: Box<[Box]>, -} - -impl CachedAcl { - fn from_snapshot(source_shard_id: ShardGroupId, snapshot: AclRecord) -> Self { - Self { - source_shard_id, - revision: snapshot.revision, - principals: snapshot - .principals - .into_iter() - .map(String::into_boxed_str) - .collect(), - } - } -} - -#[derive(Debug)] -struct AclCacheEntry { - acl: CachedAcl, - expires_at: Instant, -} - -/// Short-lived, revision-aware copies of ACL records. -/// -/// The cache has no authority of its own. A caller must supply the shard that -/// currently owns the ACL resource; an entry from a previous owner, an expired -/// entry, or no entry at all is a miss and must be refreshed or denied. -#[derive(Debug, Default)] -struct AclCache { - entries: HashMap, -} - -impl AclCache { - /// Returns `None` when there is no current cache entry for this resource. - fn authorization( - &self, - resource: &AclResource, - source_shard_id: ShardGroupId, - principal: &str, - now: Instant, - ) -> Option { - let entry = self.entries.get(resource)?; - if entry.acl.source_shard_id != source_shard_id || entry.expires_at <= now { - return None; - } - Some( - entry - .acl - .principals - .iter() - .any(|candidate| candidate.as_ref() == principal), - ) - } - - /// Retains the newest known revision from a shard and caps its authority - /// window at [`MAX_ACL_CACHE_TTL`]. A record from a newly assigned owner - /// replaces the old owner's revision, because revisions are shard-local. - fn insert(&mut self, resource: AclResource, acl: CachedAcl, now: Instant) { - let replace = match self.entries.get(&resource) { - Some(entry) if entry.acl.source_shard_id == acl.source_shard_id => { - entry.acl.revision <= acl.revision - } - Some(_) | None => true, - }; - if replace { - self.entries.insert( - resource, - AclCacheEntry { - acl, - expires_at: now + MAX_ACL_CACHE_TTL, - }, - ); - } - } -} - -/// Shareable cache boundary for concurrent client request handlers. -/// -/// The lock protects only an in-memory map lookup or replacement. -/// No network or Raft operation is performed while it is held. -/// Use of `RwLock` over lock-free data structure is jusitified because -/// the dominant cost factor won't be write lock but raft snapshot query -#[derive(Debug, Clone, Default)] -pub(crate) struct SharedAclCache(Arc>); - -impl SharedAclCache { - /// Authorizes from a fresh cache entry, or lazily refreshes the entry and - /// checks it again. The refresh future is created only after a cache miss. - /// A missing snapshot fails closed. - pub(crate) async fn authorize_or_refresh( - &self, - resource: &AclResource, - source_shard_id: ShardGroupId, - principal: &str, - refresh: F, - ) -> Result<(), ServerError> - where - F: AsyncFnOnce() -> Option, - { - if let Some(authorized) = self - .cached_authorization(resource, source_shard_id, principal) - .await - { - return if authorized { - Ok(()) - } else { - Err(ServerError::Unauthorized) - }; - } - - // Cache Miss Case - let Some(snapshot) = refresh().await else { - return Err(ServerError::Unauthorized); - }; - self.0.write().await.insert( - resource.clone(), - CachedAcl::from_snapshot(source_shard_id, snapshot), - Instant::now(), - ); - match self - .cached_authorization(resource, source_shard_id, principal) - .await - { - Some(true) => Ok(()), - Some(false) | None => Err(ServerError::Unauthorized), - } - } - - async fn cached_authorization( - &self, - resource: &AclResource, - source_shard_id: ShardGroupId, - principal: &str, - ) -> Option { - self.0 - .read() - .await - .authorization(resource, source_shard_id, principal, Instant::now()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control_plane::metadata::TopicId; - - fn resource() -> AclResource { - AclResource::TopicData(TopicId(7)) - } - - fn acl(source_shard_id: ShardGroupId, revision: u64, principals: &[&str]) -> CachedAcl { - CachedAcl { - source_shard_id, - revision, - principals: principals - .iter() - .map(|principal| (*principal).into()) - .collect(), - } - } - - fn snapshot(resource: AclResource, revision: u64, principals: &[&str]) -> AclRecord { - AclRecord { - resource, - revision, - principals: principals - .iter() - .map(|principal| (*principal).to_owned()) - .collect(), - } - } - - #[test] - fn authorizes_and_denies_exact_principals_from_a_fresh_entry() { - let now = Instant::now(); - let mut cache = AclCache::default(); - cache.insert( - resource(), - acl(ShardGroupId(3), 4, &["orders-service"]), - now, - ); - - assert_eq!( - cache.authorization(&resource(), ShardGroupId(3), "orders-service", now), - Some(true) - ); - assert_eq!( - cache.authorization(&resource(), ShardGroupId(3), "billing-service", now), - Some(false) - ); - } - - #[test] - fn expires_entries_after_the_bounded_ttl() { - let now = Instant::now(); - let mut cache = AclCache::default(); - cache.insert( - resource(), - acl(ShardGroupId(3), 4, &["orders-service"]), - now, - ); - - assert_eq!( - cache.authorization( - &resource(), - ShardGroupId(3), - "orders-service", - now + MAX_ACL_CACHE_TTL + Duration::from_millis(1), - ), - None - ); - } - - #[test] - fn rejects_an_entry_from_a_previous_owner_shard() { - let now = Instant::now(); - let mut cache = AclCache::default(); - cache.insert( - resource(), - acl(ShardGroupId(3), 4, &["orders-service"]), - now, - ); - - assert_eq!( - cache.authorization(&resource(), ShardGroupId(4), "orders-service", now), - None - ); - } - - #[test] - fn does_not_replace_a_newer_revision_from_the_same_shard() { - let now = Instant::now(); - let mut cache = AclCache::default(); - cache.insert( - resource(), - acl(ShardGroupId(3), 5, &["orders-service"]), - now, - ); - cache.insert( - resource(), - acl(ShardGroupId(3), 4, &["billing-service"]), - now, - ); - - assert_eq!( - cache.authorization(&resource(), ShardGroupId(3), "orders-service", now), - Some(true) - ); - assert_eq!( - cache.authorization(&resource(), ShardGroupId(3), "billing-service", now), - Some(false) - ); - } - - #[tokio::test] - async fn refreshes_a_miss_then_reuses_the_fresh_entry() { - let cache = SharedAclCache::default(); - let resource = resource(); - - assert_eq!( - cache - .authorize_or_refresh(&resource, ShardGroupId(3), "orders-service", || async { - Some(snapshot(resource.clone(), 1, &["orders-service"])) - }) - .await, - Ok(()) - ); - assert_eq!( - cache - .authorize_or_refresh(&resource, ShardGroupId(3), "orders-service", || async { - panic!("fresh entry must not refresh") - }) - .await, - Ok(()) - ); - } - - #[tokio::test] - async fn missing_snapshot_leaves_a_miss_for_fail_closed_handling() { - let cache = SharedAclCache::default(); - - assert_eq!( - cache - .authorize_or_refresh(&resource(), ShardGroupId(3), "orders-service", || async { - None - },) - .await, - Err(ServerError::Unauthorized) - ); - } -} From 940712f4c4fd3b10d626bd5327434d6d5a16b5fd Mon Sep 17 00:00:00 2001 From: Migorithm Date: Fri, 31 Jul 2026 10:30:03 +0400 Subject: [PATCH 45/56] channel method --- src/control_plane/consensus/multi_raft.rs | 12 +++++----- src/control_plane/consensus/raft/state.rs | 6 ++--- src/control_plane/membership/tests.rs | 4 ++-- src/control_plane/membership/topology.rs | 28 +++++++++++------------ src/it/raft/election.rs | 11 +++++++-- src/it/raft/leader_event.rs | 11 +++++++-- src/it/raft/membership_change.rs | 11 +++++++-- src/it/raft/mod.rs | 4 ++-- src/security/mod.rs | 10 ++++---- 9 files changed, 57 insertions(+), 40 deletions(-) diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index 029a5e8f..cd0611d2 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -1075,7 +1075,7 @@ mod tests { } fn new_store(node_id: NodeId, storage: Box) -> MultiRaft { - use crate::control_plane::membership::{Topology, TopologyConfig, topology_channel}; + use crate::control_plane::membership::{Topology, TopologyConfig}; // Tests just need a valid topology reader; empty topology is fine — // these tests don't exercise reconciliation or ring picks. let topology = Topology::new( @@ -1085,7 +1085,7 @@ mod tests { replication_factor: 1, }, ); - let (_pub_handle, reader) = topology_channel(topology); + let (_pub_handle, reader) = topology.channel(); MultiRaft::new(node_id, 0, storage, reader, 10000) } @@ -1645,7 +1645,7 @@ mod tests { storage: Box, all_nodes: &[NodeId], ) -> MultiRaft { - use crate::control_plane::membership::{Topology, TopologyConfig, topology_channel}; + use crate::control_plane::membership::{Topology, TopologyConfig}; let topology = Topology::new( all_nodes.iter().cloned(), TopologyConfig { @@ -1653,7 +1653,7 @@ mod tests { replication_factor: 3, }, ); - let (_pub_handle, reader) = topology_channel(topology); + let (_pub_handle, reader) = topology.channel(); MultiRaft::new(node_id, 0, storage, reader, 10000) } @@ -1668,7 +1668,7 @@ mod tests { MultiRaft, std::sync::Arc>, ) { - use crate::control_plane::membership::{Topology, TopologyConfig, topology_channel}; + use crate::control_plane::membership::{Topology, TopologyConfig}; let topology = Topology::new( all_nodes.iter().cloned(), TopologyConfig { @@ -1676,7 +1676,7 @@ mod tests { replication_factor: 3, }, ); - let (pub_handle, reader) = topology_channel(topology); + let (pub_handle, reader) = topology.channel(); ( MultiRaft::new(node_id, 0, storage, reader, 10000), pub_handle, diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index 51bc06bb..5e67d039 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -3618,9 +3618,7 @@ mod tests { // verify the proposals appear in the log. // ------------------------------------------------------------------- - use crate::control_plane::membership::{ - Topology, TopologyConfig, TopologyReader, topology_channel, - }; + use crate::control_plane::membership::{Topology, TopologyConfig, TopologyReader}; /// Build a `TopologyReader` seeded with `nodes` as live members. The /// publisher half is dropped on return — the reader's own Arc keeps the @@ -3633,7 +3631,7 @@ mod tests { replication_factor: 3, }, ); - let (_pub_handle, reader) = topology_channel(topology); + let (_pub_handle, reader) = topology.channel(); reader } diff --git a/src/control_plane/membership/tests.rs b/src/control_plane/membership/tests.rs index 5681c43f..8c4273c8 100644 --- a/src/control_plane/membership/tests.rs +++ b/src/control_plane/membership/tests.rs @@ -7,7 +7,7 @@ use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::peer_discovery::JoinConfig; use crate::control_plane::membership::swim::Swim; use crate::control_plane::membership::{ - self, DIRECT_ACK_TIMEOUT_TICKS, OutboundPacket, QueryCommand, SwimActorCommand, SwimCommand, + DIRECT_ACK_TIMEOUT_TICKS, OutboundPacket, QueryCommand, SwimActorCommand, SwimCommand, SwimHeader, SwimPacket, SwimTimer, Topology, TopologyConfig, }; use crate::control_plane::{NodeAddress, NodeId, SwimNode, SwimNodeState}; @@ -162,7 +162,7 @@ async fn setup_with_config(port: u32, join_config: JoinConfig) -> TestHarness { // Topology publish/read channel — this harness has no reader-side // consumer, but SwimActor still needs the writer half to publish into. - let (topology_pub, _topology_reader) = membership::topology_channel(swim.topology.clone()); + let (topology_pub, _topology_reader) = swim.topology.clone().channel(); let ticker_tx = spawn_scheduling_actor( tx_in.clone(), diff --git a/src/control_plane/membership/topology.rs b/src/control_plane/membership/topology.rs index 1ccba559..07f2f753 100644 --- a/src/control_plane/membership/topology.rs +++ b/src/control_plane/membership/topology.rs @@ -149,17 +149,6 @@ impl TopologyReader { } } -/// Construct a (publisher, reader) pair sharing one underlying `ArcSwap`. -/// -/// The publisher half stays with `SwimActor` (single writer); the reader can -/// be cloned freely to any number of consumers. Both see the same atomic slot; -/// what differs is the API surface — readers can only `load()`. -pub(crate) fn topology_channel(initial: Topology) -> (Arc>, TopologyReader) { - let arc = Arc::new(ArcSwap::from_pointee(initial)); - let reader = TopologyReader(arc.clone()); - (arc, reader) -} - impl Topology { pub fn new(nodes: impl IntoIterator, config: TopologyConfig) -> Self { let mut topology = Self { @@ -181,6 +170,17 @@ impl Topology { topology } + /// Construct a (publisher, reader) pair sharing one underlying `ArcSwap`. + /// + /// The publisher half stays with `SwimActor` (single writer); the reader can + /// be cloned freely to any number of consumers. Both see the same atomic slot; + /// what differs is the API surface — readers can only `load()`. + pub(crate) fn channel(self) -> (Arc>, TopologyReader) { + let arc = Arc::new(ArcSwap::from_pointee(self)); + let reader = TopologyReader(arc.clone()); + (arc, reader) + } + /// Consume the dirty flag. Returns true if the topology has been mutated /// since the last `take_dirty()` call (or since construction). The owning /// SwimActor calls this at the end of each event-loop iteration to decide @@ -838,7 +838,7 @@ mod tests { replication_factor: 2, }, ); - let (_pub_handle, reader) = topology_channel(topology); + let (_pub_handle, reader) = topology.channel(); assert_eq!(reader.live_nodes().len(), 2); } @@ -852,7 +852,7 @@ mod tests { replication_factor: 1, }, ); - let (pub_handle, reader) = topology_channel(topology); + let (pub_handle, reader) = topology.channel(); assert_eq!(reader.live_nodes().len(), 1); // Simulate what SwimActor does at the end of an iteration: build a fresh @@ -878,7 +878,7 @@ mod tests { replication_factor: 1, }, ); - let (pub_handle, reader1) = topology_channel(topology); + let (pub_handle, reader1) = topology.channel(); let reader2 = reader1.clone(); pub_handle.store(Arc::new(topology_from( diff --git a/src/it/raft/election.rs b/src/it/raft/election.rs index deba834e..e83d1156 100644 --- a/src/it/raft/election.rs +++ b/src/it/raft/election.rs @@ -9,12 +9,13 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::{EnsureGroup, MultiRaftActorCommand, RaftTimer}; -use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; +use crate::control_plane::consensus::transport::RaftTransportActor; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ShardGroup, ShardGroupId}; use crate::control_plane::{NodeId, Replicas}; use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TcpStream}; +use crate::security::{NodeTransportSecurity, SecurityActor}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; @@ -103,13 +104,19 @@ async fn run_raft_node( TICK_PERIOD_100_MS, Some(PROBE_INTERVAL_TICKS), ); + let security = SecurityActor::spawn( + node_id.clone(), + swim_tx.clone(), + raft_tx.clone(), + NodeTransportSecurity::TrustedDevelopment, + ); tokio::spawn(RaftTransportActor::run( node_id.clone(), listener, raft_tx.clone(), transport_rx, swim_tx.clone(), - ClusterSecurity::TrustedDevelopment, + security, )); let db = MetadataStorage::open(std::env::temp_dir().join(uuid::Uuid::new_v4().to_string())); let election_jitter_seed = { diff --git a/src/it/raft/leader_event.rs b/src/it/raft/leader_event.rs index dfbc2f86..192ed053 100644 --- a/src/it/raft/leader_event.rs +++ b/src/it/raft/leader_event.rs @@ -9,7 +9,7 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::*; -use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; +use crate::control_plane::consensus::transport::RaftTransportActor; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ QueryCommand, ShardGroup, ShardGroupId, SwimActorCommand, SwimCommand, @@ -19,6 +19,7 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; +use crate::security::{NodeTransportSecurity, SecurityActor}; use super::CLUSTER_PORT; @@ -114,13 +115,19 @@ fn leader_election_emits_leader_change_event() -> turmoil::Result { TICK_PERIOD_100_MS, Some(PROBE_INTERVAL_TICKS), ); + let security = SecurityActor::spawn( + node_id.clone(), + swim_tx.clone(), + raft_tx.clone(), + NodeTransportSecurity::TrustedDevelopment, + ); tokio::spawn(RaftTransportActor::run( node_id.clone(), listener, raft_tx.clone(), transport_rx, swim_tx.clone(), - ClusterSecurity::TrustedDevelopment, + security, )); let db = MetadataStorage::open( std::env::temp_dir().join(uuid::Uuid::new_v4().to_string()), diff --git a/src/it/raft/membership_change.rs b/src/it/raft/membership_change.rs index 41b91437..8371bd0a 100644 --- a/src/it/raft/membership_change.rs +++ b/src/it/raft/membership_change.rs @@ -11,7 +11,7 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::{MultiRaftActor, MutlRaftSender}; use crate::control_plane::consensus::messages::*; -use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; +use crate::control_plane::consensus::transport::RaftTransportActor; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ShardGroup, ShardGroupId, Topology}; use crate::control_plane::{NodeId, Replicas, SwimNodeState}; @@ -20,6 +20,7 @@ use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; use crate::schedulers::ticker_message::{SchedulerSender, TickerCommand}; +use crate::security::{NodeTransportSecurity, SecurityActor}; use super::{CLUSTER_PORT, QUERY_PORT, mock_swim_handler}; @@ -68,13 +69,19 @@ async fn start_raft_node( Some(PROBE_INTERVAL_TICKS), ); + let security = SecurityActor::spawn( + node_id.clone(), + swim_tx.clone(), + raft_tx.clone(), + NodeTransportSecurity::TrustedDevelopment, + ); tokio::spawn(RaftTransportActor::run( node_id.clone(), listener, raft_tx.clone(), transport_rx, swim_tx.clone(), - ClusterSecurity::TrustedDevelopment, + security, )); let db = MetadataStorage::open(std::env::temp_dir().join(uuid::Uuid::new_v4().to_string())); let election_jitter_seed = { diff --git a/src/it/raft/mod.rs b/src/it/raft/mod.rs index 1cb1582d..be960e28 100644 --- a/src/it/raft/mod.rs +++ b/src/it/raft/mod.rs @@ -6,7 +6,7 @@ use arc_swap::ArcSwap; use tokio::sync::mpsc; use crate::control_plane::membership::{ - QueryCommand, SwimActorCommand, Topology, TopologyConfig, TopologyReader, topology_channel, + QueryCommand, SwimActorCommand, Topology, TopologyConfig, TopologyReader, }; use crate::control_plane::{NodeAddress, NodeId}; @@ -54,7 +54,7 @@ pub(super) fn stub_topology_channel( replication_factor: 3, }, ); - topology_channel(topology) + topology.channel() } mod election; diff --git a/src/security/mod.rs b/src/security/mod.rs index bc0db18c..aa3e043c 100644 --- a/src/security/mod.rs +++ b/src/security/mod.rs @@ -1,13 +1,11 @@ -#![allow(dead_code)] - -pub(crate) mod acl_cache; - +mod actor; mod admission_proof; mod certificates; +mod message; +mod state; mod transport; +pub(crate) use actor::{SecurityActor, SecurityHandle}; pub(crate) use admission_proof::AdmissionProof; -#[cfg(test)] -pub(crate) use admission_proof::ProcessSigningKey; pub(crate) use certificates::{client_certificate_principal, node_certificate_principal}; pub(crate) use transport::{NodeTransportSecurity, SecureNodeTransport, TransportIdentity}; From d52f99a07124acaa3d40d5b50226a435eb981ffc Mon Sep 17 00:00:00 2001 From: Migorithm Date: Fri, 31 Jul 2026 11:50:28 +0400 Subject: [PATCH 46/56] security actor injection, replace TransportIdentity with CertificatePrincipal --- src/connections/controller.rs | 148 ++++++++-------------------- src/control_plane/metadata/mod.rs | 21 ++-- src/data_plane/transport/reader.rs | 13 ++- src/data_plane/transport/writers.rs | 8 +- src/it/raft/snapshot.rs | 11 ++- src/lib.rs | 34 +++---- src/security/mod.rs | 2 +- 7 files changed, 90 insertions(+), 147 deletions(-) diff --git a/src/connections/controller.rs b/src/connections/controller.rs index c76ca970..f5683752 100644 --- a/src/connections/controller.rs +++ b/src/connections/controller.rs @@ -3,7 +3,6 @@ use crate::connections::writer::ClientRawWriter; use crate::connections::{protocol::*, run_client_writer}; use crate::control_plane::NodeAddressInfo; use crate::control_plane::consensus::raft::errors::ProposalError; -use crate::control_plane::consensus::transport::AclSnapshotSender; use crate::control_plane::metadata::{ AclResource, ConsumerGroupResource, OpenProducerSession, ProducerSessionOwner, RangeMeta, TopicState, UpdateConsumerGroupMember, UpdateConsumerGroupMemberRequest, @@ -31,8 +30,7 @@ use crate::data_plane::messages::query::{ DataPlaneQuery, Fetch, ListOffsets, ReadConsumerOffset, ReadConsumerOffsetResult, }; use crate::net::TransportTcpStream; -use crate::security::TransportIdentity; -use crate::security::acl_cache::SharedAclCache; +use crate::security::{CertificatePrincipal, SecurityHandle}; use tokio::sync::mpsc; /// # Client ↔ Server request_id protocol @@ -51,33 +49,30 @@ use tokio::sync::mpsc; /// with responses arriving in any order. #[derive(Clone)] pub struct ClientController { - transport_identity: TransportIdentity, + certificate_principal: Option, node_id: NodeId, swim_sender: SwimSender, raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, - acl_cache: SharedAclCache, - acl_snapshot_sender: AclSnapshotSender, + security: SecurityHandle, } impl ClientController { fn new( - transport_identity: TransportIdentity, + certificate_principal: Option, node_id: NodeId, swim_sender: SwimSender, raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, - acl_cache: SharedAclCache, - acl_snapshot_sender: AclSnapshotSender, + security: SecurityHandle, ) -> Self { Self { - transport_identity, + certificate_principal, node_id, swim_sender, raft_sender, data_plane_tx, - acl_cache, - acl_snapshot_sender, + security, } } @@ -117,7 +112,7 @@ impl ClientController { pub async fn dispatch(&self, request: ClientRequest) -> ClientResponse { tracing::trace!( - transport_identity = ?self.transport_identity, + certificate_principal = ?self.certificate_principal, "dispatching client request" ); match request { @@ -151,7 +146,7 @@ impl ClientController { // Convert ephemeral authentication evidence into the durable owner // carried by the Raft command. The persisted type intentionally does // not depend on TLS or stream implementation details. - let owner = ProducerSessionOwner::from(&self.transport_identity); + let owner = ProducerSessionOwner::from(self.certificate_principal.as_ref()); let command: OpenProducerSession = req.into_command(owner.clone()); @@ -304,36 +299,9 @@ impl ClientController { } async fn authorize_acl_resource(&self, resource: AclResource) -> Result<(), ServerError> { - let TransportIdentity::CertificatePrincipal(principal) = &self.transport_identity else { - return Ok(()); - }; - - match self.route(resource.routing_key()).await? { - ShardRouting::Local(group) => { - self.acl_cache - .authorize_or_refresh(&resource, group.id, principal, || { - self.raft_sender - .get_acl_snapshot(group.id, resource.clone()) - }) - .await - } - ShardRouting::Redirect(Some(remote)) => { - let Some(owner) = remote.member else { - return Err(ServerError::Unauthorized); - }; - self.acl_cache - .authorize_or_refresh(&resource, remote.group_id, principal, || { - self.acl_snapshot_sender.fetch( - self.node_id.clone(), - owner, - remote.group_id, - resource.clone(), - ) - }) - .await - } - ShardRouting::Redirect(None) => Err(ServerError::Unauthorized), - } + self.security + .authorize(self.certificate_principal.as_ref(), resource) + .await } /// Structural redirect for a control-plane op that isn't local: to the member if @@ -712,20 +680,18 @@ pub async fn handle_client_stream( swim_sender: SwimSender, raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, - acl_cache: SharedAclCache, - acl_snapshot_sender: AclSnapshotSender, + security: SecurityHandle, ) { - let transport_identity = stream.peer_identity(); + let certificate_principal = stream.peer_principal(); let (read_half, write_half) = stream.into_split(); let (writer_tx, writer_rx) = mpsc::channel(128); let handler = ClientController::new( - transport_identity, + certificate_principal, node_id, swim_sender, raft_sender, data_plane_tx, - acl_cache, - acl_snapshot_sender, + security, ); tokio::spawn(run_client_writer( ClientRawWriter::new(write_half), @@ -746,9 +712,7 @@ mod tests { use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::MultiRaftActorCommand; use crate::control_plane::consensus::raft::states::security::AclRecord; - use crate::control_plane::consensus::transport::{ - AclSnapshotActor, ClusterSecurity, RaftTransportActor, - }; + use crate::control_plane::consensus::transport::RaftTransportActor; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ QueryCommand, ShardGroup, ShardGroupId, ShardLeaderEntry, SwimActorCommand, @@ -763,6 +727,7 @@ mod tests { use crate::data_plane::messages::DataPlaneMessage; use crate::data_plane::messages::command::{DataPlaneCommand, ProduceAck}; use crate::net::TcpListener; + use crate::security::{NodeTransportSecurity, SecurityActor}; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -836,15 +801,19 @@ mod tests { raft_sender: MutlRaftSender, data_plane_tx: DataPlaneSender, ) -> ClientController { - let acl_snapshot_sender = AclSnapshotActor::spawn(ClusterSecurity::TrustedDevelopment); + let security = SecurityActor::spawn( + node_id.clone(), + swim_sender.clone(), + raft_sender.clone(), + NodeTransportSecurity::TrustedDevelopment, + ); ClientController::new( - TransportIdentity::TrustedDevelopment, + None, node_id, swim_sender, raft_sender, data_plane_tx, - SharedAclCache::default(), - acl_snapshot_sender, + security, ) } @@ -854,31 +823,19 @@ mod tests { swim_sender: SwimSender, raft_sender: MutlRaftSender, ) -> ClientController { - authenticated_controller_with_cache( - principal, - node_id, - swim_sender, - raft_sender, - SharedAclCache::default(), - ) - } - - fn authenticated_controller_with_cache( - principal: &str, - node_id: NodeId, - swim_sender: SwimSender, - raft_sender: MutlRaftSender, - acl_cache: SharedAclCache, - ) -> ClientController { - let acl_snapshot_sender = AclSnapshotActor::spawn(ClusterSecurity::TrustedDevelopment); + let security = SecurityActor::spawn( + node_id.clone(), + swim_sender.clone(), + raft_sender.clone(), + NodeTransportSecurity::TrustedDevelopment, + ); ClientController::new( - TransportIdentity::CertificatePrincipal(principal.into()), + Some(CertificatePrincipal::new(principal)), node_id, swim_sender, raft_sender, dp_stub(), - acl_cache, - acl_snapshot_sender, + security, ) } @@ -949,35 +906,6 @@ mod tests { ); } - #[tokio::test] - async fn fresh_acl_cache_authorizes_without_a_raft_query() { - let group = test_shard_group(); - let swim = swim_sender_with(move |cmd| { - if let SwimActorCommand::Query(QueryCommand::ResolveShardGroup { reply, .. }) = cmd { - let _ = reply.send(Some(group.clone())); - } - }); - let cache = SharedAclCache::default(); - let resource = AclResource::TopicData(TopicId(7)); - assert_eq!( - cache - .authorize_or_refresh(&resource, ShardGroupId(42), "orders-service", || async { - Some(acl_snapshot(resource.clone(), &["orders-service"])) - }) - .await, - Ok(()) - ); - let controller = authenticated_controller_with_cache( - "orders-service", - node_id("node-1"), - swim, - raft_sender_with(|_| panic!("fresh ACL cache must not query Raft")), - cache, - ); - - assert_eq!(controller.authorize_acl_resource(resource).await, Ok(())); - } - #[tokio::test] async fn acl_cache_miss_reads_the_local_snapshot_once() { let group = test_shard_group(); @@ -1228,13 +1156,19 @@ mod tests { let (raft_tx, mut raft_rx) = MultiRaftActor::channel(8); let (_transport_tx, transport_rx) = mpsc::channel(1); let (swim_tx, _swim_rx) = SwimActor::channel(1); + let security = SecurityActor::spawn( + node_id("owner"), + swim_tx.clone(), + raft_tx.clone(), + NodeTransportSecurity::TrustedDevelopment, + ); tokio::spawn(RaftTransportActor::run( node_id("owner"), listener, raft_tx, transport_rx, swim_tx, - ClusterSecurity::TrustedDevelopment, + security, )); let Some(MultiRaftActorCommand::GetAclSnapshot(query)) = raft_rx.recv().await diff --git a/src/control_plane/metadata/mod.rs b/src/control_plane/metadata/mod.rs index cb48ca58..fb92a4c5 100644 --- a/src/control_plane/metadata/mod.rs +++ b/src/control_plane/metadata/mod.rs @@ -24,7 +24,7 @@ pub(crate) use consumer_group::{ConsumerGroupAssignment, ConsumerGroupMeta, Cons pub(crate) use segment::*; -use crate::{impl_new_struct_wrapper, security::TransportIdentity}; +use crate::{impl_new_struct_wrapper, security::CertificatePrincipal}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Ser, Deser)] pub struct TopicId(pub(crate) u64); @@ -55,22 +55,21 @@ pub struct ProducerSessionResource { /// Durable owner of one producer session. /// -/// This is deliberately separate from the connection's transport identity: -/// Raft snapshots retain ownership after the TLS connection disappears, and -/// transport refactors must not change the persisted metadata schema. +/// This is deliberately separate from the connection's optional certificate +/// principal: Raft snapshots retain ownership after the TLS connection +/// disappears, and trusted-development ownership must remain explicit in +/// persisted metadata rather than being encoded as an absent value. #[derive(Debug, Clone, PartialEq, Eq, Hash, Ser, Deser)] pub enum ProducerSessionOwner { CertificatePrincipal(Box), TrustedDevelopment, } -impl ProducerSessionOwner { - pub(crate) fn from(transport_identity: &TransportIdentity) -> Self { - match transport_identity { - TransportIdentity::CertificatePrincipal(principal) => { - ProducerSessionOwner::CertificatePrincipal(principal.clone()) - } - TransportIdentity::TrustedDevelopment => ProducerSessionOwner::TrustedDevelopment, +impl From> for ProducerSessionOwner { + fn from(certificate_principal: Option<&CertificatePrincipal>) -> Self { + match certificate_principal { + Some(principal) => Self::CertificatePrincipal(principal.as_ref().into()), + None => Self::TrustedDevelopment, } } } diff --git a/src/data_plane/transport/reader.rs b/src/data_plane/transport/reader.rs index a21f0ab4..608bf18f 100644 --- a/src/data_plane/transport/reader.rs +++ b/src/data_plane/transport/reader.rs @@ -5,21 +5,24 @@ use crate::control_plane::NodeId; use crate::data_plane::actor::DataPlaneSender; use crate::data_plane::messages::command::{DataPlaneCommand, ReceivePeerMessage}; use crate::net::TransportReadHalf; -use crate::security::TransportIdentity; +use crate::security::CertificatePrincipal; const NODE_ID_FRAME_MAX: usize = 1024; const DATA_FRAME_MAX: usize = 64 * 1024 * 1024; pub(super) struct DataReader { read_half: TransportReadHalf, - transport_identity: TransportIdentity, + certificate_principal: Option, } impl DataReader { - pub(super) fn new(read_half: TransportReadHalf, transport_identity: TransportIdentity) -> Self { + pub(super) fn new( + read_half: TransportReadHalf, + certificate_principal: Option, + ) -> Self { Self { read_half, - transport_identity, + certificate_principal, } } @@ -39,7 +42,7 @@ impl DataReader { #[tracing::instrument( level = "trace", skip_all, - fields(peer = %peer, transport_identity = ?self.transport_identity) + fields(peer = %peer, certificate_principal = ?self.certificate_principal) )] pub(crate) async fn run( mut self, diff --git a/src/data_plane/transport/writers.rs b/src/data_plane/transport/writers.rs index 037d0067..558863f9 100644 --- a/src/data_plane/transport/writers.rs +++ b/src/data_plane/transport/writers.rs @@ -44,9 +44,9 @@ impl TransportState { &mut self, stream: TransportTcpStream, ) -> anyhow::Result<(NodeId, DataReader)> { - let transport_identity = stream.peer_identity(); + let certificate_principal = stream.peer_principal(); let (read_half, write_half) = stream.into_split(); - let mut reader = DataReader::new(read_half, transport_identity); + let mut reader = DataReader::new(read_half, certificate_principal); let peer_id = reader.read_node_id().await?; @@ -142,7 +142,7 @@ impl TransportState { .context("connect timed out")? .context("TCP connect failed")?; - let transport_identity = stream.peer_identity(); + let certificate_principal = stream.peer_principal(); let (read_half, write_half) = stream.into_split(); self.writers.insert(target_id.clone(), write_half); @@ -156,7 +156,7 @@ impl TransportState { return Err(e).context("initial write failed"); } - Ok(DataReader::new(read_half, transport_identity)) + Ok(DataReader::new(read_half, certificate_principal)) } pub fn disconnect(&mut self, peer_id: NodeId) { diff --git a/src/it/raft/snapshot.rs b/src/it/raft/snapshot.rs index aa356d6f..33a8e005 100644 --- a/src/it/raft/snapshot.rs +++ b/src/it/raft/snapshot.rs @@ -9,7 +9,7 @@ use turmoil::Builder; use crate::control_plane::consensus::actor::MultiRaftActor; use crate::control_plane::consensus::messages::EnsureGroup; -use crate::control_plane::consensus::transport::{ClusterSecurity, RaftTransportActor}; +use crate::control_plane::consensus::transport::RaftTransportActor; use crate::control_plane::membership::actor::SwimActor; use crate::control_plane::membership::{ShardGroup, ShardGroupId}; use crate::control_plane::metadata::CreateTopic; @@ -19,6 +19,7 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TcpStream}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; +use crate::security::{NodeTransportSecurity, SecurityActor}; use super::{CLUSTER_PORT, mock_swim_handler}; @@ -49,13 +50,19 @@ async fn run_node(name: &'static str, ordinal: u16) -> Result<(), Box>(100); - let state = self.env.swim(self.rng_seed); - let node_id = state.node_id.clone(); + let swim = self.env.swim(self.rng_seed); + let node_id = swim.node_id.clone(); + let security_handle = SecurityActor::spawn( + node_id.clone(), + swim_sender.clone(), + raft_tx.clone(), + security.clone(), + ); // Topology snapshot channel: SwimActor publishes, all other actors read. // Single-writer / many-readers via ArcSwap — no locks, no contention. - let (topology_pub, topology_reader) = topology_channel(state.topology.clone()); + let (topology_pub, topology_reader) = swim.topology.clone().channel(); // Recover local durable state before this node serves or joins the // cluster: scan + replay the WAL into the segment files, then clear the @@ -115,14 +119,14 @@ impl StartUp { raft_tx.clone(), raft_transport_rx, swim_sender.clone(), - security.clone(), + security_handle.clone(), )); // Protocol actors (each spawns its own scheduler internally) SwimActor::spawn( swim_sender.clone(), swim_mailbox, - state, + swim, tx_outbound, raft_tx.clone().into(), topology_pub, @@ -168,8 +172,6 @@ impl StartUp { self.env.raft_snapshot_entry_threshold, ); - let acl_snapshot_sender = AclSnapshotActor::spawn(security.clone()); - // Client handler let _ = self .receive_client_streams( @@ -178,7 +180,7 @@ impl StartUp { raft_tx, data_plane_tx, security, - acl_snapshot_sender, + security_handle, ) .await; Ok(()) @@ -191,9 +193,8 @@ impl StartUp { raft_tx: MutlRaftSender, data_plane_tx: DataPlaneSender, security: NodeTransportSecurity, - acl_snapshot_sender: AclSnapshotSender, + security_handle: SecurityHandle, ) { - let acl_cache = SharedAclCache::default(); let addr = self.env.bind_addr(); let listener = TcpListener::bind(&addr).await.unwrap(); tracing::info!( @@ -214,7 +215,7 @@ impl StartUp { let swim_tx = swim_sender.clone(); let raft = raft_tx.clone(); let dp = data_plane_tx.clone(); - let acl_sender = acl_snapshot_sender.clone(); + let security_handle = security_handle.clone(); tokio::spawn(handle_client_stream( stream, @@ -222,8 +223,7 @@ impl StartUp { swim_tx, raft, dp, - acl_cache.clone(), - acl_sender, + security_handle, )); } } diff --git a/src/security/mod.rs b/src/security/mod.rs index aa3e043c..53588124 100644 --- a/src/security/mod.rs +++ b/src/security/mod.rs @@ -8,4 +8,4 @@ mod transport; pub(crate) use actor::{SecurityActor, SecurityHandle}; pub(crate) use admission_proof::AdmissionProof; pub(crate) use certificates::{client_certificate_principal, node_certificate_principal}; -pub(crate) use transport::{NodeTransportSecurity, SecureNodeTransport, TransportIdentity}; +pub(crate) use transport::{CertificatePrincipal, NodeCredentials, NodeTransportSecurity}; From 31c0c63d2b0b153c78701470a7cb4ccd8f6e8261 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Fri, 31 Jul 2026 13:54:34 +0400 Subject: [PATCH 47/56] Box -> CertificatePrincipal --- src/control_plane/consensus/actor.rs | 3 +- src/control_plane/consensus/messages/actor.rs | 3 +- src/control_plane/consensus/multi_raft.rs | 3 +- src/control_plane/consensus/raft/state.rs | 6 +- .../consensus/raft/states/metadata_state.rs | 13 ++- .../consensus/raft/states/security.rs | 24 +++-- src/macros/mod.rs | 1 - src/security/certificates.rs | 18 ++-- src/security/transport.rs | 87 ++++++++++++++----- 9 files changed, 117 insertions(+), 41 deletions(-) diff --git a/src/control_plane/consensus/actor.rs b/src/control_plane/consensus/actor.rs index 0d4d14e9..e7c87426 100644 --- a/src/control_plane/consensus/actor.rs +++ b/src/control_plane/consensus/actor.rs @@ -17,6 +17,7 @@ use crate::control_plane::metadata::{ }; use crate::data_plane::transport::command::DataTransportCommand; use crate::schedulers::ticker_message::{SchedulerSender, TickerCommand}; +use crate::security::CertificatePrincipal; use tokio::sync::mpsc; use uuid::Uuid; @@ -304,7 +305,7 @@ impl MutlRaftSender { pub(crate) async fn get_admission( &self, shard_group_id: ShardGroupId, - node_certificate_principal: Box, + node_certificate_principal: CertificatePrincipal, ) -> Result, ServerError> { let (reply, recv) = tokio::sync::oneshot::channel(); self.send(GetAdmission { diff --git a/src/control_plane/consensus/messages/actor.rs b/src/control_plane/consensus/messages/actor.rs index 946605f5..df40f623 100644 --- a/src/control_plane/consensus/messages/actor.rs +++ b/src/control_plane/consensus/messages/actor.rs @@ -10,6 +10,7 @@ use crate::control_plane::metadata::{AclResource, ConsumerGroupAssignment, Topic use crate::data_plane::messages::command::{ DurableSegmentEndReported, SegmentCaughtUp, SegmentPlaced, }; +use crate::security::CertificatePrincipal; use super::command::{ EnsureGroup, InboundRaftRpc, MetadataProposal, ProposeSegmentRoll, RaftProtocolMessage, @@ -91,7 +92,7 @@ pub struct GetAclSnapshot { /// Returns one admission record from a metadata shard hosted by this node. pub struct GetAdmission { pub(crate) shard_group_id: ShardGroupId, - pub(crate) node_certificate_principal: Box, + pub(crate) node_certificate_principal: CertificatePrincipal, pub(crate) reply: oneshot::Sender, ServerError>>, } diff --git a/src/control_plane/consensus/multi_raft.rs b/src/control_plane/consensus/multi_raft.rs index cd0611d2..f63dcb3b 100644 --- a/src/control_plane/consensus/multi_raft.rs +++ b/src/control_plane/consensus/multi_raft.rs @@ -25,6 +25,7 @@ use crate::data_plane::messages::command::{ DurableSegmentEndReported, RequestDurableSegmentEnd, SegmentCaughtUp, SegmentPlaced, }; use crate::data_plane::transport::command::DataTransportCommand; +use crate::security::CertificatePrincipal; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::hash::{Hash, Hasher}; use tokio::sync::oneshot; @@ -516,7 +517,7 @@ impl MultiRaft { fn admission( &self, shard_group_id: ShardGroupId, - node_certificate_principal: &str, + node_certificate_principal: &CertificatePrincipal, ) -> Result, ServerError> { let raft = self .groups diff --git a/src/control_plane/consensus/raft/state.rs b/src/control_plane/consensus/raft/state.rs index 5e67d039..ecf2aa04 100644 --- a/src/control_plane/consensus/raft/state.rs +++ b/src/control_plane/consensus/raft/state.rs @@ -25,6 +25,7 @@ use crate::data_plane::SegmentKey; use crate::data_plane::messages::command::{PlaceSegment, SegmentCaughtUp, SegmentPlaced}; use crate::data_plane::transport::command::DataTransportCommand; use crate::schedulers::ticker_message::TimerCommand; +use crate::security::CertificatePrincipal; #[cfg(any(test, debug_assertions))] use crate::test_traits::TAssertInvariant; use std::collections::{BTreeSet, HashSet}; @@ -166,7 +167,10 @@ impl Raft { self.metadata.acl_snapshot(resource) } - pub(crate) fn admission(&self, node_certificate_principal: &str) -> Option { + pub(crate) fn admission( + &self, + node_certificate_principal: &CertificatePrincipal, + ) -> Option { self.metadata.admission(node_certificate_principal) } diff --git a/src/control_plane/consensus/raft/states/metadata_state.rs b/src/control_plane/consensus/raft/states/metadata_state.rs index eaf2e97d..252d1ee7 100644 --- a/src/control_plane/consensus/raft/states/metadata_state.rs +++ b/src/control_plane/consensus/raft/states/metadata_state.rs @@ -15,6 +15,7 @@ use crate::control_plane::metadata::{ AclResource, EntryId, RangeId, SegmentId, TopicId, error::MetadataError, }; use crate::data_plane::SegmentKey; +use crate::security::CertificatePrincipal; #[cfg(any(test, debug_assertions))] use crate::test_traits::TAssertInvariant; use MetadataError::*; @@ -97,7 +98,10 @@ impl MetadataState { self.security.acl_snapshot(resource) } - pub(crate) fn admission(&self, node_certificate_principal: &str) -> Option { + pub(crate) fn admission( + &self, + node_certificate_principal: &CertificatePrincipal, + ) -> Option { self.security.admission(node_certificate_principal) } @@ -643,7 +647,7 @@ mod tests { fn security_records_survive_snapshot_restore() { let mut state = MetadataState::new(ShardGroupId(1)); let admission = AdmissionRecord { - node_certificate_principal: "broker-a".to_string(), + node_certificate_principal: CertificatePrincipal::new("broker-a"), revision: 3, epoch: 2, node_id: NodeId::new("broker-a::process-2"), @@ -679,7 +683,10 @@ mod tests { let restored = MetadataState::from_snapshot(snapshot, 9); assert_eq!( - restored.security.admissions.get("broker-a"), + restored + .security + .admissions + .get(&CertificatePrincipal::new("broker-a")), Some(&admission) ); assert_eq!(restored.security.acls.get(&acl.resource), Some(&acl)); diff --git a/src/control_plane/consensus/raft/states/security.rs b/src/control_plane/consensus/raft/states/security.rs index ec81cd90..e2fbfd47 100644 --- a/src/control_plane/consensus/raft/states/security.rs +++ b/src/control_plane/consensus/raft/states/security.rs @@ -4,6 +4,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use crate::control_plane::NodeId; use crate::control_plane::metadata::AclResource; +use crate::security::CertificatePrincipal; /// Security records replicated by one metadata shard. /// @@ -11,7 +12,7 @@ use crate::control_plane::metadata::AclResource; /// indexes do not enlarge every variant of the Raft snapshot state. #[derive(Debug, Clone, Default, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct SecurityState { - pub(super) admissions: HashMap, + pub(super) admissions: HashMap, pub(super) acls: HashMap, pub(super) revocations: HashMap<(String, Box<[u8]>), RevocationRecord>, } @@ -22,7 +23,7 @@ pub(crate) struct SecurityState { /// accept SWIM facts only when the epoch, node ID, and process key match it. #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct AdmissionRecord { - pub node_certificate_principal: String, + pub node_certificate_principal: CertificatePrincipal, pub revision: u64, pub epoch: u64, pub node_id: NodeId, @@ -54,7 +55,10 @@ pub(crate) struct RevocationRecord { } impl SecurityState { - pub(crate) fn admission(&self, node_certificate_principal: &str) -> Option { + pub(crate) fn admission( + &self, + node_certificate_principal: &CertificatePrincipal, + ) -> Option { self.admissions.get(node_certificate_principal).cloned() } @@ -151,7 +155,7 @@ mod tests { #[test] fn security_records_round_trip() { round_trip(&AdmissionRecord { - node_certificate_principal: "broker-a".to_string(), + node_certificate_principal: CertificatePrincipal::new("broker-a"), revision: 3, epoch: 2, node_id: NodeId::new("broker-a::process-2"), @@ -174,7 +178,7 @@ mod tests { fn admission_lookup_uses_the_certificate_principal() { let mut security = SecurityState::default(); let admission = AdmissionRecord { - node_certificate_principal: "broker-a".to_string(), + node_certificate_principal: CertificatePrincipal::new("broker-a"), revision: 3, epoch: 2, node_id: NodeId::new("broker-a::process-2"), @@ -185,8 +189,14 @@ mod tests { admission.clone(), ); - assert_eq!(security.admission("broker-a"), Some(admission)); - assert_eq!(security.admission("broker-b"), None); + assert_eq!( + security.admission(&CertificatePrincipal::new("broker-a")), + Some(admission) + ); + assert_eq!( + security.admission(&CertificatePrincipal::new("broker-b")), + None + ); } #[test] diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 08d0b4c3..b9e400d1 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -32,7 +32,6 @@ macro_rules! impl_new_struct_wrapper { $wrapper_name(val) } } - $crate::smart_pointer!($wrapper_name, $type); }; } diff --git a/src/security/certificates.rs b/src/security/certificates.rs index cf5e9d22..0f57cb48 100644 --- a/src/security/certificates.rs +++ b/src/security/certificates.rs @@ -3,6 +3,8 @@ use rustls::pki_types::CertificateDer; use x509_parser::extensions::GeneralName; use x509_parser::prelude::{FromDer, X509Certificate}; +use crate::security::CertificatePrincipal; + /// Reads the stable node principal from a leaf certificate's URI Subject /// Alternative Name. /// @@ -10,7 +12,9 @@ use x509_parser::prelude::{FromDer, X509Certificate}; /// `urn:eastguard:node:`. The text after that prefix is the principal used as /// the admission-record key. This function only parses the certificate; callers /// must use it after rustls has authenticated the peer's certificate chain. -pub(crate) fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Result { +pub(crate) fn node_certificate_principal( + certificate: &CertificateDer<'_>, +) -> Result { certificate_principal( certificate, "urn:eastguard:node:", @@ -24,7 +28,9 @@ pub(crate) fn node_certificate_principal(certificate: &CertificateDer<'_>) -> Re /// /// TLS authentication must succeed before callers use this parsed identity for /// authorization. -pub(crate) fn client_certificate_principal(certificate: &CertificateDer<'_>) -> Result { +pub(crate) fn client_certificate_principal( + certificate: &CertificateDer<'_>, +) -> Result { certificate_principal( certificate, "urn:eastguard:client:", @@ -38,7 +44,7 @@ fn certificate_principal( uri_prefix: &str, certificate_kind: &str, principal_name: &str, -) -> Result { +) -> Result { let (_, certificate) = X509Certificate::from_der(certificate.as_ref()).context("invalid X.509 certificate")?; let subject_alt_name = certificate @@ -65,7 +71,7 @@ fn certificate_principal( principals.next().is_none(), "{certificate_kind} certificate has multiple {principal_name}s" ); - Ok(principal.to_string()) + Ok(CertificatePrincipal::new(principal)) } #[cfg(test)] @@ -90,7 +96,7 @@ mod tests { certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:node:broker-a"]); assert_eq!( - node_certificate_principal(&certificate).unwrap(), + node_certificate_principal(&certificate).unwrap().as_ref(), "broker-a" ); } @@ -121,7 +127,7 @@ mod tests { certificate_with_uris(&["urn:example:unrelated", "urn:eastguard:client:producer-a"]); assert_eq!( - client_certificate_principal(&certificate).unwrap(), + client_certificate_principal(&certificate).unwrap().as_ref(), "producer-a" ); } diff --git a/src/security/transport.rs b/src/security/transport.rs index ff2a57d1..42edc352 100644 --- a/src/security/transport.rs +++ b/src/security/transport.rs @@ -4,7 +4,12 @@ use std::io::BufReader; use std::path::Path; use std::sync::Arc; +use super::admission_proof::{AdmissionProof, ProcessSigningKey}; +use super::certificates::node_certificate_principal; +use crate::config::{Environment, SecurityMode}; +use crate::control_plane::NodeId; use anyhow::{Context, Result}; +use borsh::{BorshDeserialize, BorshSerialize}; use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; use rustls::client::verify_server_cert_signed_by_trust_anchor; use rustls::crypto::{WebPkiSupportedAlgorithms, verify_tls12_signature, verify_tls13_signature}; @@ -16,9 +21,6 @@ use rustls::{ RootCertStore, ServerConfig, SignatureScheme, }; -use super::certificates::node_certificate_principal; -use crate::config::{Environment, SecurityMode}; - /// Verifies certificates presented to EastGuard's outbound node connections. /// /// The shared client config uses this verifier when Raft or data transport @@ -105,18 +107,7 @@ impl ServerCertVerifier for NodeServerCertVerifier { #[derive(Clone)] pub(crate) enum NodeTransportSecurity { - Secure { - server: Arc, - client: Arc, - }, - TrustedDevelopment, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum TransportIdentity { - /// Principal authenticated for this live TLS connection. This is transport - /// evidence, not a durable authorization or ownership record. - CertificatePrincipal(Box), + Secure(NodeCredentials), TrustedDevelopment, } @@ -136,7 +127,11 @@ impl NodeTransportSecurity { .trust_root_path .as_deref() .context("trust_root_path is required in secure mode")?; - Self::load_from_paths(certificate_chain, private_key_path, trust_roots) + Ok(Self::Secure(Self::load_from_paths( + certificate_chain, + private_key_path, + trust_roots, + )?)) } SecurityMode::TrustedDevelopment => Ok(Self::TrustedDevelopment), } @@ -146,12 +141,12 @@ impl NodeTransportSecurity { certificate_chain_path: &Path, private_key_path: &Path, trust_root_path: &Path, - ) -> Result { + ) -> Result { let certificate_chain = Self::load_certificates(certificate_chain_path, "certificate chain")?; + let node_certificate_principal = node_certificate_principal(&certificate_chain[0])?; let private_key = Self::load_private_key(private_key_path)?; let trust_roots = Arc::new(Self::load_trust_roots(trust_root_path)?); - let client_verifier = WebPkiClientVerifier::builder(trust_roots.clone()).build()?; let server = ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) .with_client_cert_verifier(client_verifier) @@ -161,14 +156,16 @@ impl NodeTransportSecurity { .with_custom_certificate_verifier(Arc::new(NodeServerCertVerifier::new(trust_roots))) .with_client_auth_cert(certificate_chain, private_key)?; - Ok(Self::Secure { + Ok(NodeCredentials { server: Arc::new(server), client: Arc::new(client), + node_certificate_principal, + process_signing_key: Arc::new(ProcessSigningKey::generate()?), }) } pub(crate) fn is_secure(&self) -> bool { - matches!(self, Self::Secure { .. }) + matches!(self, Self::Secure(_)) } fn load_certificates(path: &Path, kind: &'static str) -> Result>> { @@ -216,6 +213,56 @@ impl NodeTransportSecurity { } } +/// TLS credentials and process identity loaded for secure mode. +#[derive(Clone)] +pub(crate) struct NodeCredentials { + server: Arc, + client: Arc, + node_certificate_principal: CertificatePrincipal, + process_signing_key: Arc, +} + +#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, BorshSerialize, BorshDeserialize)] +pub(crate) struct CertificatePrincipal(Box); + +impl CertificatePrincipal { + pub(crate) fn new(principal: impl Into>) -> Self { + Self(principal.into()) + } +} + +impl AsRef for CertificatePrincipal { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl NodeCredentials { + pub(crate) fn server_config(&self) -> Arc { + self.server.clone() + } + + pub(crate) fn client_config(&self) -> Arc { + self.client.clone() + } + + /// Creates proof that a node connection belongs to this exact process. + /// + /// The TLS session binding makes the proof unique to one connection. Only + /// the process public key is stored in metadata Raft. + pub(crate) fn create_admission_proof( + &self, + node_id: &NodeId, + tls_session_binding: &[u8], + ) -> Result { + self.process_signing_key.sign( + &self.node_certificate_principal, + node_id, + tls_session_binding, + ) + } +} + #[cfg(test)] mod tests { use super::*; From d31e26589309d94f31862d1d6d1acdb36a1810f8 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sat, 1 Aug 2026 22:09:38 +0400 Subject: [PATCH 48/56] accept node/client with principal read --- src/net/tcp.rs | 124 ++++++++++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 54 deletions(-) diff --git a/src/net/tcp.rs b/src/net/tcp.rs index 11b8869d..bdddbab4 100644 --- a/src/net/tcp.rs +++ b/src/net/tcp.rs @@ -9,10 +9,10 @@ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio_rustls::{TlsAcceptor, TlsConnector, TlsStream}; use super::inner; -use crate::security::{ - NodeTransportSecurity, TransportIdentity, client_certificate_principal, - node_certificate_principal, -}; +use crate::security::{CertificatePrincipal, NodeTransportSecurity, node_certificate_principal}; + +const NODE_ADMISSION_EXPORTER_LABEL: &[u8] = b"EXPORTER-EastGuard-node-admission-v1"; +const NODE_ADMISSION_BINDING_BYTES: usize = 32; macro_rules! tcp_wrapper { ($name:ident) => { @@ -45,25 +45,11 @@ tcp_wrapper!(OwnedWriteHalf); /// Node Certificate Principal. Admission later binds that stable principal to /// the process-specific `NodeId` carried by the transport handshake. pub struct AuthenticatedTcpStream { - peer_principal: String, + peer_principal: CertificatePrincipal, stream: TlsStream, } impl AuthenticatedTcpStream { - pub async fn accept(stream: TcpStream, config: Arc) -> Result { - let stream = TlsAcceptor::from(config).accept(stream).await?; - - Self::from_tls_stream(stream.into(), node_certificate_principal) - } - - pub async fn accept_client( - stream: TcpStream, - config: Arc, - ) -> Result { - let stream = TlsAcceptor::from(config).accept(stream).await?; - Self::from_tls_stream(stream.into(), client_certificate_principal) - } - pub async fn connect( addr: A, config: Arc, @@ -80,7 +66,7 @@ impl AuthenticatedTcpStream { fn from_tls_stream( stream: TlsStream, - read_principal: fn(&rustls::pki_types::CertificateDer<'_>) -> Result, + read_principal: fn(&rustls::pki_types::CertificateDer<'_>) -> Result, ) -> Result { let certificate = stream .get_ref() @@ -95,10 +81,31 @@ impl AuthenticatedTcpStream { }) } - pub fn peer_principal(&self) -> &str { + pub fn peer_principal(&self) -> &CertificatePrincipal { &self.peer_principal } + /// Derives a value unique to this completed TLS session. + /// + /// Both peers derive the same bytes. Signing them binds a process-admission + /// proof to this connection, so a captured proof cannot be replayed. + fn admission_binding(&self) -> Result<[u8; NODE_ADMISSION_BINDING_BYTES]> { + let output = [0; NODE_ADMISSION_BINDING_BYTES]; + match &self.stream { + TlsStream::Client(stream) => stream.get_ref().1.export_keying_material( + output, + NODE_ADMISSION_EXPORTER_LABEL, + None, + ), + TlsStream::Server(stream) => stream.get_ref().1.export_keying_material( + output, + NODE_ADMISSION_EXPORTER_LABEL, + None, + ), + } + .context("failed to derive node-admission TLS session binding") + } + fn into_split( self, ) -> ( @@ -139,26 +146,17 @@ impl From for TransportWriteHalf { } impl TransportTcpStream { - pub async fn accept_node(stream: TcpStream, security: &NodeTransportSecurity) -> Result { - match security { - NodeTransportSecurity::Secure { server, .. } => { - AuthenticatedTcpStream::accept(stream, server.clone()) - .await - .map(Box::new) - .map(Self::Secure) - } - NodeTransportSecurity::TrustedDevelopment => Ok(Self::TrustedDevelopment(stream)), - } - } - - pub async fn accept_client( + pub async fn accept( stream: TcpStream, security: &NodeTransportSecurity, + read_principal: fn(&rustls::pki_types::CertificateDer<'_>) -> Result, ) -> Result { match security { - NodeTransportSecurity::Secure { server, .. } => { - AuthenticatedTcpStream::accept_client(stream, server.clone()) - .await + NodeTransportSecurity::Secure(security) => { + let stream = TlsAcceptor::from(security.server_config()) + .accept(stream) + .await?; + AuthenticatedTcpStream::from_tls_stream(stream.into(), read_principal) .map(Box::new) .map(Self::Secure) } @@ -171,8 +169,8 @@ impl TransportTcpStream { security: &NodeTransportSecurity, ) -> Result { match security { - NodeTransportSecurity::Secure { client, .. } => { - AuthenticatedTcpStream::connect(addr, client.clone()) + NodeTransportSecurity::Secure(security) => { + AuthenticatedTcpStream::connect(addr, security.client_config()) .await .map(Box::new) .map(Self::Secure) @@ -184,12 +182,19 @@ impl TransportTcpStream { } } - pub fn peer_identity(&self) -> TransportIdentity { + pub fn peer_principal(&self) -> Option { match self { - Self::Secure(stream) => { - TransportIdentity::CertificatePrincipal(stream.peer_principal().into()) + Self::Secure(stream) => Some(stream.peer_principal().clone()), + Self::TrustedDevelopment(_) => None, + } + } + + pub(crate) fn admission_binding(&self) -> Result<[u8; NODE_ADMISSION_BINDING_BYTES]> { + match self { + Self::Secure(stream) => stream.admission_binding(), + Self::TrustedDevelopment(_) => { + anyhow::bail!("trusted-development connections have no TLS session binding") } - Self::TrustedDevelopment(_) => TransportIdentity::TrustedDevelopment, } } @@ -417,6 +422,7 @@ impl TcpStream { #[cfg(test)] mod tests { use super::*; + use crate::security::client_certificate_principal; use rcgen::string::Ia5String; use rcgen::{CertificateParams, KeyPair, SanType}; use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; @@ -473,7 +479,7 @@ mod tests { } #[test] - fn mutual_tls_exposes_peer_principals_under_turmoil() -> turmoil::Result { + fn mutual_tls_shares_session_binding_under_turmoil() -> turmoil::Result { let (server_config, client_config) = tls_configs(); let mut sim = Builder::new().build(); @@ -482,14 +488,18 @@ mod tests { async move { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; - let mut stream = AuthenticatedTcpStream::accept(stream, server_config) - .await - .unwrap(); - assert_eq!(stream.peer_principal(), "broker-client"); + let stream = TlsAcceptor::from(server_config).accept(stream).await?; + let mut stream = AuthenticatedTcpStream::from_tls_stream( + stream.into(), + node_certificate_principal, + )?; + assert_eq!(stream.peer_principal().as_ref(), "broker-client"); + let session_binding = stream.admission_binding().unwrap(); let mut message = [0; 4]; stream.read_exact(&mut message).await?; assert_eq!(&message, b"ping"); stream.write_all(b"pong").await?; + stream.write_all(&session_binding).await?; Ok(()) } }); @@ -499,11 +509,15 @@ mod tests { AuthenticatedTcpStream::connect((turmoil::lookup("server"), 9000), client_config) .await .unwrap(); - assert_eq!(stream.peer_principal(), "broker-server"); + assert_eq!(stream.peer_principal().as_ref(), "broker-server"); + let session_binding = stream.admission_binding().unwrap(); stream.write_all(b"ping").await?; let mut message = [0; 4]; stream.read_exact(&mut message).await?; assert_eq!(&message, b"pong"); + let mut peer_session_binding = [0; NODE_ADMISSION_BINDING_BYTES]; + stream.read_exact(&mut peer_session_binding).await?; + assert_eq!(peer_session_binding, session_binding); Ok(()) }); @@ -543,10 +557,12 @@ mod tests { async move { let listener = TcpListener::bind("0.0.0.0:9000").await?; let (stream, _) = listener.accept().await?; - let stream = AuthenticatedTcpStream::accept_client(stream, server_config) - .await - .unwrap(); - assert_eq!(stream.peer_principal(), "producer-a"); + let stream = TlsAcceptor::from(server_config).accept(stream).await?; + let stream = AuthenticatedTcpStream::from_tls_stream( + stream.into(), + client_certificate_principal, + )?; + assert_eq!(stream.peer_principal().as_ref(), "producer-a"); Ok(()) } }); @@ -555,7 +571,7 @@ mod tests { AuthenticatedTcpStream::connect((turmoil::lookup("server"), 9000), client_config) .await .unwrap(); - assert_eq!(stream.peer_principal(), "broker-server"); + assert_eq!(stream.peer_principal().as_ref(), "broker-server"); Ok(()) }); From f8c21ce87075ee8d361b74e96445532b51aca175 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sat, 1 Aug 2026 22:10:59 +0400 Subject: [PATCH 49/56] signing and verifying admission proof --- src/data_plane/transport/mod.rs | 8 +- src/security/admission_proof.rs | 152 ++++++++++++++++---------------- 2 files changed, 83 insertions(+), 77 deletions(-) diff --git a/src/data_plane/transport/mod.rs b/src/data_plane/transport/mod.rs index f5d89088..5f87367c 100644 --- a/src/data_plane/transport/mod.rs +++ b/src/data_plane/transport/mod.rs @@ -10,7 +10,7 @@ use crate::control_plane::membership::actor::SwimSender; use crate::data_plane::actor::DataPlaneSender; use crate::net::{TcpListener, TransportTcpStream}; -use crate::security::NodeTransportSecurity; +use crate::security::{NodeTransportSecurity, node_certificate_principal}; use command::DataTransportCommand; use writers::TransportState; @@ -69,7 +69,11 @@ impl DataTransportActor { } Ok((stream, _)) = listener.accept() => { - let stream = match TransportTcpStream::accept_node(stream, &security).await { + let stream = match TransportTcpStream::accept( + stream, + &security, + node_certificate_principal, + ).await { Ok(stream) => stream, Err(error) => { tracing::debug!("Data TLS accept rejected: {error}"); diff --git a/src/security/admission_proof.rs b/src/security/admission_proof.rs index 710856a7..687a9dbc 100644 --- a/src/security/admission_proof.rs +++ b/src/security/admission_proof.rs @@ -1,38 +1,23 @@ use anyhow::{Context, Result}; use borsh::{BorshDeserialize, BorshSerialize}; +#[cfg(test)] +use ring::signature::KeyPair; use ring::{ - rand::{SecureRandom, SystemRandom}, - signature::{ED25519, Ed25519KeyPair, KeyPair, UnparsedPublicKey}, + rand::SystemRandom, + signature::{ED25519, Ed25519KeyPair, UnparsedPublicKey}, }; -use crate::control_plane::NodeId; +use crate::control_plane::consensus::raft::states::security::AdmissionRecord; +use crate::{control_plane::NodeId, security::CertificatePrincipal}; const ADMISSION_PROOF_DOMAIN: &str = "eastguard-node-admission-v1"; -const ADMISSION_CHALLENGE_BYTES: usize = 32; const ED25519_SIGNATURE_BYTES: usize = 64; -/// Fresh value supplied by the accepting broker for one connection. -/// -/// A proof from an earlier connection cannot be replayed because it was signed -/// for a different challenge. -#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] -pub(crate) struct AdmissionChallenge([u8; ADMISSION_CHALLENGE_BYTES]); - -impl AdmissionChallenge { - pub(crate) fn generate() -> Result { - let mut bytes = [0; ADMISSION_CHALLENGE_BYTES]; - SystemRandom::new() - .fill(&mut bytes) - .map_err(|_| anyhow::anyhow!("failed to generate admission challenge"))?; - Ok(Self(bytes)) - } -} - /// Private signing key generated for one broker process. /// /// Only its public key is committed in the admission record. The private key -/// remains in this process and proves that a connection belongs to the exact -/// process approved for the current admission epoch. +/// remains in this process and proves that a connection belongs to the +/// currently admitted process. pub(crate) struct ProcessSigningKey(Ed25519KeyPair); impl ProcessSigningKey { @@ -45,19 +30,22 @@ impl ProcessSigningKey { Ok(Self(key)) } - pub(crate) fn public_key(&self) -> Box<[u8]> { + #[cfg(test)] + fn public_key(&self) -> Box<[u8]> { self.0.public_key().as_ref().into() } pub(crate) fn sign( &self, - node_certificate_principal: &str, + node_certificate_principal: &CertificatePrincipal, node_id: &NodeId, - epoch: u64, - challenge: &AdmissionChallenge, + tls_session_binding: &[u8], ) -> Result { - let message = - admission_proof_message(node_certificate_principal, node_id, epoch, challenge)?; + let message = AdmissionProof::signing_message( + node_certificate_principal, + node_id, + tls_session_binding, + )?; let signature = self.0.sign(&message); let signature = signature .as_ref() @@ -65,21 +53,18 @@ impl ProcessSigningKey { .expect("Ed25519 signatures are always 64 bytes"); Ok(AdmissionProof { node_id: node_id.clone(), - epoch, signature, }) } } -/// Claim sent by a broker process after it receives a fresh challenge. +/// Proof that this TLS connection belongs to an admitted broker process. /// -/// The receiver still checks `node_id`, `epoch`, and the public key against the -/// current admission record. The signature makes that comparison meaningful: -/// a process holding only the reusable node certificate cannot forge the proof. +/// The receiver checks the node ID and signature against the current admission +/// record. Binding the signature to this TLS session prevents replay. #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub(crate) struct AdmissionProof { pub(crate) node_id: NodeId, - pub(crate) epoch: u64, signature: [u8; ED25519_SIGNATURE_BYTES], } @@ -87,14 +72,13 @@ impl AdmissionProof { pub(crate) fn verify( &self, process_public_key: &[u8], - node_certificate_principal: &str, - challenge: &AdmissionChallenge, + node_certificate_principal: &CertificatePrincipal, + tls_session_binding: &[u8], ) -> bool { - let Ok(message) = admission_proof_message( + let Ok(message) = Self::signing_message( node_certificate_principal, &self.node_id, - self.epoch, - challenge, + tls_session_binding, ) else { return false; }; @@ -102,22 +86,39 @@ impl AdmissionProof { .verify(&message, &self.signature) .is_ok() } -} -fn admission_proof_message( - node_certificate_principal: &str, - node_id: &NodeId, - epoch: u64, - challenge: &AdmissionChallenge, -) -> Result> { - borsh::to_vec(&( - ADMISSION_PROOF_DOMAIN, - node_certificate_principal, - node_id, - epoch, - challenge, - )) - .context("failed to encode admission proof") + pub(crate) fn verify_admission( + &self, + admission: &AdmissionRecord, + node_certificate_principal: &CertificatePrincipal, + tls_session_binding: &[u8], + ) -> Result { + anyhow::ensure!( + admission.node_certificate_principal == *node_certificate_principal + && admission.node_id == self.node_id + && self.verify( + &admission.process_public_key, + node_certificate_principal, + tls_session_binding, + ), + "node admission proof does not match the current record" + ); + Ok(self.node_id.clone()) + } + + fn signing_message( + node_certificate_principal: &CertificatePrincipal, + node_id: &NodeId, + tls_session_binding: &[u8], + ) -> Result> { + borsh::to_vec(&( + ADMISSION_PROOF_DOMAIN, + node_certificate_principal.as_ref(), + node_id, + tls_session_binding, + )) + .context("failed to encode admission proof") + } } #[cfg(test)] @@ -128,22 +129,25 @@ mod tests { fn current_process_proves_its_admitted_identity() { let key = ProcessSigningKey::generate().unwrap(); let node_id = NodeId::new("broker-a::process-2"); - let challenge = AdmissionChallenge::generate().unwrap(); - let proof = key.sign("broker-a", &node_id, 8, &challenge).unwrap(); + let binding = [7; 32]; + let principal = CertificatePrincipal::new("broker-a"); + let proof = key.sign(&principal, &node_id, &binding).unwrap(); - assert!(proof.verify(&key.public_key(), "broker-a", &challenge)); + assert!(proof.verify(&key.public_key(), &principal, &binding)); } #[test] - fn proof_is_bound_to_the_certificate_principal_and_challenge() { + fn proof_is_bound_to_the_certificate_principal_and_tls_session() { let key = ProcessSigningKey::generate().unwrap(); let node_id = NodeId::new("broker-a::process-2"); - let challenge = AdmissionChallenge::generate().unwrap(); - let other_challenge = AdmissionChallenge::generate().unwrap(); - let proof = key.sign("broker-a", &node_id, 8, &challenge).unwrap(); - - assert!(!proof.verify(&key.public_key(), "broker-b", &challenge)); - assert!(!proof.verify(&key.public_key(), "broker-a", &other_challenge)); + let binding = [7; 32]; + let other_binding = [8; 32]; + let broker_a = CertificatePrincipal::new("broker-a"); + let broker_b = CertificatePrincipal::new("broker-b"); + let proof = key.sign(&broker_a, &node_id, &binding).unwrap(); + + assert!(!proof.verify(&key.public_key(), &broker_b, &binding)); + assert!(!proof.verify(&key.public_key(), &broker_a, &other_binding)); } #[test] @@ -151,24 +155,22 @@ mod tests { let current_key = ProcessSigningKey::generate().unwrap(); let old_key = ProcessSigningKey::generate().unwrap(); let node_id = NodeId::new("broker-a::process-2"); - let challenge = AdmissionChallenge::generate().unwrap(); - let proof = old_key.sign("broker-a", &node_id, 8, &challenge).unwrap(); + let binding = [7; 32]; + let principal = CertificatePrincipal::new("broker-a"); + let proof = old_key.sign(&principal, &node_id, &binding).unwrap(); - assert!(!proof.verify(¤t_key.public_key(), "broker-a", &challenge)); + assert!(!proof.verify(¤t_key.public_key(), &principal, &binding)); } #[test] fn changing_the_claimed_identity_invalidates_the_signature() { let key = ProcessSigningKey::generate().unwrap(); let node_id = NodeId::new("broker-a::process-2"); - let challenge = AdmissionChallenge::generate().unwrap(); - let mut proof = key.sign("broker-a", &node_id, 8, &challenge).unwrap(); + let binding = [7; 32]; + let principal = CertificatePrincipal::new("broker-a"); + let mut proof = key.sign(&principal, &node_id, &binding).unwrap(); proof.node_id = NodeId::new("broker-a::process-3"); - assert!(!proof.verify(&key.public_key(), "broker-a", &challenge)); - - proof.node_id = node_id; - proof.epoch = 9; - assert!(!proof.verify(&key.public_key(), "broker-a", &challenge)); + assert!(!proof.verify(&key.public_key(), &principal, &binding)); } } From 1309866c2b22befadb93e29794c9308312c7188e Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sat, 1 Aug 2026 22:11:23 +0400 Subject: [PATCH 50/56] pass client certificate principal read --- src/lib.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2271b70d..599aa6e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,9 @@ use crate::impls::metadata_storage::MetadataStorage; use crate::net::{TcpListener, TransportTcpStream, UdpSocket}; use crate::schedulers::actor::spawn_scheduling_actor; use crate::schedulers::ticker::{PROBE_INTERVAL_TICKS, TICK_PERIOD_100_MS}; -use crate::security::{NodeTransportSecurity, SecurityActor, SecurityHandle}; +use crate::security::{ + NodeTransportSecurity, SecurityActor, SecurityHandle, client_certificate_principal, +}; use crate::{ config::ENV, control_plane::membership::{actor::SwimActor, transport::SwimTransportActor}, @@ -204,13 +206,16 @@ impl StartUp { ); while let Ok((stream, _)) = listener.accept().await { - let stream = match TransportTcpStream::accept_client(stream, &security).await { - Ok(stream) => stream, - Err(error) => { - tracing::debug!("client authentication failed: {error}"); - continue; - } - }; + let stream = + match TransportTcpStream::accept(stream, &security, client_certificate_principal) + .await + { + Ok(stream) => stream, + Err(error) => { + tracing::debug!("client authentication failed: {error}"); + continue; + } + }; let node_id = node_id.clone(); let swim_tx = swim_sender.clone(); let raft = raft_tx.clone(); From eb21fd559d051824663a03963258419a05849eac Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sat, 1 Aug 2026 22:33:14 +0400 Subject: [PATCH 51/56] transport protocol done --- .claude/rules/code-convention.md | 2 +- .claude/rules/raft-transport.md | 6 +- .../consensus/transport/protocol.rs | 67 ++++++++++--------- src/security/mod.rs | 2 +- 4 files changed, 41 insertions(+), 36 deletions(-) diff --git a/.claude/rules/code-convention.md b/.claude/rules/code-convention.md index f272c3db..6399db5b 100644 --- a/.claude/rules/code-convention.md +++ b/.claude/rules/code-convention.md @@ -6,7 +6,7 @@ Functions taking `&self`, `&mut self`, or a reference to a struct as their first ## Enum struct pattern -Enum variants that carry data must use the tuple-variant + named-struct pattern. Never use inline fields on enum variants. See `/.claude/skills/enum-struct-pattern/SKILL.md` for the full pattern including `impl_from_variant!` usage. +Enum variants that carry data must use the tuple-variant + named-struct pattern. Prefer not using inline fields on enum variants. See `/.claude/skills/enum-struct-pattern/SKILL.md` for the full pattern including `impl_from_variant!` usage. ## Result over Option for observability diff --git a/.claude/rules/raft-transport.md b/.claude/rules/raft-transport.md index 2b4e642f..d17ab2f1 100644 --- a/.claude/rules/raft-transport.md +++ b/.claude/rules/raft-transport.md @@ -27,8 +27,8 @@ cluster listener (TCP) Length-prefixed Borsh frames: 1. **Secure initial message:** either `AdmissionLookup(AdmissionRecordKey)` or - `ProcessAdmissionRequest`, which contains the dialer's process proof plus a - Raft message or ACL snapshot request. + `ProcessAdmission(AdmissionRequest)`. The admission request contains the + dialer's process proof and one `ClusterRequest`: Raft or ACL snapshot. 2. **Mutual admission:** the acceptor verifies the dialer, then replies with its own `AdmissionProof`. Both proofs sign the same TLS exporter value and are checked against the peer's current admission record. The exporter lets both @@ -40,7 +40,7 @@ Length-prefixed Borsh frames: - An ACL snapshot request carries its requesting node, shard, and resource. Its response is one `AclSnapshotResponse`, then the connection closes. 4. **Trusted-development initial message:** no cryptographic admission exchange; - the first frame remains either a Raft message or ACL snapshot request. + `Request(ClusterRequest)` directly carries the Raft or ACL snapshot request. ## Invariants diff --git a/src/control_plane/consensus/transport/protocol.rs b/src/control_plane/consensus/transport/protocol.rs index b4acaa82..0046af96 100644 --- a/src/control_plane/consensus/transport/protocol.rs +++ b/src/control_plane/consensus/transport/protocol.rs @@ -6,60 +6,65 @@ use crate::control_plane::consensus::messages::WireRaftMessage; use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::AclResource; -use crate::impl_from_variant; +use crate::security::{AdmissionProof, CertificatePrincipal}; /// The first frame on a cluster TCP connection. /// -/// A Raft connection begins with its first Raft message, which already names -/// its sender. ACL and admission lookup connections contain one read request. -/// Later Raft frames are raw; lookup connections return one response and close. +/// Secure Raft and ACL requests carry a process proof bound to their TLS +/// session. The limited admission lookup is the only secure request allowed +/// without that proof. Trusted-development connections carry a direct request. #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] -pub(super) enum InitialClusterMessage { +pub(crate) enum InitialClusterMessage { + AdmissionLookup(AdmissionRecordKey), + Request(ClusterRequest), + ProcessAdmission(AdmissionRequest), +} + +/// Requests process admission and carries the first protected cluster request. +#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] +pub(crate) struct AdmissionRequest { + pub(crate) proof: AdmissionProof, + pub(crate) request: ClusterRequest, +} + +/// Cluster requests that require current process admission in secure mode. +#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] +pub(crate) enum ClusterRequest { Raft(WireRaftMessage), AclSnapshot(AclSnapshotRequest), - AdmissionLookup(AdmissionLookupRequest), } /// One read of a committed ACL record from a shard host. #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] -pub(super) struct AclSnapshotRequest { - pub(super) requester_node_id: NodeId, - pub(super) shard_group_id: ShardGroupId, - pub(super) resource: AclResource, +pub(crate) struct AclSnapshotRequest { + pub(crate) requester_node_id: NodeId, + pub(crate) shard_group_id: ShardGroupId, + pub(crate) resource: AclResource, } /// The response to one ACL snapshot request on its dedicated connection. #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] -pub(super) struct AclSnapshotResponse { - pub(super) snapshot: Option, +pub(crate) struct AclSnapshotResponse { + pub(crate) snapshot: Option, } -/// Limited bootstrap read of one admission record from its metadata shard. +/// Identifies one admission record in its metadata shard. /// -/// In secure mode TLS authenticates the caller's node certificate, but this -/// request intentionally does not require process admission: admission is the -/// record the caller is trying to resolve. It cannot carry Raft, ACL, or client -/// data. -#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] -pub(super) struct AdmissionLookupRequest { - pub(super) shard_group_id: ShardGroupId, - pub(super) node_certificate_principal: Box, +/// The admission lookup actor uses the same value for routing, cache +/// coalescing, and the limited wire request. +#[derive(Debug, Clone, PartialEq, Eq, Hash, BorshSerialize, BorshDeserialize)] +pub(crate) struct AdmissionRecordKey { + pub(crate) shard_group_id: ShardGroupId, + pub(crate) node_certificate_principal: CertificatePrincipal, } /// Response to one limited admission lookup, after which the connection closes. #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] -pub(super) struct AdmissionLookupResponse { - pub(super) admission: Option, +pub(crate) struct AdmissionLookupResponse { + pub(crate) admission: Option, } -impl_from_variant!( - InitialClusterMessage, - Raft(WireRaftMessage), - AclSnapshot(AclSnapshotRequest), - AdmissionLookup(AdmissionLookupRequest), -); - -pub(super) fn encode_frame(value: &impl BorshSerialize) -> Result> { +pub(crate) fn encode_frame(value: &impl BorshSerialize) -> Result> { let bytes = borsh::to_vec(value)?; let len = u32::try_from(bytes.len())?; let mut frame = Vec::with_capacity(std::mem::size_of::() + bytes.len()); diff --git a/src/security/mod.rs b/src/security/mod.rs index 53588124..1c84ec37 100644 --- a/src/security/mod.rs +++ b/src/security/mod.rs @@ -8,4 +8,4 @@ mod transport; pub(crate) use actor::{SecurityActor, SecurityHandle}; pub(crate) use admission_proof::AdmissionProof; pub(crate) use certificates::{client_certificate_principal, node_certificate_principal}; -pub(crate) use transport::{CertificatePrincipal, NodeCredentials, NodeTransportSecurity}; +pub(crate) use transport::{CertificatePrincipal, NodeTransportSecurity}; From 4659d03960fa03b5ebc011d9505510affcbfa57c Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sat, 1 Aug 2026 22:52:58 +0400 Subject: [PATCH 52/56] simplify dispatcher accept, rename dial to connection --- .claude/rules/raft-transport.md | 7 +- .../consensus/transport/outbound.rs | 331 +++++++++--------- 2 files changed, 164 insertions(+), 174 deletions(-) diff --git a/.claude/rules/raft-transport.md b/.claude/rules/raft-transport.md index d17ab2f1..2735a70b 100644 --- a/.claude/rules/raft-transport.md +++ b/.claude/rules/raft-transport.md @@ -28,9 +28,10 @@ Length-prefixed Borsh frames: 1. **Secure initial message:** either `AdmissionLookup(AdmissionRecordKey)` or `ProcessAdmission(AdmissionRequest)`. The admission request contains the - dialer's process proof and one `ClusterRequest`: Raft or ACL snapshot. -2. **Mutual admission:** the acceptor verifies the dialer, then replies with its - own `AdmissionProof`. Both proofs sign the same TLS exporter value and are + connecting node's process proof and one `ClusterRequest`: Raft or ACL + snapshot. +2. **Mutual admission:** the acceptor verifies the connecting node, then replies + with its own `AdmissionProof`. Both proofs sign the same TLS exporter value and are checked against the peer's current admission record. The exporter lets both ends derive identical connection-specific bytes without sending those bytes; another TLS connection derives a different value. diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index 5b3717b4..8e346ec8 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -11,20 +11,19 @@ use crate::control_plane::consensus::messages::{ }; use crate::control_plane::NodeId; -use crate::control_plane::consensus::transport::ClusterMessageReader; +use crate::control_plane::consensus::transport::{AcceptedRaftConnection, ClusterMessageReader}; use crate::control_plane::membership::actor::SwimSender; use crate::net::{TransportTcpStream, TransportWriteHalf}; -use crate::security::{NodeTransportSecurity, TransportIdentity}; +use crate::security::{AdmissionProof, CertificatePrincipal}; -use super::protocol::{ - AclSnapshotRequest, AclSnapshotResponse, AdmissionLookupRequest, AdmissionLookupResponse, - InitialClusterMessage, encode_frame, -}; +use super::protocol::{AdmissionRequest, ClusterRequest, InitialClusterMessage, encode_frame}; +use crate::security::SecurityHandle; const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); -/// Upper bound on messages buffered per peer while its dial is in flight; +const ADMISSION_HANDSHAKE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(8); +/// Upper bound on messages buffered per peer while its connection is in flight; /// overflow is dropped (raft retries by timer). -const PENDING_DIAL_BUFFER_CAP: usize = 256; +const CONNECTING_PEER_BUFFER_CAP: usize = 256; /// Manages peer connections, address resolution, and dead-peer tracking. /// @@ -46,84 +45,59 @@ pub(super) struct RaftRpcDispatcher { /// the transport's select loop and stall flush_events in MultiRaftActor. connect_backoffs: HashMap, /// Messages buffered for peers whose connection is being established on a - /// background task; flushed (or dropped on failure) in `on_dial_result`. - pending_dials: HashMap>, - dial_tx: mpsc::Sender, - security: NodeTransportSecurity, + /// background task; flushed (or dropped on failure) in `on_connection_result`. + connecting_peers: HashMap>, + connection_result_tx: mpsc::Sender, + security: SecurityHandle, } -/// Result of a background dial attempt, delivered back to the transport loop. -pub(super) struct DialOutcome { +/// Result of a background connection attempt, delivered to the transport loop. +pub(super) struct ConnectionAttemptResult { target: NodeId, - outcome: anyhow::Result<(ClusterMessageReader, TransportWriteHalf)>, + result: anyhow::Result<(ClusterMessageReader, TransportWriteHalf)>, } impl RaftRpcDispatcher { pub(super) fn new( node_id: NodeId, - dial_tx: mpsc::Sender, - security: NodeTransportSecurity, + connection_result_tx: mpsc::Sender, + security: SecurityHandle, ) -> Self { Self { node_id, writers: HashMap::new(), dead_peers: HashSet::new(), connect_backoffs: HashMap::new(), - pending_dials: HashMap::new(), - dial_tx, + connecting_peers: HashMap::new(), + connection_result_tx, security, } } - pub(super) async fn accept(&mut self, stream: TransportTcpStream, raft_tx: &MutlRaftSender) { - let transport_identity = stream.peer_identity(); - let (read_half, write_half) = stream.into_split(); - let mut reader = ClusterMessageReader::new(read_half, transport_identity.clone()); + pub(super) fn accept(&mut self, connection: AcceptedRaftConnection, raft_tx: &MutlRaftSender) { + let AcceptedRaftConnection { + peer_id: connection_peer_id, + initial_message, + reader, + writer, + } = connection; - let Ok(initial_message) = reader.read_initial_message().await else { - tracing::debug!("cluster connection closed before its initial message"); - return; + let initial_rpc = InboundRaftRpc { + shard_group_id: initial_message.shard_group_id, + peer_id: connection_peer_id, + rpc: initial_message.rpc, }; - - match initial_message { - InitialClusterMessage::AdmissionLookup(request) => { - tokio::spawn(serve_admission_lookup_request( - transport_identity, - request, - raft_tx.clone(), - write_half, - )); - } - InitialClusterMessage::AclSnapshot(request) => { - let peer_id = request.requester_node_id.clone(); - tokio::spawn(serve_acl_snapshot_request( - peer_id, - request, - raft_tx.clone(), - write_half, - )); - } - InitialClusterMessage::Raft(initial_raft_message) => { - let initial_rpc = InboundRaftRpc { - shard_group_id: initial_raft_message.shard_group_id, - peer_id: initial_raft_message.sender, - rpc: initial_raft_message.rpc, - }; - if self.writers.contains_key(&initial_rpc.peer_id) - && initial_rpc.peer_id > self.node_id - { - // simultaneous connect: dropping accepted connection - return; - } - self.writers.insert(initial_rpc.peer_id.clone(), write_half); - let raft_tx = raft_tx.clone(); - tokio::spawn(async move { - let peer_id = initial_rpc.peer_id.clone(); - let _ = raft_tx.send(initial_rpc).await; - reader.run(raft_tx, peer_id).await; - }); - } + if self.writers.contains_key(&initial_rpc.peer_id) && initial_rpc.peer_id > self.node_id { + // simultaneous connect: dropping accepted connection + return; } + self.writers.insert(initial_rpc.peer_id.clone(), writer); + let raft_tx = raft_tx.clone(); + tokio::spawn(async move { + let reader_peer_id = initial_rpc.peer_id.clone(); + let _ = raft_tx.send(initial_rpc).await; + reader.run(raft_tx, reader_peer_id).await; + }); } pub(super) async fn send(&mut self, packets: Vec, swim_tx: &SwimSender) { @@ -170,46 +144,40 @@ impl RaftRpcDispatcher { { return; } - // No usable writer: hand the messages to the in-flight dial (if any) - // or start one on a background task. Dials must never run inline — a - // hung connect (crashed peer; acceptor starved because *its* loop is - // mid-dial) blocks this select loop for the full connect timeout, - // stalling every queued batch and the accept arm with it (#133). - if let Some(buffered) = self.pending_dials.get_mut(&target_id) { - if buffered.len() + msgs.len() <= PENDING_DIAL_BUFFER_CAP { + // No usable writer: hand the messages to the in-flight connection (if + // any) or start one on a background task. Connection attempts must + // never run inline: a hung connect blocks this select loop for the full + // timeout, stalling every queued batch and the accept arm with it (#133). + if let Some(buffered) = self.connecting_peers.get_mut(&target_id) { + if buffered.len() + msgs.len() <= CONNECTING_PEER_BUFFER_CAP { buffered.extend(msgs); } return; } let initial_raft_message = msgs.remove(0); - self.pending_dials.insert(target_id.clone(), msgs); - let dial_task = dial( - target_id.clone(), + self.connecting_peers.insert(target_id.clone(), msgs); + + tokio::spawn(connect_peer( + target_id, swim_tx.clone(), self.security.clone(), initial_raft_message, - ); - - let dial_tx = self.dial_tx.clone(); - tokio::spawn(async move { - let outcome = dial_task.await; - let _ = dial_tx - .send(DialOutcome { - target: target_id, - outcome, - }) - .await; - }); + self.connection_result_tx.clone(), + )); } - /// Installs (or discards, per the NodeId tie-break) a completed dial and - /// flushes any messages buffered while it was in flight. - pub(super) async fn on_dial_result(&mut self, result: DialOutcome, raft_tx: &MutlRaftSender) { - let DialOutcome { target, outcome } = result; - let buffered = self.pending_dials.remove(&target).unwrap_or_default(); + /// Installs (or discards, per the NodeId tie-break) a completed connection + /// and flushes any messages buffered while it was in flight. + pub(super) async fn on_connection_result( + &mut self, + attempt: ConnectionAttemptResult, + raft_tx: &MutlRaftSender, + ) { + let ConnectionAttemptResult { target, result } = attempt; + let buffered = self.connecting_peers.remove(&target).unwrap_or_default(); - let Ok((reader, write_half)) = outcome.inspect_err(|err| { - tracing::warn!(peer = %target, "dial failed: {err}"); + let Ok((reader, write_half)) = result.inspect_err(|err| { + tracing::warn!(peer = %target, "connection attempt failed: {err}"); }) else { self.connect_backoffs.insert(target, Instant::now()); return; @@ -226,7 +194,7 @@ impl RaftRpcDispatcher { tracing::debug!( peer = %target, buffered = buffered.len(), - "simultaneous connect: discarding our dial, peer's connection \ + "simultaneous connect: discarding our connection, peer's connection \ wins the tie-break (lower NodeId); rerouting buffered messages", ); if !buffered.is_empty() { @@ -244,7 +212,7 @@ impl RaftRpcDispatcher { pub(super) fn disconnect(&mut self, peer_id: NodeId) { self.writers.remove(&peer_id); - self.pending_dials.remove(&peer_id); + self.connecting_peers.remove(&peer_id); tracing::info!("[{}] Disconnected dead peer {:?}", self.node_id, peer_id); self.dead_peers.insert(peer_id); } @@ -283,88 +251,109 @@ impl RaftRpcDispatcher { } } -/// Resolve, connect (3secs cap), and send the opening Raft message on a spawned -/// task, so a hung connect can never block the transport select loop. The loop -/// installs the writer and flushes buffered messages in `on_dial_result`. -// ! never inline this. Actor Model should only do work whose duration it controls. -// ! Anything whose latency the outside actor controls must not be awaited in the handler. -async fn dial( +/// Connects, performs the opening Raft exchange, and reports the result. +async fn connect_peer( target_id: NodeId, swim_tx: SwimSender, - security: NodeTransportSecurity, + security: SecurityHandle, initial_raft_message: WireRaftMessage, -) -> anyhow::Result<(ClusterMessageReader, TransportWriteHalf)> { - let Some(addr) = swim_tx.resolve_address(target_id.clone()).await? else { - anyhow::bail!("cannot resolve address for {target_id}"); - }; - - let stream = tokio::time::timeout( - std::time::Duration::from_secs(3), - TransportTcpStream::connect_node(addr.cluster_addr(), &security), - ) - .await??; - - let transport_identity = stream.peer_identity(); - let (read_half, mut write_half) = stream.into_split(); - write_half - .write_all(&encode_frame(&InitialClusterMessage::Raft( - initial_raft_message, - ))?) - .await?; - Ok(( - ClusterMessageReader::new(read_half, transport_identity), - write_half, - )) + connection_result_tx: mpsc::Sender, +) { + let result = async { + let Some(addr) = swim_tx.resolve_address(target_id.clone()).await? else { + anyhow::bail!("cannot resolve address for {target_id}"); + }; + + let stream = tokio::time::timeout( + std::time::Duration::from_secs(3), + security.connect_cluster(addr.cluster_addr()), + ) + .await??; + + let mut connection = OutboundClusterConnection::new(stream, target_id.clone())?; + tokio::time::timeout( + ADMISSION_HANDSHAKE_TIMEOUT, + connection.send_initial_request(&security, ClusterRequest::Raft(initial_raft_message)), + ) + .await??; + Ok(connection.into_parts()) + } + .await; + + let _ = connection_result_tx + .send(ConnectionAttemptResult { + target: target_id, + result, + }) + .await; } -async fn serve_admission_lookup_request( - transport_identity: TransportIdentity, - request: AdmissionLookupRequest, - raft_tx: MutlRaftSender, - mut writer: TransportWriteHalf, -) { - let admission = match raft_tx - .get_admission(request.shard_group_id, request.node_certificate_principal) - .await - { - Ok(admission) => admission, - Err(error) => { - tracing::debug!( - ?transport_identity, - "admission lookup failed before response: {error}" - ); - return; - } - }; - let Ok(frame) = encode_frame(&AdmissionLookupResponse { admission }) else { - tracing::debug!( - ?transport_identity, - "failed to encode admission lookup response" - ); - return; - }; - if let Err(error) = writer.write_all(&frame).await { - tracing::debug!( - ?transport_identity, - "failed to send admission lookup response: {error}" +/// One outbound cluster stream before its first Raft or ACL message. +/// +/// It owns the expected peer identity and both stream halves so callers cannot +/// accidentally perform only one side of the mutual admission exchange. +pub(crate) struct OutboundClusterConnection { + pub(crate) reader: ClusterMessageReader, + writer: TransportWriteHalf, + expected_peer_id: NodeId, + /// Present only for a TLS stream; trusted-development streams skip + /// admission entirely. + tls_peer: Option<(CertificatePrincipal, [u8; 32])>, +} + +impl OutboundClusterConnection { + pub(crate) fn new( + stream: TransportTcpStream, + expected_peer_id: NodeId, + ) -> anyhow::Result { + let certificate_principal = stream.peer_principal(); + let tls_peer = match &certificate_principal { + Some(principal) => Some((principal.clone(), stream.admission_binding()?)), + None => None, + }; + + let (read_half, writer) = stream.into_split(); + Ok(Self { + reader: ClusterMessageReader::new(read_half, certificate_principal), + writer, + expected_peer_id, + tls_peer, + }) + } + + pub(crate) async fn send_initial_request( + &mut self, + security: &SecurityHandle, + request: ClusterRequest, + ) -> anyhow::Result<()> { + let Some((peer_principal, tls_session_binding)) = self.tls_peer.as_ref() else { + let initial = InitialClusterMessage::Request(request); + self.writer.write_all(&encode_frame(&initial)?).await?; + return Ok(()); + }; + let local_proof = security.create_admission_proof(tls_session_binding)?; + let process_admission = InitialClusterMessage::ProcessAdmission(AdmissionRequest { + proof: local_proof, + request, + }); + self.writer + .write_all(&encode_frame(&process_admission)?) + .await?; + let peer_admission = security.lookup_admission(peer_principal).await?; + let peer_proof = self + .reader + .read_frame::(4 * 1024, "admission proof") + .await?; + let admitted_peer = + peer_proof.verify_admission(&peer_admission, peer_principal, tls_session_binding)?; + anyhow::ensure!( + admitted_peer == self.expected_peer_id, + "connected broker differs from the expected admitted process" ); + Ok(()) } -} -async fn serve_acl_snapshot_request( - peer_id: NodeId, - request: AclSnapshotRequest, - raft_tx: MutlRaftSender, - mut writer: TransportWriteHalf, -) { - let snapshot = raft_tx - .get_acl_snapshot(request.shard_group_id, request.resource) - .await; - let Ok(frame) = encode_frame(&AclSnapshotResponse { snapshot }) else { - tracing::debug!(peer = %peer_id, "failed to encode ACL snapshot response"); - return; - }; - if let Err(error) = writer.write_all(&frame).await { - tracing::debug!(peer = %peer_id, "failed to send ACL snapshot response: {error}"); + fn into_parts(self) -> (ClusterMessageReader, TransportWriteHalf) { + (self.reader, self.writer) } } From 9ae9306d19adbd18987951ec676a6f31d906c97c Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sat, 1 Aug 2026 23:10:20 +0400 Subject: [PATCH 53/56] in --- src/control_plane/consensus/transport/outbound.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index 8e346ec8..8e392ba6 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -266,7 +266,7 @@ async fn connect_peer( let stream = tokio::time::timeout( std::time::Duration::from_secs(3), - security.connect_cluster(addr.cluster_addr()), + TransportTcpStream::connect_node(addr.cluster_addr(), security.node_transport()), ) .await??; From e641cb8bf0abe2e1a5a869454898c285bbe6c2c9 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sun, 2 Aug 2026 13:03:09 +0400 Subject: [PATCH 54/56] sender -> peer id --- src/control_plane/consensus/messages/rpc.rs | 2 +- src/control_plane/consensus/transport/outbound.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control_plane/consensus/messages/rpc.rs b/src/control_plane/consensus/messages/rpc.rs index eff5194b..0eb7a93f 100644 --- a/src/control_plane/consensus/messages/rpc.rs +++ b/src/control_plane/consensus/messages/rpc.rs @@ -102,6 +102,6 @@ impl OutboundRaftPacket { #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] pub struct WireRaftMessage { pub shard_group_id: ShardGroupId, - pub sender: NodeId, + pub peer_id: NodeId, pub rpc: RaftRpc, } diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index 8e392ba6..b54e5e4b 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -120,7 +120,7 @@ impl RaftRpcDispatcher { .or_default() .push(WireRaftMessage { shard_group_id: pkt.shard_group_id, - sender: self.node_id.clone(), + peer_id: self.node_id.clone(), rpc: pkt.rpc, }); } From 2877097021278e0e9633893b12fdfefc039f2652 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sun, 2 Aug 2026 13:16:04 +0400 Subject: [PATCH 55/56] rm AdmissionRequest, ProcessAdmission --- .claude/rules/raft-transport.md | 9 +++---- .../consensus/transport/outbound.rs | 25 +++++++++++-------- .../consensus/transport/protocol.rs | 21 ++++++---------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/.claude/rules/raft-transport.md b/.claude/rules/raft-transport.md index 2735a70b..2de0e2cd 100644 --- a/.claude/rules/raft-transport.md +++ b/.claude/rules/raft-transport.md @@ -26,10 +26,9 @@ cluster listener (TCP) Length-prefixed Borsh frames: -1. **Secure initial message:** either `AdmissionLookup(AdmissionRecordKey)` or - `ProcessAdmission(AdmissionRequest)`. The admission request contains the - connecting node's process proof and one `ClusterRequest`: Raft or ACL - snapshot. +1. **Initial message:** `InitialClusterMessage` carries one `ClusterRequest` and + an optional process-admission proof. Secure Raft and ACL requests include the + proof; the limited admission lookup omits it. 2. **Mutual admission:** the acceptor verifies the connecting node, then replies with its own `AdmissionProof`. Both proofs sign the same TLS exporter value and are checked against the peer's current admission record. The exporter lets both @@ -41,7 +40,7 @@ Length-prefixed Borsh frames: - An ACL snapshot request carries its requesting node, shard, and resource. Its response is one `AclSnapshotResponse`, then the connection closes. 4. **Trusted-development initial message:** no cryptographic admission exchange; - `Request(ClusterRequest)` directly carries the Raft or ACL snapshot request. + the Raft or ACL request carries no admission proof. ## Invariants diff --git a/src/control_plane/consensus/transport/outbound.rs b/src/control_plane/consensus/transport/outbound.rs index b54e5e4b..d24a19e9 100644 --- a/src/control_plane/consensus/transport/outbound.rs +++ b/src/control_plane/consensus/transport/outbound.rs @@ -16,7 +16,7 @@ use crate::control_plane::membership::actor::SwimSender; use crate::net::{TransportTcpStream, TransportWriteHalf}; use crate::security::{AdmissionProof, CertificatePrincipal}; -use super::protocol::{AdmissionRequest, ClusterRequest, InitialClusterMessage, encode_frame}; +use super::protocol::{ClusterRequest, InitialClusterMessage, encode_frame}; use crate::security::SecurityHandle; const CONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); @@ -326,19 +326,22 @@ impl OutboundClusterConnection { security: &SecurityHandle, request: ClusterRequest, ) -> anyhow::Result<()> { - let Some((peer_principal, tls_session_binding)) = self.tls_peer.as_ref() else { - let initial = InitialClusterMessage::Request(request); - self.writer.write_all(&encode_frame(&initial)?).await?; - return Ok(()); + let admission_proof = match self.tls_peer.as_ref() { + Some((_, tls_session_binding)) => { + Some(security.create_admission_proof(tls_session_binding)?) + } + None => None, }; - let local_proof = security.create_admission_proof(tls_session_binding)?; - let process_admission = InitialClusterMessage::ProcessAdmission(AdmissionRequest { - proof: local_proof, - request, - }); self.writer - .write_all(&encode_frame(&process_admission)?) + .write_all(&encode_frame(&InitialClusterMessage { + admission_proof, + request, + })?) .await?; + + let Some((peer_principal, tls_session_binding)) = self.tls_peer.as_ref() else { + return Ok(()); + }; let peer_admission = security.lookup_admission(peer_principal).await?; let peer_proof = self .reader diff --git a/src/control_plane/consensus/transport/protocol.rs b/src/control_plane/consensus/transport/protocol.rs index 0046af96..04f666c6 100644 --- a/src/control_plane/consensus/transport/protocol.rs +++ b/src/control_plane/consensus/transport/protocol.rs @@ -6,30 +6,23 @@ use crate::control_plane::consensus::messages::WireRaftMessage; use crate::control_plane::consensus::raft::states::security::{AclRecord, AdmissionRecord}; use crate::control_plane::membership::ShardGroupId; use crate::control_plane::metadata::AclResource; + use crate::security::{AdmissionProof, CertificatePrincipal}; /// The first frame on a cluster TCP connection. /// -/// Secure Raft and ACL requests carry a process proof bound to their TLS -/// session. The limited admission lookup is the only secure request allowed -/// without that proof. Trusted-development connections carry a direct request. -#[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] -pub(crate) enum InitialClusterMessage { - AdmissionLookup(AdmissionRecordKey), - Request(ClusterRequest), - ProcessAdmission(AdmissionRequest), -} - -/// Requests process admission and carries the first protected cluster request. +/// Secure Raft and ACL requests include a process proof bound to their TLS +/// session. Admission lookups and trusted-development requests omit it. #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] -pub(crate) struct AdmissionRequest { - pub(crate) proof: AdmissionProof, +pub(crate) struct InitialClusterMessage { + pub(crate) admission_proof: Option, pub(crate) request: ClusterRequest, } -/// Cluster requests that require current process admission in secure mode. +/// Requests accepted as the first frame on a cluster connection. #[derive(Debug, Clone, BorshSerialize, BorshDeserialize)] pub(crate) enum ClusterRequest { + AdmissionLookup(AdmissionRecordKey), Raft(WireRaftMessage), AclSnapshot(AclSnapshotRequest), } From 3f499aa9e6d682f7448babc5833e9000f6f4cf19 Mon Sep 17 00:00:00 2001 From: Migorithm Date: Sun, 2 Aug 2026 13:52:15 +0400 Subject: [PATCH 56/56] feat: accept_cluster_connection --- .../consensus/transport/inbound.rs | 196 ++++++++++++++---- 1 file changed, 159 insertions(+), 37 deletions(-) diff --git a/src/control_plane/consensus/transport/inbound.rs b/src/control_plane/consensus/transport/inbound.rs index 455b3b05..0f1bd3f9 100644 --- a/src/control_plane/consensus/transport/inbound.rs +++ b/src/control_plane/consensus/transport/inbound.rs @@ -2,53 +2,35 @@ use crate::control_plane::NodeId; use crate::control_plane::consensus::actor::MutlRaftSender; use crate::control_plane::consensus::messages::InboundRaftRpc; use crate::control_plane::consensus::messages::WireRaftMessage; -use crate::net::TransportReadHalf; -use crate::security::TransportIdentity; +use crate::net::{TcpStream, TransportReadHalf, TransportTcpStream, TransportWriteHalf}; +use crate::security::{ + AdmissionProof, CertificatePrincipal, SecurityHandle, node_certificate_principal, +}; use borsh::BorshDeserialize; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use super::protocol::{AclSnapshotResponse, AdmissionLookupResponse, InitialClusterMessage}; +use super::protocol::{ + AclSnapshotResponse, AdmissionLookupResponse, ClusterRequest, InitialClusterMessage, + encode_frame, +}; -pub(super) struct ClusterMessageReader { +pub(crate) struct ClusterMessageReader { read_half: TransportReadHalf, - transport_identity: TransportIdentity, + certificate_principal: Option, } impl ClusterMessageReader { - pub(super) fn new( + pub(crate) fn new( read_half: impl Into, - transport_identity: TransportIdentity, + certificate_principal: Option, ) -> Self { Self { read_half: read_half.into(), - transport_identity, + certificate_principal, } } - pub(super) async fn read_initial_message(&mut self) -> anyhow::Result { - self.read_frame(4 * 1024 * 1024, "initial cluster message") - .await - } - - pub(super) async fn read_raft_message(&mut self) -> anyhow::Result { - self.read_frame(4 * 1024 * 1024, "Raft message").await - } - - pub(super) async fn read_acl_snapshot_response( - &mut self, - ) -> anyhow::Result { - self.read_frame(4 * 1024 * 1024, "ACL snapshot response") - .await - } - - pub(super) async fn read_admission_lookup_response( - &mut self, - ) -> anyhow::Result { - self.read_frame(4 * 1024 * 1024, "admission lookup response") - .await - } - - async fn read_frame( + pub(crate) async fn read_frame( &mut self, maximum_size: usize, frame_name: &str, @@ -66,16 +48,19 @@ impl ClusterMessageReader { #[tracing::instrument( level = "trace", skip_all, - fields(peer = %peer, transport_identity = ?self.transport_identity) + fields(peer = %peer, certificate_principal = ?self.certificate_principal) )] pub(super) async fn run(mut self, tx: MutlRaftSender, peer: NodeId) { loop { - match self.read_raft_message().await { + match self + .read_frame::(4 * 1024 * 1024, "Raft message") + .await + { Ok(message) => { - if message.sender != peer { + if message.peer_id != peer { tracing::warn!( transport_peer = %peer, - claimed_sender = %message.sender, + claimed_sender = %message.peer_id, "rejected Raft message whose sender differs from the connection peer", ); break; @@ -96,3 +81,140 @@ impl ClusterMessageReader { } } } + +/// A Raft connection ready to enter the dispatcher writer map. +pub(super) struct AcceptedRaftConnection { + pub(super) peer_id: NodeId, + pub(super) initial_message: WireRaftMessage, + pub(super) reader: ClusterMessageReader, + pub(super) writer: TransportWriteHalf, +} + +impl AcceptedRaftConnection { + fn new( + initial_message: WireRaftMessage, + reader: ClusterMessageReader, + writer: TransportWriteHalf, + ) -> Self { + Self { + peer_id: initial_message.peer_id.clone(), + initial_message, + reader, + writer, + } + } +} + +/// Authenticates a cluster stream and handles its first request. +/// +/// Admission lookups and ACL reads finish here. Only a verified Raft stream is +/// returned to the persistent connection dispatcher. +pub(super) async fn accept_cluster_connection( + stream: TcpStream, + security: SecurityHandle, +) -> anyhow::Result> { + let stream = TransportTcpStream::accept( + stream, + security.node_transport(), + node_certificate_principal, + ) + .await?; + + let certificate_principal = stream.peer_principal(); + let tls_session_binding = if certificate_principal.is_some() { + Some(stream.admission_binding()?) + } else { + None + }; + let (read_half, mut write_half) = stream.into_split(); + let mut reader = ClusterMessageReader::new(read_half, certificate_principal.clone()); + let InitialClusterMessage { + admission_proof, + request, + } = reader + .read_frame::(4 * 1024 * 1024, "initial cluster message") + .await?; + + let tls_session_binding = tls_session_binding + .as_ref() + .map(|binding| binding.as_slice()); + + match request { + ClusterRequest::AdmissionLookup(request) => { + anyhow::ensure!( + admission_proof.is_none(), + "admission lookup carried a process proof" + ); + let admission = security.read_admission(request).await?; + write_half + .write_all(&encode_frame(&AdmissionLookupResponse { admission })?) + .await?; + Ok(None) + } + ClusterRequest::Raft(message) => { + verify_requester( + certificate_principal.as_ref(), + admission_proof.as_ref(), + &message.peer_id, + tls_session_binding, + &security, + &mut write_half, + ) + .await?; + Ok(Some(AcceptedRaftConnection::new( + message, reader, write_half, + ))) + } + ClusterRequest::AclSnapshot(snapshot_req) => { + verify_requester( + certificate_principal.as_ref(), + admission_proof.as_ref(), + &snapshot_req.requester_node_id, + tls_session_binding, + &security, + &mut write_half, + ) + .await?; + let snapshot = security.read_acl(snapshot_req).await?; + write_half + .write_all(&encode_frame(&AclSnapshotResponse { snapshot })?) + .await?; + Ok(None) + } + } +} + +async fn verify_requester( + certificate_principal: Option<&CertificatePrincipal>, + admission_proof: Option<&AdmissionProof>, + requester_node_id: &NodeId, + tls_session_binding: Option<&[u8]>, + security: &SecurityHandle, + write_half: &mut TransportWriteHalf, +) -> anyhow::Result<()> { + let Some(principal) = certificate_principal else { + anyhow::ensure!( + admission_proof.is_none(), + "trusted-development connection carried an admission proof" + ); + return Ok(()); + }; + let Some(proof) = admission_proof else { + anyhow::bail!("secure cluster connection omitted process admission"); + }; + let Some(tls_session_binding) = tls_session_binding else { + anyhow::bail!("secure connection has no TLS session binding"); + }; + let admission = security.lookup_admission(principal).await?; + let peer_id = proof.verify_admission(&admission, principal, tls_session_binding)?; + anyhow::ensure!( + requester_node_id == &peer_id, + "cluster requester differs from admitted process" + ); + write_half + .write_all(&encode_frame( + &security.create_admission_proof(tls_session_binding)?, + )?) + .await?; + Ok(()) +}