From c4391a3c9f560028e077d155c1083670a8c8a87a Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Sat, 18 Jul 2026 22:33:11 +0700 Subject: [PATCH 1/8] support DNS names in coins-chain peer addresses --- examples/coins/chain/src/testnet.rs | 81 +++- examples/coins/chain/src/tests/testnet.rs | 476 +++++++++++++++++++++- 2 files changed, 541 insertions(+), 16 deletions(-) diff --git a/examples/coins/chain/src/testnet.rs b/examples/coins/chain/src/testnet.rs index a1fc1ea..8fc8049 100644 --- a/examples/coins/chain/src/testnet.rs +++ b/examples/coins/chain/src/testnet.rs @@ -29,7 +29,7 @@ use commonware_p2p::{ use commonware_runtime::{ tokio, Clock as _, Handle, Runner as _, Spawner as _, Strategizer as _, Supervisor as _, }; -use commonware_utils::{ordered::Set, N3f1, NZUsize, NZU32}; +use commonware_utils::{ordered::Set, Hostname, N3f1, NZUsize, NZU32}; use governor::Quota; use nunchi_dkg::{ ContinueOnUpdate, PeerConfig, Storage as DkgStorage, StorageKey, StorageProtector, @@ -124,7 +124,12 @@ pub struct NodeConfig { pub share: String, pub peer_config: PeerConfig, pub listen_address: SocketAddr, - pub dialable_address: SocketAddr, + /// Address advertised to peers. IP literals use socket syntax (with brackets around IPv6), + /// while DNS names use `hostname:port` without a URL scheme or path. DNS is resolved again + /// whenever a disconnected peer retries this node, so operators should use a stable hostname + /// and an appropriate TTL. + #[serde(with = "ingress_serde")] + pub dialable_address: Ingress, pub rpc_address: SocketAddr, pub metrics_address: SocketAddr, pub bootstrappers: Vec, @@ -155,7 +160,64 @@ impl NodeConfig { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct BootstrapperConfig { pub public_key: String, - pub address: SocketAddr, + /// Address used to dial the bootstrapper. IP literals use socket syntax (with brackets around + /// IPv6), while DNS names use `hostname:port` without a URL scheme or path. DNS is resolved on + /// each reconnect attempt rather than continuously while a connection is healthy. + #[serde(with = "ingress_serde")] + pub address: Ingress, +} + +mod ingress_serde { + use super::*; + use serde::{de::Error as _, Deserializer, Serializer}; + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + if let Ok(address) = SocketAddr::from_str(&value) { + return Ok(Ingress::Socket(address)); + } + if value.contains("://") { + return Err(D::Error::custom( + "peer address must not contain a URL scheme or path", + )); + } + + let (host, port) = value + .rsplit_once(':') + .ok_or_else(|| D::Error::custom("peer address is missing a port"))?; + if host.is_empty() { + return Err(D::Error::custom("peer address is missing a host")); + } + if port.is_empty() { + return Err(D::Error::custom("peer address is missing a port")); + } + if host.contains(':') { + return Err(D::Error::custom( + "IPv6 peer addresses must use bracketed socket syntax", + )); + } + let port = port + .parse::() + .map_err(|error| D::Error::custom(format!("invalid peer address port: {error}")))?; + let host = Hostname::new(host) + .map_err(|error| D::Error::custom(format!("invalid peer address hostname: {error}")))?; + Ok(Ingress::Dns { host, port }) + } + + pub fn serialize(ingress: &Ingress, serializer: S) -> Result + where + S: Serializer, + { + match ingress { + Ingress::Socket(address) => serializer.serialize_str(&address.to_string()), + Ingress::Dns { host, port } => { + serializer.serialize_str(&format!("{host}:{port}")) + } + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -284,14 +346,14 @@ pub fn generate_local_testnet(config: LocalTestnetConfig) -> Result Result, Error>>()?; @@ -477,7 +539,7 @@ async fn start_node( .map(|bootstrapper| { Ok(( decode_unit::(&bootstrapper.public_key, "bootstrapper.public_key")?, - Ingress::from(bootstrapper.address), + bootstrapper.address.clone(), )) }) .collect::, Error>>()?; @@ -486,6 +548,7 @@ async fn start_node( node = %config.name, public_key = %public_key, listen = %config.listen_address, + dialable = ?config.dialable_address, rpc = %config.rpc_address, metrics = %config.metrics_address, "starting coins-chain validator" @@ -495,7 +558,7 @@ async fn start_node( private_key.clone(), NAMESPACE, config.listen_address, - config.dialable_address, + config.dialable_address.clone(), bootstrappers, config.networking.max_message_size, ); diff --git a/examples/coins/chain/src/tests/testnet.rs b/examples/coins/chain/src/tests/testnet.rs index ac240e4..cd701fb 100644 --- a/examples/coins/chain/src/tests/testnet.rs +++ b/examples/coins/chain/src/tests/testnet.rs @@ -5,16 +5,24 @@ //! as `narae` consume. [`run_node`] boots a single validator from one of those configs on the //! tokio runtime with authenticated peer discovery, and serves the aggregated JSON-RPC module. -use commonware_cryptography::{bls12381::primitives::group, ed25519}; +use commonware_cryptography::{bls12381::primitives::group, ed25519, Signer as _}; +use commonware_macros::select; +use commonware_p2p::{ + authenticated::discovery::{self, Network}, + Ingress, Manager, Receiver as _, Recipients, Sender as _, +}; +use commonware_runtime::{deterministic, Clock as _, Quota, Runner as _, Supervisor as _}; +use commonware_utils::{ordered::Set, Hostname, NZU32}; use std::collections::HashSet; use std::{ fs, - net::{IpAddr, Ipv4Addr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, num::NonZeroU32, path::PathBuf, + time::Duration, }; -use crate::testnet::*; +use crate::{testnet::*, NAMESPACE}; #[test] fn generated_testnet_has_unique_ports_dirs_and_complete_peer_sets() { @@ -124,17 +132,20 @@ fn generated_testnet_can_advertise_remote_hosts() { let first = NodeConfig::read(&manifest.nodes[0].config_path).expect("read first config"); let second = NodeConfig::read(&manifest.nodes[1].config_path).expect("read second config"); + let first_raw = fs::read_to_string(&manifest.nodes[0].config_path).expect("read first config"); assert_eq!(first.listen_address.ip(), IpAddr::V4(Ipv4Addr::UNSPECIFIED)); - assert_eq!(first.dialable_address.ip(), public_ips[0]); - assert_eq!(first.bootstrappers[0].address.ip(), public_ips[1]); + assert_eq!(first.dialable_address.ip(), Some(public_ips[0])); + assert_eq!(first.bootstrappers[0].address.ip(), Some(public_ips[1])); + assert!(first_raw.contains("dialable_address = \"192.0.2.10:30000\"")); + assert!(first_raw.contains("address = \"192.0.2.11:30001\"")); assert_eq!(first.storage_dir, storage_dir); assert_eq!( first.indexer_url.as_deref(), Some("https://indexer.example.com/coins-chain") ); - assert_eq!(second.dialable_address.ip(), public_ips[1]); - assert_eq!(second.bootstrappers[0].address.ip(), public_ips[0]); + assert_eq!(second.dialable_address.ip(), Some(public_ips[1])); + assert_eq!(second.bootstrappers[0].address.ip(), Some(public_ips[0])); assert_eq!( second.indexer_url.as_deref(), Some("https://indexer.example.com/coins-chain") @@ -142,3 +153,454 @@ fn generated_testnet_can_advertise_remote_hosts() { let _ = fs::remove_dir_all(dir); } + +#[test] +fn peer_addresses_round_trip_dns_and_ipv6_as_strings() { + let dir = std::env::temp_dir().join(format!( + "coins-chain-address-round-trip-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 2, + base_port: 32_000, + base_rpc_port: 33_000, + base_metrics_port: 34_000, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed: 9, + }) + .expect("generate testnet"); + let path = &manifest.nodes[0].config_path; + let mut config = NodeConfig::read(path).expect("read generated config"); + config.dialable_address = Ingress::Dns { + host: Hostname::new("validator-0.example.com").unwrap(), + port: 30_000, + }; + config.bootstrappers[0].address = Ingress::Dns { + host: Hostname::new("validator-1.example.com").unwrap(), + port: 30_001, + }; + config.write(path).expect("write DNS config"); + + let raw = fs::read_to_string(path).expect("read DNS config"); + assert!(raw.contains("dialable_address = \"validator-0.example.com:30000\"")); + assert!(raw.contains("address = \"validator-1.example.com:30001\"")); + let decoded = NodeConfig::read(path).expect("read DNS config"); + assert_eq!(decoded.dialable_address, config.dialable_address); + assert_eq!( + decoded.bootstrappers[0].address, + config.bootstrappers[0].address + ); + + config.dialable_address = Ingress::Socket(SocketAddr::new( + IpAddr::V6("2001:db8::10".parse::().unwrap()), + 30_000, + )); + config.write(path).expect("write IPv6 config"); + let raw = fs::read_to_string(path).expect("read IPv6 config"); + assert!(raw.contains("dialable_address = \"[2001:db8::10]:30000\"")); + assert_eq!( + NodeConfig::read(path) + .expect("read IPv6 config") + .dialable_address, + config.dialable_address + ); + + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn invalid_peer_addresses_are_rejected_through_node_config_read() { + let dir = std::env::temp_dir().join(format!( + "coins-chain-invalid-addresses-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 1, + base_port: 35_000, + base_rpc_port: 36_000, + base_metrics_port: 37_000, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed: 10, + }) + .expect("generate testnet"); + let valid_path = &manifest.nodes[0].config_path; + let valid = fs::read_to_string(valid_path).expect("read generated config"); + let valid_address = "dialable_address = \"127.0.0.1:35000\""; + assert!(valid.contains(valid_address)); + + let cases = [ + ("validator.example.com", "missing a port"), + ("validator.example.com:not-a-port", "invalid peer address port"), + ("validator.example.com:65536", "invalid peer address port"), + ("-validator.example.com:30000", "invalid peer address hostname"), + ( + "https://validator.example.com:30000", + "must not contain a URL scheme or path", + ), + ("2001:db8::10:30000", "must use bracketed socket syntax"), + ]; + for (index, (address, expected)) in cases.into_iter().enumerate() { + let path = dir.join(format!("invalid-{index}.toml")); + let invalid = valid.replace( + valid_address, + &format!("dialable_address = \"{address}\""), + ); + fs::write(&path, invalid).expect("write invalid config"); + let error = NodeConfig::read(&path).expect_err("invalid address should fail"); + assert!( + error.to_string().contains(expected), + "unexpected error for {address}: {error}" + ); + } + + let _ = fs::remove_dir_all(dir); +} + +fn parse_dns_address_through_node_config(address: &str, seed: u64) -> Ingress { + let dir = std::env::temp_dir().join(format!( + "coins-chain-parse-dns-{seed}-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 1, + base_port: 38_000, + base_rpc_port: 38_100, + base_metrics_port: 38_200, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed, + }) + .expect("generate parser fixture"); + let path = &manifest.nodes[0].config_path; + let valid = fs::read_to_string(path).expect("read parser fixture"); + let dns = valid.replace( + "dialable_address = \"127.0.0.1:38000\"", + &format!("dialable_address = \"{address}\""), + ); + fs::write(path, dns).expect("write parser fixture"); + let ingress = NodeConfig::read(path) + .expect("parse DNS address") + .dialable_address; + let _ = fs::remove_dir_all(dir); + ingress +} + +fn reconnect_config( + key: ed25519::PrivateKey, + listen: SocketAddr, + dialable: Ingress, + bootstrappers: Vec<(ed25519::PublicKey, Ingress)>, +) -> discovery::Config { + let mut config = discovery::Config::local( + key, + NAMESPACE, + listen, + dialable, + bootstrappers, + 1024 * 1024, + ); + config.peer_connection_cooldown = Duration::from_millis(100); + config.dial_frequency = Duration::from_millis(50); + config.gossip_bit_vec_frequency = Duration::from_millis(100); + config +} + +fn run_dns_bootstrapper_redeployment(seed: u64, dns: Ingress) -> String { + let runner = deterministic::Runner::new( + deterministic::Config::new() + .with_seed(seed) + .with_timeout(Some(Duration::from_secs(20))), + ); + runner.start(|context| async move { + let key_a = ed25519::PrivateKey::from_seed(100); + let key_b = ed25519::PrivateKey::from_seed(101); + let public_a = key_a.public_key(); + let public_b = key_b.public_key(); + let peers = Set::try_from(vec![public_a.clone(), public_b.clone()]).unwrap(); + let socket_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 10)), 39_000); + let socket_b1 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 11)), 39_000); + let socket_b2 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 12)), 39_000); + let unreachable_a = Ingress::Socket(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 0, 250)), + 49_000, + )); + context.resolver_register("validator-b-bootstrap.test", Some(vec![socket_b1.ip()])); + + let config_b = reconnect_config(key_b.clone(), socket_b1, dns.clone(), vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_initial").child("network"), config_b); + oracle_b.track(0, peers.clone()); + let (mut sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let handle_b = network_b.start(); + + let config_a = reconnect_config( + key_a, + socket_a, + unreachable_a, + vec![(public_b.clone(), dns.clone())], + ); + assert!(config_a.allow_dns); + let (mut network_a, mut oracle_a) = + Network::new(context.child("a").child("network"), config_a); + oracle_a.track(0, peers.clone()); + let (mut sender_a, mut receiver_a) = + network_a.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_a = network_a.start(); + + let initial_exchange = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-before".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + loop { + sender_b.send( + Recipients::One(public_a.clone()), + b"b-before".to_vec(), + true, + ); + select! { + result = receiver_a.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_b); + assert_eq!(message.as_ref(), b"b-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = initial_exchange => {}, + _ = context.sleep(Duration::from_secs(5)) => panic!("initial DNS connection timed out"), + } + + handle_b.abort(); + drop(sender_b); + drop(receiver_b); + context.resolver_register("validator-b-bootstrap.test", Some(vec![socket_b2.ip()])); + + // The restarted peer has no bootstrappers and A advertises an unreachable address, so B + // cannot initiate the replacement connection. + let config_b = reconnect_config(key_b, socket_b2, dns, vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_restarted").child("network"), config_b); + oracle_b.track(0, peers); + let (_sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_b = network_b.start(); + + let reconnected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-after".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-after"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = reconnected => {}, + _ = context.sleep(Duration::from_secs(5)) => panic!("DNS bootstrapper did not reconnect"), + } + context.auditor().state() + }) +} + +#[test] +fn dns_bootstrapper_is_resolved_again_after_redeployment() { + let dns = parse_dns_address_through_node_config("validator-b-bootstrap.test:39000", 11); + let first = run_dns_bootstrapper_redeployment(42, dns.clone()); + let second = run_dns_bootstrapper_redeployment(42, dns); + assert_eq!(first, second); +} + +fn run_discovered_dns_redeployment(seed: u64, dns: Ingress) -> String { + let runner = deterministic::Runner::new( + deterministic::Config::new() + .with_seed(seed) + .with_timeout(Some(Duration::from_secs(20))), + ); + runner.start(|context| async move { + let key_a = ed25519::PrivateKey::from_seed(200); + let key_b = ed25519::PrivateKey::from_seed(201); + let key_c = ed25519::PrivateKey::from_seed(202); + let public_a = key_a.public_key(); + let public_b = key_b.public_key(); + let public_c = key_c.public_key(); + let peers = Set::try_from(vec![ + public_a.clone(), + public_b.clone(), + public_c.clone(), + ]) + .unwrap(); + let socket_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 10)), 40_000); + let socket_b1 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 11)), 40_000); + let socket_b2 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 12)), 40_000); + let socket_c = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 13)), 40_000); + let unreachable_a = Ingress::Socket(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 1, 250)), + 50_000, + )); + context.resolver_register("validator-b-discovered.test", Some(vec![socket_b1.ip()])); + + let config_c = reconnect_config(key_c, socket_c, socket_c.into(), vec![]); + let (mut network_c, mut oracle_c) = + Network::new(context.child("c").child("network"), config_c); + oracle_c.track(0, peers.clone()); + let (_sender_c, _receiver_c) = + network_c.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_c = network_c.start(); + + let config_b = reconnect_config( + key_b.clone(), + socket_b1, + dns.clone(), + vec![(public_c.clone(), socket_c.into())], + ); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_initial").child("network"), config_b); + oracle_b.track(0, peers.clone()); + let (mut sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let handle_b = network_b.start(); + + // A only knows C initially. It must learn B's DNS ingress through discovery. + let config_a = reconnect_config( + key_a, + socket_a, + unreachable_a, + vec![(public_c, socket_c.into())], + ); + let (mut network_a, mut oracle_a) = + Network::new(context.child("a").child("network"), config_a); + oracle_a.track(0, peers.clone()); + let (mut sender_a, mut receiver_a) = + network_a.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_a = network_a.start(); + + let learned_and_connected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-before".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + loop { + sender_b.send( + Recipients::One(public_a.clone()), + b"b-before".to_vec(), + true, + ); + select! { + result = receiver_a.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_b); + assert_eq!(message.as_ref(), b"b-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = learned_and_connected => {}, + _ = context.sleep(Duration::from_secs(7)) => panic!("A did not discover B's DNS ingress"), + } + + handle_b.abort(); + drop(sender_b); + drop(receiver_b); + context.resolver_register("validator-b-discovered.test", Some(vec![socket_b2.ip()])); + + // B has no bootstrapper after restart and A's advertised address is unreachable. A must + // retain the discovered hostname, resolve it again, and dial B at its replacement IP. + let config_b = reconnect_config(key_b, socket_b2, dns, vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_restarted").child("network"), config_b); + oracle_b.track(0, peers); + let (_sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_b = network_b.start(); + + let reconnected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-after".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-after"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = reconnected => {}, + _ = context.sleep(Duration::from_secs(7)) => panic!("discovered DNS peer did not reconnect"), + } + context.auditor().state() + }) +} + +#[test] +fn discovered_dns_address_is_resolved_again_after_redeployment() { + let dns = parse_dns_address_through_node_config("validator-b-discovered.test:40000", 12); + let first = run_discovered_dns_redeployment(84, dns.clone()); + let second = run_discovered_dns_redeployment(84, dns); + assert_eq!(first, second); +} From 830c649f026330a872c293059f96534f05014876 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Wed, 22 Jul 2026 09:05:33 +0700 Subject: [PATCH 2/8] feat: Bound validator disk growth with prunable history and cleanup --- dkg/src/orchestrator/actor.rs | 152 +++++- examples/coins/chain/src/engine.rs | 303 ++++++----- examples/coins/chain/src/history.rs | 466 +++++++++++++++++ .../chain/src/indexer/backfiller/consumer.rs | 483 +++++++----------- .../chain/src/indexer/backfiller/producer.rs | 243 +++++---- .../chain/src/indexer/backfiller/state.rs | 182 +++++-- examples/coins/chain/src/indexer/metrics.rs | 140 ++--- examples/coins/chain/src/indexer/mod.rs | 109 +++- examples/coins/chain/src/lib.rs | 1 + examples/coins/chain/src/testnet.rs | 41 +- examples/coins/chain/src/tests/indexer.rs | 37 +- examples/coins/chain/tests/common/network.rs | 4 +- 12 files changed, 1446 insertions(+), 715 deletions(-) create mode 100644 examples/coins/chain/src/history.rs diff --git a/dkg/src/orchestrator/actor.rs b/dkg/src/orchestrator/actor.rs index f815252..5a250d0 100644 --- a/dkg/src/orchestrator/actor.rs +++ b/dkg/src/orchestrator/actor.rs @@ -26,9 +26,14 @@ use commonware_parallel::Strategy; use commonware_runtime::{ buffer::paged::CacheRef, spawn_cell, - telemetry::metrics::{Gauge, GaugeExt, MetricsExt as _}, + telemetry::metrics::{ + histogram::Buckets, CounterFamily, EncodeLabelSet, Gauge, GaugeExt, Histogram, + MetricsExt as _, + }, BufferPooler, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage, }; +use commonware_storage::metadata::{self, Metadata}; +use commonware_utils::sequence::U64; use commonware_utils::{vec::NonEmptyVec, NZUsize, NZU16}; use rand::CryptoRng; use std::{ @@ -39,6 +44,13 @@ use std::{ }; use tracing::{debug, info, warn}; +const CLEANUP_WATERMARK_KEY: U64 = U64::new(0); + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] +struct CleanupStatusLabel { + status: &'static str, +} + /// Reporter that discards consensus activity. pub struct NoopReporter { _phantom: PhantomData, @@ -146,6 +158,10 @@ where page_cache_ref: CacheRef, latest_epoch: Gauge, + partition_cleanup_total: CounterFamily, + partition_cleanup_watermark: Gauge, + partitions_active: Gauge, + partition_cleanup_duration: Histogram, _phantom: PhantomData, } @@ -180,6 +196,23 @@ where // Register latest_epoch gauge for Grafana integration let latest_epoch = context.gauge("latest_epoch", "current epoch"); + let partition_cleanup_total = context.family( + "consensus_partition_cleanup", + "Total number of consensus partition cleanup outcomes", + ); + let partition_cleanup_watermark = context.gauge( + "consensus_partition_cleanup_watermark", + "Next consensus epoch partition requiring cleanup", + ); + let partitions_active = context.gauge( + "consensus_partitions_active", + "Number of consensus epoch partitions with active engines", + ); + let partition_cleanup_duration = context.histogram( + "consensus_partition_cleanup_duration_seconds", + "Duration of consensus epoch partition cleanup operations", + Buckets::LOCAL, + ); ( Self { @@ -200,6 +233,10 @@ where recovered_floor: config.recovered_floor, page_cache_ref, latest_epoch, + partition_cleanup_total, + partition_cleanup_watermark, + partitions_active, + partition_cleanup_duration, _phantom: PhantomData, }, Mailbox::new(sender), @@ -268,6 +305,24 @@ where // Wait for instructions to transition epochs. let epocher = FixedEpocher::new(self.epoch_length); let mut engines: BTreeMap> = BTreeMap::new(); + let cleanup_partition = format!("{}_cleanup-v1", self.partition_prefix); + let mut cleanup_metadata = Metadata::::init( + self.context.child("partition_cleanup_metadata"), + metadata::Config { + partition: cleanup_partition, + codec_config: (), + }, + ) + .await + .expect("failed to initialize consensus partition cleanup metadata"); + let mut next_epoch_to_clean = cleanup_metadata + .get(&CLEANUP_WATERMARK_KEY) + .cloned() + .unwrap_or_else(|| U64::new(0)); + let _ = self + .partition_cleanup_watermark + .try_set(u64::from(&next_epoch_to_clean)); + let mut startup_cleanup_done = false; select_loop! { self.context, @@ -317,6 +372,28 @@ where continue; } + if !startup_cleanup_done { + assert!( + u64::from(&next_epoch_to_clean) <= transition.epoch.get(), + "consensus partition cleanup watermark {} is ahead of first entering epoch {}", + u64::from(&next_epoch_to_clean), + transition.epoch, + ); + if let Some(previous) = transition.epoch.previous() { + assert!( + !engines.keys().any(|epoch| *epoch <= previous), + "refusing to clean a consensus partition with an active engine" + ); + self.cleanup_through( + &mut cleanup_metadata, + &mut next_epoch_to_clean, + previous, + ) + .await; + } + startup_cleanup_done = true; + } + let floor = if self .recovered_floor .as_ref() @@ -364,6 +441,7 @@ where .await; engines.insert(transition.epoch, handle); let _ = self.latest_epoch.try_set(transition.epoch.get()); + let _ = self.partitions_active.try_set(engines.len()); info!(epoch = %transition.epoch, "entered epoch"); } @@ -374,16 +452,88 @@ where continue; }; handle.abort(); + match handle.await { + Ok(()) | Err(commonware_runtime::Error::Aborted) => {} + Err(error) => { + panic!("consensus engine for epoch {epoch} failed while stopping: {error}") + } + } + let _ = self.partitions_active.try_set(engines.len()); // Unregister the signing scheme for the epoch. assert!(self.provider.unregister(&epoch)); + if u64::from(&next_epoch_to_clean) <= epoch.get() { + assert!( + !engines.keys().any(|active| *active <= epoch), + "refusing to clean a consensus partition with an active engine" + ); + self.cleanup_through( + &mut cleanup_metadata, + &mut next_epoch_to_clean, + epoch, + ) + .await; + } + info!(%epoch, "exited epoch"); } }, } } + async fn cleanup_through( + &mut self, + metadata: &mut Metadata, + next_epoch_to_clean: &mut U64, + through: Epoch, + ) { + while u64::from(&*next_epoch_to_clean) <= through.get() { + let epoch = Epoch::new(u64::from(&*next_epoch_to_clean)); + let partition = format!("{}_consensus_{}", self.partition_prefix, epoch); + let started = std::time::Instant::now(); + let status = match self.context.remove(&partition, None).await { + Ok(()) => { + info!(%epoch, %partition, "removed retired consensus partition"); + "removed" + } + Err(commonware_runtime::Error::PartitionMissing(_)) => { + info!(%epoch, %partition, "retired consensus partition already missing"); + "missing" + } + Err(error) => { + self.partition_cleanup_total + .get_or_create(&CleanupStatusLabel { status: "failed" }) + .inc(); + self.partition_cleanup_duration + .observe(started.elapsed().as_secs_f64()); + warn!(%epoch, %partition, %error, "failed to remove retired consensus partition"); + panic!("failed to remove retired consensus partition {partition}: {error}"); + } + }; + self.partition_cleanup_total + .get_or_create(&CleanupStatusLabel { status }) + .inc(); + self.partition_cleanup_duration + .observe(started.elapsed().as_secs_f64()); + + let next = epoch + .get() + .checked_add(1) + .expect("consensus cleanup watermark overflow"); + metadata + .put_sync(CLEANUP_WATERMARK_KEY, U64::new(next)) + .await + .unwrap_or_else(|error| { + panic!( + "failed to persist consensus cleanup watermark after removing {partition}: {error}" + ) + }); + *next_epoch_to_clean = U64::new(next); + let _ = self.partition_cleanup_watermark.try_set(next); + } + } + // Epoch zero uses genesis as its floor; every later epoch is anchored by the // last finalized block from the previous epoch. fn floor_boundary(epocher: &FixedEpocher, epoch: Epoch) -> Option { diff --git a/examples/coins/chain/src/engine.rs b/examples/coins/chain/src/engine.rs index 00a47dd..3247e10 100644 --- a/examples/coins/chain/src/engine.rs +++ b/examples/coins/chain/src/engine.rs @@ -1,9 +1,10 @@ use crate::application::{self, Application}; use crate::execution::NodeHandle; use crate::genesis::{genesis_target, state_commitment, ChainGenesis}; +use crate::history::{self, BlocksArchive, FinalizationsArchive}; use crate::indexer; use crate::{ - Block, EpochProvider, Finalization, Provider, PublicKey, Scheme, Transaction, BLOCKS_PER_EPOCH, + Block, EpochProvider, Provider, PublicKey, Scheme, Transaction, BLOCKS_PER_EPOCH, NAMESPACE, }; use commonware_broadcast::buffered; @@ -13,7 +14,7 @@ use commonware_consensus::{ core::Actor as MarshalActor, resolver, standard::{Inline, Standard}, - store::{Blocks as _, Certificates}, + store::Certificates, }, simplex::elector::Random, types::{Epoch, FixedEpocher, Height, ViewDelta}, @@ -24,7 +25,6 @@ use commonware_cryptography::{ dkg::feldman_desmedt::Output, primitives::{group, variant::MinSig}, }, - certificate::Verifier as _, ed25519::{self, Batch}, sha256::Digest, BatchVerifier, Digestible, Signer, @@ -41,12 +41,8 @@ use commonware_runtime::{ buffer::paged::CacheRef, spawn_cell, BufferPooler, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage, Strategizer, }; -use commonware_storage::{ - archive::{immutable, Identifier as ArchiveIdentifier}, - metadata::{self, Metadata}, - queue, -}; -use commonware_utils::{sequence::U64, union, NZDuration, NZU64}; +use commonware_storage::{archive::{Archive as ArchiveStore, Identifier as ArchiveIdentifier}, metadata::{self, Metadata}, queue}; +use commonware_utils::{sequence::U64, union, NZDuration}; use futures::lock::Mutex as AsyncMutex; use governor::clock::Clock as GClock; use nunchi_chain::engine::*; @@ -84,8 +80,6 @@ pub struct Config, P: Manager, @@ -100,6 +94,7 @@ pub struct Config, P: Manager, pub indexer: Option, + pub indexer_spool_limits: indexer::SpoolLimits, } type DkgActor = nunchi_chain::DkgActor; @@ -110,8 +105,6 @@ type LimitedStatefulAppMailbox = VerifyLimiter>; type InlineApp = Inline, Block, FixedEpocher>; type Marshaled = BoxedAutomaton>; type SchemeProvider = Provider; -type FinalizationsArchive = immutable::Archive; -type BlocksArchive = immutable::Archive; type Marshal = MarshalActor< E, Standard, @@ -167,6 +160,7 @@ where stateful: StatefulApp, stateful_mailbox: StatefulAppMailbox, indexer_producer: Option, + indexer_producer_handle: Option>, indexer_consumer: Option>, } @@ -251,88 +245,36 @@ where ); let start = Instant::now(); - let finalizations_by_height = immutable::Archive::init( - context.child("finalizations_by_height"), - immutable::Config { - metadata_partition: format!( - "{}-finalizations-by-height-metadata", - config.partition_prefix - ), - freezer_table_partition: format!( - "{}-finalizations-by-height-freezer-table", - config.partition_prefix - ), - freezer_table_initial_size: config.finalized_freezer_table_initial_size, - freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY, - freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE, - freezer_key_partition: format!( - "{}-finalizations-by-height-freezer-key", - config.partition_prefix - ), - freezer_key_page_cache: page_cache.clone(), - freezer_key_write_buffer: WRITE_BUFFER, - freezer_value_partition: format!( - "{}-finalizations-by-height-freezer-value", - config.partition_prefix - ), - freezer_value_write_buffer: WRITE_BUFFER, - freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE, - freezer_value_compression: FREEZER_VALUE_COMPRESSION, - ordinal_partition: format!( - "{}-finalizations-by-height-ordinal", - config.partition_prefix - ), - ordinal_write_buffer: WRITE_BUFFER, - items_per_section: IMMUTABLE_ITEMS_PER_SECTION, - codec_config: Scheme::certificate_codec_config_unbounded(), - replay_buffer: REPLAY_BUFFER, - }, - ) - .await - .expect("failed to initialize finalizations by height archive"); - info!(elapsed = ?start.elapsed(), "restored finalizations by height archive"); - - let start = Instant::now(); - let finalized_blocks = immutable::Archive::init( - context.child("finalized_blocks"), - immutable::Config { - metadata_partition: format!( - "{}-finalized_blocks-metadata", - config.partition_prefix - ), - freezer_table_partition: format!( - "{}-finalized_blocks-freezer-table", - config.partition_prefix - ), - freezer_table_initial_size: config.blocks_freezer_table_initial_size, - freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY, - freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE, - freezer_key_partition: format!( - "{}-finalized_blocks-freezer-key", - config.partition_prefix - ), - freezer_key_page_cache: page_cache.clone(), - freezer_key_write_buffer: WRITE_BUFFER, - freezer_value_partition: format!( - "{}-finalized_blocks-freezer-value", - config.partition_prefix - ), - freezer_value_write_buffer: WRITE_BUFFER, - freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE, - freezer_value_compression: FREEZER_VALUE_COMPRESSION, - ordinal_partition: format!("{}-finalized_blocks-ordinal", config.partition_prefix), - ordinal_write_buffer: WRITE_BUFFER, - items_per_section: IMMUTABLE_ITEMS_PER_SECTION, - codec_config: block_codec_config, - replay_buffer: REPLAY_BUFFER, - }, + let opened_history = history::open( + &context, + &config.partition_prefix, + page_cache.clone(), + block_codec_config, ) .await - .expect("failed to initialize finalized blocks archive"); - info!(elapsed = ?start.elapsed(), "restored finalized blocks archive"); + .unwrap_or_else(|error| panic!("failed to initialize bounded finalized history: {error}")); + let history::Opened { + finalizations: mut finalizations_by_height, + blocks: mut finalized_blocks, + partitions: history_partitions, + marker_status, + } = opened_history; + info!( + elapsed = ?start.elapsed(), + format_version = history::FORMAT_VERSION, + finalizations_key = %history_partitions.finalizations_key, + finalizations_value = %history_partitions.finalizations_value, + blocks_key = %history_partitions.blocks_key, + blocks_value = %history_partitions.blocks_value, + logical_retention = history::LOGICAL_RETENTION, + items_per_section = history::PRUNABLE_ITEMS_PER_SECTION.get(), + maximum_section_and_cadence_slack = history::MAX_RETAINED_HEIGHTS - history::LOGICAL_RETENTION, + ?marker_status, + "restored prunable finalized history" + ); let recovered_height = Certificates::last_index(&finalizations_by_height); - let mut recovered_floor = if let Some(height) = recovered_height { + let recovered_floor = if let Some(height) = recovered_height { Certificates::get( &finalizations_by_height, ArchiveIdentifier::Index(height.get()), @@ -425,15 +367,35 @@ where .expect("failed to initialize state database for startup probe"); state.sync_target() }; - if repair_marshal_progress_if_state_is_behind( + if let Some(processed_height) = validate_marshal_progress_against_state( &context, &config.partition_prefix, &finalized_blocks, ¤t_state_target, ) - .await + .await { + validate_history_through_processed( + &finalizations_by_height, + &finalized_blocks, + processed_height, + ) + .await; + if let Some(tip) = ArchiveStore::last_index(&finalized_blocks) { + let policy_floor = history::policy_floor(tip, history::LOGICAL_RETENTION); + let safe_floor = policy_floor.min(processed_height.get()); + finalizations_by_height + .prune(safe_floor) + .await + .expect("failed to startup-prune finalized certificates"); + finalized_blocks + .prune(safe_floor) + .await + .expect("failed to startup-prune finalized blocks"); + } + } else if ArchiveStore::last_index(&finalized_blocks).is_some() + || ArchiveStore::last_index(&finalizations_by_height).is_some() { - recovered_floor = None; + panic!("finalized history exists without persisted marshal processed height; rebuild or perform verified peer state sync"); } let applied_height = Arc::new(AsyncMutex::new(Height::zero())); let app = Application::with_consensus( @@ -554,7 +516,7 @@ where FixedEpocher::new(BLOCKS_PER_EPOCH), )); - let (indexer_producer, indexer_pusher, indexer_consumer) = + let (indexer_producer, indexer_producer_handle, indexer_pusher, indexer_consumer) = if let Some(client) = config.indexer.clone() { let indexer_context = context.child("indexer"); let indexer_metrics = indexer::IndexerMetrics::register(&indexer_context); @@ -562,10 +524,13 @@ where let queue = queue::shared::init( context.child("indexer_queue"), queue::Config { - partition: format!("{}-indexer-finalized-queue", config.partition_prefix), - items_per_section: NZU64!(128), + partition: format!( + "{}-indexer-finalized-payload-queue-v1", + config.partition_prefix + ), + items_per_section: indexer::SPOOL_ITEMS_PER_SECTION, compression: None, - codec_config: (), + codec_config: block_codec_config, page_cache: page_cache.clone(), write_buffer: WRITE_BUFFER, }, @@ -581,14 +546,20 @@ where mailbox_size: MAILBOX_SIZE, backfiller_max_active: commonware_utils::NZUsize!(16), backfiller_retry: Duration::from_millis(500), + spool_limits: config.indexer_spool_limits, metrics: indexer_metrics, }, ) .await; - let (producer, pusher, consumer) = indexer.split(); - (Some(producer), Some(pusher), Some(consumer)) + let (producer, producer_handle, pusher, consumer) = indexer.split(); + ( + Some(producer), + Some(producer_handle), + Some(pusher), + Some(consumer), + ) } else { - (None, None, None) + (None, None, None, None) }; let (orchestrator, orchestrator_mailbox) = orchestrator::Actor::new( @@ -629,6 +600,7 @@ where stateful, stateful_mailbox, indexer_producer, + indexer_producer_handle, indexer_consumer, }; (engine, node_handle) @@ -747,10 +719,13 @@ where .mempool .start_p2p(self.context.child("mempool"), mempool); let indexer_consumer_handle = self.indexer_consumer.map(indexer::Consumer::start); + let indexer_producer_handle = self.indexer_producer_handle; let clob_handle = self.clob.start_p2p(self.context.child("clob"), clob); let mut shutdown = self.context.stopped(); - if let Some(indexer_consumer_handle) = indexer_consumer_handle { + if let (Some(indexer_producer_handle), Some(indexer_consumer_handle)) = + (indexer_producer_handle, indexer_consumer_handle) + { commonware_macros::select! { stopped = &mut shutdown => match stopped { Ok(0) => { @@ -769,6 +744,7 @@ where result = orchestrator_handle => unexpected_exit("orchestrator", result), result = mempool_handle => unexpected_exit("mempool", result), result = clob_handle => unexpected_exit("clob", result), + result = indexer_producer_handle => unexpected_exit("indexer_producer", result), result = indexer_consumer_handle => unexpected_exit("indexer_consumer", result), } } else { @@ -806,14 +782,13 @@ fn unexpected_exit( } const MARSHAL_PROCESSED_KEY: U64 = U64::new(0xFF); -const STATE_SYNC_METADATA_SUFFIX: &str = "state_sync_metadata"; -async fn repair_marshal_progress_if_state_is_behind( +async fn validate_marshal_progress_against_state( context: &E, partition_prefix: &str, finalized_blocks: &BlocksArchive, current_target: &commonware_storage::qmdb::sync::Target, -) -> bool +) -> Option where E: BufferPooler + Clock @@ -837,63 +812,87 @@ where ) .await .expect("failed to initialize marshal progress metadata probe"); - let Some(processed_height) = metadata.get(&MARSHAL_PROCESSED_KEY).copied() else { - return false; - }; + let processed_height = metadata.get(&MARSHAL_PROCESSED_KEY).copied()?; drop(metadata); - let Some(processed_block) = finalized_blocks - .get(ArchiveIdentifier::Index(processed_height.get())) - .await - .expect("failed to read processed block for state repair") - else { - warn!( - %processed_height, - "marshal progress references a missing finalized block; clearing startup metadata to replay" - ); - remove_partition_if_exists(context, &marshal_metadata_partition).await; - clear_disabled_state_sync_metadata(context, partition_prefix).await; - return true; + let Some(tip) = ArchiveStore::last_index(finalized_blocks) else { + panic!("marshal progress references height {processed_height}, but finalized block history is empty; rebuild or perform verified peer state sync"); }; - let processed_target = >::sync_targets(&processed_block); - let current_size = current_target.range.end().as_u64(); - let processed_size = processed_target.range.end().as_u64(); - if current_size >= processed_size { - return false; + let search_end = tip.min( + processed_height + .get() + .saturating_add(MAX_PENDING_ACKS.get() as u64), + ); + for height in processed_height.get()..=search_end { + let Some(block) = ArchiveStore::get( + finalized_blocks, + ArchiveIdentifier::Index(height), + ) + .await + .expect("failed to read finalized block while validating QMDB startup target") + else { + continue; + }; + let target = >::sync_targets(&block); + if &target == current_target { + return Some(processed_height); + } } - warn!( - %processed_height, - current_size, - processed_size, - "state database is behind marshal progress; clearing startup metadata to replay finalized blocks" + panic!( + "QMDB committed target does not exactly match canonical finalized history from processed height {} through {}; rebuild or perform verified peer state sync", + processed_height, + search_end, ); - remove_partition_if_exists(context, &marshal_metadata_partition).await; - remove_partition_if_exists( - context, - &format!("{partition_prefix}{STATE_SYNC_METADATA_SUFFIX}"), - ) - .await; - true } -async fn clear_disabled_state_sync_metadata(context: &E, partition_prefix: &str) +async fn validate_history_through_processed( + finalizations: &FinalizationsArchive, + blocks: &BlocksArchive, + processed_height: Height, +) where - E: Storage, + E: BufferPooler + Metrics + Storage, { - remove_partition_if_exists( - context, - &format!("{partition_prefix}{STATE_SYNC_METADATA_SUFFIX}"), - ) - .await; -} + let processed = processed_height.get(); + let first = match (finalizations.first_index(), blocks.first_index()) { + (Some(finalization), Some(block)) => finalization.max(block), + _ => panic!( + "marshal processed height {processed_height} has incomplete finalized history; rebuild or perform verified peer state sync" + ), + }; + if first > processed { + panic!( + "marshal processed height {processed_height} is below retained finalized history floor {first}; rebuild or perform verified peer state sync" + ); + } -async fn remove_partition_if_exists(context: &E, partition: &str) -where - E: Storage, -{ - match context.remove(partition, None).await { - Ok(()) | Err(commonware_runtime::Error::PartitionMissing(_)) => {} - Err(error) => panic!("failed to remove partition {partition}: {error}"), + for height in first..=processed { + let certificate = finalizations + .get(ArchiveIdentifier::Index(height)) + .await + .expect("failed to validate finalized certificate archive"); + let block = blocks + .get(ArchiveIdentifier::Index(height)) + .await + .expect("failed to validate finalized block archive"); + if height == processed { + assert!( + block.is_some(), + "marshal processed height {processed_height} references a missing finalized block; rebuild or perform verified peer state sync" + ); + } + if let (Some(certificate), Some(block)) = (certificate, block) { + assert_eq!( + certificate.proposal.payload, + block.digest(), + "finalized block and certificate conflict at height {height}" + ); + assert_eq!( + certificate.proposal.round.epoch(), + block.context.round.epoch(), + "finalized block and certificate epochs conflict at height {height}" + ); + } } } diff --git a/examples/coins/chain/src/history.rs b/examples/coins/chain/src/history.rs new file mode 100644 index 0000000..402af05 --- /dev/null +++ b/examples/coins/chain/src/history.rs @@ -0,0 +1,466 @@ +//! Bounded finalized-history storage for coins-chain validators. + +use crate::{Block, Finalization}; +use commonware_codec::Read; +use commonware_cryptography::sha256::Digest; +use commonware_runtime::{buffer::paged::CacheRef, BufferPooler, Clock, Metrics, Spawner, Storage}; +use commonware_storage::{ + archive::{prunable, Archive as _}, + metadata::{self, Metadata}, + translator::EightCap, +}; +use commonware_utils::{sequence::U64, NZU64}; +use nunchi_chain::engine::{ + MAX_PENDING_ACKS, PRUNE_MAINTENANCE_INTERVAL, PRUNE_RETAINED_MARSHAL_BLOCKS, REPLAY_BUFFER, + WRITE_BUFFER, +}; +use std::num::NonZeroU64; + +pub(crate) const FORMAT_VERSION: u64 = 1; +pub(crate) const PRUNABLE_ITEMS_PER_SECTION: NonZeroU64 = NZU64!(4_096); +pub(crate) const LOGICAL_RETENTION: u64 = + PRUNE_RETAINED_MARSHAL_BLOCKS as u64 + MAX_PENDING_ACKS.get() as u64 + 1; +pub(crate) const MAX_RETAINED_HEIGHTS: u64 = LOGICAL_RETENTION + + PRUNABLE_ITEMS_PER_SECTION.get() + - 1 + + PRUNE_MAINTENANCE_INTERVAL.get() as u64 + - 1; + +const MARKER_KEY: U64 = U64::new(0); + +pub(crate) type FinalizationsArchive = + prunable::Archive; +pub(crate) type BlocksArchive = prunable::Archive; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MarkerStatus { + Created, + RecoveredPartialInitialization, + Present, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum HistoryError { + #[error("legacy finalized-history partition exists: {partition}; stop the devnet and clear the complete validator storage directory before relaunch")] + LegacyPartition { partition: String }, + #[error("history format marker exists but required partition is missing: {partition}; clear the complete devnet validator storage directory before relaunch")] + MarkedPartitionMissing { partition: String }, + #[error("history format marker is missing but {partition} contains finalized data at height {height}; refusing to modify unmarked storage")] + NonEmptyUnmarked { partition: String, height: u64 }, + #[error("invalid history format marker in {partition}: expected version {expected}, found {found:?}")] + InvalidMarker { + partition: String, + expected: u64, + found: Option, + }, + #[error("failed to inspect history partition {partition}: {source}")] + Inspect { + partition: String, + #[source] + source: commonware_runtime::Error, + }, + #[error("failed to open history archive partition {partition}: {source}")] + Archive { + partition: String, + #[source] + source: commonware_storage::archive::Error, + }, + #[error("failed to access history marker partition {partition}: {source}")] + Marker { + partition: String, + #[source] + source: commonware_storage::metadata::Error, + }, + #[error("invalid history retention configuration: {0}")] + Retention(&'static str), +} + +#[derive(Clone, Debug)] +pub(crate) struct Partitions { + pub(crate) finalizations_key: String, + pub(crate) finalizations_value: String, + pub(crate) blocks_key: String, + pub(crate) blocks_value: String, + pub(crate) metadata: String, +} + +impl Partitions { + pub(crate) fn new(prefix: &str) -> Self { + Self { + finalizations_key: format!( + "{prefix}-finalizations-by-height-prunable-v1-key" + ), + finalizations_value: format!( + "{prefix}-finalizations-by-height-prunable-v1-value" + ), + blocks_key: format!("{prefix}-finalized-blocks-prunable-v1-key"), + blocks_value: format!("{prefix}-finalized-blocks-prunable-v1-value"), + metadata: format!("{prefix}-history-prunable-v1-metadata"), + } + } + + fn archives(&self) -> [&str; 4] { + [ + &self.finalizations_key, + &self.finalizations_value, + &self.blocks_key, + &self.blocks_value, + ] + } +} + +pub(crate) struct Opened +where + E: BufferPooler + Clock + Storage + Metrics, +{ + pub(crate) finalizations: FinalizationsArchive, + pub(crate) blocks: BlocksArchive, + pub(crate) partitions: Partitions, + pub(crate) marker_status: MarkerStatus, +} + +pub(crate) const fn policy_floor(tip: u64, retained: u64) -> u64 { + tip.saturating_add(1).saturating_sub(retained) +} + +pub(crate) fn validate_production_retention() -> Result<(), HistoryError> { + if MAX_PENDING_ACKS.get() != 16 { + return Err(HistoryError::Retention( + "MAX_PENDING_ACKS changed; update the mandatory rewind assertion", + )); + } + if LOGICAL_RETENTION != 217 { + return Err(HistoryError::Retention( + "logical finalized-history retention must be 217", + )); + } + if MAX_RETAINED_HEIGHTS != 4_343 { + return Err(HistoryError::Retention( + "section-granular finalized-history bound must be 4,343", + )); + } + Ok(()) +} + +pub(crate) async fn open( + context: &E, + prefix: &str, + page_cache: CacheRef, + block_codec_config: ::Cfg, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + validate_production_retention()?; + reject_legacy(context, prefix).await?; + + let partitions = Partitions::new(prefix); + let marker_existed = partition_exists(context, &partitions.metadata).await?; + let mut archive_existed = [false; 4]; + for (exists, partition) in archive_existed + .iter_mut() + .zip(partitions.archives().into_iter()) + { + *exists = partition_exists(context, partition).await?; + } + let mut finalizations = + init_finalizations(context, &partitions, page_cache.clone()).await?; + let mut blocks = init_blocks( + context, + &partitions, + page_cache.clone(), + block_codec_config, + ) + .await?; + + let marker_status = if marker_existed { + validate_marker(context, &partitions.metadata).await?; + let archives_empty = finalizations.first_index().is_none() && blocks.first_index().is_none(); + let all_lazy = archive_existed.iter().all(|exists| !exists); + if !(archive_existed.iter().all(|exists| *exists) || archives_empty && all_lazy) { + let partition = partitions + .archives() + .into_iter() + .zip(archive_existed) + .find_map(|(partition, exists)| (!exists).then_some(partition)) + .expect("at least one marked archive partition is missing"); + return Err(HistoryError::MarkedPartitionMissing { + partition: partition.to_string(), + }); + } + MarkerStatus::Present + } else { + if let Some(height) = finalizations.first_index() { + return Err(HistoryError::NonEmptyUnmarked { + partition: partitions.finalizations_key.clone(), + height, + }); + } + if let Some(height) = blocks.first_index() { + return Err(HistoryError::NonEmptyUnmarked { + partition: partitions.blocks_key.clone(), + height, + }); + } + + // Ensure all newly-created archive blobs are durable and structurally reopenable before + // committing the format marker. + finalizations.sync().await.map_err(|source| HistoryError::Archive { + partition: partitions.finalizations_key.clone(), + source, + })?; + blocks.sync().await.map_err(|source| HistoryError::Archive { + partition: partitions.blocks_key.clone(), + source, + })?; + drop(finalizations); + drop(blocks); + finalizations = init_finalizations(context, &partitions, page_cache.clone()).await?; + blocks = init_blocks(context, &partitions, page_cache, block_codec_config).await?; + write_marker(context, &partitions.metadata).await?; + + if archive_existed.into_iter().any(|exists| exists) { + MarkerStatus::RecoveredPartialInitialization + } else { + MarkerStatus::Created + } + }; + + Ok(Opened { + finalizations, + blocks, + partitions, + marker_status, + }) +} + +async fn init_finalizations( + context: &E, + partitions: &Partitions, + page_cache: CacheRef, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + prunable::Archive::init( + context.child("finalizations_by_height"), + prunable::Config { + translator: EightCap, + key_partition: partitions.finalizations_key.clone(), + key_page_cache: page_cache, + value_partition: partitions.finalizations_value.clone(), + compression: Some(3), + codec_config: (), + items_per_section: PRUNABLE_ITEMS_PER_SECTION, + key_write_buffer: WRITE_BUFFER, + value_write_buffer: WRITE_BUFFER, + replay_buffer: REPLAY_BUFFER, + }, + ) + .await + .map_err(|source| HistoryError::Archive { + partition: partitions.finalizations_key.clone(), + source, + }) +} + +async fn init_blocks( + context: &E, + partitions: &Partitions, + page_cache: CacheRef, + codec_config: ::Cfg, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + prunable::Archive::init( + context.child("finalized_blocks"), + prunable::Config { + translator: EightCap, + key_partition: partitions.blocks_key.clone(), + key_page_cache: page_cache, + value_partition: partitions.blocks_value.clone(), + compression: Some(3), + codec_config, + items_per_section: PRUNABLE_ITEMS_PER_SECTION, + key_write_buffer: WRITE_BUFFER, + value_write_buffer: WRITE_BUFFER, + replay_buffer: REPLAY_BUFFER, + }, + ) + .await + .map_err(|source| HistoryError::Archive { + partition: partitions.blocks_key.clone(), + source, + }) +} + +async fn partition_exists(context: &E, partition: &str) -> Result { + match context.scan(partition).await { + Ok(_) => Ok(true), + Err(commonware_runtime::Error::PartitionMissing(_)) => Ok(false), + Err(source) => Err(HistoryError::Inspect { + partition: partition.to_string(), + source, + }), + } +} + +async fn validate_marker(context: &E, partition: &str) -> Result<(), HistoryError> +where + E: BufferPooler + Clock + Storage + Metrics + Spawner, +{ + let marker = Metadata::::init( + context.child("history_marker"), + metadata::Config { + partition: partition.to_string(), + codec_config: (), + }, + ) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + })?; + let found = marker.get(&MARKER_KEY).copied(); + if found != Some(FORMAT_VERSION) { + return Err(HistoryError::InvalidMarker { + partition: partition.to_string(), + expected: FORMAT_VERSION, + found, + }); + } + Ok(()) +} + +async fn write_marker(context: &E, partition: &str) -> Result<(), HistoryError> +where + E: BufferPooler + Clock + Storage + Metrics + Spawner, +{ + let mut marker = Metadata::::init( + context.child("history_marker"), + metadata::Config { + partition: partition.to_string(), + codec_config: (), + }, + ) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + })?; + marker + .put_sync(MARKER_KEY, FORMAT_VERSION) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + }) +} + +async fn reject_legacy(context: &E, prefix: &str) -> Result<(), HistoryError> { + const FINALIZATION_SUFFIXES: [&str; 5] = [ + "finalizations-by-height-metadata", + "finalizations-by-height-freezer-table", + "finalizations-by-height-freezer-key", + "finalizations-by-height-freezer-value", + "finalizations-by-height-ordinal", + ]; + const BLOCK_SUFFIXES: [&str; 5] = [ + "finalized_blocks-metadata", + "finalized_blocks-freezer-table", + "finalized_blocks-freezer-key", + "finalized_blocks-freezer-value", + "finalized_blocks-ordinal", + ]; + + for suffix in FINALIZATION_SUFFIXES.into_iter().chain(BLOCK_SUFFIXES) { + let partition = format!("{prefix}-{suffix}"); + if partition_exists(context, &partition).await? { + return Err(HistoryError::LegacyPartition { partition }); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use commonware_runtime::{buffer::paged::CacheRef, deterministic, Runner as _}; + use commonware_utils::{NZU16, NZU32, NZUsize}; + + #[test] + fn production_retention_arithmetic() { + validate_production_retention().unwrap(); + assert_eq!(policy_floor(216, LOGICAL_RETENTION), 0); + assert_eq!(policy_floor(217, LOGICAL_RETENTION), 1); + assert_eq!(LOGICAL_RETENTION, 217); + assert_eq!(MAX_RETAINED_HEIGHTS, 4_343); + } + + #[test] + fn fresh_format_marker_is_durable_and_legacy_is_not_created() { + deterministic::Runner::default().start(|context| async move { + let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(32)); + let first = open( + &context, + "validator", + page_cache.clone(), + (NZU32!(1), ()), + ) + .await + .expect("initialize fresh history"); + assert_eq!(first.marker_status, MarkerStatus::Created); + assert_eq!(first.finalizations.first_index(), None); + assert_eq!(first.blocks.first_index(), None); + drop(first); + + let reopened = open( + &context, + "validator", + page_cache, + (NZU32!(1), ()), + ) + .await + .expect("reopen marked history"); + assert_eq!(reopened.marker_status, MarkerStatus::Present); + + for suffix in [ + "finalizations-by-height-metadata", + "finalizations-by-height-freezer-table", + "finalizations-by-height-freezer-key", + "finalizations-by-height-freezer-value", + "finalizations-by-height-ordinal", + "finalized_blocks-metadata", + "finalized_blocks-freezer-table", + "finalized_blocks-freezer-key", + "finalized_blocks-freezer-value", + "finalized_blocks-ordinal", + ] { + assert!(matches!( + context.scan(&format!("validator-{suffix}")).await, + Err(commonware_runtime::Error::PartitionMissing(_)) + )); + } + }); + } + + #[test] + fn refuses_known_legacy_partition() { + deterministic::Runner::default().start(|context| async move { + context + .open("validator-finalized_blocks-ordinal", b"legacy") + .await + .expect("create legacy partition"); + let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(32)); + let result = open( + &context, + "validator", + page_cache, + (NZU32!(1), ()), + ) + .await; + assert!(matches!(result, Err(HistoryError::LegacyPartition { .. }))); + }); + } +} diff --git a/examples/coins/chain/src/indexer/backfiller/consumer.rs b/examples/coins/chain/src/indexer/backfiller/consumer.rs index e714eb2..0bd66e8 100644 --- a/examples/coins/chain/src/indexer/backfiller/consumer.rs +++ b/examples/coins/chain/src/indexer/backfiller/consumer.rs @@ -1,17 +1,14 @@ -use super::{Decision, Entry, SharedState}; +use super::{ + producer::{Admission, AdmissionReceiver}, + Decision, Entry, SharedState, +}; use crate::indexer::{ metrics::{ - estimated_finalized_bytes, BackfillDecision, BackfillPhase, BackfillResetReason, - BackfillUploadGuard, BackfillWaitReason, BlockMetricSource, QueueReadSource, QueueStatus, - SharedCacheSource, + estimated_finalized_bytes, BackfillDecision, BackfillPhase, BackfillWaitReason, + BlockMetricSource, ProducerStatus, QueueReadSource, QueueStatus, }, - Client, IndexerMetrics, -}; -use crate::{Block, Finalized, Scheme}; -use commonware_consensus::marshal::{ - core::Mailbox as MarshalMailbox, standard::Standard, Identifier, + Client, IndexerMetrics, SpoolLimits, }; -use commonware_consensus::types::Height; use commonware_cryptography::sha256::Digest; use commonware_macros::select_loop; use commonware_runtime::{ @@ -22,7 +19,7 @@ use commonware_storage::queue; use commonware_utils::futures::{OptionFuture, Pool}; use std::{ num::NonZeroUsize, - time::{Duration, Instant}, + time::{Duration, Instant, UNIX_EPOCH}, }; use tracing::{debug, warn}; @@ -45,47 +42,39 @@ enum Completion { Skipped { position: u64, height: u64, - }, - Stale { - reason: BackfillResetReason, - first_height: u64, - first_digest: Digest, - attempts: u64, - elapsed: Duration, + digest: Digest, }, } pub(crate) struct Config { pub(crate) max_active: NonZeroUsize, pub(crate) retry: Duration, - pub(crate) missing_finalization_grace: Duration, - pub(crate) mismatched_finalization_grace: Duration, + pub(crate) spool_limits: SpoolLimits, } pub struct Consumer { context: ContextCell, client: C, - marshal: MarshalMailbox>, metrics: IndexerMetrics, upload_results: status::Counter, uploads: SharedState, writer: queue::Writer, reader: queue::Reader, + admission: AdmissionReceiver, active: Pool, max_active: NonZeroUsize, retry: Duration, - missing_finalization_grace: Duration, - mismatched_finalization_grace: Duration, + spool_limits: SpoolLimits, } impl Consumer { pub fn new( context: E, client: C, - marshal: MarshalMailbox>, metrics: IndexerMetrics, uploads: SharedState, backfiller: (queue::Writer, queue::Reader), + admission: AdmissionReceiver, config: Config, ) -> Self { let upload_results = context.register( @@ -96,25 +85,23 @@ impl Consumer< let Config { max_active, retry, - missing_finalization_grace, - mismatched_finalization_grace, + spool_limits, } = config; metrics.backfill_configured(max_active.get()); let (writer, reader) = backfiller; Self { context: ContextCell::new(context), client, - marshal, metrics, upload_results, uploads, writer, reader, + admission, active: Pool::default(), max_active, retry, - missing_finalization_grace, - mismatched_finalization_grace, + spool_limits, } } @@ -127,25 +114,24 @@ impl Consumer< self.context, on_start => { self.fill_slots().await; - if self.active.is_empty() { - let item = Self::recv_queue(&mut self.reader, self.metrics.clone()).await; - let Some((position, entry)) = item else { - warn!("consumer queue closed"); - break; - }; - self.start_upload(position, entry).await; - continue; - } let item = OptionFuture::from( (self.active.len() < self.max_active.get()).then(|| { Self::recv_queue(&mut self.reader, self.metrics.clone()) }), ); + let admission = self.admission.recv(); }, on_stopped => {}, completion = self.active.next_completed() => { self.complete(completion).await; }, + request = admission => { + let Some(request) = request else { + warn!("indexer spool admission coordinator closed"); + break; + }; + self.admit(request).await; + }, item = item => { match item { Some((position, entry)) => { @@ -160,6 +146,87 @@ impl Consumer< } } + async fn admit(&mut self, admission: Admission) { + let Admission { entry, response } = admission; + loop { + let now_millis: u64 = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + let next = { + let uploads = self.uploads.lock(); + let (entries, bytes) = uploads.spool_usage(); + uploads.oldest_queued().and_then( + |(digest, position, height, old_bytes, enqueued_at)| { + let age = Duration::from_millis( + now_millis.saturating_sub(enqueued_at), + ); + let status = if age >= self.spool_limits.max_age { + Some(ProducerStatus::ExpiredAge) + } else if entries >= self.spool_limits.max_entries { + Some(ProducerStatus::ExpiredEntries) + } else if bytes.saturating_add(entry.encoded_len) + > self.spool_limits.max_bytes + { + Some(ProducerStatus::ExpiredBytes) + } else { + None + }; + status.map(|status| { + (digest, position, height, old_bytes, age, status) + }) + }, + ) + }; + let Some((digest, position, height, encoded_len, age, status)) = next else { + break; + }; + + // Advancing the floor invalidates positions held by upload tasks. Cancel all of them + // and replay the still-live suffix so a stale completion cannot be applied twice. + self.active.cancel_all(); + self.reader + .ack_up_to(position.saturating_add(1)) + .await + .unwrap_or_else(|error| panic!("failed to expire indexer spool floor: {error:?}")); + let sync_started = Instant::now(); + self.writer.sync().await.unwrap_or_else(|error| { + panic!("failed to durably sync expired indexer spool floor: {error:?}") + }); + self.metrics + .queue_synced(QueueStatus::Success, sync_started.elapsed()); + self.reader.reset().await; + let expired = self.uploads.lock().expire_through(position); + for (_, _, _) in &expired { + self.metrics.producer_recorded(status, Duration::ZERO); + } + warn!( + height, + ?digest, + encoded_len, + ?age, + ?status, + expired_entries = expired.len(), + "terminally expired oldest finalized indexer payload before admission" + ); + } + + self.metrics.queue_entry(entry.height()); + let position = self.writer.enqueue(entry.clone()).await.unwrap_or_else(|error| { + self.metrics.queue_enqueued(QueueStatus::Failure); + panic!("failed to enqueue finalized indexer payload: {error:?}") + }); + self.uploads.lock().mark_queued(position, &entry); + self.metrics.queue_enqueued(QueueStatus::Success); + response + .send(()) + .expect("indexer spool producer stopped before admission completed"); + } + async fn fill_slots(&mut self) { while self.active.len() < self.max_active.get() { let item = Self::try_recv_queue(&mut self.reader, self.metrics.clone()).await; @@ -177,7 +244,7 @@ impl Consumer< match reader.recv().await { Ok(Some((position, entry))) => { metrics.queue_read(QueueReadSource::Recv, QueueStatus::Success); - metrics.queue_entry(entry.height); + metrics.queue_entry(entry.height()); Some((position, entry)) } Ok(None) => { @@ -198,7 +265,7 @@ impl Consumer< match reader.try_recv().await { Ok(Some((position, entry))) => { metrics.queue_read(QueueReadSource::TryRecv, QueueStatus::Success); - metrics.queue_entry(entry.height); + metrics.queue_entry(entry.height()); Some((position, entry)) } Ok(None) => { @@ -213,7 +280,26 @@ impl Consumer< } async fn start_upload(&mut self, position: u64, entry: Entry) { - let Entry { height, digest } = entry; + let height = entry.height(); + let digest = entry.digest(); + let enqueued_at_millis = entry.enqueued_at_millis; + let finalized = entry.finalized; + if self.payload_age(enqueued_at_millis) >= self.spool_limits.max_age { + self.metrics + .producer_recorded(ProducerStatus::ExpiredAge, Duration::ZERO); + warn!( + height, + ?digest, + "terminally expiring finalized indexer payload at configured age bound" + ); + self.complete(Completion::Skipped { + position, + height, + digest, + }) + .await; + return; + } let decision = self.uploads.lock().should_upload(&digest); self.metrics .backfill_decision((&decision).into(), BackfillPhase::Start); @@ -222,7 +308,11 @@ impl Consumer< let mut upload = self.metrics.start_backfill_upload(); upload.skipped(); } - self.complete(Completion::Skipped { position, height }) + self.complete(Completion::Skipped { + position, + height, + digest, + }) .await; debug!(?digest, "consumer skipping already-uploaded block"); return; @@ -235,82 +325,35 @@ impl Consumer< .with_attribute("digest", digest) .with_attribute("height", height); let client = self.client.clone(); - let marshal = self.marshal.clone(); let metrics = self.metrics.clone(); let upload_results = self.upload_results.clone(); let uploads = self.uploads.clone(); let retry = self.retry; - let missing_finalization_grace = self.missing_finalization_grace; - let mismatched_finalization_grace = self.mismatched_finalization_grace; + let max_age = self.spool_limits.max_age; async move { let mut upload = metrics.start_backfill_upload(); - let Some(block) = - Self::wait_for_uploadable_block( - &context, &marshal, &metrics, &uploads, &mut upload, digest, retry, - ) - .await - else { - upload.skipped(); - debug!(?digest, "skipping previously uploaded block"); - return Completion::Skipped { position, height }; - }; + metrics.observe_block(BlockMetricSource::ConsumerCached, &finalized.block); + upload.hold_block(&finalized.block); - let mut proof_failure = ProofFailure::default(); - let finalized = loop { - let Some(proof) = marshal.get_finalization(Height::new(height)).await else { - if let Some(stale) = proof_failure.observe( - &context, - BackfillResetReason::MissingFinalization, - missing_finalization_grace, - height, - digest, - ) { - upload.abandoned(); - return stale; - } - warn!( - height, - ?digest, - "consumer could not find finalization, retrying" - ); - Self::wait( - &context, - &metrics, - BackfillWaitReason::MissingFinalization, - retry, - ) - .await; - continue; - }; - if proof.proposal.payload != digest { - if let Some(stale) = proof_failure.observe( - &context, - BackfillResetReason::MismatchedFinalization, - mismatched_finalization_grace, + loop { + let now_millis: u64 = context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + // Backward wall-clock movement produces age zero until the clock catches up. + let age = Duration::from_millis(now_millis.saturating_sub(enqueued_at_millis)); + if age >= max_age { + metrics.producer_recorded(ProducerStatus::ExpiredAge, Duration::ZERO); + warn!(height, ?digest, ?age, "terminally expiring active finalized indexer upload at configured age bound"); + return Completion::Skipped { + position, height, digest, - ) { - upload.abandoned(); - return stale; - } - warn!( - height, - ?digest, - "consumer found mismatched finalization, retrying" - ); - Self::wait( - &context, - &metrics, - BackfillWaitReason::MismatchedFinalization, - retry, - ) - .await; - continue; + }; } - break Finalized::new(proof, block.clone()); - }; - - loop { let decision = { let uploads = uploads.lock(); uploads.should_upload(&digest) @@ -320,7 +363,11 @@ impl Consumer< Decision::Skip => { upload.skipped(); debug!(?digest, "skipping previously uploaded block"); - return Completion::Skipped { position, height }; + return Completion::Skipped { + position, + height, + digest, + }; } Decision::Wait => { Self::wait( @@ -363,86 +410,17 @@ impl Consumer< }); } - async fn wait_for_uploadable_block( - context: &E, - marshal: &MarshalMailbox>, - metrics: &IndexerMetrics, - uploads: &SharedState, - upload: &mut BackfillUploadGuard, - digest: Digest, - retry: Duration, - ) -> Option { - enum NextBlock { - AlreadyUploaded, - WaitForCertificate, - Ready(Box), - FetchFromMarshal, - } - - loop { - let next = { - let uploads = uploads.lock(); - match uploads.should_upload(&digest) { - Decision::Skip => { - metrics.backfill_decision( - BackfillDecision::Skip, - BackfillPhase::BeforeBlock, - ); - NextBlock::AlreadyUploaded - } - Decision::Wait => { - metrics.backfill_decision( - BackfillDecision::Wait, - BackfillPhase::BeforeBlock, - ); - NextBlock::WaitForCertificate - } - Decision::Proceed => { - metrics.backfill_decision( - BackfillDecision::Proceed, - BackfillPhase::BeforeBlock, - ); - uploads - .cached_block(&digest) - .map(|block| NextBlock::Ready(Box::new(block))) - .unwrap_or(NextBlock::FetchFromMarshal) - } - } - }; - - match next { - NextBlock::AlreadyUploaded => return None, - NextBlock::WaitForCertificate => { - Self::wait( - context, - metrics, - BackfillWaitReason::CertificateUpload, - retry, - ) - .await; - } - NextBlock::Ready(block) => { - metrics.observe_block(BlockMetricSource::ConsumerCached, &block); - upload.hold_block(&block); - return Some(*block); - } - NextBlock::FetchFromMarshal => { - if let Some(block) = marshal.get_block(Identifier::Digest(digest)).await { - metrics.observe_block(BlockMetricSource::ConsumerMarshal, &block); - uploads - .lock() - .cache_block(block.clone(), SharedCacheSource::ConsumerMarshal); - upload.hold_block(&block); - return Some(block); - } - warn!( - ?digest, - "consumer could not find block in marshal, retrying" - ); - Self::wait(context, metrics, BackfillWaitReason::MissingBlock, retry).await; - } - } - } + fn payload_age(&self, enqueued_at_millis: u64) -> Duration { + let now_millis: u64 = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + // Backward wall-clock movement produces age zero until the clock catches up. + Duration::from_millis(now_millis.saturating_sub(enqueued_at_millis)) } async fn wait( @@ -462,28 +440,22 @@ impl Consumer< } async fn complete(&mut self, completion: Completion) { - let (position, height) = match completion { + let (position, height, digest) = match completion { Completion::Uploaded { position, height, digest, } => { self.uploads.lock().mark_uploaded(digest, height); - (position, height) - } - Completion::Skipped { position, height } => (position, height), - Completion::Stale { - reason, - first_height, - first_digest, - attempts, - elapsed, - } => { - self.reset_stale_queue(reason, first_height, first_digest, attempts, elapsed) - .await; - return; + (position, height, digest) } + Completion::Skipped { + position, + height, + digest, + } => (position, height, digest), }; + self.uploads.lock().finish_queued(&digest); let floor = self.reader.ack_floor().await; match self.reader.ack(position).await { @@ -511,109 +483,4 @@ impl Consumer< } } - async fn reset_stale_queue( - &mut self, - reason: BackfillResetReason, - first_height: u64, - first_digest: Digest, - attempts: u64, - elapsed: Duration, - ) { - let queue_size = self.writer.size().await; - let ack_floor = self.reader.ack_floor().await; - let abandoned_entries = queue_size.saturating_sub(ack_floor); - let tip = Self::latest_proof_bearing_tip(self.marshal.clone()).await; - let (tip_height, tip_digest) = tip.unwrap_or((0, first_digest)); - let abandoned_height_span = tip_height.saturating_sub(first_height); - - warn!( - ?reason, - first_height, - ?first_digest, - tip_height, - ?tip_digest, - attempts, - ?elapsed, - abandoned_entries, - abandoned_height_span, - "resetting stale durable backfill queue" - ); - - self.active.cancel_all(); - match self.reader.ack_up_to(queue_size).await { - Ok(()) => self.metrics.queue_acked(QueueStatus::Success), - Err(err) => { - self.metrics.queue_acked(QueueStatus::Failure); - panic!("failed to ack stale finalized queue: {err:?}"); - } - } - let sync_started = Instant::now(); - match self.writer.sync().await { - Ok(()) => self - .metrics - .queue_synced(QueueStatus::Success, sync_started.elapsed()), - Err(err) => { - self.metrics - .queue_synced(QueueStatus::Failure, sync_started.elapsed()); - panic!("failed to sync stale finalized queue reset: {err:?}"); - } - } - self.reader.reset().await; - self.metrics.queue_ack_floor(tip_height); - self.metrics.backfill_queue_reset( - reason, - abandoned_entries, - abandoned_height_span, - ); - self.uploads.lock().advance_queue_floor(tip_height); - self.uploads.lock().restart_above(tip_height); - } - - async fn latest_proof_bearing_tip( - marshal: MarshalMailbox>, - ) -> Option<(u64, Digest)> { - let (height, _) = marshal.get_info(Identifier::Latest).await?; - for height in (0..=height.get()).rev() { - if let Some(proof) = marshal.get_finalization(Height::new(height)).await { - return Some((height, proof.proposal.payload)); - } - } - None - } -} - -#[derive(Default)] -struct ProofFailure { - reason: Option, - first_seen: Option, - attempts: u64, -} - -impl ProofFailure { - fn observe( - &mut self, - context: &E, - reason: BackfillResetReason, - grace: Duration, - height: u64, - digest: Digest, - ) -> Option { - if self.reason != Some(reason) { - self.reason = Some(reason); - self.first_seen = Some(context.current()); - self.attempts = 0; - } - self.attempts = self.attempts.saturating_add(1); - let elapsed = context - .current() - .duration_since(self.first_seen.expect("proof failure start")) - .unwrap_or_default(); - (elapsed >= grace).then_some(Completion::Stale { - reason, - first_height: height, - first_digest: digest, - attempts: self.attempts, - elapsed, - }) - } } diff --git a/examples/coins/chain/src/indexer/backfiller/producer.rs b/examples/coins/chain/src/indexer/backfiller/producer.rs index 88b01e4..fccdb5d 100644 --- a/examples/coins/chain/src/indexer/backfiller/producer.rs +++ b/examples/coins/chain/src/indexer/backfiller/producer.rs @@ -3,23 +3,32 @@ use crate::{ indexer::{ metrics::{ estimated_block_bytes, BlockMetricSource, ProducerActivity, ProducerStatus, - QueueStatus, }, - IndexerMetrics, + IndexerMetrics, SpoolLimits, }, - Block, + Block, Finalized, Scheme, }; use commonware_actor::{ mailbox::{self, Overflow, Policy}, Feedback, }; -use commonware_consensus::{marshal::Update, Reporter}; +use commonware_consensus::{ + marshal::{core::Mailbox as MarshalMailbox, standard::Standard, Update}, + types::Height, + Reporter, +}; use commonware_runtime::{ spawn_cell, BufferPooler, Clock, ContextCell, Handle, Metrics, Spawner, Storage, }; -use commonware_storage::queue; use commonware_utils::{acknowledgement::Exact, Acknowledgement}; -use std::{collections::VecDeque, num::NonZeroUsize, sync::Arc, time::Instant}; +use commonware_utils::channel::oneshot; +use std::{ + collections::VecDeque, + num::NonZeroUsize, + sync::Arc, + time::{Duration, Instant, UNIX_EPOCH}, +}; +use tracing::warn; #[derive(Clone)] pub struct Producer { @@ -27,6 +36,22 @@ pub struct Producer { metrics: IndexerMetrics, } +pub(crate) struct Admission { + pub(crate) entry: Entry, + pub(crate) response: oneshot::Sender<()>, +} + +impl Policy for Admission { + type Overflow = VecDeque; + + fn handle(overflow: &mut Self::Overflow, message: Self) { + overflow.push_back(message); + } +} + +pub(crate) type AdmissionSender = mailbox::Sender; +pub(crate) type AdmissionReceiver = mailbox::Receiver; + struct Message { block: Arc, block_estimated_bytes: u64, @@ -87,8 +112,21 @@ struct Actor { context: ContextCell, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, + marshal: MarshalMailbox>, + admission: AdmissionSender, receiver: mailbox::Receiver, + retry: Duration, + missing_finalization_grace: Duration, + mismatched_finalization_grace: Duration, + spool_limits: SpoolLimits, +} + +pub(crate) struct Config { + pub(crate) mailbox_size: NonZeroUsize, + pub(crate) retry: Duration, + pub(crate) missing_finalization_grace: Duration, + pub(crate) mismatched_finalization_grace: Duration, + pub(crate) spool_limits: SpoolLimits, } impl Reporter for Producer { @@ -123,16 +161,29 @@ impl Actor { context: E, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, - mailbox_size: NonZeroUsize, + marshal: MarshalMailbox>, + admission: AdmissionSender, + config: Config, ) -> (Self, Producer) { + let Config { + mailbox_size, + retry, + missing_finalization_grace, + mismatched_finalization_grace, + spool_limits, + } = config; let (sender, receiver) = mailbox::new(context.child("mailbox"), mailbox_size); let actor = Self { context: ContextCell::new(context), uploads, metrics: metrics.clone(), - writer, + marshal, + admission, receiver, + retry, + missing_finalization_grace, + mismatched_finalization_grace, + spool_limits, }; ( actor, @@ -158,25 +209,91 @@ impl Actor { let started = Instant::now(); self.metrics .observe_block(BlockMetricSource::ProducerRecord, block); - let Some(entry) = self.uploads.lock().record(block) else { + let Some(candidate) = self.uploads.lock().record(block) else { self.metrics.producer_recorded( ProducerStatus::AlreadyUploaded, started.elapsed(), ); return; }; - self.metrics.queue_entry(entry.height); - match self.writer.enqueue(entry).await { - Ok(_) => { - self.metrics.queue_enqueued(QueueStatus::Success); - self.metrics - .producer_recorded(ProducerStatus::Recorded, started.elapsed()); - } - Err(err) => { - self.metrics.queue_enqueued(QueueStatus::Failure); - panic!("failed to enqueue finalized digest: {err:?}"); + let first_seen = self.context.current(); + self.metrics.certificate_request_started(); + let proof = loop { + let Some(proof) = self + .marshal + .get_finalization(Height::new(candidate.height)) + .await + else { + let elapsed = self + .context + .current() + .duration_since(first_seen) + .unwrap_or_default(); + assert!( + elapsed < self.missing_finalization_grace, + "marshal has no finalization certificate for durable indexer payload at height {} after {:?}", + candidate.height, + elapsed, + ); + warn!(height = candidate.height, ?elapsed, "waiting for finalized certificate before spooling indexer payload"); + self.context.sleep(self.retry).await; + continue; + }; + if proof.proposal.payload != candidate.digest + || proof.proposal.round.epoch() != block.context.round.epoch() + { + let elapsed = self + .context + .current() + .duration_since(first_seen) + .unwrap_or_default(); + assert!( + elapsed < self.mismatched_finalization_grace, + "marshal finalization certificate conflicts with block at height {} after {:?}", + candidate.height, + elapsed, + ); + warn!(height = candidate.height, ?elapsed, "waiting for matching finalized certificate before spooling indexer payload"); + self.context.sleep(self.retry).await; + continue; } + break proof; + }; + self.metrics.certificate_request_finished(); + let enqueued_at_millis = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + let entry = Entry::new(enqueued_at_millis, Finalized::new(proof, block.clone())); + let expiry = if entry.encoded_len > self.spool_limits.max_payload_bytes + || entry.encoded_len > self.spool_limits.max_bytes + { + Some(ProducerStatus::ExpiredOversized) + } else { + None + }; + if let Some(status) = expiry { + self.metrics.producer_recorded(status, started.elapsed()); + warn!( + height = entry.height(), + digest = ?entry.digest(), + encoded_len = entry.encoded_len, + ?status, + "terminally expiring finalized indexer payload to preserve spool availability bound" + ); + return; } + let (response, completed) = oneshot::channel(); + let _ = self.admission.enqueue(Admission { entry, response }); + completed + .await + .expect("indexer spool admission coordinator stopped"); + self.metrics + .producer_recorded(ProducerStatus::Recorded, started.elapsed()); } } @@ -184,15 +301,23 @@ pub fn init( context: E, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, - mailbox_size: NonZeroUsize, -) -> Producer + marshal: MarshalMailbox>, + admission: AdmissionSender, + config: Config, +) -> (Producer, Handle<()>) where E: BufferPooler + Clock + Storage + Metrics + Spawner, { - let (actor, producer) = Actor::new(context, uploads, metrics, writer, mailbox_size); - actor.start(); - producer + let (actor, producer) = Actor::new( + context, + uploads, + metrics, + marshal, + admission, + config, + ); + let handle = actor.start(); + (producer, handle) } #[cfg(test)] @@ -200,16 +325,12 @@ mod tests { use super::*; use crate::{StateCommitment, Transaction, EPOCH}; use commonware_consensus::types::{Height, Round, View}; - use commonware_cryptography::{ed25519, Digestible, Hasher, Sha256, Signer}; - use commonware_runtime::{ - buffer::paged::CacheRef, deterministic, Runner as _, Supervisor as _, - }; + use commonware_cryptography::{ed25519, Hasher, Sha256, Signer}; + use commonware_runtime::{deterministic, Runner as _, Supervisor as _}; use commonware_storage::mmr::Location; use commonware_utils::{ - acknowledgement::Exact, range::NonEmptyRange, sync::Mutex, NZUsize, NZU16, NZU64, + acknowledgement::Exact, range::NonEmptyRange, NZUsize, }; - use futures::FutureExt; - use std::{sync::Arc, time::Duration}; fn state(height: u64) -> StateCommitment { StateCommitment { @@ -239,60 +360,6 @@ mod tests { ) } - #[test] - fn queues_finalized_block_before_acknowledging() { - deterministic::Runner::default().start(|context| async move { - let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(128)); - let (writer, mut reader) = queue::shared::init( - context.child("queue"), - queue::Config { - partition: "indexer-producer-test".to_string(), - items_per_section: NZU64!(16), - compression: None, - codec_config: (), - page_cache, - write_buffer: NZUsize!(1024), - }, - ) - .await - .expect("init queue"); - let uploads = Arc::new(Mutex::new(crate::indexer::backfiller::State::new())); - let metrics = IndexerMetrics::register(&context.child("indexer")); - let mut producer = init(context.child("producer"), uploads, metrics, writer, NZUsize!(4)); - let block = block(2, 2, b"block"); - let digest = block.digest(); - let (ack, waiter) = Exact::handle(); - - assert!(producer - .report(commonware_consensus::marshal::Update::Block( - block.into(), - ack, - )) - .accepted()); - commonware_macros::select! { - result = waiter.fuse() => result.expect("acknowledged"), - _ = context.sleep(Duration::from_secs(1)) => panic!("ack timed out"), - } - - let (_, entry) = reader - .recv() - .await - .expect("read queue") - .expect("queued entry"); - assert_eq!(entry.height, 2); - assert_eq!(entry.digest, digest); - - let encoded = context.encode(); - assert!(encoded.contains( - "indexer_producer_report_total{activity=\"block\",status=\"enqueued\"} 1", - )); - assert!(encoded.contains( - "indexer_producer_report_total{activity=\"block\",status=\"recorded\"} 1", - )); - assert!(encoded.contains("indexer_producer_record_duration_seconds_bucket")); - }); - } - #[test] fn mailbox_overflow_tracks_retained_block_pressure() { deterministic::Runner::default().start(|context| async move { diff --git a/examples/coins/chain/src/indexer/backfiller/state.rs b/examples/coins/chain/src/indexer/backfiller/state.rs index 217b70f..e52c24c 100644 --- a/examples/coins/chain/src/indexer/backfiller/state.rs +++ b/examples/coins/chain/src/indexer/backfiller/state.rs @@ -5,10 +5,10 @@ use crate::{ }, IndexerMetrics, }, - Block, + Block, Finalized, }; use bytes::{Buf, BufMut}; -use commonware_codec::{self, FixedSize, Read, Write}; +use commonware_codec::{self, EncodeSize, FixedSize, Read, Write}; use commonware_cryptography::{sha256::Digest, Digestible}; use commonware_utils::{sync::Mutex, PrioritySet}; use std::{collections::BTreeMap, sync::Arc}; @@ -19,30 +19,84 @@ pub enum Decision { Proceed, } -#[derive(Clone, Copy, PartialEq, Eq)] -pub struct Entry { +pub struct Candidate { pub height: u64, pub digest: Digest, } -impl FixedSize for Entry { - const SIZE: usize = u64::SIZE + Digest::SIZE; +const ENTRY_VERSION: u8 = 1; + +#[derive(Clone)] +pub struct Entry { + pub version: u8, + pub enqueued_at_millis: u64, + pub encoded_len: u64, + pub finalized: Finalized, +} + +impl Entry { + pub fn new(enqueued_at_millis: u64, finalized: Finalized) -> Self { + let mut entry = Self { + version: ENTRY_VERSION, + enqueued_at_millis, + encoded_len: 0, + finalized, + }; + entry.encoded_len = entry.encode_size() as u64; + entry + } + + pub fn height(&self) -> u64 { + self.finalized.block.height.get() + } + + pub fn digest(&self) -> Digest { + self.finalized.block.digest() + } } impl Write for Entry { fn write(&self, buf: &mut impl BufMut) { - self.height.write(buf); - self.digest.write(buf); + self.version.write(buf); + self.enqueued_at_millis.write(buf); + self.encoded_len.write(buf); + self.finalized.write(buf); } } impl Read for Entry { - type Cfg = (); + type Cfg = ::Cfg; + + fn read_cfg(buf: &mut impl Buf, cfg: &Self::Cfg) -> Result { + let version = u8::read_cfg(buf, &())?; + if version != ENTRY_VERSION { + return Err(commonware_codec::Error::Invalid( + "indexer spool entry", + "unsupported version", + )); + } + let enqueued_at_millis = u64::read_cfg(buf, &())?; + let encoded_len = u64::read_cfg(buf, &())?; + let finalized = Finalized::read_cfg(buf, cfg)?; + let entry = Self { + version, + enqueued_at_millis, + encoded_len, + finalized, + }; + if entry.encode_size() as u64 != encoded_len { + return Err(commonware_codec::Error::Invalid( + "indexer spool entry", + "encoded length mismatch", + )); + } + Ok(entry) + } +} - fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { - let height = u64::read_cfg(buf, &())?; - let digest = Digest::read_cfg(buf, &())?; - Ok(Self { height, digest }) +impl EncodeSize for Entry { + fn encode_size(&self) -> usize { + u8::SIZE + u64::SIZE + u64::SIZE + self.finalized.encode_size() } } @@ -55,6 +109,8 @@ pub struct State { cached_block_estimated_bytes: u64, certificate_uploads: BTreeMap, certificate_upload_refs: usize, + queued: BTreeMap, + queued_bytes: u64, metrics: Option, } @@ -74,6 +130,8 @@ impl State { cached_block_estimated_bytes: 0, certificate_uploads: BTreeMap::new(), certificate_upload_refs: 0, + queued: BTreeMap::new(), + queued_bytes: 0, metrics: None, } } @@ -89,8 +147,8 @@ impl State { self.uploaded.contains(digest) } - pub fn record(&mut self, block: &Block) -> Option { - let entry = Entry { + pub fn record(&mut self, block: &Block) -> Option { + let entry = Candidate { height: block.height.get(), digest: block.digest(), }; @@ -99,7 +157,7 @@ impl State { self.sync_metrics(); return None; } - if self.is_uploaded(&entry.digest) { + if self.is_uploaded(&entry.digest) || self.queued(&entry.digest) { self.sync_metrics(); return None; } @@ -108,12 +166,74 @@ impl State { Some(entry) } + pub fn queued(&self, digest: &Digest) -> bool { + self.queued.contains_key(digest) + } + + pub fn mark_queued(&mut self, position: u64, entry: &Entry) { + if let Some((_, _, old_bytes, _)) = + self.queued + .insert( + entry.digest(), + ( + position, + entry.height(), + entry.encoded_len, + entry.enqueued_at_millis, + ), + ) + { + self.queued_bytes = self.queued_bytes.saturating_sub(old_bytes); + } + self.queued_bytes = self.queued_bytes.saturating_add(entry.encoded_len); + self.sync_metrics(); + } + + pub fn recover_queued(&mut self, position: u64, entry: &Entry) { + self.observe_finalization(entry.height()); + self.mark_queued(position, entry); + } + + pub fn finish_queued(&mut self, digest: &Digest) { + if let Some((_, _, bytes, _)) = self.queued.remove(digest) { + self.queued_bytes = self.queued_bytes.saturating_sub(bytes); + } + self.sync_metrics(); + } + + pub fn spool_usage(&self) -> (u64, u64) { + (self.queued.len() as u64, self.queued_bytes) + } + + pub fn oldest_queued(&self) -> Option<(Digest, u64, u64, u64, u64)> { + self.queued + .iter() + .min_by_key(|(_, (position, _, _, _))| *position) + .map(|(digest, (position, height, bytes, enqueued_at))| { + (*digest, *position, *height, *bytes, *enqueued_at) + }) + } + + pub fn expire_through(&mut self, position: u64) -> Vec<(Digest, u64, u64)> { + let expired: Vec<_> = self + .queued + .iter() + .filter(|(_, (queued_position, _, _, _))| *queued_position <= position) + .map(|(digest, (_, height, bytes, _))| (*digest, *height, *bytes)) + .collect(); + for (digest, _, _) in &expired { + self.finish_queued(digest); + } + expired + } + pub fn advance_queue_floor(&mut self, height: u64) { self.acked_through = self.acked_through.max(height); self.prune(); self.sync_metrics(); } + #[cfg(test)] pub fn restart_above(&mut self, height: u64) { self.restart_watermark = self.restart_watermark.max(height); let pruned = self @@ -145,6 +265,7 @@ impl State { self.sync_metrics(); } + #[cfg(test)] pub fn cached_block(&self, digest: &Digest) -> Option { self.cached_blocks .get(digest) @@ -279,6 +400,20 @@ impl State { uploaded_digests: self.uploaded.len(), latest_finalized_height: self.latest_finalized, acked_through_height: self.acked_through, + spool_entries: self.queued.len(), + spool_logical_bytes: self.queued_bytes, + spool_oldest_height: self + .queued + .values() + .map(|(_, height, _, _)| *height) + .min() + .unwrap_or(0), + spool_oldest_enqueued_at_millis: self + .queued + .values() + .map(|(_, _, _, enqueued_at)| *enqueued_at) + .min() + .unwrap_or(0), }); } } @@ -290,7 +425,6 @@ pub type SharedState = Arc>; mod tests { use super::*; use crate::{StateCommitment, Transaction, EPOCH}; - use commonware_codec::{DecodeExt, Encode}; use commonware_consensus::types::{Height, Round, View}; use commonware_cryptography::{ed25519, Hasher, Sha256, Signer}; use commonware_runtime::{deterministic, Metrics as _, Runner as _, Supervisor as _}; @@ -325,20 +459,6 @@ mod tests { ) } - #[test] - fn entry_codec_roundtrips() { - let entry = Entry { - height: 7, - digest: Sha256::hash(b"block"), - }; - - let encoded = entry.encode(); - let decoded = Entry::decode(encoded).expect("decode entry"); - - assert_eq!(decoded.height, entry.height); - assert_eq!(decoded.digest, entry.digest); - } - #[test] fn record_caches_block_until_uploaded() { let mut state = State::new(); diff --git a/examples/coins/chain/src/indexer/metrics.rs b/examples/coins/chain/src/indexer/metrics.rs index 18d2f08..ca211ea 100644 --- a/examples/coins/chain/src/indexer/metrics.rs +++ b/examples/coins/chain/src/indexer/metrics.rs @@ -163,7 +163,6 @@ pub(crate) enum BlockMetricSource { ProducerRecord, LiveCertificate, ConsumerCached, - ConsumerMarshal, } impl BlockMetricSource { @@ -172,7 +171,6 @@ impl BlockMetricSource { Self::ProducerRecord => "producer_record", Self::LiveCertificate => "live_certificate", Self::ConsumerCached => "consumer_cached", - Self::ConsumerMarshal => "consumer_marshal", } } } @@ -192,7 +190,6 @@ impl EncodeLabelValueTrait for BlockMetricSource { pub(crate) enum SharedCacheSource { ProducerRecord, LiveCertificate, - ConsumerMarshal, } impl SharedCacheSource { @@ -200,7 +197,6 @@ impl SharedCacheSource { match self { Self::ProducerRecord => "producer_record", Self::LiveCertificate => "live_certificate", - Self::ConsumerMarshal => "consumer_marshal", } } } @@ -248,7 +244,6 @@ impl EncodeLabelValueTrait for SharedRetentionReason { pub(crate) enum BackfillStatus { Uploaded, Skipped, - Abandoned, } impl BackfillStatus { @@ -256,7 +251,6 @@ impl BackfillStatus { match self { Self::Uploaded => "uploaded", Self::Skipped => "skipped", - Self::Abandoned => "abandoned", } } } @@ -303,7 +297,6 @@ impl EncodeLabelValueTrait for BackfillDecision { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum BackfillPhase { Start, - BeforeBlock, BeforeAttempt, } @@ -311,7 +304,6 @@ impl BackfillPhase { const fn as_str(self) -> &'static str { match self { Self::Start => "start", - Self::BeforeBlock => "before_block", Self::BeforeAttempt => "before_attempt", } } @@ -331,9 +323,6 @@ impl EncodeLabelValueTrait for BackfillPhase { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum BackfillWaitReason { CertificateUpload, - MissingBlock, - MissingFinalization, - MismatchedFinalization, HttpError, } @@ -341,40 +330,11 @@ impl BackfillWaitReason { const fn as_str(self) -> &'static str { match self { Self::CertificateUpload => "certificate_upload", - Self::MissingBlock => "missing_block", - Self::MissingFinalization => "missing_finalization", - Self::MismatchedFinalization => "mismatched_finalization", Self::HttpError => "http_error", } } } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub(crate) enum BackfillResetReason { - MissingFinalization, - MismatchedFinalization, -} - -impl BackfillResetReason { - const fn as_str(self) -> &'static str { - match self { - Self::MissingFinalization => "missing_finalization", - Self::MismatchedFinalization => "mismatched_finalization", - } - } -} - -impl EncodeLabelValueTrait for BackfillResetReason { - fn encode( - &self, - encoder: &mut commonware_runtime::telemetry::metrics::LabelValueEncoder, - ) -> Result<(), fmt::Error> { - use fmt::Write as _; - - encoder.write_str(self.as_str()) - } -} - impl EncodeLabelValueTrait for BackfillWaitReason { fn encode( &self, @@ -475,6 +435,10 @@ pub(crate) enum ProducerStatus { Ignored, Recorded, AlreadyUploaded, + ExpiredOversized, + ExpiredEntries, + ExpiredBytes, + ExpiredAge, } impl ProducerStatus { @@ -485,6 +449,10 @@ impl ProducerStatus { Self::Ignored => "ignored", Self::Recorded => "recorded", Self::AlreadyUploaded => "already_uploaded", + Self::ExpiredOversized => "expired_oversized", + Self::ExpiredEntries => "expired_entries", + Self::ExpiredBytes => "expired_bytes", + Self::ExpiredAge => "expired_age", } } } @@ -538,6 +506,10 @@ pub(crate) struct SharedStateSnapshot { pub(crate) uploaded_digests: usize, pub(crate) latest_finalized_height: u64, pub(crate) acked_through_height: u64, + pub(crate) spool_entries: usize, + pub(crate) spool_logical_bytes: u64, + pub(crate) spool_oldest_height: u64, + pub(crate) spool_oldest_enqueued_at_millis: u64, } #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] @@ -599,11 +571,6 @@ struct BackfillWaitReasonLabel { reason: BackfillWaitReason, } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] -struct BackfillResetReasonLabel { - reason: BackfillResetReason, -} - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] struct QueueStatusLabel { status: QueueStatus, @@ -666,6 +633,11 @@ pub(crate) struct Inner { shared_uploaded_digests: Gauge, shared_latest_finalized_height: Gauge, shared_acked_through_height: Gauge, + spool_entries: Gauge, + spool_logical_bytes: Gauge, + spool_oldest_height: Gauge, + spool_oldest_enqueued_at_millis: Gauge, + producer_certificate_requests: Gauge, shared_pruned: CounterFamily, shared_cache_inserted: CounterFamily, shared_cache_removed: CounterFamily, @@ -677,9 +649,6 @@ pub(crate) struct Inner { backfill_decision: CounterFamily, backfill_wait_duration: HistogramFamily, backfill_retry: CounterFamily, - backfill_queue_reset: CounterFamily, - backfill_queue_reset_abandoned_entries: HistogramFamily, - backfill_queue_reset_abandoned_height_span: HistogramFamily, backfill_active_block_estimated_bytes: Gauge, backfill_active_body_estimated_bytes: Gauge, queue_enqueue: CounterFamily, @@ -817,6 +786,26 @@ impl IndexerMetrics { "shared_acked_through_height", "Queue acknowledgement floor tracked by shared indexer state", ), + spool_entries: context.gauge( + "spool_entries", + "Current number of self-contained finalized payloads in the durable spool", + ), + spool_logical_bytes: context.gauge( + "spool_logical_bytes", + "Current encoded bytes in self-contained finalized spool payloads", + ), + spool_oldest_height: context.gauge( + "spool_oldest_height", + "Oldest finalized height retained in the durable spool", + ), + spool_oldest_enqueued_at_millis: context.gauge( + "spool_oldest_enqueued_at_millis", + "Wall-clock Unix millisecond timestamp of the oldest durable spool payload", + ), + producer_certificate_requests: context.gauge( + "spool_certificate_requests", + "Current finalized blocks awaiting a marshal certificate before spool admission", + ), shared_pruned: context.family( "shared_prune", "Total shared indexer state retention removals by reason", @@ -867,24 +856,6 @@ impl IndexerMetrics { "backfill_retry", "Total durable backfill retries by reason", ), - backfill_queue_reset: context.family( - "backfill_queue_reset", - "Total durable backfill queue resets by reason", - ), - backfill_queue_reset_abandoned_entries: context.register( - "backfill_queue_reset_abandoned_entries", - "Durable backfill entries abandoned by queue reset", - raw::Family:: raw::Histogram>::new_with_constructor( - local_histogram, - ), - ), - backfill_queue_reset_abandoned_height_span: context.register( - "backfill_queue_reset_abandoned_height_span", - "Durable backfill height span abandoned by queue reset", - raw::Family:: raw::Histogram>::new_with_constructor( - local_histogram, - ), - ), backfill_active_block_estimated_bytes: context.gauge( "backfill_active_block_estimated_bytes", "Current estimated encoded block bytes held by durable backfill upload tasks", @@ -1074,6 +1045,16 @@ impl IndexerMetrics { let _ = self .shared_acked_through_height .try_set(snapshot.acked_through_height); + let _ = self.spool_entries.try_set(snapshot.spool_entries); + let _ = self + .spool_logical_bytes + .try_set(snapshot.spool_logical_bytes); + let _ = self + .spool_oldest_height + .try_set(snapshot.spool_oldest_height); + let _ = self + .spool_oldest_enqueued_at_millis + .try_set(snapshot.spool_oldest_enqueued_at_millis); self.queue_ack_floor(snapshot.acked_through_height); let lag = snapshot .latest_finalized_height @@ -1081,6 +1062,14 @@ impl IndexerMetrics { let _ = self.queue_lag_height.try_set(lag); } + pub(crate) fn certificate_request_started(&self) { + self.producer_certificate_requests.inc(); + } + + pub(crate) fn certificate_request_finished(&self) { + self.producer_certificate_requests.dec(); + } + pub(crate) fn shared_cache_inserted(&self, source: SharedCacheSource) { self.shared_cache_inserted .get_or_create(&SharedCacheSourceLabel { source }) @@ -1132,22 +1121,6 @@ impl IndexerMetrics { .observe(duration.as_secs_f64()); } - pub(crate) fn backfill_queue_reset( - &self, - reason: BackfillResetReason, - abandoned_entries: u64, - abandoned_height_span: u64, - ) { - let label = BackfillResetReasonLabel { reason }; - self.backfill_queue_reset.get_or_create(&label).inc(); - self.backfill_queue_reset_abandoned_entries - .get_or_create(&label) - .observe(abandoned_entries as f64); - self.backfill_queue_reset_abandoned_height_span - .get_or_create(&label) - .observe(abandoned_height_span as f64); - } - pub(crate) fn start_backfill_body(&self, bytes: u64) -> BackfillBodyGuard { let bytes = gauge_u64(bytes); self.backfill_active_body_estimated_bytes.inc_by(bytes); @@ -1403,9 +1376,6 @@ impl BackfillUploadGuard { self.status = Some(BackfillStatus::Skipped); } - pub(crate) const fn abandoned(&mut self) { - self.status = Some(BackfillStatus::Abandoned); - } } impl Drop for BackfillUploadGuard { diff --git a/examples/coins/chain/src/indexer/mod.rs b/examples/coins/chain/src/indexer/mod.rs index 2604221..fac82a2 100644 --- a/examples/coins/chain/src/indexer/mod.rs +++ b/examples/coins/chain/src/indexer/mod.rs @@ -6,9 +6,9 @@ use commonware_consensus::types::Epoch; use commonware_cryptography::bls12381::{ dkg::feldman_desmedt::Output as DkgOutput, primitives::variant::MinSig, }; -use commonware_runtime::{BufferPooler, Clock, Metrics, Spawner, Storage}; +use commonware_runtime::{BufferPooler, Clock, Handle, Metrics, Spawner, Storage}; use commonware_storage::queue; -use commonware_utils::sync::Mutex; +use commonware_utils::{sync::Mutex, NZU64}; use std::{future::Future, num::NonZeroUsize, sync::Arc, time::Duration}; use thiserror::Error; @@ -21,7 +21,7 @@ use backfiller::{SharedState, State}; pub(crate) use metrics::{DkgUploadStatus, IndexerMetrics}; #[cfg(test)] pub(crate) use metrics::{ - BackfillDecision, BackfillPhase, BackfillResetReason, BackfillWaitReason, BlockMetricSource, + BackfillDecision, BackfillPhase, BackfillWaitReason, BlockMetricSource, HttpArtifact, LiveUploadArtifact, ProducerActivity, ProducerStatus, QueueReadSource, QueueStatus, SharedCacheSource, SharedRetentionReason, SharedStateSnapshot, }; @@ -32,6 +32,64 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const MISSING_FINALIZATION_GRACE: Duration = Duration::from_secs(120); const MISMATCHED_FINALIZATION_GRACE: Duration = Duration::from_secs(15); +pub(crate) const SPOOL_ITEMS_PER_SECTION: std::num::NonZeroU64 = NZU64!(128); + +/// Availability window for durable finalized uploads while the external indexer is unavailable. +#[derive(Clone, Copy, Debug)] +pub struct SpoolLimits { + pub max_entries: u64, + pub max_bytes: u64, + pub max_payload_bytes: u64, + pub max_age: Duration, +} + +impl Default for SpoolLimits { + fn default() -> Self { + Self { + max_entries: crate::BLOCKS_PER_EPOCH.get(), + max_bytes: 2 * 1024 * 1024 * 1024, + max_payload_bytes: 16 * 1024 * 1024, + max_age: Duration::from_secs(24 * 60 * 60), + } + } +} + +impl SpoolLimits { + fn validate(self) { + assert!(self.max_entries > 0, "indexer spool max_entries must be non-zero"); + assert!(self.max_bytes > 0, "indexer spool max_bytes must be non-zero"); + assert!( + self.max_payload_bytes > 0 && self.max_payload_bytes <= self.max_bytes, + "indexer spool max_payload_bytes must be in 1..=max_bytes" + ); + assert!(!self.max_age.is_zero(), "indexer spool max_age must be non-zero"); + self.max_encoded_payload_bytes() + .expect("indexer spool encoded payload bound overflows u64"); + } + + pub fn max_encoded_payload_bytes(self) -> Option { + (SPOOL_ITEMS_PER_SECTION.get() - 1) + .checked_mul(self.max_payload_bytes)? + .checked_add(self.max_bytes) + } +} + +#[cfg(test)] +mod spool_tests { + use super::*; + + #[test] + fn default_spool_bound_includes_section_slack() { + let limits = SpoolLimits::default(); + assert_eq!(limits.max_entries, crate::BLOCKS_PER_EPOCH.get()); + assert_eq!(limits.max_bytes, 2 * 1024 * 1024 * 1024); + assert_eq!(limits.max_age, Duration::from_secs(24 * 60 * 60)); + assert_eq!( + limits.max_encoded_payload_bytes(), + Some(limits.max_bytes + 127 * limits.max_payload_bytes) + ); + } +} /// Errors returned by the HTTP indexer client. #[derive(Debug, Error)] @@ -184,11 +242,13 @@ pub(crate) struct Config { pub(crate) mailbox_size: NonZeroUsize, pub(crate) backfiller_max_active: NonZeroUsize, pub(crate) backfiller_retry: Duration, + pub(crate) spool_limits: SpoolLimits, pub(crate) metrics: IndexerMetrics, } pub(crate) struct Indexer { producer: Producer, + producer_handle: Handle<()>, pusher: Pusher, consumer: Consumer, } @@ -205,8 +265,10 @@ impl Indexer Indexer (Producer, Pusher, Consumer) { + pub(crate) fn split(self) -> (Producer, Handle<()>, Pusher, Consumer) { let Self { producer, + producer_handle, pusher, consumer, } = self; - (producer, pusher, consumer) + (producer, producer_handle, pusher, consumer) } } diff --git a/examples/coins/chain/src/lib.rs b/examples/coins/chain/src/lib.rs index b034460..359f7f8 100644 --- a/examples/coins/chain/src/lib.rs +++ b/examples/coins/chain/src/lib.rs @@ -19,6 +19,7 @@ pub mod application; pub mod engine; pub mod execution; pub mod genesis; +pub(crate) mod history; pub mod indexer; pub mod rpc; pub mod runtime; diff --git a/examples/coins/chain/src/testnet.rs b/examples/coins/chain/src/testnet.rs index 8fc8049..9d51d91 100644 --- a/examples/coins/chain/src/testnet.rs +++ b/examples/coins/chain/src/testnet.rs @@ -48,7 +48,6 @@ use std::{ }; use tracing::{info, warn, Level}; -const FREEZER_TABLE_INITIAL_SIZE: u32 = 2u32.pow(14); const DEFAULT_MAX_BLOCK_TRANSACTIONS: usize = 4_096; const DEFAULT_MAX_MESSAGE_SIZE: u32 = 1024 * 1024; const DEFAULT_CHANNEL_BACKLOG: usize = 1024; @@ -137,6 +136,18 @@ pub struct NodeConfig { pub genesis_path: Option, #[serde(default)] pub indexer_url: Option, + /// Maximum self-contained finalized payloads retained while the indexer is unavailable. + #[serde(default = "default_indexer_spool_max_entries")] + pub indexer_spool_max_entries: u64, + /// Maximum logical encoded payload bytes retained by the indexer spool. + #[serde(default = "default_indexer_spool_max_bytes")] + pub indexer_spool_max_bytes: u64, + /// Maximum encoded size accepted for one finalized payload. + #[serde(default = "default_indexer_spool_max_payload_bytes")] + pub indexer_spool_max_payload_bytes: u64, + /// Maximum payload age before visible terminal expiry. + #[serde(default = "default_indexer_spool_max_age_seconds")] + pub indexer_spool_max_age_seconds: u64, pub consensus: ConsensusConfig, pub networking: NetworkConfig, /// Enable one-time peer QMDB state sync for a fresh joining node. @@ -157,6 +168,22 @@ impl NodeConfig { } } +fn default_indexer_spool_max_entries() -> u64 { + indexer::SpoolLimits::default().max_entries +} + +fn default_indexer_spool_max_bytes() -> u64 { + indexer::SpoolLimits::default().max_bytes +} + +fn default_indexer_spool_max_payload_bytes() -> u64 { + indexer::SpoolLimits::default().max_payload_bytes +} + +fn default_indexer_spool_max_age_seconds() -> u64 { + indexer::SpoolLimits::default().max_age.as_secs() +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct BootstrapperConfig { pub public_key: String, @@ -390,6 +417,10 @@ pub fn generate_local_testnet(config: LocalTestnetConfig) -> Result> = ContinueOnUpdate::boxed(); diff --git a/examples/coins/chain/src/tests/indexer.rs b/examples/coins/chain/src/tests/indexer.rs index b26e49e..d26399b 100644 --- a/examples/coins/chain/src/tests/indexer.rs +++ b/examples/coins/chain/src/tests/indexer.rs @@ -1,6 +1,6 @@ use crate::{ indexer::{ - BackfillDecision, BackfillPhase, BackfillResetReason, BackfillWaitReason, + BackfillDecision, BackfillPhase, BackfillWaitReason, BlockMetricSource, DkgUploadStatus, HttpArtifact, IndexerMetrics, LiveUploadArtifact, ProducerActivity, ProducerStatus, QueueReadSource, QueueStatus, SharedCacheSource, SharedRetentionReason, @@ -346,10 +346,13 @@ fn shared_state_metrics_use_expected_labels() { uploaded_digests: 4, latest_finalized_height: 8, acked_through_height: 6, + spool_entries: 5, + spool_logical_bytes: 512, + spool_oldest_height: 2, + spool_oldest_enqueued_at_millis: 1_000, }); metrics.shared_cache_inserted(SharedCacheSource::ProducerRecord); metrics.shared_cache_inserted(SharedCacheSource::LiveCertificate); - metrics.shared_cache_inserted(SharedCacheSource::ConsumerMarshal); metrics.shared_cache_removed(SharedRetentionReason::Uploaded); metrics.shared_cache_removed(SharedRetentionReason::CertificateFinished); metrics.shared_cache_removed(SharedRetentionReason::Pruned); @@ -373,9 +376,6 @@ fn shared_state_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_shared_cache_insert_total{source=\"live_certificate\"} 1", )); - assert!(encoded.contains( - "indexer_shared_cache_insert_total{source=\"consumer_marshal\"} 1", - )); assert!(encoded.contains( "indexer_shared_cache_remove_total{reason=\"uploaded\"} 1", )); @@ -402,7 +402,6 @@ fn block_metrics_use_expected_sources() { metrics.observe_block(BlockMetricSource::ProducerRecord, &block); metrics.observe_block(BlockMetricSource::LiveCertificate, &block); metrics.observe_block(BlockMetricSource::ConsumerCached, &block); - metrics.observe_block(BlockMetricSource::ConsumerMarshal, &block); let encoded = context.encode(); assert!(encoded.contains("indexer_block_estimated_bytes_bucket")); @@ -410,7 +409,6 @@ fn block_metrics_use_expected_sources() { assert!(encoded.contains("source=\"producer_record\"")); assert!(encoded.contains("source=\"live_certificate\"")); assert!(encoded.contains("source=\"consumer_cached\"")); - assert!(encoded.contains("source=\"consumer_marshal\"")); }); } @@ -433,20 +431,11 @@ fn backfill_metrics_use_expected_labels() { upload.skipped(); } metrics.backfill_decision(BackfillDecision::Skip, BackfillPhase::Start); - metrics.backfill_decision(BackfillDecision::Wait, BackfillPhase::BeforeBlock); metrics.backfill_decision(BackfillDecision::Proceed, BackfillPhase::BeforeAttempt); metrics.backfill_retry(BackfillWaitReason::CertificateUpload); - metrics.backfill_retry(BackfillWaitReason::MissingBlock); - metrics.backfill_retry(BackfillWaitReason::MissingFinalization); - metrics.backfill_retry(BackfillWaitReason::MismatchedFinalization); metrics.backfill_retry(BackfillWaitReason::HttpError); metrics.backfill_waited(BackfillWaitReason::CertificateUpload, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MissingBlock, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MissingFinalization, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MismatchedFinalization, Duration::from_millis(1)); metrics.backfill_waited(BackfillWaitReason::HttpError, Duration::from_millis(1)); - metrics.backfill_queue_reset(BackfillResetReason::MissingFinalization, 32, 31); - metrics.backfill_queue_reset(BackfillResetReason::MismatchedFinalization, 2, 1); metrics.queue_enqueued(QueueStatus::Success); metrics.queue_enqueued(QueueStatus::Failure); metrics.queue_acked(QueueStatus::Success); @@ -486,9 +475,6 @@ fn backfill_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_backfill_decision_total{decision=\"skip\",phase=\"start\"} 1", )); - assert!(encoded.contains( - "indexer_backfill_decision_total{decision=\"wait\",phase=\"before_block\"} 1", - )); assert!(encoded.contains( "indexer_backfill_decision_total{decision=\"proceed\",phase=\"before_attempt\"} 1", )); @@ -496,20 +482,7 @@ fn backfill_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_backfill_retry_total{reason=\"certificate_upload\"} 1", )); - assert!(encoded.contains("indexer_backfill_retry_total{reason=\"missing_block\"} 1")); - assert!(encoded - .contains("indexer_backfill_retry_total{reason=\"missing_finalization\"} 1")); - assert!(encoded - .contains("indexer_backfill_retry_total{reason=\"mismatched_finalization\"} 1")); assert!(encoded.contains("indexer_backfill_retry_total{reason=\"http_error\"} 1")); - assert!(encoded.contains( - "indexer_backfill_queue_reset_total{reason=\"missing_finalization\"} 1", - )); - assert!(encoded.contains( - "indexer_backfill_queue_reset_total{reason=\"mismatched_finalization\"} 1", - )); - assert!(encoded.contains("indexer_backfill_queue_reset_abandoned_entries_bucket")); - assert!(encoded.contains("indexer_backfill_queue_reset_abandoned_height_span_bucket")); assert!(encoded.contains("indexer_backfill_active_block_estimated_bytes 0")); assert!(encoded.contains("indexer_backfill_active_body_estimated_bytes 0")); assert!(encoded.contains("indexer_queue_enqueue_total{status=\"success\"} 1")); diff --git a/examples/coins/chain/tests/common/network.rs b/examples/coins/chain/tests/common/network.rs index 04495cb..843caa3 100644 --- a/examples/coins/chain/tests/common/network.rs +++ b/examples/coins/chain/tests/common/network.rs @@ -38,7 +38,6 @@ use std::{ time::Duration, }; -const FREEZER_TABLE_INITIAL_SIZE: u32 = 2u32.pow(14); // 1MB const TEST_QUOTA: Quota = Quota::per_second(NZU32!(u32::MAX)); const MAX_BLOCK_TRANSACTIONS: usize = 256; @@ -512,8 +511,6 @@ async fn start_validator( blocker: oracle.control(public_key.clone()), manager: oracle.manager(), partition_prefix: uid.clone(), - blocks_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, - finalized_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, signer: signer.clone(), dkg_storage_key: [9u8; 32], output, @@ -527,6 +524,7 @@ async fn start_validator( pool_config: PoolConfig::default(), genesis: None, indexer: None, + indexer_spool_limits: Default::default(), }; let validator_context = context.child("validator").with_attribute("id", &uid); From fe267437f8c2c59b6baf9a1556191f7cfcc32a0d Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Wed, 22 Jul 2026 19:48:16 +0700 Subject: [PATCH 3/8] fix(dkg): handle closed consensus tasks at epoch rollover Treat Closed as an expected result when retiring an aborted consensus engine while preserving fatal handling for genuine task failures. Make the coins engine epoch length internally configurable and add a deterministic regression test covering epoch transition, partition cleanup, continued finalization, and clean shutdown. --- dkg/src/orchestrator/actor.rs | 5 +- examples/coins/chain/src/engine.rs | 15 ++-- examples/coins/chain/src/testnet.rs | 3 +- examples/coins/chain/tests/coins.rs | 54 ++++++++++++- examples/coins/chain/tests/common/network.rs | 85 +++++++++++++++++++- 5 files changed, 149 insertions(+), 13 deletions(-) diff --git a/dkg/src/orchestrator/actor.rs b/dkg/src/orchestrator/actor.rs index 5a250d0..060b701 100644 --- a/dkg/src/orchestrator/actor.rs +++ b/dkg/src/orchestrator/actor.rs @@ -452,8 +452,11 @@ where continue; }; handle.abort(); + // Spawned task handles close their completion channel when aborted. match handle.await { - Ok(()) | Err(commonware_runtime::Error::Aborted) => {} + Ok(()) + | Err(commonware_runtime::Error::Aborted) + | Err(commonware_runtime::Error::Closed) => {} Err(error) => { panic!("consensus engine for epoch {epoch} failed while stopping: {error}") } diff --git a/examples/coins/chain/src/engine.rs b/examples/coins/chain/src/engine.rs index 3247e10..2930cbf 100644 --- a/examples/coins/chain/src/engine.rs +++ b/examples/coins/chain/src/engine.rs @@ -3,10 +3,7 @@ use crate::execution::NodeHandle; use crate::genesis::{genesis_target, state_commitment, ChainGenesis}; use crate::history::{self, BlocksArchive, FinalizationsArchive}; use crate::indexer; -use crate::{ - Block, EpochProvider, Provider, PublicKey, Scheme, Transaction, BLOCKS_PER_EPOCH, - NAMESPACE, -}; +use crate::{Block, EpochProvider, Provider, PublicKey, Scheme, Transaction, NAMESPACE}; use commonware_broadcast::buffered; use commonware_consensus::{ marshal::{ @@ -57,6 +54,7 @@ use nunchi_mempool::{Mempool, PoolConfig}; use rand::{CryptoRng, Rng}; use std::{ marker::PhantomData, + num::NonZeroU64, sync::Arc, time::{Duration, Instant}, }; @@ -85,6 +83,7 @@ pub struct Config, P: Manager, pub share: Option, pub peer_config: PeerConfig, + pub epoch_length: NonZeroU64, pub leader_timeout: Duration, pub certification_timeout: Duration, pub strategy: S, @@ -228,7 +227,7 @@ where max_supported_mode: MAX_SUPPORTED_MODE, namespace: NAMESPACE.to_vec(), storage_protector: dkg::StorageProtector::new(config.dkg_storage_key), - epoch_length: BLOCKS_PER_EPOCH, + epoch_length: config.epoch_length, }, ); @@ -459,7 +458,7 @@ where finalized_blocks, marshal::Config { provider: provider.clone(), - epocher: FixedEpocher::new(BLOCKS_PER_EPOCH), + epocher: FixedEpocher::new(config.epoch_length), start: marshal_start, partition_prefix: format!("{}_marshal", config.partition_prefix), mailbox_size: MAILBOX_SIZE, @@ -513,7 +512,7 @@ where APPLICATION_VERIFY_CONCURRENCY, ), marshal_mailbox.clone(), - FixedEpocher::new(BLOCKS_PER_EPOCH), + FixedEpocher::new(config.epoch_length), )); let (indexer_producer, indexer_producer_handle, indexer_pusher, indexer_consumer) = @@ -576,7 +575,7 @@ where muxer_size: MAILBOX_SIZE.get(), mailbox_size: MAILBOX_SIZE, partition_prefix: format!("{}_consensus", config.partition_prefix), - epoch_length: BLOCKS_PER_EPOCH, + epoch_length: config.epoch_length, genesis_digest, recovered_floor, _phantom: PhantomData, diff --git a/examples/coins/chain/src/testnet.rs b/examples/coins/chain/src/testnet.rs index 9d51d91..feaf62a 100644 --- a/examples/coins/chain/src/testnet.rs +++ b/examples/coins/chain/src/testnet.rs @@ -10,7 +10,7 @@ use crate::{ channels, engine::{Config as EngineConfig, Engine}, genesis::{ChainGenesis, GenesisError}, - indexer, rpc, PublicKey, NAMESPACE, + indexer, rpc, PublicKey, BLOCKS_PER_EPOCH, NAMESPACE, }; use commonware_codec::{Decode, DecodeExt, Encode, EncodeSize}; use commonware_consensus::marshal; @@ -640,6 +640,7 @@ async fn start_node( output, share: Some(share), peer_config: config.peer_config.clone(), + epoch_length: BLOCKS_PER_EPOCH, leader_timeout: Duration::from_millis(config.consensus.leader_timeout_ms), certification_timeout: Duration::from_millis(config.consensus.certification_timeout_ms), strategy: context diff --git a/examples/coins/chain/tests/coins.rs b/examples/coins/chain/tests/coins.rs index 9051289..da9ec09 100644 --- a/examples/coins/chain/tests/coins.rs +++ b/examples/coins/chain/tests/coins.rs @@ -8,7 +8,8 @@ use commonware_cryptography::Signer as _; use commonware_cryptography::{Hasher, Sha256}; use commonware_macros::{select, test_traced}; use commonware_p2p::simulated::Link; -use commonware_runtime::{deterministic, Clock, Runner as _}; +use commonware_runtime::{deterministic, Clock, Runner as _, Spawner as _, Supervisor as _}; +use commonware_utils::NZU64; use nunchi_authority::{ proposal_id, AuthorityOperation, MultisigPolicy, RegistryChange, Transaction as AuthorityTransaction, @@ -104,6 +105,56 @@ fn reaches_height_100() { }); } +#[test_traced] +fn crosses_epoch_boundary_and_reclaims_retired_partition() { + with_large_stack(|| { + let executor = deterministic::Runner::timed(Duration::from_secs(60)); + executor.start(|mut context| async move { + let cfg = ValidatorConfig { + epoch_length: NZU64!(20), + ..ValidatorConfig::default() + }; + let mut network = TestNetworkBuilder::new(VALIDATORS) + .with_initial_link(reliable_link()) + .with_validator_config(cfg) + .build(&mut context) + .await; + network.start_all().await; + + let validator_task_prefix = "validator"; + assert!(network.running_tasks(validator_task_prefix) > 0); + + network.run_until_height(25).await; + + network.assert_validator_metric("engine_orchestrator_latest_epoch", &[], 1); + network.assert_validator_metric( + "engine_orchestrator_consensus_partitions_active", + &[], + 1, + ); + network.assert_validator_metric( + "engine_orchestrator_consensus_partition_cleanup_watermark", + &[], + 1, + ); + network.assert_validator_metric( + "engine_orchestrator_consensus_partition_cleanup_total", + &[("status", "removed")], + 1, + ); + network.assert_consensus_partition_missing(0).await; + assert!(network.running_tasks(validator_task_prefix) > 0); + + let shutdown = network.context().child("shutdown"); + shutdown + .stop(0, Some(Duration::from_secs(10))) + .await + .expect("validator tasks should stop cleanly"); + assert_eq!(network.running_tasks(validator_task_prefix), 0); + }); + }); +} + #[test_traced] fn state_syncs_late_validator() { with_large_stack(|| { @@ -154,6 +205,7 @@ fn recovers_unclean_shutdown() { let cfg = ValidatorConfig { leader_timeout: Duration::from_millis(250), certification_timeout: Duration::from_millis(500), + ..ValidatorConfig::default() }; let wait = diff --git a/examples/coins/chain/tests/common/network.rs b/examples/coins/chain/tests/common/network.rs index 843caa3..21d473f 100644 --- a/examples/coins/chain/tests/common/network.rs +++ b/examples/coins/chain/tests/common/network.rs @@ -15,7 +15,7 @@ use commonware_p2p::{ use commonware_parallel::Sequential; use commonware_runtime::{ deterministic::{self, Runner}, - Clock, Metrics, Runner as _, Supervisor, + Clock, Error as RuntimeError, Metrics, Runner as _, Storage, Supervisor, }; use commonware_utils::{ ordered::{Map, Set}, @@ -27,7 +27,7 @@ use nunchi_coins::{Address, Ledger}; use nunchi_coins_chain::{ engine::{Config, Engine}, execution::NodeHandle, - PublicKey, Transaction, + PublicKey, Transaction, BLOCKS_PER_EPOCH, }; use nunchi_common::QmdbReader; use nunchi_dkg::{ContinueOnUpdate, PeerConfig}; @@ -35,6 +35,7 @@ use nunchi_mempool::{MempoolHandle, PoolConfig}; use nunchi_oracle::OracleLedger; use std::{ collections::{HashMap, HashSet}, + num::NonZeroU64, time::Duration, }; @@ -108,6 +109,7 @@ pub(crate) fn lossy_link() -> Link { #[derive(Clone)] pub(crate) struct ValidatorConfig { + pub(crate) epoch_length: NonZeroU64, pub(crate) leader_timeout: Duration, pub(crate) certification_timeout: Duration, } @@ -115,6 +117,7 @@ pub(crate) struct ValidatorConfig { impl Default for ValidatorConfig { fn default() -> Self { Self { + epoch_length: BLOCKS_PER_EPOCH, leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(2), } @@ -355,6 +358,83 @@ impl TestNetwork<'_> { } } + pub(crate) fn assert_validator_metric( + &self, + suffix: &str, + required_labels: &[(&str, &str)], + expected_value: u64, + ) { + let expected = self.started_validator_ids(); + let mut observed = HashSet::new(); + let metrics = self.context.encode(); + for line in metrics.lines() { + let Some((metric, labels, value)) = validator_metric_sample(line) else { + continue; + }; + if !metric.ends_with(suffix) + || !required_labels + .iter() + .all(|(name, expected)| metric_label(labels, name) == Some(*expected)) + { + continue; + } + let Some(id) = metric_label(labels, "id") else { + continue; + }; + if !expected.contains(id) { + continue; + } + assert_eq!( + value.parse::().unwrap(), + expected_value, + "unexpected {suffix} value for {id}: {line}", + ); + assert!( + observed.insert(id.to_string()), + "duplicate {suffix} for {id}" + ); + } + assert_eq!( + observed, expected, + "missing {suffix} samples in metrics: {metrics}", + ); + } + + pub(crate) async fn assert_consensus_partition_missing(&self, epoch: u64) { + for signer in &self.private_keys { + let public_key = signer.public_key(); + if self.registrations.contains_key(&public_key) { + continue; + } + let partition = format!("validator_{public_key}_consensus_consensus_{epoch}"); + let result = self.context.scan(&partition).await; + assert!( + matches!( + result, + Err(RuntimeError::PartitionMissing(ref missing)) if missing == &partition + ), + "retired consensus partition {partition} still exists: {result:?}", + ); + } + } + + pub(crate) fn running_tasks(&self, prefix: &str) -> usize { + self.context + .encode() + .lines() + .filter_map(|line| { + if !line.starts_with("runtime_tasks_running{") { + return None; + } + let name = line.split("name=\"").nth(1)?.split('"').next()?; + if !name.starts_with(prefix) { + return None; + } + line.rsplit(' ').next()?.parse::().ok() + }) + .sum() + } + fn started_validator_ids(&self) -> HashSet { self.private_keys .iter() @@ -516,6 +596,7 @@ async fn start_validator( output, share: Some(share), peer_config, + epoch_length: cfg.epoch_length, leader_timeout: cfg.leader_timeout, certification_timeout: cfg.certification_timeout, strategy: Sequential, From 0e95fef07895ecdc7bd2ab7b8d59dee97458e5e8 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Thu, 23 Jul 2026 03:03:30 +0700 Subject: [PATCH 4/8] fix clippy --- examples/coins/chain/src/history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/coins/chain/src/history.rs b/examples/coins/chain/src/history.rs index 402af05..6d39607 100644 --- a/examples/coins/chain/src/history.rs +++ b/examples/coins/chain/src/history.rs @@ -159,7 +159,7 @@ where let mut archive_existed = [false; 4]; for (exists, partition) in archive_existed .iter_mut() - .zip(partitions.archives().into_iter()) + .zip(partitions.archives()) { *exists = partition_exists(context, partition).await?; } From bc93e270729f6b67f8c4b27395529b591f969fe7 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Thu, 23 Jul 2026 12:05:20 +0700 Subject: [PATCH 5/8] feat: authenticate DKG state for cross-epoch recovery - commit canonical DKG checkpoints and dealer logs to coins-chain state - bind startup to an exact certified QMDB target - reconcile protected DKG storage with crash-safe import markers - support shareless validators until resharing provides a valid share - seed authenticated DKG state in both genesis modes - reject unsupported bridge-chain peer state sync - add transition, recovery, genesis, and startup validation tests --- chain/src/application.rs | 133 +++++- chain/src/consensus.rs | 1 + chain/src/consensus/dkg_state.rs | 338 ++++++++++++++ chain/src/execution.rs | 10 +- chain/src/lib.rs | 2 + chain/src/startup.rs | 260 +++++++++++ dkg/src/actor.rs | 418 +++++++++++++++-- dkg/src/lib.rs | 12 +- dkg/src/public.rs | 528 ++++++++++++++++++++++ dkg/src/state.rs | 102 +++++ dkg/src/tests/mod.rs | 1 + dkg/src/tests/public.rs | 187 ++++++++ dkg/src/tests/state.rs | 66 ++- examples/bridge-chain/src/engine.rs | 18 +- examples/bridge-chain/src/execution.rs | 4 +- examples/bridge-chain/src/rpc.rs | 8 +- examples/bridge-chain/src/testnet.rs | 4 +- examples/bridge-chain/src/tests/mod.rs | 9 + examples/coins/chain/src/engine.rs | 259 +++++++++-- examples/coins/chain/src/execution.rs | 20 +- examples/coins/chain/src/genesis.rs | 33 ++ examples/coins/chain/src/tests/genesis.rs | 100 +++- 22 files changed, 2369 insertions(+), 144 deletions(-) create mode 100644 chain/src/consensus/dkg_state.rs create mode 100644 chain/src/startup.rs create mode 100644 dkg/src/public.rs create mode 100644 dkg/src/tests/public.rs diff --git a/chain/src/application.rs b/chain/src/application.rs index a25332b..5fd6d3c 100644 --- a/chain/src/application.rs +++ b/chain/src/application.rs @@ -18,7 +18,7 @@ use futures::{lock::Mutex as AsyncMutex, StreamExt}; use nunchi_common::{Overlay, QmdbBatch, QmdbDatabaseSet, QmdbMerkleized, Runtime, RuntimeContext}; use nunchi_dkg::{Context, Scheme}; use nunchi_mempool::{MempoolHandle, PoolTransaction}; -use rand::Rng; +use rand::{CryptoRng, Rng}; use std::{ collections::HashMap, marker::PhantomData, @@ -29,8 +29,8 @@ use std::{ use tracing::{debug, error}; use crate::{ - Block, ConsensusExtension, DkgMailbox, EventConsumer, NoConsensusExtension, NoopEventConsumer, - StateCommitment, TransactionEventContext, + Block, ConsensusExtension, DkgMailbox, DkgState, EventConsumer, NoConsensusExtension, + NoopEventConsumer, StateCommitment, TransactionEventContext, }; /// The height of the last finalized block applied to a node's ledger. @@ -67,6 +67,7 @@ where submitter: MempoolHandle, max_block_transactions: usize, dkg: Option>, + dkg_state: Option, consensus: Ext, events: Events, applied_height: SharedAppliedHeight, @@ -83,6 +84,7 @@ struct ApplicationMetrics { proposal_validate_duration: Histogram, proposal_merkleize_duration: Histogram, apply_transactions_duration: Histogram, + dkg_state_duration: Histogram, apply_merkleize_duration: Histogram, } @@ -109,6 +111,11 @@ impl ApplicationMetrics { "duration spent applying block transactions", Buckets::LOCAL, ), + dkg_state_duration: context.histogram( + "dkg_public_state_duration_seconds", + "duration spent validating and transitioning authenticated public DKG state", + Buckets::LOCAL, + ), apply_merkleize_duration: context.histogram( "apply_merkleize_duration_seconds", "duration spent merkleizing applied block state", @@ -162,6 +169,7 @@ where submitter, max_block_transactions, dkg, + dkg_state: None, consensus, events, applied_height, @@ -273,17 +281,21 @@ where Some((included, merkleized)) } - async fn build_proposal_state( + async fn build_proposal_state< + E: BufferPooler + Storage + Clock + Metrics + CryptoRng, + >( &mut self, - timing: Option<(&E, &ApplicationMetrics)>, + timing: Option<&ApplicationMetrics>, batches: as DatabaseSet>::Unmerkleized, context: RuntimeContext, candidates: Vec, + reshare_log: Option<&nunchi_dkg::DealerLog>, + rng: &mut E, ) -> Option<(Vec, Ext::Payload, QmdbMerkleized)> { let mut batch = QmdbBatch::new(batches); let mut included = Vec::new(); - let validate_start = timing.map(|(runtime_context, _)| runtime_context.current()); + let validate_start = timing.map(|_| rng.current()); for transaction in candidates { if included.len() == self.max_block_transactions { break; @@ -303,10 +315,10 @@ where } } } - if let Some(((runtime_context, metrics), validate_start)) = timing.zip(validate_start) { + if let Some((metrics, validate_start)) = timing.zip(validate_start) { metrics .proposal_validate_duration - .observe_between(validate_start, runtime_context.current()); + .observe_between(validate_start, rng.current()); } let extension = self.consensus.propose().await; @@ -317,8 +329,28 @@ where { return None; } + let dkg_start = timing.map(|_| rng.current()); + if let Some(dkg_state) = &self.dkg_state { + if let Err(error) = dkg_state + .apply_block( + &mut batch, + Height::new(context.height), + reshare_log, + rng, + ) + .await + { + debug!(?error, "invalid authenticated DKG public-state update"); + return None; + } + } + if let Some((metrics, dkg_start)) = timing.zip(dkg_start) { + metrics + .dkg_state_duration + .observe_between(dkg_start, rng.current()); + } - let merkleize_start = timing.map(|(runtime_context, _)| runtime_context.current()); + let merkleize_start = timing.map(|_| rng.current()); let merkleized = match batch.merkleize().await { Ok(merkleized) => merkleized, Err(error) => { @@ -326,10 +358,10 @@ where return None; } }; - if let Some(((runtime_context, metrics), merkleize_start)) = timing.zip(merkleize_start) { + if let Some((metrics, merkleize_start)) = timing.zip(merkleize_start) { metrics .proposal_merkleize_duration - .observe_between(merkleize_start, runtime_context.current()); + .observe_between(merkleize_start, rng.current()); } Some((included, extension, merkleized)) } @@ -337,17 +369,18 @@ where #[allow(clippy::too_many_arguments)] async fn execute_block( &mut self, - runtime_context: &E, + runtime_context: &mut E, metrics: &ApplicationMetrics, batches: as DatabaseSet>::Unmerkleized, context: RuntimeContext, transactions: &[R::Transaction], extension: &Ext::Payload, + reshare_log: Option<&nunchi_dkg::DealerLog>, events: &EventHandler, commit_extension: bool, ) -> Option> where - E: BufferPooler + Storage + Clock + Metrics, + E: BufferPooler + Storage + Clock + Metrics + CryptoRng, EventHandler: EventConsumer, { events.begin_block(context).await; @@ -389,6 +422,27 @@ where .commit_payload(&mut batch, context, extension) .await; } + let dkg_start = runtime_context.current(); + if let Some(dkg_state) = &self.dkg_state { + if let Err(error) = dkg_state + .apply_block( + &mut batch, + Height::new(context.height), + reshare_log, + runtime_context, + ) + .await + { + debug!(?error, "invalid authenticated DKG public-state update"); + if let Some(digest) = context.block_digest { + events.discard_block(digest).await; + } + return None; + } + } + metrics + .dkg_state_duration + .observe_between(dkg_start, runtime_context.current()); metrics .apply_transactions_duration .observe_between(apply_start, runtime_context.current()); @@ -476,6 +530,31 @@ where dkg, ) } + + /// Construct an application whose DKG progress is authenticated by QMDB. + #[allow(clippy::too_many_arguments)] + pub fn with_authenticated_dkg( + submitter: MempoolHandle, + max_block_transactions: usize, + consensus: Ext, + dkg: DkgMailbox, + dkg_state: DkgState, + applied_height: SharedAppliedHeight, + genesis_state: StateCommitment, + genesis_payload: sha256::Digest, + ) -> Self { + let mut application = Self::with_consensus( + submitter, + max_block_transactions, + consensus, + Some(dkg), + applied_height, + genesis_state, + genesis_payload, + ); + application.dkg_state = Some(dkg_state); + application + } } impl Application @@ -590,7 +669,7 @@ where impl StatefulApplication for Application where - E: Rng + Spawner + Metrics + Clock + Storage + BufferPooler, + E: Rng + CryptoRng + Spawner + Metrics + Clock + Storage + BufferPooler, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -612,7 +691,7 @@ where async fn propose( &mut self, - (runtime_context, context): (E, Self::Context), + (mut runtime_context, context): (E, Self::Context), ancestry: impl futures::Stream> + Send, batches: >::Unmerkleized, input: &mut Self::InputProvider, @@ -631,19 +710,23 @@ where parent.height.next(), timestamp, ); + // Obtain the optional dealer log before executing the speculative + // state batch so that it is committed by the resulting state root. + let reshare_log = match &mut self.dkg { + Some(dkg) => dkg.act().await, + None => None, + }; let (transactions, extension, merkleized) = self .build_proposal_state( - Some((&runtime_context, &metrics)), + Some(&metrics), batches, execution_context, candidates, + reshare_log.as_ref(), + &mut runtime_context, ) .await?; let state_range = Self::state_range(&merkleized); - let reshare_log = match &mut self.dkg { - Some(dkg) => dkg.act().await, - None => None, - }; let block = Block::new( context, parent.digest(), @@ -662,7 +745,7 @@ where async fn verify( &mut self, - (runtime_context, _): (E, Self::Context), + (mut runtime_context, _): (E, Self::Context), ancestry: impl futures::Stream> + Send, batches: >::Unmerkleized, ) -> Option<>::Merkleized> { @@ -700,12 +783,13 @@ where let execution_context = Self::block_runtime_context(&block); let merkleized = self .execute_block( - &runtime_context, + &mut runtime_context, &metrics, batches, execution_context, &block.transactions, &block.extension, + block.reshare_log.as_ref(), &NoopEventConsumer, false, ) @@ -719,7 +803,7 @@ where async fn apply( &mut self, - (runtime_context, _): (E, Self::Context), + (mut runtime_context, _): (E, Self::Context), block: &Self::Block, batches: >::Unmerkleized, ) -> >::Merkleized { @@ -728,12 +812,13 @@ where let events = self.events.clone(); let merkleized = self .execute_block( - &runtime_context, + &mut runtime_context, &metrics, batches, execution_context, &block.transactions, &block.extension, + block.reshare_log.as_ref(), &events, true, ) diff --git a/chain/src/consensus.rs b/chain/src/consensus.rs index e38198f..7b1ce37 100644 --- a/chain/src/consensus.rs +++ b/chain/src/consensus.rs @@ -1,4 +1,5 @@ mod dkg; +pub mod dkg_state; mod extension; pub use dkg::{dkg_reporters, DkgActor, DkgMailbox, DkgReporters}; diff --git a/chain/src/consensus/dkg_state.rs b/chain/src/consensus/dkg_state.rs new file mode 100644 index 0000000..a223448 --- /dev/null +++ b/chain/src/consensus/dkg_state.rs @@ -0,0 +1,338 @@ +//! Authenticated public DKG state stored in the chain QMDB. + +use commonware_codec::{Encode, Read}; +use commonware_consensus::types::{Epocher, FixedEpocher, Height}; +use commonware_cryptography::{ + bls12381::primitives::variant::MinSig, + ed25519::{self, Batch}, +}; +use commonware_parallel::Sequential; +use nunchi_common::{ + CommitState, Namespace, StateError, StateStore, MAX_STATE_VALUE_SIZE, +}; +use nunchi_dkg::{ + public_transition, DkgProtocolConfig, PublicCheckpoint, STATE_FORMAT_VERSION, +}; +use rand::CryptoRng; + +const DKG_NAMESPACE: Namespace = Namespace::new(b"_NUNCHI_CHAIN_DKG_STATE"); + +#[repr(u8)] +enum Table { + Marker = 0, + Checkpoint = 1, + Log = 2, +} + +impl From for u8 { + fn from(value: Table) -> Self { + value as Self + } +} + +/// Authenticated DKG state configuration installed in a DKG-enabled application. +#[derive(Clone)] +pub struct DkgState { + config: DkgProtocolConfig, + max_participants: std::num::NonZeroU32, +} + +impl DkgState { + pub fn new(config: DkgProtocolConfig) -> Result { + config.validate()?; + let max_participants = std::num::NonZeroU32::new( + config.max_participants_per_round(), + ) + .ok_or(Error::InvalidConfiguration)?; + validate_size_bounds(max_participants)?; + Ok(Self { + config, + max_participants, + }) + } + + pub const fn config(&self) -> &DkgProtocolConfig { + &self.config + } + + /// Stage the format marker and epoch-zero checkpoint. + pub async fn seed( + &self, + state: &mut S, + empty_root: commonware_cryptography::sha256::Digest, + initial_output: commonware_cryptography::bls12381::dkg::feldman_desmedt::Output< + MinSig, + ed25519::PublicKey, + >, + ) -> Result { + let expected = PublicCheckpoint::genesis(&self.config, initial_output)?; + let marker = self.marker_value()?; + match state.get(&marker_key()).await? { + Some(existing) if existing != marker => return Err(Error::MismatchedMarker), + Some(_) => { + let checkpoint = self.load_checkpoint(state).await?; + if checkpoint.epoch == commonware_consensus::types::Epoch::zero() + && checkpoint != expected + { + return Err(Error::MismatchedInitialCheckpoint); + } + return Ok(checkpoint); + } + None if state.root() != empty_root => return Err(Error::UnmarkedState), + None => {} + } + put_bounded(state, checkpoint_key(), expected.encode().to_vec())?; + put_bounded(state, marker_key(), marker)?; + Ok(expected) + } + + /// Require the current state format marker and load its checkpoint. + pub async fn load_checkpoint( + &self, + state: &S, + ) -> Result { + let marker = state + .get(&marker_key()) + .await? + .ok_or(Error::MissingMarker)?; + if marker != self.marker_value()? { + return Err(Error::MismatchedMarker); + } + let raw = state + .get(&checkpoint_key()) + .await? + .ok_or(Error::MissingCheckpoint)?; + let checkpoint = decode_checkpoint(&raw, self.max_participants)?; + self.config.validate_checkpoint(&checkpoint)?; + Ok(checkpoint) + } + + /// Validate and stage one block's public DKG effects. + pub async fn apply_block( + &self, + state: &mut S, + height: Height, + signed_log: Option<&nunchi_dkg::DealerLog>, + rng: &mut R, + ) -> Result<(), Error> + where + S: StateStore + Send + Sync, + R: CryptoRng, + { + let checkpoint = self.load_checkpoint(state).await?; + let info = self.config.round_info(&checkpoint)?; + if let Some(signed_log) = signed_log { + let (dealer, _) = signed_log + .clone() + .check(&info) + .ok_or(Error::InvalidDealerLog)?; + let eligible = self + .config + .participants_for_round(checkpoint.successful_round); + if eligible.position(&dealer).is_none() { + return Err(Error::IneligibleDealer); + } + let key = log_key(checkpoint.epoch.get(), &dealer); + let encoded = signed_log.encode(); + if let Some(existing) = state.get(&key).await? { + let existing = decode_log(&existing, self.max_participants)?; + if existing != *signed_log { + return Err(Error::ConflictingDealerLog); + } + } else { + put_bounded(state, key, encoded.to_vec())?; + } + } + + let epocher = FixedEpocher::new(self.config.epoch_length); + if epocher.last(checkpoint.epoch) != Some(height) { + return Ok(()); + } + + let eligible = self + .config + .participants_for_round(checkpoint.successful_round); + let mut logs = Vec::new(); + for dealer in &eligible { + let key = log_key(checkpoint.epoch.get(), dealer); + if let Some(raw) = state.get(&key).await? { + logs.push(decode_log(&raw, self.max_participants)?); + } + } + let next = public_transition::( + &self.config, + &checkpoint, + logs, + height, + rng, + &Sequential, + )?; + + // The checkpoint and current-log set change atomically in this batch. + for dealer in &eligible { + state.remove(log_key(checkpoint.epoch.get(), dealer)); + } + put_bounded(state, checkpoint_key(), next.checkpoint.encode().to_vec())?; + Ok(()) + } + + /// Load all current finalized signed logs from authenticated state. + pub async fn load_logs( + &self, + state: &S, + ) -> Result, Error> { + let checkpoint = self.load_checkpoint(state).await?; + let eligible = self + .config + .participants_for_round(checkpoint.successful_round); + let mut logs = Vec::new(); + for dealer in &eligible { + if let Some(raw) = state + .get(&log_key(checkpoint.epoch.get(), dealer)) + .await? + { + logs.push(decode_log(&raw, self.max_participants)?); + } + } + Ok(logs) + } + + fn marker_value(&self) -> Result, Error> { + Ok((STATE_FORMAT_VERSION, self.config.digest()?).encode().to_vec()) + } +} + +fn validate_size_bounds(max_participants: std::num::NonZeroU32) -> Result<(), Error> { + let participants = usize::try_from(max_participants.get()) + .map_err(|_| Error::InvalidConfiguration)?; + // Conservative codec formulas use the larger BLS element size and charge + // every player result for an identity, tag, and Ed25519 signature. + let polynomial = participants + .checked_mul(96) + .and_then(|size| size.checked_add(16)) + .ok_or(Error::InvalidConfiguration)?; + let ordered_set = participants + .checked_mul(32) + .and_then(|size| size.checked_add(10)) + .ok_or(Error::InvalidConfiguration)?; + let checkpoint = polynomial + .checked_add( + ordered_set + .checked_mul(3) + .ok_or(Error::InvalidConfiguration)?, + ) + .and_then(|size| size.checked_add(128)) + .ok_or(Error::InvalidConfiguration)?; + let dealer_log = polynomial + .checked_add( + participants + .checked_mul(32 + 1 + 64) + .ok_or(Error::InvalidConfiguration)?, + ) + .and_then(|size| size.checked_add(32 + 64 + 32)) + .ok_or(Error::InvalidConfiguration)?; + if checkpoint > MAX_STATE_VALUE_SIZE || dealer_log > MAX_STATE_VALUE_SIZE { + return Err(Error::ConfigurationTooLarge { + checkpoint, + dealer_log, + maximum: MAX_STATE_VALUE_SIZE, + }); + } + Ok(()) +} + +fn put_bounded( + state: &mut S, + key: commonware_cryptography::sha256::Digest, + value: Vec, +) -> Result<(), Error> { + if value.len() > MAX_STATE_VALUE_SIZE { + return Err(Error::ValueTooLarge(value.len())); + } + state.set(key, value); + Ok(()) +} + +fn decode_checkpoint( + raw: &[u8], + max_participants: std::num::NonZeroU32, +) -> Result { + let mut buf = raw; + let checkpoint = PublicCheckpoint::read_cfg( + &mut buf, + &(max_participants, nunchi_dkg::MAX_SUPPORTED_MODE), + )?; + if !buf.is_empty() { + return Err(Error::TrailingBytes); + } + Ok(checkpoint) +} + +fn decode_log( + raw: &[u8], + max_participants: std::num::NonZeroU32, +) -> Result { + let mut buf = raw; + let log = nunchi_dkg::DealerLog::read_cfg(&mut buf, &max_participants)?; + if !buf.is_empty() { + return Err(Error::TrailingBytes); + } + Ok(log) +} + +fn marker_key() -> commonware_cryptography::sha256::Digest { + DKG_NAMESPACE.key(Table::Marker, &[]) +} + +fn checkpoint_key() -> commonware_cryptography::sha256::Digest { + DKG_NAMESPACE.key(Table::Checkpoint, &[]) +} + +fn log_key( + epoch: u64, + dealer: &ed25519::PublicKey, +) -> commonware_cryptography::sha256::Digest { + let mut logical = epoch.encode().to_vec(); + logical.extend_from_slice(&dealer.encode()); + DKG_NAMESPACE.key(Table::Log, &logical) +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid DKG state configuration")] + InvalidConfiguration, + #[error( + "DKG configuration exceeds QMDB value bounds: checkpoint {checkpoint}, dealer log {dealer_log}, maximum {maximum}" + )] + ConfigurationTooLarge { + checkpoint: usize, + dealer_log: usize, + maximum: usize, + }, + #[error("authenticated state error: {0}")] + State(#[from] StateError), + #[error("public DKG state error: {0}")] + Public(#[from] nunchi_dkg::public::Error), + #[error("public DKG state codec error: {0}")] + Codec(#[from] commonware_codec::Error), + #[error("public DKG state value contains trailing bytes")] + TrailingBytes, + #[error("authenticated state has no DKG format marker; re-genesis or perform verified peer state sync")] + MissingMarker, + #[error("authenticated state is non-empty but has no DKG format marker; re-genesis or perform verified peer state sync")] + UnmarkedState, + #[error("authenticated state DKG format or protocol configuration differs; re-genesis required")] + MismatchedMarker, + #[error("authenticated state has no DKG checkpoint")] + MissingCheckpoint, + #[error("existing authenticated DKG genesis checkpoint differs")] + MismatchedInitialCheckpoint, + #[error("dealer log is invalid for the authenticated checkpoint")] + InvalidDealerLog, + #[error("dealer is not eligible for the authenticated checkpoint round")] + IneligibleDealer, + #[error("dealer submitted a conflicting finalized log")] + ConflictingDealerLog, + #[error("authenticated DKG value size {0} exceeds the QMDB limit")] + ValueTooLarge(usize), +} diff --git a/chain/src/execution.rs b/chain/src/execution.rs index dd41431..39fbd82 100644 --- a/chain/src/execution.rs +++ b/chain/src/execution.rs @@ -18,7 +18,7 @@ use crate::{ #[derive(Clone)] pub struct NodeHandle where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -31,7 +31,7 @@ where impl NodeHandle where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -58,7 +58,7 @@ where /// Read-only queries answered from the stateful actor's committed databases. pub struct StatefulQuery where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -69,7 +69,7 @@ where impl Clone for StatefulQuery where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -84,7 +84,7 @@ where impl StatefulQuery where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, diff --git a/chain/src/lib.rs b/chain/src/lib.rs index 74244a4..a69a12f 100644 --- a/chain/src/lib.rs +++ b/chain/src/lib.rs @@ -8,6 +8,7 @@ pub mod engine; pub mod events; pub mod execution; pub mod state_sync; +pub mod startup; mod macros; #[cfg(test)] mod tests; @@ -18,6 +19,7 @@ pub use consensus::{ dkg_reporters, BlockExtension, Composite, ConsensusExtension, DkgActor, DkgMailbox, DkgReporters, NoConsensusExtension, }; +pub use consensus::dkg_state::{DkgState, Error as DkgStateError}; pub use events::{ EventConsumer, FinalizedEvents, InMemoryEventConsumer, IndexedEvent, NoopEventConsumer, TransactionEventContext, TransactionEvents, diff --git a/chain/src/startup.rs b/chain/src/startup.rs new file mode 100644 index 0000000..fa7bc36 --- /dev/null +++ b/chain/src/startup.rs @@ -0,0 +1,260 @@ +//! Exact binding between an attached QMDB and a certified startup block. + +use commonware_actor::Feedback; +use commonware_consensus::{ + marshal::Update, + types::Height, + Block, Heightable, Reporter, +}; +use commonware_cryptography::{sha256::Digest, Digestible}; +use commonware_storage::{ + mmr::Family, + qmdb::sync::Target, +}; +use commonware_utils::Acknowledgement as _; +use std::{ + collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, + sync::{Arc, Mutex}, +}; + +/// A bounded startup candidate recorded from canonical local history or a +/// certificate-verified peer-sync floor. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StartupCandidate { + pub height: Height, + pub digest: Digest, + pub state_target: Target, + pub certificate_payload: Option, + pub genesis: bool, +} + +/// The single certified block whose complete target equals attached QMDB. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StartupArtifact { + pub anchor_height: Height, + pub anchor_digest: Digest, + pub anchor_state_target: Target, +} + +/// Bounded candidate collector used only during startup. +pub struct StartupCoordinator { + capacity: NonZeroUsize, + candidates: BTreeMap<(Height, Digest), StartupCandidate>, + failure: Option, +} + +impl StartupCoordinator { + pub fn new(capacity: NonZeroUsize) -> Self { + Self { + capacity, + candidates: BTreeMap::new(), + failure: None, + } + } + + /// Record a candidate, deduplicating the same height and digest. + pub fn record(&mut self, candidate: StartupCandidate) -> Result<(), Error> { + let key = (candidate.height, candidate.digest); + if let Some(existing) = self.candidates.get(&key) { + return (existing == &candidate) + .then_some(()) + .ok_or(Error::ConflictingCandidate); + } + if self.candidates.len() == self.capacity.get() { + return Err(Error::CapacityExceeded); + } + self.candidates.insert(key, candidate); + Ok(()) + } + + /// Match the complete root and operation range and require exactly one + /// certificate-bound candidate. + pub fn resolve( + &mut self, + attached: &Target, + ) -> Result { + if let Some(error) = self.failure.take() { + return Err(error); + } + let mut matches = self + .candidates + .values() + .filter(|candidate| &candidate.state_target == attached); + let candidate = matches.next().ok_or(Error::NoExactMatch)?; + if matches.next().is_some() { + return Err(Error::MultipleExactMatches); + } + if candidate.genesis { + if candidate.height != Height::zero() + || candidate.certificate_payload.is_some() + { + return Err(Error::InvalidGenesisCandidate); + } + } else if candidate.certificate_payload != Some(candidate.digest) { + return Err(Error::CertificatePayloadMismatch); + } + let artifact = StartupArtifact { + anchor_height: candidate.height, + anchor_digest: candidate.digest, + anchor_state_target: candidate.state_target.clone(), + }; + self.candidates.clear(); + Ok(artifact) + } + + fn fail(&mut self, error: Error) { + self.failure.get_or_insert(error); + } +} + +/// Reporter that records selected certified startup blocks and immediately +/// acknowledges its `Exact` clone. +#[derive(Clone)] +pub struct StartupReporter { + coordinator: Arc>, + certified_payloads: BTreeSet, + target: fn(&B) -> Target, +} + +impl> StartupReporter { + pub fn new( + coordinator: Arc>, + certified_payloads: impl IntoIterator, + target: fn(&B) -> Target, + ) -> Self { + Self { + coordinator, + certified_payloads: certified_payloads.into_iter().collect(), + target, + } + } +} + +impl Reporter for StartupReporter +where + B: Block + + Heightable + + Digestible + + Send + + Sync + + 'static, +{ + type Activity = Update; + + fn report(&mut self, update: Self::Activity) -> Feedback { + if let Update::Block(block, acknowledgement) = update { + let digest = block.digest(); + if self.certified_payloads.contains(&digest) { + let result = self + .coordinator + .lock() + .expect("startup coordinator lock poisoned") + .record(StartupCandidate { + height: block.height(), + digest, + state_target: (self.target)(&block), + certificate_payload: Some(digest), + genesis: false, + }); + if let Err(error) = result { + self.coordinator + .lock() + .expect("startup coordinator lock poisoned") + .fail(error); + } + } + acknowledgement.acknowledge(); + } + Feedback::Ok + } +} + +#[derive(Clone, Debug, thiserror::Error, Eq, PartialEq)] +pub enum Error { + #[error("startup candidate capacity exceeded")] + CapacityExceeded, + #[error("same startup height and digest were recorded with conflicting data")] + ConflictingCandidate, + #[error("attached QMDB has no exact certified startup candidate")] + NoExactMatch, + #[error("attached QMDB matches multiple startup candidates")] + MultipleExactMatches, + #[error("startup certificate payload does not equal its block digest")] + CertificatePayloadMismatch, + #[error("genesis startup candidate is malformed")] + InvalidGenesisCandidate, +} + +#[cfg(test)] +mod tests { + use super::*; + use commonware_storage::mmr::Location; + use commonware_utils::{non_empty_range, NZUsize}; + + fn target(root: u8, start: u64, end: u64) -> Target { + Target::new( + Digest([root; 32]), + non_empty_range!(Location::new(start), Location::new(end)), + ) + } + + fn candidate( + height: u64, + digest: u8, + state_target: Target, + ) -> StartupCandidate { + let digest = Digest([digest; 32]); + StartupCandidate { + height: Height::new(height), + digest, + state_target, + certificate_payload: Some(digest), + genesis: false, + } + } + + #[test] + fn exact_target_and_certificate_payload_are_required() { + let exact = target(1, 4, 9); + let mut coordinator = StartupCoordinator::new(NZUsize!(4)); + coordinator + .record(candidate(7, 2, exact.clone())) + .unwrap(); + assert_eq!( + coordinator.resolve(&target(1, 4, 8)), + Err(Error::NoExactMatch) + ); + assert_eq!( + coordinator.resolve(&target(1, 5, 9)), + Err(Error::NoExactMatch) + ); + let artifact = coordinator.resolve(&exact).unwrap(); + assert_eq!(artifact.anchor_height, Height::new(7)); + assert_eq!(artifact.anchor_state_target, exact); + } + + #[test] + fn root_only_range_only_and_conflicting_matches_are_rejected() { + let exact = target(3, 10, 20); + let mut coordinator = StartupCoordinator::new(NZUsize!(4)); + coordinator + .record(candidate(1, 1, target(3, 1, 2))) + .unwrap(); + coordinator + .record(candidate(2, 2, target(4, 10, 20))) + .unwrap(); + assert_eq!(coordinator.resolve(&exact), Err(Error::NoExactMatch)); + + coordinator + .record(candidate(3, 3, exact.clone())) + .unwrap(); + coordinator + .record(candidate(4, 4, exact.clone())) + .unwrap(); + assert_eq!( + coordinator.resolve(&exact), + Err(Error::MultipleExactMatches) + ); + } +} diff --git a/dkg/src/actor.rs b/dkg/src/actor.rs index 354d846..259ace2 100644 --- a/dkg/src/actor.rs +++ b/dkg/src/actor.rs @@ -1,16 +1,20 @@ use super::{ - state::{Dealer, Epoch as EpochState, Player, Storage}, + state::{ + Dealer, Epoch as EpochState, Player, Reconciliation, + ReconciliationPhase, Storage, + }, Mailbox, Message as MailboxMessage, PostUpdate, Update, UpdateCallBack, }; use crate::{ orchestrator::{self, EpochTransition}, protector::StorageProtector, + public::{transition_logs, DkgProtocolConfig, PublicCheckpoint, N3F1_FAULT_MODEL}, setup::PeerConfig, - ReshareBlock, + validate_share, ReshareBlock, STATE_FORMAT_VERSION, }; use commonware_actor::mailbox::{self, Receiver as ActorReceiver}; use commonware_codec::{Encode, EncodeSize, Error as CodecError, Read, ReadExt, Write}; -use commonware_consensus::types::{Epoch, EpochPhase, Epocher, FixedEpocher}; +use commonware_consensus::types::{Epoch, EpochPhase, Epocher, FixedEpocher, Height}; use commonware_cryptography::{ bls12381::{ dkg::feldman_desmedt::{ @@ -23,8 +27,9 @@ use commonware_cryptography::{ }, }, ed25519::{self, Batch}, + sha256::Sha256, transcript::Summary, - BatchVerifier, PublicKey, Signer, + BatchVerifier, Hasher, PublicKey, Signer, }; use commonware_macros::select_loop; use commonware_math::algebra::Random; @@ -38,7 +43,10 @@ use commonware_runtime::{ }; use commonware_utils::{ordered::Set, Acknowledgement as _, N3f1, NZU32}; use rand::CryptoRng; -use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize}; +use std::{ + collections::BTreeMap, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, +}; use tracing::{debug, error, info, warn}; /// Per-peer label. @@ -113,6 +121,23 @@ pub struct Config

{ pub epoch_length: NonZeroU64, } +/// Authenticated public state used to reconcile protected DKG storage before +/// the actor enters consensus. +pub struct AuthenticatedBootstrap { + pub config: DkgProtocolConfig, + pub checkpoint: PublicCheckpoint, + pub logs: Vec, + pub initial_share: Option, +} + +enum Bootstrap { + Legacy { + output: Option>, + share: Option, + }, + Authenticated(Box), +} + /// Execution mode for the DKG actor. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum Execution { @@ -202,7 +227,7 @@ where /// Start the DKG actor. pub fn start( - mut self, + self, output: Option>, share: Option, orchestrator: orchestrator::Mailbox, @@ -211,17 +236,55 @@ where impl Receiver, ), callback: Box>, + ) -> Handle<()> { + self.start_inner( + Bootstrap::Legacy { output, share }, + orchestrator, + dkg, + callback, + ) + } + + /// Start after reconciling protected storage with authenticated QMDB + /// checkpoint and log state. + pub fn start_authenticated( + self, + bootstrap: AuthenticatedBootstrap, + orchestrator: orchestrator::Mailbox, + dkg: ( + impl Sender, + impl Receiver, + ), + callback: Box>, + ) -> Handle<()> { + self.start_inner( + Bootstrap::Authenticated(Box::new(bootstrap)), + orchestrator, + dkg, + callback, + ) + } + + fn start_inner( + mut self, + bootstrap: Bootstrap, + orchestrator: orchestrator::Mailbox, + dkg: ( + impl Sender, + impl Receiver, + ), + callback: Box>, ) -> Handle<()> { match self.execution { Execution::Shared => spawn_cell!( self.context, - self.run(output, share, orchestrator, dkg, callback) + self.run(bootstrap, orchestrator, dkg, callback) ), Execution::Dedicated => { let context = self.context.take(); context.dedicated().spawn(move |context| { self.context.restore(context); - self.run(output, share, orchestrator, dkg, callback) + self.run(bootstrap, orchestrator, dkg, callback) }) } } @@ -229,8 +292,7 @@ where async fn run( mut self, - output: Option>, - share: Option, + bootstrap: Bootstrap, mut orchestrator: orchestrator::Mailbox, (sender, receiver): ( impl Sender, @@ -260,16 +322,29 @@ where return; } }; - if storage.epoch().is_none() { - let initial_state = EpochState { - round: 0, - rng_seed: Summary::random(self.context.as_present_mut()), - output, - share, - }; - if let Err(err) = storage.set_epoch(Epoch::zero(), initial_state).await { - error!(%err, "failed to persist initial DKG epoch"); - return; + match bootstrap { + Bootstrap::Legacy { output, share } => { + if storage.epoch().is_none() { + let initial_state = EpochState { + round: 0, + rng_seed: Summary::random(self.context.as_present_mut()), + output, + share, + }; + if let Err(err) = storage.set_epoch(Epoch::zero(), initial_state).await { + error!(%err, "failed to persist initial DKG epoch"); + return; + } + } + } + Bootstrap::Authenticated(bootstrap) => { + if let Err(err) = self + .reconcile_authenticated(&mut storage, *bootstrap, &self_pk) + .await + { + error!(%err, "failed to reconcile authenticated DKG state"); + return; + } } } @@ -366,7 +441,8 @@ where .expect("round info configuration should be correct"); // Initialize dealer state if we are a dealer (factory handles log submission check) - let mut dealer_state: Option> = am_dealer + let mut dealer_state: Option> = (am_dealer + && (is_dkg || epoch_state.share.is_some())) .then(|| { storage.create_dealer::( epoch, @@ -546,43 +622,149 @@ where // Finalize the round before acknowledging // // TODO(#3453): Minimize end-of-epoch processing via pre-verify - let mut logs = Logs::<_, _, N3f1>::new(round.clone()); - for (dealer, log) in storage.logs(epoch) { - logs.record(dealer, log); - } + let checked_logs = storage.logs(epoch); let (success, next_round, next_output, next_share) = - if let Some(ps) = player_state.take() { - match ps.finalize::( + if let Some(previous_output) = epoch_state.output.as_ref() { + let protocol_config = DkgProtocolConfig { + state_format_version: STATE_FORMAT_VERSION, + namespace: self.namespace.clone(), + epoch_length: self.epoch_length, + participants: self.peer_config.participants.clone(), + num_participants_per_round: self + .peer_config + .num_participants_per_round + .clone(), + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: N3F1_FAULT_MODEL, + trusted_initial_identity: *previous_output.public().public(), + }; + let checkpoint = PublicCheckpoint { + format_version: STATE_FORMAT_VERSION, + protocol_config_digest: protocol_config + .digest() + .expect("actor DKG configuration should be valid"), + epoch, + successful_round: epoch_state.round, + activation_height: epoch + .previous() + .and_then(|previous| epocher.last(previous)) + .unwrap_or(Height::zero()), + output: previous_output.clone(), + }; + let public = match transition_logs::<_, _, Batch>( + &protocol_config, + &checkpoint, + checked_logs.clone(), + block.height(), self.context.as_present_mut(), - logs, &Sequential, ) { - Ok((new_output, new_share)) => ( - true, - epoch_state.round + 1, - Some(new_output), - Some(new_share), - ), - Err(_) => ( + Ok(public) => public, + Err(err) => { + error!(%epoch, %err, "failed public DKG transition"); + break 'actor; + } + }; + if !public.succeeded { + ( false, epoch_state.round, epoch_state.output.clone(), epoch_state.share.clone(), - ), + ) + } else if let Some(ps) = player_state.take() { + let mut player_logs = Logs::<_, _, N3f1>::new(round.clone()); + for (dealer, log) in checked_logs { + player_logs.record(dealer, log); + } + match ps.finalize::( + self.context.as_present_mut(), + player_logs, + &Sequential, + ) { + Ok((player_output, player_share)) + if player_output == public.checkpoint.output => + { + if let Err(err) = validate_share( + &player_output, + &self_pk, + &player_share, + ) { + error!(%epoch, %err, "derived DKG share is invalid"); + break 'actor; + } + ( + true, + public.checkpoint.successful_round, + Some(player_output), + Some(player_share), + ) + } + Err( + commonware_cryptography::bls12381::dkg::feldman_desmedt::Error::MissingPlayerDealing, + ) => ( + true, + public.checkpoint.successful_round, + Some(public.checkpoint.output), + None, + ), + Ok(_) | Err(_) => { + error!(%epoch, "player result conflicts with public DKG transition"); + break 'actor; + } + } + } else { + ( + true, + public.checkpoint.successful_round, + Some(public.checkpoint.output), + None, + ) } } else { - match observe::<_, _, N3f1, Batch>( - self.context.as_present_mut(), - logs, - &Sequential, - ) { - Ok(output) => (true, epoch_state.round + 1, Some(output), None), - Err(_) => ( - false, - epoch_state.round, - epoch_state.output.clone(), - epoch_state.share.clone(), - ), + let mut logs = Logs::<_, _, N3f1>::new(round.clone()); + for (dealer, log) in checked_logs { + logs.record(dealer, log); + } + if let Some(ps) = player_state.take() { + match ps.finalize::( + self.context.as_present_mut(), + logs, + &Sequential, + ) { + Ok((new_output, new_share)) => ( + true, + epoch_state.round + 1, + Some(new_output), + Some(new_share), + ), + Err(_) => ( + false, + epoch_state.round, + epoch_state.output.clone(), + epoch_state.share.clone(), + ), + } + } else { + match observe::<_, _, N3f1, Batch>( + self.context.as_present_mut(), + logs, + &Sequential, + ) { + Ok(output) => ( + true, + epoch_state.round + 1, + Some(output), + None, + ), + Err(_) => ( + false, + epoch_state.round, + epoch_state.output.clone(), + epoch_state.share.clone(), + ), + } } }; if success { @@ -651,6 +833,126 @@ where info!("exiting DKG actor"); } + async fn reconcile_authenticated( + &mut self, + storage: &mut Storage, + bootstrap: AuthenticatedBootstrap, + self_pk: &ed25519::PublicKey, + ) -> Result<(), ReconciliationError> { + bootstrap + .config + .validate_checkpoint(&bootstrap.checkpoint)?; + if bootstrap.config.namespace != self.namespace + || bootstrap.config.epoch_length != self.epoch_length + || bootstrap.config.participants != self.peer_config.participants + || bootstrap.config.num_participants_per_round + != self.peer_config.num_participants_per_round + { + return Err(ReconciliationError::LocalConfigurationMismatch); + } + let info = bootstrap.config.round_info(&bootstrap.checkpoint)?; + let checkpoint_digest = Sha256::hash(&bootstrap.checkpoint.encode()); + if let Some(reconciliation) = storage.reconciliation() { + if reconciliation.phase == ReconciliationPhase::Importing + && (reconciliation.checkpoint_digest != checkpoint_digest + || reconciliation.target_epoch != bootstrap.checkpoint.epoch) + { + return Err(ReconciliationError::ImportInProgressConflict); + } + } + let importing = Reconciliation { + format_version: STATE_FORMAT_VERSION, + checkpoint_digest, + target_epoch: bootstrap.checkpoint.epoch, + phase: ReconciliationPhase::Importing, + }; + let complete = Reconciliation { + phase: ReconciliationPhase::Complete, + ..importing.clone() + }; + let mut logs = BTreeMap::new(); + for signed in bootstrap.logs { + let (dealer, log) = signed + .check(&info) + .ok_or(ReconciliationError::InvalidAuthenticatedLog)?; + if let Some(existing) = logs.insert(dealer, log.clone()) { + if existing != log { + return Err(ReconciliationError::ConflictingAuthenticatedLog); + } + } + } + + let target_epoch = bootstrap.checkpoint.epoch; + let mut rewrite_state = false; + let share = match storage.epoch() { + None if target_epoch == Epoch::zero() => { + if let Some(share) = bootstrap.initial_share { + validate_share(&bootstrap.checkpoint.output, self_pk, &share)?; + Some(share) + } else { + None + } + } + None => None, + Some((local_epoch, _)) if local_epoch > target_epoch => { + return Err(ReconciliationError::LocalStateAhead) + } + Some((local_epoch, _)) if local_epoch < target_epoch => { + rewrite_state = true; + None + } + Some((_, local)) => { + if local.round != bootstrap.checkpoint.successful_round + || local.output.as_ref() != Some(&bootstrap.checkpoint.output) + { + return Err(ReconciliationError::MatchingEpochConflict); + } + if let Some(share) = local.share.as_ref() { + validate_share(&bootstrap.checkpoint.output, self_pk, share)?; + } + for (dealer, log) in &logs { + if let Some(existing) = storage.logs(target_epoch).get(dealer) { + if existing != log { + return Err(ReconciliationError::ConflictingAuthenticatedLog); + } + } + } + storage.set_reconciliation(importing).await?; + for (dealer, log) in logs { + storage.append_log(target_epoch, dealer, log).await?; + } + storage.set_reconciliation(complete).await?; + return Ok(()); + } + }; + + storage.set_reconciliation(importing).await?; + for (dealer, log) in logs { + if let Some(existing) = storage.logs(target_epoch).get(&dealer) { + if existing != &log { + return Err(ReconciliationError::ConflictingAuthenticatedLog); + } + continue; + } + storage.append_log(target_epoch, dealer, log).await?; + } + if rewrite_state || storage.epoch().is_none() { + storage + .set_epoch( + target_epoch, + EpochState { + round: bootstrap.checkpoint.successful_round, + rng_seed: Summary::random(self.context.as_present_mut()), + output: Some(bootstrap.checkpoint.output), + share, + }, + ) + .await?; + } + storage.set_reconciliation(complete).await?; + Ok(()) + } + async fn distribute_shares>( self_pk: &ed25519::PublicKey, storage: &mut Storage, @@ -691,3 +993,23 @@ where } } } + +#[derive(Debug, thiserror::Error)] +enum ReconciliationError { + #[error("authenticated public DKG state error: {0}")] + Public(#[from] crate::public::Error), + #[error("protected DKG storage error: {0}")] + Storage(#[from] crate::state::Error), + #[error("local DKG protocol configuration differs from authenticated state")] + LocalConfigurationMismatch, + #[error("authenticated dealer log is invalid")] + InvalidAuthenticatedLog, + #[error("authenticated dealer logs conflict")] + ConflictingAuthenticatedLog, + #[error("protected DKG state is ahead of authenticated QMDB state")] + LocalStateAhead, + #[error("protected DKG state conflicts with authenticated QMDB at the same epoch")] + MatchingEpochConflict, + #[error("a different authenticated DKG checkpoint import is already in progress")] + ImportInProgressConflict, +} diff --git a/dkg/src/lib.rs b/dkg/src/lib.rs index 4d92d01..1bfea95 100644 --- a/dkg/src/lib.rs +++ b/dkg/src/lib.rs @@ -9,6 +9,7 @@ mod actor; mod consensus; mod egress; mod ingress; +pub mod public; pub mod protector; pub mod orchestrator; mod setup; @@ -16,7 +17,7 @@ mod state; #[cfg(test)] mod tests; -pub use actor::{Actor, Config, Execution}; +pub use actor::{Actor, AuthenticatedBootstrap, Config, Execution}; pub use consensus::{ Activity, Context, EdScheme, EpochProvider, Finalization, Identity, Notarization, Provider, PublicKey, Scheme, Seed, Seedable, Signature, ThresholdScheme, @@ -24,8 +25,15 @@ pub use consensus::{ pub use egress::{ContinueOnUpdate, PostUpdate, Update, UpdateCallBack}; pub use ingress::{Mailbox, Message}; pub use protector::{StorageKey, StorageProtector}; +pub use public::{ + checked_threshold_scheme, transition as public_transition, transition_logs, validate_anchor, + validate_share, DkgProtocolConfig, PublicCheckpoint, PublicTransition, + ProtocolConfigReadCfg, STATE_FORMAT_VERSION, +}; pub use setup::PeerConfig; -pub use state::{Epoch as StoredEpoch, Storage}; +pub use state::{ + Epoch as StoredEpoch, Reconciliation, ReconciliationPhase, Storage, +}; pub type DealerLog = SignedDealerLog; diff --git a/dkg/src/public.rs b/dkg/src/public.rs new file mode 100644 index 0000000..5616176 --- /dev/null +++ b/dkg/src/public.rs @@ -0,0 +1,528 @@ +//! Authenticated public state for DKG resharing. +//! +//! This module contains the canonical protocol configuration, the public +//! checkpoint committed by an application, and the deterministic transition +//! used by both applications and DKG actors. + +use commonware_codec::{Encode, EncodeSize, Error as CodecError, RangeCfg, Read, ReadExt, Write}; +use commonware_consensus::types::{Epoch, Epocher, FixedEpocher, Height}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::{ + observe, DealerLog, Info, Logs, Output, SignedDealerLog, + }, + primitives::{ + group::Share, + sharing::{Mode, ModeVersion}, + variant::{MinSig, Variant}, + }, + }, + ed25519, + sha256::{Digest, Sha256}, + BatchVerifier, Hasher, PublicKey, Signer, +}; +use commonware_parallel::Strategy; +use commonware_utils::{ordered::Set, N3f1, Participant, TryCollect}; +use rand::{rngs::StdRng, seq::IteratorRandom, CryptoRng, SeedableRng}; +use std::{collections::BTreeMap, num::{NonZeroU32, NonZeroU64}}; + +/// Current authenticated DKG state format. +pub const STATE_FORMAT_VERSION: u16 = 1; + +/// Identifier committed for the `N3f1` fault model. +pub const N3F1_FAULT_MODEL: u8 = 1; + +const CONFIG_DOMAIN: &[u8] = b"NUNCHI_DKG_PROTOCOL_CONFIG"; + +/// Canonical protocol configuration committed by every public checkpoint. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DkgProtocolConfig { + pub state_format_version: u16, + pub namespace: Vec, + pub epoch_length: NonZeroU64, + pub participants: Set

, + pub num_participants_per_round: Vec, + pub mode: Mode, + pub mode_version: u8, + pub fault_model: u8, + pub trusted_initial_identity: V::Public, +} + +impl DkgProtocolConfig { + /// Validate semantic constraints that are not expressible through the codec. + pub fn validate(&self) -> Result<(), Error> { + if self.state_format_version != STATE_FORMAT_VERSION { + return Err(Error::UnsupportedFormat(self.state_format_version)); + } + if self.namespace.is_empty() { + return Err(Error::EmptyNamespace); + } + if self.participants.is_empty() { + return Err(Error::EmptyParticipants); + } + if self.num_participants_per_round.is_empty() { + return Err(Error::EmptyRoundSchedule); + } + if self.mode_version != 0 { + return Err(Error::UnsupportedModeVersion(self.mode_version)); + } + if self.fault_model != N3F1_FAULT_MODEL { + return Err(Error::UnsupportedFaultModel(self.fault_model)); + } + let total = self.participants.len(); + if self + .num_participants_per_round + .iter() + .any(|&count| count == 0 || count as usize > total) + { + return Err(Error::InvalidRoundSchedule); + } + Ok(()) + } + + /// Hash the canonical Commonware codec encoding. + pub fn digest(&self) -> Result { + self.validate()?; + Ok(Sha256::hash(&self.encode())) + } + + /// Return the maximum configured participant count. + pub fn max_participants_per_round(&self) -> u32 { + self.num_participants_per_round + .iter() + .copied() + .max() + .expect("validated schedule is non-empty") + } + + /// Select the participants for a successful-round number. + pub fn participants_for_round(&self, round: u64) -> Set

{ + let schedule_index = (round % self.num_participants_per_round.len() as u64) as usize; + let count = self.num_participants_per_round[schedule_index] as usize; + let participants = self.participants.iter().cloned(); + if round == 0 { + return participants.take(count).try_collect().unwrap(); + } + let mut rng = StdRng::seed_from_u64(round); + participants + .sample(&mut rng, count) + .into_iter() + .try_collect() + .unwrap() + } + + /// Reconstruct the exact public round information for a checkpoint. + pub fn round_info( + &self, + checkpoint: &PublicCheckpoint, + ) -> Result, Error> { + self.validate_checkpoint(checkpoint)?; + let dealers = self.participants_for_round(checkpoint.successful_round); + let players = self.participants_for_round(checkpoint.successful_round + 1); + Info::new::( + &self.namespace, + checkpoint.epoch.get(), + Some(checkpoint.output.clone()), + self.mode, + dealers, + players, + ) + .map_err(|error| Error::RoundInfo(error.to_string())) + } + + /// Validate that a checkpoint belongs to this protocol configuration. + pub fn validate_checkpoint( + &self, + checkpoint: &PublicCheckpoint, + ) -> Result<(), Error> { + let expected = self.digest()?; + if checkpoint.format_version != self.state_format_version { + return Err(Error::UnsupportedFormat(checkpoint.format_version)); + } + if checkpoint.protocol_config_digest != expected { + return Err(Error::ProtocolConfigMismatch); + } + if checkpoint.output.public().public() != &self.trusted_initial_identity { + return Err(Error::ThresholdIdentityMismatch); + } + Ok(()) + } +} + +impl EncodeSize for DkgProtocolConfig { + fn encode_size(&self) -> usize { + CONFIG_DOMAIN.len() + + self.state_format_version.encode_size() + + self.namespace.encode_size() + + self.epoch_length.encode_size() + + self.participants.encode_size() + + self.num_participants_per_round.encode_size() + + self.mode.encode_size() + + self.mode_version.encode_size() + + self.fault_model.encode_size() + + self.trusted_initial_identity.encode_size() + } +} + +impl Write for DkgProtocolConfig { + fn write(&self, buf: &mut impl bytes::BufMut) { + buf.put_slice(CONFIG_DOMAIN); + self.state_format_version.write(buf); + self.namespace.write(buf); + self.epoch_length.write(buf); + self.participants.write(buf); + self.num_participants_per_round.write(buf); + self.mode.write(buf); + self.mode_version.write(buf); + self.fault_model.write(buf); + self.trusted_initial_identity.write(buf); + } +} + +/// Codec limits for a protocol configuration. +#[derive(Clone, Copy)] +pub struct ProtocolConfigReadCfg { + pub max_namespace_len: usize, + pub max_participants: NonZeroU32, + pub max_round_schedule_len: usize, + pub max_supported_mode: ModeVersion, +} + +impl Read for DkgProtocolConfig { + type Cfg = ProtocolConfigReadCfg; + + fn read_cfg( + buf: &mut impl bytes::Buf, + cfg: &Self::Cfg, + ) -> Result { + let domain: [u8; CONFIG_DOMAIN.len()] = ReadExt::read(buf)?; + if domain.as_slice() != CONFIG_DOMAIN { + return Err(CodecError::Invalid("DkgProtocolConfig", "invalid domain")); + } + let value = Self { + state_format_version: ReadExt::read(buf)?, + namespace: Vec::::read_cfg( + buf, + &(RangeCfg::from(1..=cfg.max_namespace_len), ()), + )?, + epoch_length: ReadExt::read(buf)?, + participants: Set::

::read_cfg( + buf, + &(RangeCfg::from(1..=cfg.max_participants.get() as usize), ()), + )?, + num_participants_per_round: Vec::::read_cfg( + buf, + &(RangeCfg::from(1..=cfg.max_round_schedule_len), ()), + )?, + mode: Mode::read_cfg(buf, &cfg.max_supported_mode)?, + mode_version: ReadExt::read(buf)?, + fault_model: ReadExt::read(buf)?, + trusted_initial_identity: ReadExt::read(buf)?, + }; + value + .validate() + .map_err(|_| CodecError::Invalid("DkgProtocolConfig", "invalid configuration"))?; + Ok(value) + } +} + +/// Public DKG state authenticated by the application state root. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PublicCheckpoint { + pub format_version: u16, + pub protocol_config_digest: Digest, + pub epoch: Epoch, + pub successful_round: u64, + pub activation_height: Height, + pub output: Output, +} + +impl PublicCheckpoint { + /// Construct the epoch-zero checkpoint from trusted configuration. + pub fn genesis( + config: &DkgProtocolConfig, + output: Output, + ) -> Result { + let checkpoint = Self { + format_version: config.state_format_version, + protocol_config_digest: config.digest()?, + epoch: Epoch::zero(), + successful_round: 0, + activation_height: Height::zero(), + output, + }; + config.validate_checkpoint(&checkpoint)?; + Ok(checkpoint) + } +} + +impl EncodeSize for PublicCheckpoint { + fn encode_size(&self) -> usize { + self.format_version.encode_size() + + self.protocol_config_digest.encode_size() + + self.epoch.encode_size() + + self.successful_round.encode_size() + + self.activation_height.encode_size() + + self.output.encode_size() + } +} + +impl Write for PublicCheckpoint { + fn write(&self, buf: &mut impl bytes::BufMut) { + self.format_version.write(buf); + self.protocol_config_digest.write(buf); + self.epoch.write(buf); + self.successful_round.write(buf); + self.activation_height.write(buf); + self.output.write(buf); + } +} + +impl Read for PublicCheckpoint { + type Cfg = (NonZeroU32, ModeVersion); + + fn read_cfg(buf: &mut impl bytes::Buf, cfg: &Self::Cfg) -> Result { + let format_version = u16::read(buf)?; + if format_version != STATE_FORMAT_VERSION { + return Err(CodecError::Invalid( + "PublicCheckpoint", + "unsupported format version", + )); + } + Ok(Self { + format_version, + protocol_config_digest: ReadExt::read(buf)?, + epoch: ReadExt::read(buf)?, + successful_round: ReadExt::read(buf)?, + activation_height: ReadExt::read(buf)?, + output: Output::read_cfg(buf, cfg)?, + }) + } +} + +/// Result of a deterministic public transition. +pub struct PublicTransition { + pub checkpoint: PublicCheckpoint, + pub succeeded: bool, +} + +/// Execute the public transition committed at an epoch boundary. +pub fn transition( + config: &DkgProtocolConfig, + checkpoint: &PublicCheckpoint, + signed_logs: impl IntoIterator>, + boundary_height: Height, + rng: &mut impl CryptoRng, + strategy: &impl Strategy, +) -> Result, Error> +where + V: Variant, + P: PublicKey, + S: Signer, + B: BatchVerifier, +{ + config.validate_checkpoint(checkpoint)?; + let epocher = FixedEpocher::new(config.epoch_length); + if epocher.last(checkpoint.epoch) != Some(boundary_height) { + return Err(Error::WrongBoundaryHeight); + } + let info = config.round_info(checkpoint)?; + let eligible = config.participants_for_round(checkpoint.successful_round); + let mut unique = BTreeMap::>::new(); + for signed in signed_logs { + let checked = signed.clone().check(&info).ok_or(Error::InvalidSignedLog)?; + let dealer = checked.0; + if eligible.position(&dealer).is_none() { + return Err(Error::IneligibleDealer); + } + if let Some(existing) = unique.get(&dealer) { + if existing != &signed { + return Err(Error::ConflictingDealerLog); + } + continue; + } + unique.insert(dealer, signed); + } + + let mut checked = BTreeMap::new(); + for signed in unique.into_values() { + let (dealer, log) = signed.check(&info) + .ok_or(Error::InvalidSignedLog)?; + checked.insert(dealer, log); + } + transition_logs::( + config, + checkpoint, + checked, + boundary_height, + rng, + strategy, + ) +} + +/// Execute a transition from logs whose dealer signatures were already +/// authenticated while ingesting finalized blocks. +/// +/// This is the actor-facing half of [`transition`]. Applications should use +/// [`transition`] so signatures and duplicate submissions are checked in the +/// same call. +pub fn transition_logs( + config: &DkgProtocolConfig, + checkpoint: &PublicCheckpoint, + checked_logs: BTreeMap>, + boundary_height: Height, + rng: &mut impl CryptoRng, + strategy: &impl Strategy, +) -> Result, Error> +where + V: Variant, + P: PublicKey, + B: BatchVerifier, +{ + config.validate_checkpoint(checkpoint)?; + let epocher = FixedEpocher::new(config.epoch_length); + if epocher.last(checkpoint.epoch) != Some(boundary_height) { + return Err(Error::WrongBoundaryHeight); + } + let info = config.round_info(checkpoint)?; + let eligible = config.participants_for_round(checkpoint.successful_round); + let mut logs = Logs::::new(info); + for (dealer, log) in checked_logs { + if eligible.position(&dealer).is_none() { + return Err(Error::IneligibleDealer); + } + logs.record(dealer, log); + } + let observed = observe::(rng, logs, strategy); + let (succeeded, successful_round, output) = match observed { + Ok(output) => (true, checkpoint.successful_round + 1, output), + Err(_) => ( + false, + checkpoint.successful_round, + checkpoint.output.clone(), + ), + }; + Ok(PublicTransition { + checkpoint: PublicCheckpoint { + format_version: checkpoint.format_version, + protocol_config_digest: checkpoint.protocol_config_digest, + epoch: checkpoint.epoch.next(), + successful_round, + activation_height: boundary_height, + output, + }, + succeeded, + }) +} + +/// Validate exact checkpoint semantics at a certified startup anchor. +pub fn validate_anchor( + config: &DkgProtocolConfig, + checkpoint: &PublicCheckpoint, + anchor_height: Height, +) -> Result<(), Error> { + config.validate_checkpoint(checkpoint)?; + if anchor_height == Height::zero() { + return (checkpoint.epoch == Epoch::zero() + && checkpoint.activation_height == Height::zero()) + .then_some(()) + .ok_or(Error::CheckpointHeightMismatch); + } + let epocher = FixedEpocher::new(config.epoch_length); + let bounds = epocher + .containing(anchor_height) + .ok_or(Error::CheckpointHeightMismatch)?; + let expected = if anchor_height == bounds.last() { + (bounds.epoch().next(), anchor_height) + } else { + let epoch = bounds.epoch(); + let activation = epoch + .previous() + .and_then(|previous| epocher.last(previous)) + .unwrap_or(Height::zero()); + (epoch, activation) + }; + (checkpoint.epoch == expected.0 && checkpoint.activation_height == expected.1) + .then_some(()) + .ok_or(Error::CheckpointHeightMismatch) +} + +/// Validate a protected private share against authenticated public state. +pub fn validate_share( + output: &Output, + participant: &P, + share: &Share, +) -> Result<(), Error> { + let index = output + .players() + .position(participant) + .map(Participant::from_usize) + .ok_or(Error::ParticipantHasNoShare)?; + if share.index != index { + return Err(Error::InvalidLocalShare); + } + let expected = output + .public() + .partial_public(index) + .map_err(|_| Error::InvalidLocalShare)?; + if expected != share.public::() { + return Err(Error::InvalidLocalShare); + } + Ok(()) +} + +/// Construct a threshold scheme only after validating its private share. +pub fn checked_threshold_scheme( + namespace: &[u8], + output: &Output, + participant: &ed25519::PublicKey, + share: Share, +) -> Result { + validate_share(output, participant, &share)?; + crate::ThresholdScheme::signer( + namespace, + output.players().clone(), + output.public().clone(), + share, + ) + .ok_or(Error::InvalidLocalShare) +} + +/// Public-state validation failures. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("unsupported DKG state format version {0}")] + UnsupportedFormat(u16), + #[error("DKG namespace must not be empty")] + EmptyNamespace, + #[error("DKG participant set must not be empty")] + EmptyParticipants, + #[error("DKG round schedule must not be empty")] + EmptyRoundSchedule, + #[error("DKG round schedule contains an invalid participant count")] + InvalidRoundSchedule, + #[error("unsupported DKG mode version {0}")] + UnsupportedModeVersion(u8), + #[error("unsupported DKG fault model {0}")] + UnsupportedFaultModel(u8), + #[error("checkpoint protocol configuration digest does not match")] + ProtocolConfigMismatch, + #[error("checkpoint threshold identity does not match")] + ThresholdIdentityMismatch, + #[error("failed to construct DKG round info: {0}")] + RoundInfo(String), + #[error("transition height is not the current epoch boundary")] + WrongBoundaryHeight, + #[error("signed dealer log is invalid for the checkpoint")] + InvalidSignedLog, + #[error("dealer is not eligible in this DKG round")] + IneligibleDealer, + #[error("dealer submitted conflicting logs")] + ConflictingDealerLog, + #[error("checkpoint does not match the certified anchor height")] + CheckpointHeightMismatch, + #[error("validator is not a player in the checkpoint output")] + ParticipantHasNoShare, + #[error("protected local share does not match authenticated public state")] + InvalidLocalShare, +} diff --git a/dkg/src/state.rs b/dkg/src/state.rs index d0c309e..c11b85f 100644 --- a/dkg/src/state.rs +++ b/dkg/src/state.rs @@ -15,6 +15,7 @@ use commonware_cryptography::{ }, primitives::{group::Share, sharing::ModeVersion, variant::Variant}, }, + sha256::Digest, transcript::{Summary, Transcript}, BatchVerifier, PublicKey, Signer, }; @@ -45,6 +46,81 @@ const READ_BUFFER: NonZeroUsize = NZUsize!(1 << 20); const RECORD_AD_DOMAIN: &[u8] = b"nunchi-dkg-storage"; const RECORD_KIND_EPOCH: u8 = 0; const RECORD_KIND_EVENT: u8 = 1; +const RECONCILIATION_KEY: u8 = 0; + +/// Durable authenticated-state import phase. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReconciliationPhase { + Importing, + Complete, +} + +impl EncodeSize for ReconciliationPhase { + fn encode_size(&self) -> usize { + 1 + } +} + +impl Write for ReconciliationPhase { + fn write(&self, buf: &mut impl BufMut) { + match self { + Self::Importing => 0u8.write(buf), + Self::Complete => 1u8.write(buf), + } + } +} + +impl Read for ReconciliationPhase { + type Cfg = (); + + fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { + match u8::read(buf)? { + 0 => Ok(Self::Importing), + 1 => Ok(Self::Complete), + other => Err(CodecError::InvalidEnum(other)), + } + } +} + +/// Crash-recovery marker for an authenticated checkpoint import. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Reconciliation { + pub format_version: u16, + pub checkpoint_digest: Digest, + pub target_epoch: EpochNum, + pub phase: ReconciliationPhase, +} + +impl EncodeSize for Reconciliation { + fn encode_size(&self) -> usize { + self.format_version.encode_size() + + self.checkpoint_digest.encode_size() + + self.target_epoch.encode_size() + + self.phase.encode_size() + } +} + +impl Write for Reconciliation { + fn write(&self, buf: &mut impl BufMut) { + self.format_version.write(buf); + self.checkpoint_digest.write(buf); + self.target_epoch.write(buf); + self.phase.write(buf); + } +} + +impl Read for Reconciliation { + type Cfg = (); + + fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { + Ok(Self { + format_version: ReadExt::read(buf)?, + checkpoint_digest: ReadExt::read(buf)?, + target_epoch: ReadExt::read(buf)?, + phase: ReadExt::read(buf)?, + }) + } +} /// Errors returned by DKG persistent storage. #[derive(Debug, thiserror::Error)] @@ -204,6 +280,7 @@ where public_key: P, states: Metadata, + reconciliation: Metadata, msgs: SVJournal, // In-memory state @@ -238,6 +315,14 @@ where }, ) .await?; + let reconciliation = Metadata::init( + context.child("reconciliation"), + MetadataConfig { + partition: format!("{partition_prefix}_reconciliation"), + codec_config: (), + }, + ) + .await?; let mut msgs = SVJournal::init( context.child("msgs"), @@ -311,6 +396,7 @@ where namespace, public_key, states, + reconciliation, msgs, current, epochs, @@ -459,6 +545,22 @@ where self.current.as_ref().map(|(e, s)| (*e, s.clone())) } + /// Return the durable authenticated-state reconciliation marker. + pub fn reconciliation(&self) -> Option { + self.reconciliation.get(&RECONCILIATION_KEY).cloned() + } + + /// Persist and sync an authenticated-state reconciliation phase. + pub async fn set_reconciliation( + &mut self, + reconciliation: Reconciliation, + ) -> Result<(), Error> { + self.reconciliation + .put(RECONCILIATION_KEY, reconciliation); + self.reconciliation.sync().await?; + Ok(()) + } + fn get_or_create_epoch(&mut self, epoch: EpochNum) -> &mut EpochCache { self.epochs.entry(epoch).or_default() } diff --git a/dkg/src/tests/mod.rs b/dkg/src/tests/mod.rs index 8172f8b..ae41927 100644 --- a/dkg/src/tests/mod.rs +++ b/dkg/src/tests/mod.rs @@ -1,3 +1,4 @@ mod actor; mod protector; +mod public; mod state; diff --git a/dkg/src/tests/public.rs b/dkg/src/tests/public.rs new file mode 100644 index 0000000..a7ff59d --- /dev/null +++ b/dkg/src/tests/public.rs @@ -0,0 +1,187 @@ +use crate::{ + public_transition, transition_logs, validate_anchor, validate_share, + DkgProtocolConfig, PublicCheckpoint, STATE_FORMAT_VERSION, +}; +use commonware_codec::{Encode, Read}; +use commonware_consensus::types::{Epoch, Epocher, FixedEpocher, Height}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::{deal, Dealer, Player, Verdict}, + primitives::{ + group::{Private, Scalar, Share}, + sharing::Mode, + variant::MinSig, + }, + }, + ed25519::{self, Batch}, + Signer, +}; +use commonware_math::algebra::Ring; +use commonware_parallel::Sequential; +use commonware_utils::{ + ordered::Set, test_rng, N3f1, Participant, TestRng, NZU64, +}; + +fn setup() -> ( + DkgProtocolConfig, + Vec, + PublicCheckpoint, + commonware_utils::ordered::Map, +) { + let signers = (0..4) + .map(ed25519::PrivateKey::from_seed) + .collect::>(); + let participants = + Set::from_iter_dedup(signers.iter().map(Signer::public_key)); + let (output, shares) = deal::( + test_rng(), + Mode::NonZeroCounter, + participants.clone(), + ) + .unwrap(); + let config = DkgProtocolConfig { + state_format_version: STATE_FORMAT_VERSION, + namespace: b"nunchi-public-state-test".to_vec(), + epoch_length: NZU64!(10), + participants, + num_participants_per_round: vec![4], + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: crate::public::N3F1_FAULT_MODEL, + trusted_initial_identity: *output.public().public(), + }; + let checkpoint = PublicCheckpoint::genesis(&config, output).unwrap(); + (config, signers, checkpoint, shares) +} + +#[test] +fn protocol_digest_is_canonical_and_binds_configuration() { + let (config, _, _, _) = setup(); + let mut same = config.clone(); + assert_eq!(config.digest().unwrap(), same.digest().unwrap()); + same.epoch_length = NZU64!(11); + assert_ne!(config.digest().unwrap(), same.digest().unwrap()); + assert_eq!( + commonware_formatting::hex(&config.digest().unwrap()), + "cef10ca10a298de2fc304c09729e8e846ed682db984a66a8d5c09cd8a1be8b0d" + ); +} + +#[test] +fn checkpoint_codec_is_bounded_and_rejects_wrong_format() { + let (config, _, checkpoint, _) = setup(); + let encoded = checkpoint.encode(); + let mut input = encoded.as_ref(); + let decoded = PublicCheckpoint::read_cfg( + &mut input, + &( + commonware_utils::NZU32!(config.max_participants_per_round()), + crate::MAX_SUPPORTED_MODE, + ), + ) + .unwrap(); + assert_eq!(decoded, checkpoint); + assert!(input.is_empty()); + + let mut malformed = encoded.to_vec(); + malformed[0] ^= 1; + assert!(PublicCheckpoint::::read_cfg( + &mut malformed.as_slice(), + &( + commonware_utils::NZU32!(config.max_participants_per_round()), + crate::MAX_SUPPORTED_MODE, + ), + ) + .is_err()); +} + +#[test] +fn failed_public_transition_advances_epoch_only() { + let (config, _, checkpoint, _) = setup(); + let boundary = FixedEpocher::new(config.epoch_length) + .last(Epoch::zero()) + .unwrap(); + let transition = transition_logs::<_, _, Batch>( + &config, + &checkpoint, + Default::default(), + boundary, + &mut test_rng(), + &Sequential, + ) + .unwrap(); + assert!(!transition.succeeded); + assert_eq!(transition.checkpoint.epoch, Epoch::new(1)); + assert_eq!( + transition.checkpoint.successful_round, + checkpoint.successful_round + ); + assert_eq!(transition.checkpoint.output, checkpoint.output); + assert_eq!(transition.checkpoint.activation_height, boundary); + validate_anchor(&config, &transition.checkpoint, boundary).unwrap(); +} + +#[test] +fn successful_transition_and_identical_duplicate_are_deterministic() { + let (config, signers, checkpoint, shares) = setup(); + let info = config.round_info(&checkpoint).unwrap(); + let mut logs = Vec::new(); + for (dealer_index, signer) in signers.iter().enumerate() { + let dealer_pk = signer.public_key(); + let (mut dealer, public, private) = Dealer::::start::( + TestRng::new(100 + dealer_index as u64), + info.clone(), + signer.clone(), + shares.get_value(&dealer_pk).cloned(), + ) + .unwrap(); + for (player_pk, private) in private { + let player_signer = signers + .iter() + .find(|candidate| candidate.public_key() == player_pk) + .unwrap() + .clone(); + let mut player = Player::new(info.clone(), player_signer).unwrap(); + let Verdict::Valid(ack) = + player.dealer_message::(dealer_pk.clone(), public.clone(), private) + else { + panic!("valid dealing should be acknowledged"); + }; + dealer.receive_player_ack(player_pk, ack).unwrap(); + } + logs.push(dealer.finalize::()); + } + logs.push(logs[0].clone()); + let boundary = FixedEpocher::new(config.epoch_length) + .last(Epoch::zero()) + .unwrap(); + let transition = public_transition::<_, _, ed25519::PrivateKey, Batch>( + &config, + &checkpoint, + logs, + boundary, + &mut test_rng(), + &Sequential, + ) + .unwrap(); + assert!(transition.succeeded); + assert_eq!(transition.checkpoint.epoch, Epoch::new(1)); + assert_eq!(transition.checkpoint.successful_round, 1); +} + +#[test] +fn local_share_validation_is_checked_without_panicking() { + let (_, signers, checkpoint, shares) = setup(); + let participant = signers[0].public_key(); + let share = shares.get_value(&participant).unwrap(); + validate_share(&checkpoint.output, &participant, share).unwrap(); + + let invalid = Share::new( + Participant::new(share.index.get()), + Private::new(Scalar::one()), + ); + assert!(validate_share(&checkpoint.output, &participant, &invalid).is_err()); + let wrong_index = Share::new(Participant::new(1), share.private.clone()); + assert!(validate_share(&checkpoint.output, &participant, &wrong_index).is_err()); + assert!(validate_anchor(&setup().0, &checkpoint, Height::new(10)).is_err()); +} diff --git a/dkg/src/tests/state.rs b/dkg/src/tests/state.rs index 4a27bf8..e472721 100644 --- a/dkg/src/tests/state.rs +++ b/dkg/src/tests/state.rs @@ -1,6 +1,9 @@ use crate::{ protector::{ProtectionError, SealedRecord, StorageProtector}, - state::{Dealer, Epoch as EpochState, Error as StorageError, Storage}, + state::{ + Dealer, Epoch as EpochState, Error as StorageError, Reconciliation, + ReconciliationPhase, Storage, + }, }; use bytes::Bytes; use commonware_codec::{Encode, RangeCfg, ReadExt}; @@ -15,6 +18,7 @@ use commonware_cryptography::{ }, }, ed25519::{self}, + sha256::Digest, transcript::Summary, Signer, }; @@ -458,6 +462,66 @@ fn storage_recovers_no_share_observer_epoch() { }); } +#[test_traced] +fn storage_recovers_reconciliation_phase_from_separate_partition() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let signers = create_test_signers(4); + let public_key = signers[0].public_key(); + let partition = "reconciliation_marker"; + let marker = Reconciliation { + format_version: crate::STATE_FORMAT_VERSION, + checkpoint_digest: Digest([9; 32]), + target_epoch: Epoch::new(15), + phase: ReconciliationPhase::Importing, + }; + + let mut storage = init_storage( + context.child("storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + public_key.clone(), + ) + .await; + storage + .set_reconciliation(marker.clone()) + .await + .expect("reconciliation marker should persist"); + drop(storage); + + let mut recovered = init_storage( + context.child("recovered_storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + public_key, + ) + .await; + assert_eq!(recovered.reconciliation(), Some(marker.clone())); + + let complete = Reconciliation { + phase: ReconciliationPhase::Complete, + ..marker + }; + recovered + .set_reconciliation(complete.clone()) + .await + .expect("completed marker should persist"); + drop(recovered); + + let recovered = init_storage( + context.child("completed_storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + signers[0].public_key(), + ) + .await; + assert_eq!(recovered.reconciliation(), Some(complete)); + }); +} + #[test_traced] fn test_dealer_handle_returns_false_when_player_not_in_unsent() { let executor = deterministic::Runner::default(); diff --git a/examples/bridge-chain/src/engine.rs b/examples/bridge-chain/src/engine.rs index 3bda1e5..b459f10 100644 --- a/examples/bridge-chain/src/engine.rs +++ b/examples/bridge-chain/src/engine.rs @@ -81,6 +81,19 @@ pub struct Config, P: Manager, } +#[derive(Debug, thiserror::Error)] +pub enum StartupError { + #[error("bridge-chain does not authenticate DKG progress in QMDB; state_sync = true is unsupported")] + UnsupportedDkgStateSync, +} + +pub fn validate_state_sync(enabled: bool) -> Result<(), StartupError> { + if enabled { + return Err(StartupError::UnsupportedDkgStateSync); + } + Ok(()) +} + type DkgActor = nunchi_chain::DkgActor; type DkgMailbox = nunchi_chain::DkgMailbox; type StatefulApp = @@ -168,7 +181,8 @@ where impl Sender, impl Receiver, ), - ) -> (Self, NodeHandle) { + ) -> Result<(Self, NodeHandle), StartupError> { + validate_state_sync(config.state_sync)?; let (mempool, submitter) = Mempool::::new(config.pool_config.clone()); let mempool = mempool.start(context.child("mempool")); @@ -489,7 +503,7 @@ where stateful, stateful_mailbox, }; - (engine, node_handle) + Ok((engine, node_handle)) } #[allow(clippy::too_many_arguments)] diff --git a/examples/bridge-chain/src/execution.rs b/examples/bridge-chain/src/execution.rs index e6e6b5f..35aaa7b 100644 --- a/examples/bridge-chain/src/execution.rs +++ b/examples/bridge-chain/src/execution.rs @@ -13,7 +13,7 @@ pub use nunchi_chain::SharedAppliedHeight; #[derive(Clone)] pub struct NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub bridge: BridgeMailbox, pub stateful: StatefulMailbox, @@ -23,7 +23,7 @@ where impl NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub fn new( bridge: BridgeMailbox, diff --git a/examples/bridge-chain/src/rpc.rs b/examples/bridge-chain/src/rpc.rs index e127f05..7a34724 100644 --- a/examples/bridge-chain/src/rpc.rs +++ b/examples/bridge-chain/src/rpc.rs @@ -21,7 +21,8 @@ where + commonware_runtime::Spawner + commonware_runtime::Metrics + commonware_runtime::Clock - + rand::Rng, + + rand::Rng + + rand::CryptoRng, { node: NodeHandle, } @@ -32,7 +33,8 @@ where + commonware_runtime::Spawner + commonware_runtime::Metrics + commonware_runtime::Clock - + rand::Rng, + + rand::Rng + + rand::CryptoRng, { pub const fn new(node: NodeHandle) -> Self { Self { node } @@ -74,6 +76,7 @@ where + commonware_runtime::Metrics + commonware_runtime::Clock + rand::Rng + + rand::CryptoRng + Send + Sync + 'static, @@ -91,6 +94,7 @@ where + commonware_runtime::Metrics + commonware_runtime::Clock + rand::Rng + + rand::CryptoRng + Send + Sync + 'static, diff --git a/examples/bridge-chain/src/testnet.rs b/examples/bridge-chain/src/testnet.rs index 2c1107a..635d657 100644 --- a/examples/bridge-chain/src/testnet.rs +++ b/examples/bridge-chain/src/testnet.rs @@ -209,6 +209,8 @@ pub enum Error { TomlDeserialize(#[from] toml::de::Error), #[error("engine stopped unexpectedly: {0}")] Engine(commonware_runtime::Error), + #[error("engine startup failed: {0}")] + EngineStartup(#[from] crate::engine::StartupError), } struct Material { @@ -585,7 +587,7 @@ async fn start_node( probe, state_sync, ) - .await; + .await?; let engine_handle = engine.start( pending, recovered, diff --git a/examples/bridge-chain/src/tests/mod.rs b/examples/bridge-chain/src/tests/mod.rs index 34d6682..11dfd7c 100644 --- a/examples/bridge-chain/src/tests/mod.rs +++ b/examples/bridge-chain/src/tests/mod.rs @@ -27,6 +27,15 @@ use crate::{application, Application, Block, TxPool}; const FOREIGN_NAMESPACE: &[u8] = b"_NUNCHI_BRIDGE_CHAIN_FOREIGN"; const WRONG_NAMESPACE: &[u8] = b"_NUNCHI_BRIDGE_CHAIN_WRONG"; +#[test] +fn peer_state_sync_is_rejected_until_bridge_authenticates_dkg_state() { + assert!(matches!( + crate::engine::validate_state_sync(true), + Err(crate::engine::StartupError::UnsupportedDkgStateSync) + )); + assert!(crate::engine::validate_state_sync(false).is_ok()); +} + fn schemes(namespace: &[u8], seed: u64) -> Vec { let mut rng = TestRng::new(seed); vrf::fixture::(&mut rng, namespace, 4).schemes diff --git a/examples/coins/chain/src/engine.rs b/examples/coins/chain/src/engine.rs index 2930cbf..43dd899 100644 --- a/examples/coins/chain/src/engine.rs +++ b/examples/coins/chain/src/engine.rs @@ -1,6 +1,6 @@ use crate::application::{self, Application}; use crate::execution::NodeHandle; -use crate::genesis::{genesis_target, state_commitment, ChainGenesis}; +use crate::genesis::{authenticated_genesis_target, state_commitment, ChainGenesis}; use crate::history::{self, BlocksArchive, FinalizationsArchive}; use crate::indexer; use crate::{Block, EpochProvider, Provider, PublicKey, Scheme, Transaction, NAMESPACE}; @@ -20,7 +20,7 @@ use commonware_consensus::{ use commonware_cryptography::{ bls12381::{ dkg::feldman_desmedt::Output, - primitives::{group, variant::MinSig}, + primitives::{group, sharing::Mode, variant::MinSig}, }, ed25519::{self, Batch}, sha256::Digest, @@ -28,7 +28,7 @@ use commonware_cryptography::{ }; use commonware_glue::stateful::{ Application as StatefulApplication, - db::ManagedDb as _, + db::{DatabaseSet as _, ManagedDb as _}, probe::{Config as ProbeConfig, Probe}, Config as StatefulConfig, Mailbox as StatefulMailbox, Stateful as StatefulActor, SyncPlan, }; @@ -48,14 +48,14 @@ use nunchi_chain::state_sync::{ Mailbox as StateSyncMailbox, }; use nunchi_clob::{ClobActor, ClobConfig, ClobExtension}; -use nunchi_common::{QmdbBackend, QmdbState}; +use nunchi_common::{QmdbBackend, QmdbReader, QmdbState}; use nunchi_dkg::{self as dkg, orchestrator, PeerConfig, UpdateCallBack, MAX_SUPPORTED_MODE}; use nunchi_mempool::{Mempool, PoolConfig}; use rand::{CryptoRng, Rng}; use std::{ marker::PhantomData, num::NonZeroU64, - sync::Arc, + sync::{Arc, Mutex}, time::{Duration, Instant}, }; use tracing::{info, warn}; @@ -124,6 +124,7 @@ type Orchestrator = orchestrator::Actor< Option>, >; type IndexerConsumer = indexer::Consumer; +type StartupReporter = nunchi_chain::startup::StartupReporter; /// The engine that drives the coins-chain [Application]. #[allow(clippy::type_complexity)] @@ -147,6 +148,9 @@ where config: Config, dkg: DkgActor, dkg_mailbox: DkgMailbox, + dkg_state: nunchi_chain::DkgState, + startup_coordinator: Arc>, + startup_reporter: StartupReporter, buffer: buffered::Engine, buffered_mailbox: buffered::Mailbox, marshal: Marshal, @@ -214,6 +218,19 @@ where let num_participants = commonware_utils::NZU32!(config.peer_config.max_participants_per_round()); let block_codec_config = (num_participants, ()); + let protocol_config = dkg::DkgProtocolConfig { + state_format_version: dkg::STATE_FORMAT_VERSION, + namespace: NAMESPACE.to_vec(), + epoch_length: config.epoch_length, + participants: config.peer_config.participants.clone(), + num_participants_per_round: config.peer_config.num_participants_per_round.clone(), + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: dkg::public::N3F1_FAULT_MODEL, + trusted_initial_identity: *config.output.public().public(), + }; + let dkg_state = nunchi_chain::DkgState::new(protocol_config) + .expect("invalid authenticated DKG protocol configuration"); let (dkg, dkg_mailbox) = dkg::Actor::new( context.child("dkg"), @@ -322,41 +339,65 @@ where .expect("failed to initialize empty state database for genesis commitment"); state_commitment(empty.sync_target()) }; - let genesis_state = if let Some(genesis) = &config.genesis { - let fingerprint = commonware_formatting::hex( - &genesis - .fingerprint() - .expect("failed to fingerprint statically configured genesis"), - ); - let compute_partition = format!("{}-genesis-{}", config.partition_prefix, fingerprint); - let compute_config = - QmdbState::::config_with_page_cache(&compute_partition, page_cache.clone()); - let expected = genesis_target( - context.child("genesis_commitment"), - compute_config, - genesis, - &empty_state, + let dkg_fingerprint = commonware_formatting::hex( + &dkg_state + .config() + .digest() + .expect("failed to fingerprint authenticated DKG configuration"), + ); + let app_fingerprint = config + .genesis + .as_ref() + .map(|genesis| { + commonware_formatting::hex( + &genesis + .fingerprint() + .expect("failed to fingerprint statically configured genesis"), + ) + }) + .unwrap_or_else(|| "none".to_owned()); + let compute_partition = format!( + "{}-genesis-{}-{}", + config.partition_prefix, dkg_fingerprint, app_fingerprint + ); + let compute_config = + QmdbState::::config_with_page_cache(&compute_partition, page_cache.clone()); + let expected = authenticated_genesis_target( + context.child("genesis_commitment"), + compute_config, + &dkg_state, + config.output.clone(), + config.genesis.as_ref(), + &empty_state, + ) + .await + .expect("failed to materialize authenticated genesis commitment"); + let state_was_empty = { + let state = QmdbState::init_with_config( + context.child("genesis_existing_state_probe"), + db_config.clone(), ) .await - .expect("failed to materialize genesis commitment"); - - let mut state = - QmdbState::init_with_config(context.child("genesis_seed"), db_config.clone()) - .await - .expect("failed to initialize state database for genesis seeding"); - genesis - .apply_to_state(&mut state, &empty_state) - .await - .expect("failed to seed state database with genesis"); - let actual = state_commitment(state.sync_target()); + .expect("failed to inspect state database before genesis seeding"); + state.sync_target().root == empty_state.root + }; + let actual = authenticated_genesis_target( + context.child("genesis_seed"), + db_config.clone(), + &dkg_state, + config.output.clone(), + config.genesis.as_ref(), + &empty_state, + ) + .await + .expect("failed to seed authenticated state database"); + if state_was_empty { assert_eq!( expected, actual, - "state database genesis commitment must match the genesis block commitment" + "fresh state database genesis commitment must match the genesis block commitment" ); - expected - } else { - empty_state - }; + } + let genesis_state = expected; let current_state_target = { let state = QmdbState::init_with_config( context.child("startup_state_probe"), @@ -366,13 +407,16 @@ where .expect("failed to initialize state database for startup probe"); state.sync_target() }; - if let Some(processed_height) = validate_marshal_progress_against_state( + let local_startup_candidate = if let Some((processed_height, candidate)) = + validate_marshal_progress_against_state( &context, &config.partition_prefix, + &finalizations_by_height, &finalized_blocks, ¤t_state_target, ) - .await { + .await + { validate_history_through_processed( &finalizations_by_height, &finalized_blocks, @@ -391,17 +435,21 @@ where .await .expect("failed to startup-prune finalized blocks"); } + Some(candidate) } else if ArchiveStore::last_index(&finalized_blocks).is_some() || ArchiveStore::last_index(&finalizations_by_height).is_some() { panic!("finalized history exists without persisted marshal processed height; rebuild or perform verified peer state sync"); - } + } else { + None + }; let applied_height = Arc::new(AsyncMutex::new(Height::zero())); - let app = Application::with_consensus( + let app = Application::with_authenticated_dkg( submitter.clone(), config.max_block_transactions, ClobExtension::new(clob_mailbox.clone()), - Some(dkg_mailbox.clone()), + dkg_mailbox.clone(), + dkg_state.clone(), applied_height.clone(), genesis_state, application::genesis_payload(), @@ -449,6 +497,40 @@ where .expect("state-sync floor probe stopped"); plan = plan.with_floor(floor); } + let startup_coordinator = Arc::new(Mutex::new( + nunchi_chain::startup::StartupCoordinator::new(MAX_PENDING_ACKS), + )); + startup_coordinator + .lock() + .expect("startup coordinator lock poisoned") + .record(nunchi_chain::startup::StartupCandidate { + height: Height::zero(), + digest: genesis_digest, + state_target: >::sync_targets(&genesis), + certificate_payload: None, + genesis: true, + }) + .expect("genesis startup candidate should fit"); + if let Some(candidate) = local_startup_candidate { + startup_coordinator + .lock() + .expect("startup coordinator lock poisoned") + .record(candidate) + .expect("local startup candidate should fit"); + } + let certified_payloads = recovered_floor + .iter() + .map(|certificate| certificate.proposal.payload) + .chain( + plan.floor() + .into_iter() + .map(|certificate| certificate.proposal.payload), + ); + let startup_reporter = nunchi_chain::startup::StartupReporter::new( + startup_coordinator.clone(), + certified_payloads, + block_state_target, + ); let marshal_start = recovered_floor .clone() .map_or_else(|| plan.marshal_start(genesis), marshal::Start::Floor); @@ -587,6 +669,9 @@ where config, dkg, dkg_mailbox, + dkg_state, + startup_coordinator, + startup_reporter, buffer, buffered_mailbox, marshal, @@ -695,16 +780,15 @@ where ), callback: Box>, ) -> Result<(), EngineError> { - let dkg_handle = self.dkg.start( - Some(self.config.output), - self.config.share, - self.orchestrator_mailbox, - dkg, - callback, - ); let buffer_handle = self.buffer.start(broadcast); let reporters: Reporters<_, _, indexer::Producer> = Reporters::from(( - nunchi_chain::dkg_reporters(self.stateful_mailbox, self.dkg_mailbox), + Reporters::from(( + nunchi_chain::dkg_reporters( + self.stateful_mailbox.clone(), + self.dkg_mailbox, + ), + self.startup_reporter, + )), self.indexer_producer, )); let marshal_handle = self @@ -714,6 +798,46 @@ where let state_sync_handle = self.state_sync_handle; let stateful_handle = self.stateful.start(); let orchestrator_handle = self.orchestrator.start(votes, certificates, resolver); + // Marshal may queue one finalized block for the not-yet-started DKG + // mailbox while stateful attaches or reconstructs QMDB. Once the + // database subscription resolves, reconcile protected DKG storage + // against the authenticated checkpoint before acknowledging that block. + let databases = self.stateful_mailbox.subscribe_databases().await; + let attached_target = databases.committed_targets().await; + let artifact = self + .startup_coordinator + .lock() + .expect("startup coordinator lock poisoned") + .resolve(&attached_target) + .expect("attached QMDB does not exactly match one certified startup block"); + let reader = QmdbReader::new(databases); + let checkpoint = self + .dkg_state + .load_checkpoint(&reader) + .await + .expect("attached QMDB has invalid authenticated DKG checkpoint"); + dkg::validate_anchor( + self.dkg_state.config(), + &checkpoint, + artifact.anchor_height, + ) + .expect("authenticated DKG checkpoint does not match startup anchor height"); + let logs = self + .dkg_state + .load_logs(&reader) + .await + .expect("attached QMDB has invalid authenticated DKG logs"); + let dkg_handle = self.dkg.start_authenticated( + dkg::AuthenticatedBootstrap { + config: self.dkg_state.config().clone(), + checkpoint, + logs, + initial_share: self.config.share, + }, + self.orchestrator_mailbox, + dkg, + callback, + ); let mempool_handle = self .mempool .start_p2p(self.context.child("mempool"), mempool); @@ -780,20 +904,34 @@ fn unexpected_exit( } } +fn block_state_target( + block: &Block, +) -> commonware_storage::qmdb::sync::Target< + commonware_storage::mmr::Family, + Digest, +> { + commonware_storage::qmdb::sync::Target::new( + block.state_root, + block.state_range.clone(), + ) +} + const MARSHAL_PROCESSED_KEY: U64 = U64::new(0xFF); async fn validate_marshal_progress_against_state( context: &E, partition_prefix: &str, + finalizations: &FinalizationsArchive, finalized_blocks: &BlocksArchive, current_target: &commonware_storage::qmdb::sync::Target, -) -> Option +) -> Option<(Height, nunchi_chain::startup::StartupCandidate)> where E: BufferPooler + Clock + Metrics + Network + Rng + + CryptoRng + Spawner + Storage + Strategizer @@ -834,7 +972,28 @@ where }; let target = >::sync_targets(&block); if &target == current_target { - return Some(processed_height); + let certificate = Certificates::get( + finalizations, + ArchiveIdentifier::Index(height), + ) + .await + .expect("failed to read startup candidate finalization") + .expect("startup candidate block has no finalization"); + assert_eq!( + certificate.proposal.payload, + block.digest(), + "startup candidate certificate payload must equal block digest" + ); + return Some(( + processed_height, + nunchi_chain::startup::StartupCandidate { + height: block.height, + digest: block.digest(), + state_target: target, + certificate_payload: Some(certificate.proposal.payload), + genesis: false, + }, + )); } } diff --git a/examples/coins/chain/src/execution.rs b/examples/coins/chain/src/execution.rs index 752eee9..f6dde33 100644 --- a/examples/coins/chain/src/execution.rs +++ b/examples/coins/chain/src/execution.rs @@ -17,7 +17,7 @@ pub use nunchi_chain::SharedAppliedHeight; #[derive(Clone)] pub struct NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub submitter: MempoolHandle, pub clob: ClobMailbox, @@ -27,7 +27,7 @@ where impl NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub fn new( submitter: MempoolHandle, @@ -53,14 +53,14 @@ where /// Read-only coin queries answered from the stateful actor's committed databases. pub struct StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { stateful: StatefulMailbox, } impl Clone for StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { fn clone(&self) -> Self { Self { @@ -71,7 +71,7 @@ where impl StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub fn new(stateful: StatefulMailbox) -> Self { Self { stateful } @@ -85,7 +85,15 @@ where #[async_trait] impl CoinQuery for StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng + Send + Sync + 'static, + E: Context + + Spawner + + Metrics + + Clock + + rand::Rng + + rand::CryptoRng + + Send + + Sync + + 'static, { async fn nonce(&self, account: Address) -> Result { self.ledger().await.nonce(&account).await diff --git a/examples/coins/chain/src/genesis.rs b/examples/coins/chain/src/genesis.rs index 06b8823..6a86f51 100644 --- a/examples/coins/chain/src/genesis.rs +++ b/examples/coins/chain/src/genesis.rs @@ -1,6 +1,13 @@ use crate::StateCommitment; use commonware_codec::{DecodeExt, Encode}; use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::Output, + primitives::variant::MinSig, + }, + ed25519, +}; use commonware_storage::{mmr::Family, qmdb::sync::Target, Context}; use nunchi_authority::{AuthorityGenesis, AuthorityLedger}; use nunchi_clob::{ClobGenesis, ClobLedger}; @@ -8,6 +15,7 @@ use nunchi_coins::{CoinsGenesis, Ledger}; use nunchi_common::{ CommitState, Namespace, Overlay, QmdbConfig, QmdbState, StateError, StateStore, }; +use nunchi_chain::DkgState; use nunchi_oracle::{OracleGenesis, OracleLedger}; use serde::{Deserialize, Serialize}; use std::{fs, path::Path}; @@ -59,6 +67,8 @@ pub enum GenesisError { MismatchedGenesis, #[error("existing chain state is non-empty but has no genesis marker")] UnmarkedState, + #[error("authenticated DKG genesis error: {0}")] + Dkg(#[from] nunchi_chain::DkgStateError), } impl ChainGenesis { @@ -136,6 +146,29 @@ where Ok(state_commitment(state.sync_target())) } +/// Initialize the mandatory authenticated DKG checkpoint and optional +/// application genesis in one canonical order. +pub async fn authenticated_genesis_target( + context: E, + config: QmdbConfig, + dkg: &DkgState, + initial_output: Output, + genesis: Option<&ChainGenesis>, + empty: &StateCommitment, +) -> Result +where + E: Context + commonware_runtime::BufferPooler, +{ + let mut state = QmdbState::init_with_config(context, config).await?; + dkg.seed(&mut state, empty.root, initial_output).await?; + state.commit().await?; + let dkg_baseline = state_commitment(state.sync_target()); + if let Some(genesis) = genesis { + genesis.apply_to_state(&mut state, &dkg_baseline).await?; + } + Ok(state_commitment(state.sync_target())) +} + pub fn state_commitment(target: Target) -> StateCommitment { StateCommitment { root: target.root, diff --git a/examples/coins/chain/src/tests/genesis.rs b/examples/coins/chain/src/tests/genesis.rs index 9c48f6e..b28773a 100644 --- a/examples/coins/chain/src/tests/genesis.rs +++ b/examples/coins/chain/src/tests/genesis.rs @@ -3,12 +3,19 @@ use nunchi_authority::{AuthorityGenesis, AuthorityLedger}; use nunchi_coins::{CoinsGenesis, Ledger}; use nunchi_common::{CommitState, QmdbState}; -use commonware_cryptography::{ed25519, Signer as _}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::deal, + primitives::{sharing::Mode, variant::MinSig}, + }, + ed25519, Signer as _, +}; use commonware_runtime::{deterministic, Runner as _, Supervisor as _}; use nunchi_authority::{AuthorityDB, AuthorityOperation, Transaction as AuthorityTransaction}; use nunchi_coins::{Address, CoinDB, CoinSpec, TokenFactory, TokenName, TokenSymbol}; use nunchi_crypto::PrivateKey; use nunchi_oracle::{IntervalKey, OracleGenesis, OracleLedger}; +use commonware_utils::{ordered::Set, test_rng, N3f1, NZU64}; use crate::genesis::*; @@ -79,6 +86,35 @@ async fn empty_commitment(context: deterministic::Context, partition: &str) -> S state_commitment(state.sync_target()) } +fn authenticated_dkg() -> ( + nunchi_chain::DkgState, + commonware_cryptography::bls12381::dkg::feldman_desmedt::Output< + MinSig, + ed25519::PublicKey, + >, +) { + let participants = Set::from_iter_dedup( + (0..4) + .map(ed25519::PrivateKey::from_seed) + .map(|signer| signer.public_key()), + ); + let (output, _) = + deal::(test_rng(), Mode::NonZeroCounter, participants.clone()) + .unwrap(); + let config = nunchi_dkg::DkgProtocolConfig { + state_format_version: nunchi_dkg::STATE_FORMAT_VERSION, + namespace: crate::NAMESPACE.to_vec(), + epoch_length: NZU64!(10), + participants, + num_participants_per_round: vec![4], + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: nunchi_dkg::public::N3F1_FAULT_MODEL, + trusted_initial_identity: *output.public().public(), + }; + (nunchi_chain::DkgState::new(config).unwrap(), output) +} + #[test] fn genesis_json_roundtrips() { let genesis = sample_genesis(); @@ -87,6 +123,68 @@ fn genesis_json_roundtrips() { assert_eq!(decoded, genesis); } +#[test] +fn authenticated_checkpoint_is_seeded_with_and_without_application_genesis() { + deterministic::Runner::default().start(|context| async move { + let (dkg, output) = authenticated_dkg(); + for (suffix, empty_label, config_label, state_label, reopen_label, genesis) in [ + ( + "none", + "empty_none", + "config_none", + "state_none", + "reopen_none", + None, + ), + ( + "some", + "empty_some", + "config_some", + "state_some", + "reopen_some", + Some(sample_genesis()), + ), + ] { + let partition = format!("authenticated-genesis-{suffix}"); + let empty = empty_commitment( + context.child(empty_label), + &format!("{partition}-empty"), + ) + .await; + let config_context = context.child(config_label); + let config = QmdbState::::config( + &config_context, + &partition, + ); + let commitment = authenticated_genesis_target( + context.child(state_label), + config.clone(), + &dkg, + output.clone(), + genesis.as_ref(), + &empty, + ) + .await + .unwrap(); + assert_ne!(commitment.root, empty.root); + + let state = QmdbState::init_with_config( + context.child(reopen_label), + config, + ) + .await + .unwrap(); + let checkpoint = dkg.load_checkpoint(&state).await.unwrap(); + assert_eq!(checkpoint.epoch, commonware_consensus::types::Epoch::zero()); + assert_eq!(checkpoint.activation_height.get(), 0); + assert_eq!( + checkpoint.protocol_config_digest, + dkg.config().digest().unwrap() + ); + } + }); +} + #[test] fn genesis_json_loads_from_disk() { let genesis = ChainGenesis::from_slice(GENESIS_FIXTURE).unwrap(); From dad8b619af98988f195ae04e893b0946f1f21539 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Thu, 23 Jul 2026 19:31:06 +0700 Subject: [PATCH 6/8] fix dkg startup floor for state-sync joins --- dkg/src/orchestrator/actor.rs | 148 +++++++++++++++++++++------- dkg/src/orchestrator/mod.rs | 2 +- examples/bridge-chain/src/engine.rs | 1 + examples/coins/chain/src/engine.rs | 10 +- 4 files changed, 120 insertions(+), 41 deletions(-) diff --git a/dkg/src/orchestrator/actor.rs b/dkg/src/orchestrator/actor.rs index 060b701..ef4fbe8 100644 --- a/dkg/src/orchestrator/actor.rs +++ b/dkg/src/orchestrator/actor.rs @@ -42,7 +42,7 @@ use std::{ num::{NonZeroU64, NonZeroUsize}, time::Duration, }; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; const CLEANUP_WATERMARK_KEY: U64 = U64::new(0); @@ -113,10 +113,38 @@ where pub epoch_length: NonZeroU64, pub genesis_digest: Digest, pub recovered_floor: Option>, + pub startup_floor: Option, pub _phantom: PhantomData, } +/// A certified startup boundary that can anchor the first entered epoch when +/// local finalized block history is absent. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StartupFloor { + pub height: Height, + pub digest: Digest, +} + +impl StartupFloor { + fn digest_for_epoch(&self, epocher: &FixedEpocher, epoch: Epoch) -> Option { + Self::floor_boundary(epocher, epoch) + .filter(|boundary| *boundary == self.height) + .map(|_| self.digest) + } + + // Epoch zero uses genesis as its floor; every later epoch is anchored by the + // last finalized block from the previous epoch. + fn floor_boundary(epocher: &FixedEpocher, epoch: Epoch) -> Option { + let previous_epoch = epoch.previous()?; + Some( + epocher + .last(previous_epoch) + .expect("previous epoch should be covered by epoch strategy"), + ) + } +} + pub struct Actor>> where E: BufferPooler + Spawner + Metrics + CryptoRng + Clock + Storage + Network, @@ -155,6 +183,7 @@ where epoch_length: NonZeroU64, genesis_digest: Digest, recovered_floor: Option>, + startup_floor: Option, page_cache_ref: CacheRef, latest_epoch: Gauge, @@ -231,6 +260,7 @@ where epoch_length: config.epoch_length, genesis_digest: config.genesis_digest, recovered_floor: config.recovered_floor, + startup_floor: config.startup_floor, page_cache_ref, latest_epoch, partition_cleanup_total, @@ -243,6 +273,10 @@ where ) } + pub fn set_startup_floor(&mut self, startup_floor: StartupFloor) { + self.startup_floor = Some(startup_floor); + } + pub fn start( mut self, votes: ( @@ -394,34 +428,8 @@ where startup_cleanup_done = true; } - let floor = if self - .recovered_floor - .as_ref() - .is_some_and(|floor| floor.epoch() == transition.epoch) - { - Floor::Finalized( - self.recovered_floor - .take() - .expect("matching recovered floor must exist"), - ) - } else { - // DKG state does not persist the consensus floor; derive the epoch's - // genesis from marshal's finalized boundary block. - let digest = match Self::floor_boundary(&epocher, transition.epoch) { - Some(boundary_height) => self - .marshal - .get_block(boundary_height) - .await - .unwrap_or_else(|| { - panic!( - "missing finalized boundary block at height {} for epoch {}", - boundary_height, transition.epoch - ) - }) - .digest(), - None => self.genesis_digest, - }; - Floor::Genesis(digest) + let Some(floor) = self.resolve_floor(&epocher, transition.epoch).await else { + break; }; // Register the new signing scheme with the scheme provider. @@ -537,15 +545,45 @@ where } } - // Epoch zero uses genesis as its floor; every later epoch is anchored by the - // last finalized block from the previous epoch. - fn floor_boundary(epocher: &FixedEpocher, epoch: Epoch) -> Option { - let previous_epoch = epoch.previous()?; - Some( - epocher - .last(previous_epoch) - .expect("previous epoch should be covered by epoch strategy"), - ) + async fn resolve_floor( + &mut self, + epocher: &FixedEpocher, + epoch: Epoch, + ) -> Option> { + if self + .recovered_floor + .as_ref() + .is_some_and(|floor| floor.epoch() == epoch) + { + return Some(Floor::Finalized( + self.recovered_floor + .take() + .expect("matching recovered floor must exist"), + )); + } + + let Some(boundary_height) = StartupFloor::floor_boundary(epocher, epoch) else { + return Some(Floor::Genesis(self.genesis_digest)); + }; + + if let Some(block) = self.marshal.get_block(boundary_height).await { + return Some(Floor::Genesis(block.digest())); + } + + if let Some(digest) = self + .startup_floor + .as_ref() + .and_then(|startup_floor| startup_floor.digest_for_epoch(epocher, epoch)) + { + return Some(Floor::Genesis(digest)); + } + + error!( + %boundary_height, + %epoch, + "refusing to enter epoch without recovered floor, certified startup boundary, or local finalized boundary block" + ); + None } async fn enter_epoch( @@ -607,3 +645,37 @@ where engine.start(vote, certificate, resolver) } } + +#[cfg(test)] +mod tests { + use super::*; + use commonware_utils::NZU64; + + #[test] + fn startup_floor_only_matches_previous_epoch_boundary() { + let epocher = FixedEpocher::new(NZU64!(10)); + let digest = Digest([9; 32]); + let floor = StartupFloor { + height: Height::new(19), + digest, + }; + + assert_eq!( + floor.digest_for_epoch(&epocher, Epoch::new(2)), + Some(digest) + ); + assert_eq!(floor.digest_for_epoch(&epocher, Epoch::new(1)), None); + assert_eq!(floor.digest_for_epoch(&epocher, Epoch::zero()), None); + } + + #[test] + fn non_boundary_startup_floor_is_not_inferred() { + let epocher = FixedEpocher::new(NZU64!(10)); + let floor = StartupFloor { + height: Height::new(12), + digest: Digest([7; 32]), + }; + + assert_eq!(floor.digest_for_epoch(&epocher, Epoch::new(2)), None); + } +} diff --git a/dkg/src/orchestrator/mod.rs b/dkg/src/orchestrator/mod.rs index 36faace..f52bea3 100644 --- a/dkg/src/orchestrator/mod.rs +++ b/dkg/src/orchestrator/mod.rs @@ -1,7 +1,7 @@ //! Consensus engine orchestrator for epoch transitions. mod actor; -pub use actor::{Actor, Config, NoopReporter}; +pub use actor::{Actor, Config, NoopReporter, StartupFloor}; mod ingress; #[cfg(test)] diff --git a/examples/bridge-chain/src/engine.rs b/examples/bridge-chain/src/engine.rs index b459f10..9f05e66 100644 --- a/examples/bridge-chain/src/engine.rs +++ b/examples/bridge-chain/src/engine.rs @@ -483,6 +483,7 @@ where epoch_length: BLOCKS_PER_EPOCH, genesis_digest, recovered_floor, + startup_floor: None, _phantom: PhantomData, }, ); diff --git a/examples/coins/chain/src/engine.rs b/examples/coins/chain/src/engine.rs index 43dd899..cd863bf 100644 --- a/examples/coins/chain/src/engine.rs +++ b/examples/coins/chain/src/engine.rs @@ -660,6 +660,7 @@ where epoch_length: config.epoch_length, genesis_digest, recovered_floor, + startup_floor: None, _phantom: PhantomData, }, ); @@ -745,7 +746,7 @@ where #[allow(clippy::too_many_arguments)] async fn run( - self, + mut self, votes: ( impl Sender, impl Receiver, @@ -797,7 +798,6 @@ where let probe_handle = self.probe_handle; let state_sync_handle = self.state_sync_handle; let stateful_handle = self.stateful.start(); - let orchestrator_handle = self.orchestrator.start(votes, certificates, resolver); // Marshal may queue one finalized block for the not-yet-started DKG // mailbox while stateful attaches or reconstructs QMDB. Once the // database subscription resolves, reconcile protected DKG storage @@ -822,11 +822,17 @@ where artifact.anchor_height, ) .expect("authenticated DKG checkpoint does not match startup anchor height"); + self.orchestrator + .set_startup_floor(orchestrator::StartupFloor { + height: artifact.anchor_height, + digest: artifact.anchor_digest, + }); let logs = self .dkg_state .load_logs(&reader) .await .expect("attached QMDB has invalid authenticated DKG logs"); + let orchestrator_handle = self.orchestrator.start(votes, certificates, resolver); let dkg_handle = self.dkg.start_authenticated( dkg::AuthenticatedBootstrap { config: self.dkg_state.config().clone(), From 12e781fd1496402d2624070e371e87a5531eed82 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Thu, 23 Jul 2026 21:32:07 +0700 Subject: [PATCH 7/8] fix dkg player resume for authenticated shareless state-sync --- dkg/src/actor.rs | 48 ++++++++++--- dkg/src/state.rs | 33 +++++++-- dkg/src/tests/actor.rs | 159 ++++++++++++++++++++++++++++++++++++++++- dkg/src/tests/state.rs | 84 +++++++++++++++++++++- 4 files changed, 304 insertions(+), 20 deletions(-) diff --git a/dkg/src/actor.rs b/dkg/src/actor.rs index 259ace2..65b5e52 100644 --- a/dkg/src/actor.rs +++ b/dkg/src/actor.rs @@ -1,6 +1,6 @@ use super::{ state::{ - Dealer, Epoch as EpochState, Player, Reconciliation, + CreatePlayerError, Dealer, Epoch as EpochState, Player, Reconciliation, ReconciliationPhase, Storage, }, Mailbox, Message as MailboxMessage, PostUpdate, Update, UpdateCallBack, @@ -303,6 +303,7 @@ where let max_read_size = NZU32!(self.peer_config.max_participants_per_round()); let epocher = FixedEpocher::new(self.epoch_length); let self_pk = self.signer.public_key(); + let authenticated_bootstrap = matches!(bootstrap, Bootstrap::Authenticated(_)); // Initialize persistent state let mut storage = match Storage::init( @@ -454,16 +455,41 @@ where }) .flatten(); - // Initialize player state if we are a player - let mut player_state: Option> = am_player - .then(|| { - storage.create_player::( - epoch, - self.signer.clone(), - round.clone(), - ) - }) - .flatten(); + // Initialize player state if we are a player. + let mut player_state: Option> = if am_player { + match storage.create_player::( + epoch, + self.signer.clone(), + round.clone(), + ) { + Ok(player) => Some(player), + Err(CreatePlayerError::MissingPlayerDealing) + if authenticated_bootstrap && epoch_state.share.is_none() => + { + warn!( + %epoch, + validator = ?self_pk, + authenticated_bootstrap, + has_share = false, + "continuing without DKG player state because authenticated public logs have no local private dealings" + ); + None + } + Err(err) => { + error!( + %epoch, + validator = ?self_pk, + authenticated_bootstrap, + has_share = epoch_state.share.is_some(), + %err, + "failed to resume DKG player from protected storage" + ); + break 'actor; + } + } + } else { + None + }; select_loop! { self.context, diff --git a/dkg/src/state.rs b/dkg/src/state.rs index c11b85f..1c0474f 100644 --- a/dkg/src/state.rs +++ b/dkg/src/state.rs @@ -24,7 +24,10 @@ use commonware_runtime::{ buffer::paged::CacheRef, Buf, BufMut, BufferPooler, Clock, Metrics, Storage as RuntimeStorage, }; use commonware_storage::{ - journal::{self, segmented::variable::{Config as SVConfig, Journal as SVJournal}}, + journal::{ + self, + segmented::variable::{Config as SVConfig, Journal as SVJournal}, + }, metadata::{self, Config as MetadataConfig, Metadata}, }; use commonware_utils::{Faults, NZUsize, NZU16}; @@ -48,6 +51,17 @@ const RECORD_KIND_EPOCH: u8 = 0; const RECORD_KIND_EVENT: u8 = 1; const RECONCILIATION_KEY: u8 = 0; +/// Error returned when resuming a player from protected storage. +#[derive(Debug, thiserror::Error)] +pub enum CreatePlayerError { + /// Public dealer logs are present, but matching private dealer messages are not. + #[error("missing private player dealing")] + MissingPlayerDealing, + /// The crypto player could not be resumed from the persisted state. + #[error("failed to resume DKG player: {0}")] + Crypto(commonware_cryptography::bls12381::dkg::feldman_desmedt::Error), +} + /// Durable authenticated-state import phase. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ReconciliationPhase { @@ -732,16 +746,25 @@ where epoch: EpochNum, signer: C, round_info: Info, - ) -> Option> { + ) -> Result, CreatePlayerError> { let logs = self.logs(epoch); let dealings = self.dealings(epoch); - let (crypto_player, acks) = CryptoPlayer::resume::(round_info, signer, &logs, dealings) - .expect("should be able to resume player"); + let (crypto_player, acks) = + CryptoPlayer::resume::(round_info, signer, &logs, dealings).map_err(|err| { + if matches!( + err, + commonware_cryptography::bls12381::dkg::feldman_desmedt::Error::MissingPlayerDealing + ) { + CreatePlayerError::MissingPlayerDealing + } else { + CreatePlayerError::Crypto(err) + } + })?; for dealer in acks.keys() { debug!(?epoch, ?dealer, "restored committed dealer message"); } - Some(Player { + Ok(Player { player: crypto_player, acks, }) diff --git a/dkg/src/tests/actor.rs b/dkg/src/tests/actor.rs index 869a84e..eef84cf 100644 --- a/dkg/src/tests/actor.rs +++ b/dkg/src/tests/actor.rs @@ -7,7 +7,10 @@ use commonware_codec::{EncodeSize, Error as CodecError, Read, ReadExt, Write}; use commonware_consensus::types::Epoch; use commonware_consensus::{types::Height, Heightable}; use commonware_cryptography::{ - bls12381::{dkg::feldman_desmedt::deal, primitives::variant::MinSig}, + bls12381::{ + dkg::feldman_desmedt::{deal, Dealer, DealerLog, Player, Verdict}, + primitives::{sharing::Mode, variant::MinSig}, + }, ed25519::{PrivateKey, PublicKey as Ed25519PublicKey}, sha256, transcript::Summary, @@ -16,7 +19,7 @@ use commonware_cryptography::{ use commonware_macros::test_traced; use commonware_math::algebra::Random; use commonware_p2p::{utils::mocks::inert_channel, PeerSetSubscription, Provider}; -use commonware_runtime::{deterministic, Runner, Supervisor as _}; +use commonware_runtime::{deterministic, Clock, Runner, Supervisor as _}; use commonware_utils::{channel::mpsc, N3f1, NZUsize, TryCollect, NZU32, NZU64}; use core::marker::PhantomData; use std::collections::BTreeMap; @@ -130,6 +133,56 @@ fn peer_config( (peer_config, participants) } +fn finalized_dkg_log( + namespace: &[u8], + epoch: Epoch, + dealers: commonware_utils::ordered::Set, + players: commonware_utils::ordered::Set, + participants: &BTreeMap, + dealer_pk: &Ed25519PublicKey, +) -> DealerLog { + let round = commonware_cryptography::bls12381::dkg::feldman_desmedt::Info::new::( + namespace, + epoch.get(), + None, + Mode::NonZeroCounter, + dealers, + players, + ) + .expect("round info should be valid"); + let dealer_signer = participants + .get(dealer_pk) + .cloned() + .expect("dealer signer should exist"); + let (mut dealer, public, private) = Dealer::::start::( + commonware_utils::test_rng(), + round.clone(), + dealer_signer, + None, + ) + .expect("dealer should start"); + + for (player_pk, private) in private { + let player_signer = participants + .get(&player_pk) + .cloned() + .expect("player signer should exist"); + let mut player = Player::new(round.clone(), player_signer).expect("player should start"); + let Verdict::Valid(ack) = + player.dealer_message::(dealer_pk.clone(), public.clone(), private) + else { + panic!("valid dealing should be acknowledged"); + }; + dealer.receive_player_ack(player_pk, ack).unwrap(); + } + + dealer + .finalize::() + .check(&round) + .expect("finalized log should check") + .1 +} + fn assert_recovered_storage_controls_dkg_mode_on_restart(execution: Execution, suffix: &str) { let executor = deterministic::Runner::seeded(8); executor.start(|mut context| async move { @@ -227,3 +280,105 @@ fn default_execution_recovered_storage_controls_dkg_mode_on_restart() { fn dedicated_execution_recovered_storage_controls_dkg_mode_on_restart() { assert_recovered_storage_controls_dkg_mode_on_restart(Execution::Dedicated, "dedicated"); } + +#[test_traced] +fn legacy_missing_player_dealing_exits_actor() { + let executor = deterministic::Runner::seeded(9); + executor.start(|mut context| async move { + let namespace = b"test_dkg".to_vec(); + let epoch = Epoch::zero(); + let (peer_config, participants) = peer_config(4, vec![4]); + let self_pk = peer_config + .participants + .iter() + .next() + .cloned() + .expect("participant exists"); + let signer = participants + .get(&self_pk) + .cloned() + .expect("signer should exist"); + let dealer_pk = peer_config + .participants + .iter() + .find(|candidate| **candidate != self_pk) + .cloned() + .expect("dealer exists"); + let log = finalized_dkg_log( + &namespace, + epoch, + peer_config.participants.clone(), + peer_config.dealers(0), + &participants, + &dealer_pk, + ); + let partition_prefix = format!("legacy_missing_player_dealing_{self_pk}"); + + let mut storage = Storage::<_, MinSig, Ed25519PublicKey>::init( + context.child("seed_storage"), + &partition_prefix, + StorageProtector::new(TEST_STORAGE_KEY), + namespace.clone(), + self_pk.clone(), + NZU32!(peer_config.max_participants_per_round()), + crate::MAX_SUPPORTED_MODE, + ) + .await + .expect("storage init should succeed"); + storage + .set_epoch( + epoch, + EpochState { + round: 0, + rng_seed: Summary::random(&mut context), + output: None, + share: None, + }, + ) + .await + .expect("set epoch should succeed"); + storage + .append_log(epoch, dealer_pk, log) + .await + .expect("append log should succeed"); + drop(storage); + + let (actor, _mailbox) = Actor::<_, _, TestBlock>::new( + context.child("actor"), + Config { + manager: NoopManager::::default(), + signer, + mailbox_size: NZUsize!(8), + execution: Execution::default(), + partition_prefix, + peer_config: peer_config.clone(), + max_supported_mode: crate::MAX_SUPPORTED_MODE, + namespace, + storage_protector: StorageProtector::new(TEST_STORAGE_KEY), + epoch_length: NZU64!(200), + }, + ); + let (sender, receiver) = inert_channel(&peer_config.participants); + let (orchestrator_sender, mut orchestrator_receiver) = + mailbox::new(context.child("orchestrator_mailbox"), NZUsize!(4)); + let handle = actor.start( + None, + None, + crate::orchestrator::Mailbox::new(orchestrator_sender), + (sender, receiver), + ContinueOnUpdate::boxed(), + ); + + let Some(Message::Enter(transition)) = orchestrator_receiver.recv().await else { + panic!("actor should emit an epoch transition before detecting bad player state"); + }; + assert_eq!(transition.epoch, epoch); + + commonware_macros::select! { + _ = handle => {}, + _ = context.sleep(std::time::Duration::from_secs(1)) => { + panic!("legacy actor should fail closed instead of continuing shareless"); + }, + } + }); +} diff --git a/dkg/src/tests/state.rs b/dkg/src/tests/state.rs index e472721..2bdf01d 100644 --- a/dkg/src/tests/state.rs +++ b/dkg/src/tests/state.rs @@ -1,7 +1,7 @@ use crate::{ protector::{ProtectionError, SealedRecord, StorageProtector}, state::{ - Dealer, Epoch as EpochState, Error as StorageError, Reconciliation, + CreatePlayerError, Dealer, Epoch as EpochState, Error as StorageError, Reconciliation, ReconciliationPhase, Storage, }, }; @@ -10,7 +10,7 @@ use commonware_codec::{Encode, RangeCfg, ReadExt}; use commonware_consensus::types::Epoch; use commonware_cryptography::{ bls12381::{ - dkg::feldman_desmedt::{DealerPrivMsg, DealerPubMsg, Info, PlayerAck, Verdict}, + dkg::feldman_desmedt::{DealerPrivMsg, DealerPubMsg, Info, Player, PlayerAck, Verdict}, primitives::{ group::{Private, Scalar, Share}, sharing::Mode, @@ -121,6 +121,45 @@ fn test_dealing( (dealer_signer.public_key(), pub_msg, priv_msg) } +fn finalized_dealer_log( + signers: &[ed25519::PrivateKey], + dealer_index: usize, +) -> ( + ed25519::PublicKey, + commonware_cryptography::bls12381::dkg::feldman_desmedt::DealerLog< + MinPk, + ed25519::PublicKey, + >, +) { + let round_info = create_round_info(signers); + let dealer_signer = signers[dealer_index].clone(); + let dealer_pk = dealer_signer.public_key(); + let mut rng = test_rng(); + let (mut dealer, pub_msg, priv_msgs) = + commonware_cryptography::bls12381::dkg::feldman_desmedt::Dealer::::start::< + N3f1, + >(&mut rng, round_info.clone(), dealer_signer, None) + .expect("valid dealer"); + for (player_pk, priv_msg) in priv_msgs { + let player_signer = signers + .iter() + .find(|candidate| candidate.public_key() == player_pk) + .expect("player signer should exist") + .clone(); + let mut player = Player::new(round_info.clone(), player_signer).expect("valid player"); + let Verdict::Valid(ack) = + player.dealer_message::(dealer_pk.clone(), pub_msg.clone(), priv_msg) + else { + panic!("valid dealing should be acknowledged"); + }; + dealer.receive_player_ack(player_pk, ack).unwrap(); + } + let signed = dealer.finalize::(); + signed + .check(&round_info) + .expect("finalized log should check") +} + async fn corrupt_epoch_record(context: E, partition: &str, epoch: Epoch) where E: StorageContext, @@ -522,6 +561,47 @@ fn storage_recovers_reconciliation_phase_from_separate_partition() { }); } +#[test_traced] +fn create_player_returns_missing_dealing_for_imported_public_logs() { + let executor = deterministic::Runner::default(); + executor.start(|mut context| async move { + let signers = create_test_signers(4); + let player_signer = signers[1].clone(); + let player_pk = player_signer.public_key(); + let partition = "missing_player_dealing"; + let epoch = Epoch::zero(); + let round_info = create_round_info(&signers); + let (dealer_pk, log) = finalized_dealer_log(&signers, 0); + + let mut storage = init_storage( + context.child("storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + player_pk, + ) + .await; + storage + .set_epoch(epoch, epoch_state(&mut context, 0, None)) + .await + .expect("set epoch should succeed"); + storage + .append_log(epoch, dealer_pk, log) + .await + .expect("append log should succeed"); + + match storage.create_player::( + epoch, + player_signer, + round_info, + ) { + Err(CreatePlayerError::MissingPlayerDealing) => {} + Err(err) => panic!("expected missing private dealing, got {err}"), + Ok(_) => panic!("missing private dealing should be reported"), + } + }); +} + #[test_traced] fn test_dealer_handle_returns_false_when_player_not_in_unsent() { let executor = deterministic::Runner::default(); From 3583b2602d589e4cd15780d100c4bb099bfb3a04 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Thu, 23 Jul 2026 22:30:19 +0700 Subject: [PATCH 8/8] fix state-sync recovery from in-epoch certified anchors --- dkg/src/orchestrator/actor.rs | 23 +++++++++++ examples/bridge-chain/src/engine.rs | 1 + examples/coins/chain/src/engine.rs | 21 +++++++++- examples/coins/chain/src/testnet.rs | 12 +++++- examples/coins/chain/src/tests/testnet.rs | 49 ++++++++++++++++++++++- examples/coins/chain/tests/coins.rs | 13 +++++- 6 files changed, 113 insertions(+), 6 deletions(-) diff --git a/dkg/src/orchestrator/actor.rs b/dkg/src/orchestrator/actor.rs index ef4fbe8..1f395f8 100644 --- a/dkg/src/orchestrator/actor.rs +++ b/dkg/src/orchestrator/actor.rs @@ -113,6 +113,8 @@ where pub epoch_length: NonZeroU64, pub genesis_digest: Digest, pub recovered_floor: Option>, + /// A certificate-verified state-sync anchor for an in-progress epoch. + pub startup_finalization: Option>, pub startup_floor: Option, pub _phantom: PhantomData, @@ -183,6 +185,7 @@ where epoch_length: NonZeroU64, genesis_digest: Digest, recovered_floor: Option>, + startup_finalization: Option>, startup_floor: Option, page_cache_ref: CacheRef, @@ -260,6 +263,7 @@ where epoch_length: config.epoch_length, genesis_digest: config.genesis_digest, recovered_floor: config.recovered_floor, + startup_finalization: config.startup_finalization, startup_floor: config.startup_floor, page_cache_ref, latest_epoch, @@ -277,6 +281,12 @@ where self.startup_floor = Some(startup_floor); } + /// Installs the certificate-verified state-sync anchor used to resume an + /// in-progress epoch when the local marshal archive is empty. + pub fn set_startup_finalization(&mut self, startup_finalization: Finalization) { + self.startup_finalization = Some(startup_finalization); + } + pub fn start( mut self, votes: ( @@ -562,6 +572,18 @@ where )); } + if self + .startup_finalization + .as_ref() + .is_some_and(|finalization| finalization.epoch() == epoch) + { + return Some(Floor::Finalized( + self.startup_finalization + .take() + .expect("matching startup finalization must exist"), + )); + } + let Some(boundary_height) = StartupFloor::floor_boundary(epocher, epoch) else { return Some(Floor::Genesis(self.genesis_digest)); }; @@ -678,4 +700,5 @@ mod tests { assert_eq!(floor.digest_for_epoch(&epocher, Epoch::new(2)), None); } + } diff --git a/examples/bridge-chain/src/engine.rs b/examples/bridge-chain/src/engine.rs index 9f05e66..c61e60d 100644 --- a/examples/bridge-chain/src/engine.rs +++ b/examples/bridge-chain/src/engine.rs @@ -483,6 +483,7 @@ where epoch_length: BLOCKS_PER_EPOCH, genesis_digest, recovered_floor, + startup_finalization: None, startup_floor: None, _phantom: PhantomData, }, diff --git a/examples/coins/chain/src/engine.rs b/examples/coins/chain/src/engine.rs index cd863bf..cb9d4c6 100644 --- a/examples/coins/chain/src/engine.rs +++ b/examples/coins/chain/src/engine.rs @@ -14,8 +14,9 @@ use commonware_consensus::{ store::Certificates, }, simplex::elector::Random, + simplex::types::Finalization, types::{Epoch, FixedEpocher, Height, ViewDelta}, - Reporters, + Epochable, Reporters, }; use commonware_cryptography::{ bls12381::{ @@ -151,6 +152,7 @@ where dkg_state: nunchi_chain::DkgState, startup_coordinator: Arc>, startup_reporter: StartupReporter, + startup_finalization: Option>, buffer: buffered::Engine, buffered_mailbox: buffered::Mailbox, marshal: Marshal, @@ -497,6 +499,10 @@ where .expect("state-sync floor probe stopped"); plan = plan.with_floor(floor); } + // Preserve the certificate-verified anchor while the plan is consumed + // by the stateful actor. It is needed to resume Simplex from a QMDB + // snapshot taken within, rather than at the start of, an epoch. + let startup_finalization = plan.floor().cloned(); let startup_coordinator = Arc::new(Mutex::new( nunchi_chain::startup::StartupCoordinator::new(MAX_PENDING_ACKS), )); @@ -660,6 +666,7 @@ where epoch_length: config.epoch_length, genesis_digest, recovered_floor, + startup_finalization: None, startup_floor: None, _phantom: PhantomData, }, @@ -673,6 +680,7 @@ where dkg_state, startup_coordinator, startup_reporter, + startup_finalization, buffer, buffered_mailbox, marshal, @@ -822,6 +830,17 @@ where artifact.anchor_height, ) .expect("authenticated DKG checkpoint does not match startup anchor height"); + if let Some(finalization) = self.startup_finalization.take() { + assert_eq!( + finalization.proposal.payload, artifact.anchor_digest, + "state-sync finalization does not certify the attached QMDB anchor" + ); + assert_eq!( + finalization.epoch(), checkpoint.epoch, + "state-sync finalization epoch does not match authenticated DKG checkpoint" + ); + self.orchestrator.set_startup_finalization(finalization); + } self.orchestrator .set_startup_floor(orchestrator::StartupFloor { height: artifact.anchor_height, diff --git a/examples/coins/chain/src/testnet.rs b/examples/coins/chain/src/testnet.rs index feaf62a..556b31d 100644 --- a/examples/coins/chain/src/testnet.rs +++ b/examples/coins/chain/src/testnet.rs @@ -41,7 +41,7 @@ use serde::{Deserialize, Serialize}; use std::{ fs, net::{IpAddr, SocketAddr}, - num::{NonZeroU32, TryFromIntError}, + num::{NonZeroU32, NonZeroU64, TryFromIntError}, path::{Path, PathBuf}, str::FromStr, time::{Duration, Instant}, @@ -136,6 +136,9 @@ pub struct NodeConfig { pub genesis_path: Option, #[serde(default)] pub indexer_url: Option, + /// Number of finalized blocks in each consensus epoch. + #[serde(default = "default_epoch_length")] + pub epoch_length: NonZeroU64, /// Maximum self-contained finalized payloads retained while the indexer is unavailable. #[serde(default = "default_indexer_spool_max_entries")] pub indexer_spool_max_entries: u64, @@ -168,6 +171,10 @@ impl NodeConfig { } } +fn default_epoch_length() -> NonZeroU64 { + BLOCKS_PER_EPOCH +} + fn default_indexer_spool_max_entries() -> u64 { indexer::SpoolLimits::default().max_entries } @@ -417,6 +424,7 @@ pub fn generate_local_testnet(config: LocalTestnetConfig) -> Result