diff --git a/chain/src/application.rs b/chain/src/application.rs index a25332b..5fd6d3c 100644 --- a/chain/src/application.rs +++ b/chain/src/application.rs @@ -18,7 +18,7 @@ use futures::{lock::Mutex as AsyncMutex, StreamExt}; use nunchi_common::{Overlay, QmdbBatch, QmdbDatabaseSet, QmdbMerkleized, Runtime, RuntimeContext}; use nunchi_dkg::{Context, Scheme}; use nunchi_mempool::{MempoolHandle, PoolTransaction}; -use rand::Rng; +use rand::{CryptoRng, Rng}; use std::{ collections::HashMap, marker::PhantomData, @@ -29,8 +29,8 @@ use std::{ use tracing::{debug, error}; use crate::{ - Block, ConsensusExtension, DkgMailbox, EventConsumer, NoConsensusExtension, NoopEventConsumer, - StateCommitment, TransactionEventContext, + Block, ConsensusExtension, DkgMailbox, DkgState, EventConsumer, NoConsensusExtension, + NoopEventConsumer, StateCommitment, TransactionEventContext, }; /// The height of the last finalized block applied to a node's ledger. @@ -67,6 +67,7 @@ where submitter: MempoolHandle, max_block_transactions: usize, dkg: Option>, + dkg_state: Option, consensus: Ext, events: Events, applied_height: SharedAppliedHeight, @@ -83,6 +84,7 @@ struct ApplicationMetrics { proposal_validate_duration: Histogram, proposal_merkleize_duration: Histogram, apply_transactions_duration: Histogram, + dkg_state_duration: Histogram, apply_merkleize_duration: Histogram, } @@ -109,6 +111,11 @@ impl ApplicationMetrics { "duration spent applying block transactions", Buckets::LOCAL, ), + dkg_state_duration: context.histogram( + "dkg_public_state_duration_seconds", + "duration spent validating and transitioning authenticated public DKG state", + Buckets::LOCAL, + ), apply_merkleize_duration: context.histogram( "apply_merkleize_duration_seconds", "duration spent merkleizing applied block state", @@ -162,6 +169,7 @@ where submitter, max_block_transactions, dkg, + dkg_state: None, consensus, events, applied_height, @@ -273,17 +281,21 @@ where Some((included, merkleized)) } - async fn build_proposal_state( + async fn build_proposal_state< + E: BufferPooler + Storage + Clock + Metrics + CryptoRng, + >( &mut self, - timing: Option<(&E, &ApplicationMetrics)>, + timing: Option<&ApplicationMetrics>, batches: as DatabaseSet>::Unmerkleized, context: RuntimeContext, candidates: Vec, + reshare_log: Option<&nunchi_dkg::DealerLog>, + rng: &mut E, ) -> Option<(Vec, Ext::Payload, QmdbMerkleized)> { let mut batch = QmdbBatch::new(batches); let mut included = Vec::new(); - let validate_start = timing.map(|(runtime_context, _)| runtime_context.current()); + let validate_start = timing.map(|_| rng.current()); for transaction in candidates { if included.len() == self.max_block_transactions { break; @@ -303,10 +315,10 @@ where } } } - if let Some(((runtime_context, metrics), validate_start)) = timing.zip(validate_start) { + if let Some((metrics, validate_start)) = timing.zip(validate_start) { metrics .proposal_validate_duration - .observe_between(validate_start, runtime_context.current()); + .observe_between(validate_start, rng.current()); } let extension = self.consensus.propose().await; @@ -317,8 +329,28 @@ where { return None; } + let dkg_start = timing.map(|_| rng.current()); + if let Some(dkg_state) = &self.dkg_state { + if let Err(error) = dkg_state + .apply_block( + &mut batch, + Height::new(context.height), + reshare_log, + rng, + ) + .await + { + debug!(?error, "invalid authenticated DKG public-state update"); + return None; + } + } + if let Some((metrics, dkg_start)) = timing.zip(dkg_start) { + metrics + .dkg_state_duration + .observe_between(dkg_start, rng.current()); + } - let merkleize_start = timing.map(|(runtime_context, _)| runtime_context.current()); + let merkleize_start = timing.map(|_| rng.current()); let merkleized = match batch.merkleize().await { Ok(merkleized) => merkleized, Err(error) => { @@ -326,10 +358,10 @@ where return None; } }; - if let Some(((runtime_context, metrics), merkleize_start)) = timing.zip(merkleize_start) { + if let Some((metrics, merkleize_start)) = timing.zip(merkleize_start) { metrics .proposal_merkleize_duration - .observe_between(merkleize_start, runtime_context.current()); + .observe_between(merkleize_start, rng.current()); } Some((included, extension, merkleized)) } @@ -337,17 +369,18 @@ where #[allow(clippy::too_many_arguments)] async fn execute_block( &mut self, - runtime_context: &E, + runtime_context: &mut E, metrics: &ApplicationMetrics, batches: as DatabaseSet>::Unmerkleized, context: RuntimeContext, transactions: &[R::Transaction], extension: &Ext::Payload, + reshare_log: Option<&nunchi_dkg::DealerLog>, events: &EventHandler, commit_extension: bool, ) -> Option> where - E: BufferPooler + Storage + Clock + Metrics, + E: BufferPooler + Storage + Clock + Metrics + CryptoRng, EventHandler: EventConsumer, { events.begin_block(context).await; @@ -389,6 +422,27 @@ where .commit_payload(&mut batch, context, extension) .await; } + let dkg_start = runtime_context.current(); + if let Some(dkg_state) = &self.dkg_state { + if let Err(error) = dkg_state + .apply_block( + &mut batch, + Height::new(context.height), + reshare_log, + runtime_context, + ) + .await + { + debug!(?error, "invalid authenticated DKG public-state update"); + if let Some(digest) = context.block_digest { + events.discard_block(digest).await; + } + return None; + } + } + metrics + .dkg_state_duration + .observe_between(dkg_start, runtime_context.current()); metrics .apply_transactions_duration .observe_between(apply_start, runtime_context.current()); @@ -476,6 +530,31 @@ where dkg, ) } + + /// Construct an application whose DKG progress is authenticated by QMDB. + #[allow(clippy::too_many_arguments)] + pub fn with_authenticated_dkg( + submitter: MempoolHandle, + max_block_transactions: usize, + consensus: Ext, + dkg: DkgMailbox, + dkg_state: DkgState, + applied_height: SharedAppliedHeight, + genesis_state: StateCommitment, + genesis_payload: sha256::Digest, + ) -> Self { + let mut application = Self::with_consensus( + submitter, + max_block_transactions, + consensus, + Some(dkg), + applied_height, + genesis_state, + genesis_payload, + ); + application.dkg_state = Some(dkg_state); + application + } } impl Application @@ -590,7 +669,7 @@ where impl StatefulApplication for Application where - E: Rng + Spawner + Metrics + Clock + Storage + BufferPooler, + E: Rng + CryptoRng + Spawner + Metrics + Clock + Storage + BufferPooler, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -612,7 +691,7 @@ where async fn propose( &mut self, - (runtime_context, context): (E, Self::Context), + (mut runtime_context, context): (E, Self::Context), ancestry: impl futures::Stream> + Send, batches: >::Unmerkleized, input: &mut Self::InputProvider, @@ -631,19 +710,23 @@ where parent.height.next(), timestamp, ); + // Obtain the optional dealer log before executing the speculative + // state batch so that it is committed by the resulting state root. + let reshare_log = match &mut self.dkg { + Some(dkg) => dkg.act().await, + None => None, + }; let (transactions, extension, merkleized) = self .build_proposal_state( - Some((&runtime_context, &metrics)), + Some(&metrics), batches, execution_context, candidates, + reshare_log.as_ref(), + &mut runtime_context, ) .await?; let state_range = Self::state_range(&merkleized); - let reshare_log = match &mut self.dkg { - Some(dkg) => dkg.act().await, - None => None, - }; let block = Block::new( context, parent.digest(), @@ -662,7 +745,7 @@ where async fn verify( &mut self, - (runtime_context, _): (E, Self::Context), + (mut runtime_context, _): (E, Self::Context), ancestry: impl futures::Stream> + Send, batches: >::Unmerkleized, ) -> Option<>::Merkleized> { @@ -700,12 +783,13 @@ where let execution_context = Self::block_runtime_context(&block); let merkleized = self .execute_block( - &runtime_context, + &mut runtime_context, &metrics, batches, execution_context, &block.transactions, &block.extension, + block.reshare_log.as_ref(), &NoopEventConsumer, false, ) @@ -719,7 +803,7 @@ where async fn apply( &mut self, - (runtime_context, _): (E, Self::Context), + (mut runtime_context, _): (E, Self::Context), block: &Self::Block, batches: >::Unmerkleized, ) -> >::Merkleized { @@ -728,12 +812,13 @@ where let events = self.events.clone(); let merkleized = self .execute_block( - &runtime_context, + &mut runtime_context, &metrics, batches, execution_context, &block.transactions, &block.extension, + block.reshare_log.as_ref(), &events, true, ) diff --git a/chain/src/consensus.rs b/chain/src/consensus.rs index e38198f..7b1ce37 100644 --- a/chain/src/consensus.rs +++ b/chain/src/consensus.rs @@ -1,4 +1,5 @@ mod dkg; +pub mod dkg_state; mod extension; pub use dkg::{dkg_reporters, DkgActor, DkgMailbox, DkgReporters}; diff --git a/chain/src/consensus/dkg_state.rs b/chain/src/consensus/dkg_state.rs new file mode 100644 index 0000000..a223448 --- /dev/null +++ b/chain/src/consensus/dkg_state.rs @@ -0,0 +1,338 @@ +//! Authenticated public DKG state stored in the chain QMDB. + +use commonware_codec::{Encode, Read}; +use commonware_consensus::types::{Epocher, FixedEpocher, Height}; +use commonware_cryptography::{ + bls12381::primitives::variant::MinSig, + ed25519::{self, Batch}, +}; +use commonware_parallel::Sequential; +use nunchi_common::{ + CommitState, Namespace, StateError, StateStore, MAX_STATE_VALUE_SIZE, +}; +use nunchi_dkg::{ + public_transition, DkgProtocolConfig, PublicCheckpoint, STATE_FORMAT_VERSION, +}; +use rand::CryptoRng; + +const DKG_NAMESPACE: Namespace = Namespace::new(b"_NUNCHI_CHAIN_DKG_STATE"); + +#[repr(u8)] +enum Table { + Marker = 0, + Checkpoint = 1, + Log = 2, +} + +impl From for u8 { + fn from(value: Table) -> Self { + value as Self + } +} + +/// Authenticated DKG state configuration installed in a DKG-enabled application. +#[derive(Clone)] +pub struct DkgState { + config: DkgProtocolConfig, + max_participants: std::num::NonZeroU32, +} + +impl DkgState { + pub fn new(config: DkgProtocolConfig) -> Result { + config.validate()?; + let max_participants = std::num::NonZeroU32::new( + config.max_participants_per_round(), + ) + .ok_or(Error::InvalidConfiguration)?; + validate_size_bounds(max_participants)?; + Ok(Self { + config, + max_participants, + }) + } + + pub const fn config(&self) -> &DkgProtocolConfig { + &self.config + } + + /// Stage the format marker and epoch-zero checkpoint. + pub async fn seed( + &self, + state: &mut S, + empty_root: commonware_cryptography::sha256::Digest, + initial_output: commonware_cryptography::bls12381::dkg::feldman_desmedt::Output< + MinSig, + ed25519::PublicKey, + >, + ) -> Result { + let expected = PublicCheckpoint::genesis(&self.config, initial_output)?; + let marker = self.marker_value()?; + match state.get(&marker_key()).await? { + Some(existing) if existing != marker => return Err(Error::MismatchedMarker), + Some(_) => { + let checkpoint = self.load_checkpoint(state).await?; + if checkpoint.epoch == commonware_consensus::types::Epoch::zero() + && checkpoint != expected + { + return Err(Error::MismatchedInitialCheckpoint); + } + return Ok(checkpoint); + } + None if state.root() != empty_root => return Err(Error::UnmarkedState), + None => {} + } + put_bounded(state, checkpoint_key(), expected.encode().to_vec())?; + put_bounded(state, marker_key(), marker)?; + Ok(expected) + } + + /// Require the current state format marker and load its checkpoint. + pub async fn load_checkpoint( + &self, + state: &S, + ) -> Result { + let marker = state + .get(&marker_key()) + .await? + .ok_or(Error::MissingMarker)?; + if marker != self.marker_value()? { + return Err(Error::MismatchedMarker); + } + let raw = state + .get(&checkpoint_key()) + .await? + .ok_or(Error::MissingCheckpoint)?; + let checkpoint = decode_checkpoint(&raw, self.max_participants)?; + self.config.validate_checkpoint(&checkpoint)?; + Ok(checkpoint) + } + + /// Validate and stage one block's public DKG effects. + pub async fn apply_block( + &self, + state: &mut S, + height: Height, + signed_log: Option<&nunchi_dkg::DealerLog>, + rng: &mut R, + ) -> Result<(), Error> + where + S: StateStore + Send + Sync, + R: CryptoRng, + { + let checkpoint = self.load_checkpoint(state).await?; + let info = self.config.round_info(&checkpoint)?; + if let Some(signed_log) = signed_log { + let (dealer, _) = signed_log + .clone() + .check(&info) + .ok_or(Error::InvalidDealerLog)?; + let eligible = self + .config + .participants_for_round(checkpoint.successful_round); + if eligible.position(&dealer).is_none() { + return Err(Error::IneligibleDealer); + } + let key = log_key(checkpoint.epoch.get(), &dealer); + let encoded = signed_log.encode(); + if let Some(existing) = state.get(&key).await? { + let existing = decode_log(&existing, self.max_participants)?; + if existing != *signed_log { + return Err(Error::ConflictingDealerLog); + } + } else { + put_bounded(state, key, encoded.to_vec())?; + } + } + + let epocher = FixedEpocher::new(self.config.epoch_length); + if epocher.last(checkpoint.epoch) != Some(height) { + return Ok(()); + } + + let eligible = self + .config + .participants_for_round(checkpoint.successful_round); + let mut logs = Vec::new(); + for dealer in &eligible { + let key = log_key(checkpoint.epoch.get(), dealer); + if let Some(raw) = state.get(&key).await? { + logs.push(decode_log(&raw, self.max_participants)?); + } + } + let next = public_transition::( + &self.config, + &checkpoint, + logs, + height, + rng, + &Sequential, + )?; + + // The checkpoint and current-log set change atomically in this batch. + for dealer in &eligible { + state.remove(log_key(checkpoint.epoch.get(), dealer)); + } + put_bounded(state, checkpoint_key(), next.checkpoint.encode().to_vec())?; + Ok(()) + } + + /// Load all current finalized signed logs from authenticated state. + pub async fn load_logs( + &self, + state: &S, + ) -> Result, Error> { + let checkpoint = self.load_checkpoint(state).await?; + let eligible = self + .config + .participants_for_round(checkpoint.successful_round); + let mut logs = Vec::new(); + for dealer in &eligible { + if let Some(raw) = state + .get(&log_key(checkpoint.epoch.get(), dealer)) + .await? + { + logs.push(decode_log(&raw, self.max_participants)?); + } + } + Ok(logs) + } + + fn marker_value(&self) -> Result, Error> { + Ok((STATE_FORMAT_VERSION, self.config.digest()?).encode().to_vec()) + } +} + +fn validate_size_bounds(max_participants: std::num::NonZeroU32) -> Result<(), Error> { + let participants = usize::try_from(max_participants.get()) + .map_err(|_| Error::InvalidConfiguration)?; + // Conservative codec formulas use the larger BLS element size and charge + // every player result for an identity, tag, and Ed25519 signature. + let polynomial = participants + .checked_mul(96) + .and_then(|size| size.checked_add(16)) + .ok_or(Error::InvalidConfiguration)?; + let ordered_set = participants + .checked_mul(32) + .and_then(|size| size.checked_add(10)) + .ok_or(Error::InvalidConfiguration)?; + let checkpoint = polynomial + .checked_add( + ordered_set + .checked_mul(3) + .ok_or(Error::InvalidConfiguration)?, + ) + .and_then(|size| size.checked_add(128)) + .ok_or(Error::InvalidConfiguration)?; + let dealer_log = polynomial + .checked_add( + participants + .checked_mul(32 + 1 + 64) + .ok_or(Error::InvalidConfiguration)?, + ) + .and_then(|size| size.checked_add(32 + 64 + 32)) + .ok_or(Error::InvalidConfiguration)?; + if checkpoint > MAX_STATE_VALUE_SIZE || dealer_log > MAX_STATE_VALUE_SIZE { + return Err(Error::ConfigurationTooLarge { + checkpoint, + dealer_log, + maximum: MAX_STATE_VALUE_SIZE, + }); + } + Ok(()) +} + +fn put_bounded( + state: &mut S, + key: commonware_cryptography::sha256::Digest, + value: Vec, +) -> Result<(), Error> { + if value.len() > MAX_STATE_VALUE_SIZE { + return Err(Error::ValueTooLarge(value.len())); + } + state.set(key, value); + Ok(()) +} + +fn decode_checkpoint( + raw: &[u8], + max_participants: std::num::NonZeroU32, +) -> Result { + let mut buf = raw; + let checkpoint = PublicCheckpoint::read_cfg( + &mut buf, + &(max_participants, nunchi_dkg::MAX_SUPPORTED_MODE), + )?; + if !buf.is_empty() { + return Err(Error::TrailingBytes); + } + Ok(checkpoint) +} + +fn decode_log( + raw: &[u8], + max_participants: std::num::NonZeroU32, +) -> Result { + let mut buf = raw; + let log = nunchi_dkg::DealerLog::read_cfg(&mut buf, &max_participants)?; + if !buf.is_empty() { + return Err(Error::TrailingBytes); + } + Ok(log) +} + +fn marker_key() -> commonware_cryptography::sha256::Digest { + DKG_NAMESPACE.key(Table::Marker, &[]) +} + +fn checkpoint_key() -> commonware_cryptography::sha256::Digest { + DKG_NAMESPACE.key(Table::Checkpoint, &[]) +} + +fn log_key( + epoch: u64, + dealer: &ed25519::PublicKey, +) -> commonware_cryptography::sha256::Digest { + let mut logical = epoch.encode().to_vec(); + logical.extend_from_slice(&dealer.encode()); + DKG_NAMESPACE.key(Table::Log, &logical) +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid DKG state configuration")] + InvalidConfiguration, + #[error( + "DKG configuration exceeds QMDB value bounds: checkpoint {checkpoint}, dealer log {dealer_log}, maximum {maximum}" + )] + ConfigurationTooLarge { + checkpoint: usize, + dealer_log: usize, + maximum: usize, + }, + #[error("authenticated state error: {0}")] + State(#[from] StateError), + #[error("public DKG state error: {0}")] + Public(#[from] nunchi_dkg::public::Error), + #[error("public DKG state codec error: {0}")] + Codec(#[from] commonware_codec::Error), + #[error("public DKG state value contains trailing bytes")] + TrailingBytes, + #[error("authenticated state has no DKG format marker; re-genesis or perform verified peer state sync")] + MissingMarker, + #[error("authenticated state is non-empty but has no DKG format marker; re-genesis or perform verified peer state sync")] + UnmarkedState, + #[error("authenticated state DKG format or protocol configuration differs; re-genesis required")] + MismatchedMarker, + #[error("authenticated state has no DKG checkpoint")] + MissingCheckpoint, + #[error("existing authenticated DKG genesis checkpoint differs")] + MismatchedInitialCheckpoint, + #[error("dealer log is invalid for the authenticated checkpoint")] + InvalidDealerLog, + #[error("dealer is not eligible for the authenticated checkpoint round")] + IneligibleDealer, + #[error("dealer submitted a conflicting finalized log")] + ConflictingDealerLog, + #[error("authenticated DKG value size {0} exceeds the QMDB limit")] + ValueTooLarge(usize), +} diff --git a/chain/src/execution.rs b/chain/src/execution.rs index dd41431..39fbd82 100644 --- a/chain/src/execution.rs +++ b/chain/src/execution.rs @@ -18,7 +18,7 @@ use crate::{ #[derive(Clone)] pub struct NodeHandle where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -31,7 +31,7 @@ where impl NodeHandle where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -58,7 +58,7 @@ where /// Read-only queries answered from the stateful actor's committed databases. pub struct StatefulQuery where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -69,7 +69,7 @@ where impl Clone for StatefulQuery where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, @@ -84,7 +84,7 @@ where impl StatefulQuery where - E: StorageContext + Spawner + Metrics + Clock + rand::Rng, + E: StorageContext + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, R: Runtime + Clone + Send + Sync + 'static, R::Transaction: PoolTransaction + Sync, Ext: ConsensusExtension + Sync, diff --git a/chain/src/lib.rs b/chain/src/lib.rs index 74244a4..a69a12f 100644 --- a/chain/src/lib.rs +++ b/chain/src/lib.rs @@ -8,6 +8,7 @@ pub mod engine; pub mod events; pub mod execution; pub mod state_sync; +pub mod startup; mod macros; #[cfg(test)] mod tests; @@ -18,6 +19,7 @@ pub use consensus::{ dkg_reporters, BlockExtension, Composite, ConsensusExtension, DkgActor, DkgMailbox, DkgReporters, NoConsensusExtension, }; +pub use consensus::dkg_state::{DkgState, Error as DkgStateError}; pub use events::{ EventConsumer, FinalizedEvents, InMemoryEventConsumer, IndexedEvent, NoopEventConsumer, TransactionEventContext, TransactionEvents, diff --git a/chain/src/startup.rs b/chain/src/startup.rs new file mode 100644 index 0000000..fa7bc36 --- /dev/null +++ b/chain/src/startup.rs @@ -0,0 +1,260 @@ +//! Exact binding between an attached QMDB and a certified startup block. + +use commonware_actor::Feedback; +use commonware_consensus::{ + marshal::Update, + types::Height, + Block, Heightable, Reporter, +}; +use commonware_cryptography::{sha256::Digest, Digestible}; +use commonware_storage::{ + mmr::Family, + qmdb::sync::Target, +}; +use commonware_utils::Acknowledgement as _; +use std::{ + collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, + sync::{Arc, Mutex}, +}; + +/// A bounded startup candidate recorded from canonical local history or a +/// certificate-verified peer-sync floor. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StartupCandidate { + pub height: Height, + pub digest: Digest, + pub state_target: Target, + pub certificate_payload: Option, + pub genesis: bool, +} + +/// The single certified block whose complete target equals attached QMDB. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StartupArtifact { + pub anchor_height: Height, + pub anchor_digest: Digest, + pub anchor_state_target: Target, +} + +/// Bounded candidate collector used only during startup. +pub struct StartupCoordinator { + capacity: NonZeroUsize, + candidates: BTreeMap<(Height, Digest), StartupCandidate>, + failure: Option, +} + +impl StartupCoordinator { + pub fn new(capacity: NonZeroUsize) -> Self { + Self { + capacity, + candidates: BTreeMap::new(), + failure: None, + } + } + + /// Record a candidate, deduplicating the same height and digest. + pub fn record(&mut self, candidate: StartupCandidate) -> Result<(), Error> { + let key = (candidate.height, candidate.digest); + if let Some(existing) = self.candidates.get(&key) { + return (existing == &candidate) + .then_some(()) + .ok_or(Error::ConflictingCandidate); + } + if self.candidates.len() == self.capacity.get() { + return Err(Error::CapacityExceeded); + } + self.candidates.insert(key, candidate); + Ok(()) + } + + /// Match the complete root and operation range and require exactly one + /// certificate-bound candidate. + pub fn resolve( + &mut self, + attached: &Target, + ) -> Result { + if let Some(error) = self.failure.take() { + return Err(error); + } + let mut matches = self + .candidates + .values() + .filter(|candidate| &candidate.state_target == attached); + let candidate = matches.next().ok_or(Error::NoExactMatch)?; + if matches.next().is_some() { + return Err(Error::MultipleExactMatches); + } + if candidate.genesis { + if candidate.height != Height::zero() + || candidate.certificate_payload.is_some() + { + return Err(Error::InvalidGenesisCandidate); + } + } else if candidate.certificate_payload != Some(candidate.digest) { + return Err(Error::CertificatePayloadMismatch); + } + let artifact = StartupArtifact { + anchor_height: candidate.height, + anchor_digest: candidate.digest, + anchor_state_target: candidate.state_target.clone(), + }; + self.candidates.clear(); + Ok(artifact) + } + + fn fail(&mut self, error: Error) { + self.failure.get_or_insert(error); + } +} + +/// Reporter that records selected certified startup blocks and immediately +/// acknowledges its `Exact` clone. +#[derive(Clone)] +pub struct StartupReporter { + coordinator: Arc>, + certified_payloads: BTreeSet, + target: fn(&B) -> Target, +} + +impl> StartupReporter { + pub fn new( + coordinator: Arc>, + certified_payloads: impl IntoIterator, + target: fn(&B) -> Target, + ) -> Self { + Self { + coordinator, + certified_payloads: certified_payloads.into_iter().collect(), + target, + } + } +} + +impl Reporter for StartupReporter +where + B: Block + + Heightable + + Digestible + + Send + + Sync + + 'static, +{ + type Activity = Update; + + fn report(&mut self, update: Self::Activity) -> Feedback { + if let Update::Block(block, acknowledgement) = update { + let digest = block.digest(); + if self.certified_payloads.contains(&digest) { + let result = self + .coordinator + .lock() + .expect("startup coordinator lock poisoned") + .record(StartupCandidate { + height: block.height(), + digest, + state_target: (self.target)(&block), + certificate_payload: Some(digest), + genesis: false, + }); + if let Err(error) = result { + self.coordinator + .lock() + .expect("startup coordinator lock poisoned") + .fail(error); + } + } + acknowledgement.acknowledge(); + } + Feedback::Ok + } +} + +#[derive(Clone, Debug, thiserror::Error, Eq, PartialEq)] +pub enum Error { + #[error("startup candidate capacity exceeded")] + CapacityExceeded, + #[error("same startup height and digest were recorded with conflicting data")] + ConflictingCandidate, + #[error("attached QMDB has no exact certified startup candidate")] + NoExactMatch, + #[error("attached QMDB matches multiple startup candidates")] + MultipleExactMatches, + #[error("startup certificate payload does not equal its block digest")] + CertificatePayloadMismatch, + #[error("genesis startup candidate is malformed")] + InvalidGenesisCandidate, +} + +#[cfg(test)] +mod tests { + use super::*; + use commonware_storage::mmr::Location; + use commonware_utils::{non_empty_range, NZUsize}; + + fn target(root: u8, start: u64, end: u64) -> Target { + Target::new( + Digest([root; 32]), + non_empty_range!(Location::new(start), Location::new(end)), + ) + } + + fn candidate( + height: u64, + digest: u8, + state_target: Target, + ) -> StartupCandidate { + let digest = Digest([digest; 32]); + StartupCandidate { + height: Height::new(height), + digest, + state_target, + certificate_payload: Some(digest), + genesis: false, + } + } + + #[test] + fn exact_target_and_certificate_payload_are_required() { + let exact = target(1, 4, 9); + let mut coordinator = StartupCoordinator::new(NZUsize!(4)); + coordinator + .record(candidate(7, 2, exact.clone())) + .unwrap(); + assert_eq!( + coordinator.resolve(&target(1, 4, 8)), + Err(Error::NoExactMatch) + ); + assert_eq!( + coordinator.resolve(&target(1, 5, 9)), + Err(Error::NoExactMatch) + ); + let artifact = coordinator.resolve(&exact).unwrap(); + assert_eq!(artifact.anchor_height, Height::new(7)); + assert_eq!(artifact.anchor_state_target, exact); + } + + #[test] + fn root_only_range_only_and_conflicting_matches_are_rejected() { + let exact = target(3, 10, 20); + let mut coordinator = StartupCoordinator::new(NZUsize!(4)); + coordinator + .record(candidate(1, 1, target(3, 1, 2))) + .unwrap(); + coordinator + .record(candidate(2, 2, target(4, 10, 20))) + .unwrap(); + assert_eq!(coordinator.resolve(&exact), Err(Error::NoExactMatch)); + + coordinator + .record(candidate(3, 3, exact.clone())) + .unwrap(); + coordinator + .record(candidate(4, 4, exact.clone())) + .unwrap(); + assert_eq!( + coordinator.resolve(&exact), + Err(Error::MultipleExactMatches) + ); + } +} diff --git a/dkg/src/actor.rs b/dkg/src/actor.rs index 354d846..65b5e52 100644 --- a/dkg/src/actor.rs +++ b/dkg/src/actor.rs @@ -1,16 +1,20 @@ use super::{ - state::{Dealer, Epoch as EpochState, Player, Storage}, + state::{ + CreatePlayerError, Dealer, Epoch as EpochState, Player, Reconciliation, + ReconciliationPhase, Storage, + }, Mailbox, Message as MailboxMessage, PostUpdate, Update, UpdateCallBack, }; use crate::{ orchestrator::{self, EpochTransition}, protector::StorageProtector, + public::{transition_logs, DkgProtocolConfig, PublicCheckpoint, N3F1_FAULT_MODEL}, setup::PeerConfig, - ReshareBlock, + validate_share, ReshareBlock, STATE_FORMAT_VERSION, }; use commonware_actor::mailbox::{self, Receiver as ActorReceiver}; use commonware_codec::{Encode, EncodeSize, Error as CodecError, Read, ReadExt, Write}; -use commonware_consensus::types::{Epoch, EpochPhase, Epocher, FixedEpocher}; +use commonware_consensus::types::{Epoch, EpochPhase, Epocher, FixedEpocher, Height}; use commonware_cryptography::{ bls12381::{ dkg::feldman_desmedt::{ @@ -23,8 +27,9 @@ use commonware_cryptography::{ }, }, ed25519::{self, Batch}, + sha256::Sha256, transcript::Summary, - BatchVerifier, PublicKey, Signer, + BatchVerifier, Hasher, PublicKey, Signer, }; use commonware_macros::select_loop; use commonware_math::algebra::Random; @@ -38,7 +43,10 @@ use commonware_runtime::{ }; use commonware_utils::{ordered::Set, Acknowledgement as _, N3f1, NZU32}; use rand::CryptoRng; -use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize}; +use std::{ + collections::BTreeMap, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, +}; use tracing::{debug, error, info, warn}; /// Per-peer label. @@ -113,6 +121,23 @@ pub struct Config

{ pub epoch_length: NonZeroU64, } +/// Authenticated public state used to reconcile protected DKG storage before +/// the actor enters consensus. +pub struct AuthenticatedBootstrap { + pub config: DkgProtocolConfig, + pub checkpoint: PublicCheckpoint, + pub logs: Vec, + pub initial_share: Option, +} + +enum Bootstrap { + Legacy { + output: Option>, + share: Option, + }, + Authenticated(Box), +} + /// Execution mode for the DKG actor. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum Execution { @@ -202,7 +227,7 @@ where /// Start the DKG actor. pub fn start( - mut self, + self, output: Option>, share: Option, orchestrator: orchestrator::Mailbox, @@ -211,17 +236,55 @@ where impl Receiver, ), callback: Box>, + ) -> Handle<()> { + self.start_inner( + Bootstrap::Legacy { output, share }, + orchestrator, + dkg, + callback, + ) + } + + /// Start after reconciling protected storage with authenticated QMDB + /// checkpoint and log state. + pub fn start_authenticated( + self, + bootstrap: AuthenticatedBootstrap, + orchestrator: orchestrator::Mailbox, + dkg: ( + impl Sender, + impl Receiver, + ), + callback: Box>, + ) -> Handle<()> { + self.start_inner( + Bootstrap::Authenticated(Box::new(bootstrap)), + orchestrator, + dkg, + callback, + ) + } + + fn start_inner( + mut self, + bootstrap: Bootstrap, + orchestrator: orchestrator::Mailbox, + dkg: ( + impl Sender, + impl Receiver, + ), + callback: Box>, ) -> Handle<()> { match self.execution { Execution::Shared => spawn_cell!( self.context, - self.run(output, share, orchestrator, dkg, callback) + self.run(bootstrap, orchestrator, dkg, callback) ), Execution::Dedicated => { let context = self.context.take(); context.dedicated().spawn(move |context| { self.context.restore(context); - self.run(output, share, orchestrator, dkg, callback) + self.run(bootstrap, orchestrator, dkg, callback) }) } } @@ -229,8 +292,7 @@ where async fn run( mut self, - output: Option>, - share: Option, + bootstrap: Bootstrap, mut orchestrator: orchestrator::Mailbox, (sender, receiver): ( impl Sender, @@ -241,6 +303,7 @@ where let max_read_size = NZU32!(self.peer_config.max_participants_per_round()); let epocher = FixedEpocher::new(self.epoch_length); let self_pk = self.signer.public_key(); + let authenticated_bootstrap = matches!(bootstrap, Bootstrap::Authenticated(_)); // Initialize persistent state let mut storage = match Storage::init( @@ -260,16 +323,29 @@ where return; } }; - if storage.epoch().is_none() { - let initial_state = EpochState { - round: 0, - rng_seed: Summary::random(self.context.as_present_mut()), - output, - share, - }; - if let Err(err) = storage.set_epoch(Epoch::zero(), initial_state).await { - error!(%err, "failed to persist initial DKG epoch"); - return; + match bootstrap { + Bootstrap::Legacy { output, share } => { + if storage.epoch().is_none() { + let initial_state = EpochState { + round: 0, + rng_seed: Summary::random(self.context.as_present_mut()), + output, + share, + }; + if let Err(err) = storage.set_epoch(Epoch::zero(), initial_state).await { + error!(%err, "failed to persist initial DKG epoch"); + return; + } + } + } + Bootstrap::Authenticated(bootstrap) => { + if let Err(err) = self + .reconcile_authenticated(&mut storage, *bootstrap, &self_pk) + .await + { + error!(%err, "failed to reconcile authenticated DKG state"); + return; + } } } @@ -366,7 +442,8 @@ where .expect("round info configuration should be correct"); // Initialize dealer state if we are a dealer (factory handles log submission check) - let mut dealer_state: Option> = am_dealer + let mut dealer_state: Option> = (am_dealer + && (is_dkg || epoch_state.share.is_some())) .then(|| { storage.create_dealer::( epoch, @@ -378,16 +455,41 @@ where }) .flatten(); - // Initialize player state if we are a player - let mut player_state: Option> = am_player - .then(|| { - storage.create_player::( - epoch, - self.signer.clone(), - round.clone(), - ) - }) - .flatten(); + // Initialize player state if we are a player. + let mut player_state: Option> = if am_player { + match storage.create_player::( + epoch, + self.signer.clone(), + round.clone(), + ) { + Ok(player) => Some(player), + Err(CreatePlayerError::MissingPlayerDealing) + if authenticated_bootstrap && epoch_state.share.is_none() => + { + warn!( + %epoch, + validator = ?self_pk, + authenticated_bootstrap, + has_share = false, + "continuing without DKG player state because authenticated public logs have no local private dealings" + ); + None + } + Err(err) => { + error!( + %epoch, + validator = ?self_pk, + authenticated_bootstrap, + has_share = epoch_state.share.is_some(), + %err, + "failed to resume DKG player from protected storage" + ); + break 'actor; + } + } + } else { + None + }; select_loop! { self.context, @@ -546,43 +648,149 @@ where // Finalize the round before acknowledging // // TODO(#3453): Minimize end-of-epoch processing via pre-verify - let mut logs = Logs::<_, _, N3f1>::new(round.clone()); - for (dealer, log) in storage.logs(epoch) { - logs.record(dealer, log); - } + let checked_logs = storage.logs(epoch); let (success, next_round, next_output, next_share) = - if let Some(ps) = player_state.take() { - match ps.finalize::( + if let Some(previous_output) = epoch_state.output.as_ref() { + let protocol_config = DkgProtocolConfig { + state_format_version: STATE_FORMAT_VERSION, + namespace: self.namespace.clone(), + epoch_length: self.epoch_length, + participants: self.peer_config.participants.clone(), + num_participants_per_round: self + .peer_config + .num_participants_per_round + .clone(), + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: N3F1_FAULT_MODEL, + trusted_initial_identity: *previous_output.public().public(), + }; + let checkpoint = PublicCheckpoint { + format_version: STATE_FORMAT_VERSION, + protocol_config_digest: protocol_config + .digest() + .expect("actor DKG configuration should be valid"), + epoch, + successful_round: epoch_state.round, + activation_height: epoch + .previous() + .and_then(|previous| epocher.last(previous)) + .unwrap_or(Height::zero()), + output: previous_output.clone(), + }; + let public = match transition_logs::<_, _, Batch>( + &protocol_config, + &checkpoint, + checked_logs.clone(), + block.height(), self.context.as_present_mut(), - logs, &Sequential, ) { - Ok((new_output, new_share)) => ( - true, - epoch_state.round + 1, - Some(new_output), - Some(new_share), - ), - Err(_) => ( + Ok(public) => public, + Err(err) => { + error!(%epoch, %err, "failed public DKG transition"); + break 'actor; + } + }; + if !public.succeeded { + ( false, epoch_state.round, epoch_state.output.clone(), epoch_state.share.clone(), - ), + ) + } else if let Some(ps) = player_state.take() { + let mut player_logs = Logs::<_, _, N3f1>::new(round.clone()); + for (dealer, log) in checked_logs { + player_logs.record(dealer, log); + } + match ps.finalize::( + self.context.as_present_mut(), + player_logs, + &Sequential, + ) { + Ok((player_output, player_share)) + if player_output == public.checkpoint.output => + { + if let Err(err) = validate_share( + &player_output, + &self_pk, + &player_share, + ) { + error!(%epoch, %err, "derived DKG share is invalid"); + break 'actor; + } + ( + true, + public.checkpoint.successful_round, + Some(player_output), + Some(player_share), + ) + } + Err( + commonware_cryptography::bls12381::dkg::feldman_desmedt::Error::MissingPlayerDealing, + ) => ( + true, + public.checkpoint.successful_round, + Some(public.checkpoint.output), + None, + ), + Ok(_) | Err(_) => { + error!(%epoch, "player result conflicts with public DKG transition"); + break 'actor; + } + } + } else { + ( + true, + public.checkpoint.successful_round, + Some(public.checkpoint.output), + None, + ) } } else { - match observe::<_, _, N3f1, Batch>( - self.context.as_present_mut(), - logs, - &Sequential, - ) { - Ok(output) => (true, epoch_state.round + 1, Some(output), None), - Err(_) => ( - false, - epoch_state.round, - epoch_state.output.clone(), - epoch_state.share.clone(), - ), + let mut logs = Logs::<_, _, N3f1>::new(round.clone()); + for (dealer, log) in checked_logs { + logs.record(dealer, log); + } + if let Some(ps) = player_state.take() { + match ps.finalize::( + self.context.as_present_mut(), + logs, + &Sequential, + ) { + Ok((new_output, new_share)) => ( + true, + epoch_state.round + 1, + Some(new_output), + Some(new_share), + ), + Err(_) => ( + false, + epoch_state.round, + epoch_state.output.clone(), + epoch_state.share.clone(), + ), + } + } else { + match observe::<_, _, N3f1, Batch>( + self.context.as_present_mut(), + logs, + &Sequential, + ) { + Ok(output) => ( + true, + epoch_state.round + 1, + Some(output), + None, + ), + Err(_) => ( + false, + epoch_state.round, + epoch_state.output.clone(), + epoch_state.share.clone(), + ), + } } }; if success { @@ -651,6 +859,126 @@ where info!("exiting DKG actor"); } + async fn reconcile_authenticated( + &mut self, + storage: &mut Storage, + bootstrap: AuthenticatedBootstrap, + self_pk: &ed25519::PublicKey, + ) -> Result<(), ReconciliationError> { + bootstrap + .config + .validate_checkpoint(&bootstrap.checkpoint)?; + if bootstrap.config.namespace != self.namespace + || bootstrap.config.epoch_length != self.epoch_length + || bootstrap.config.participants != self.peer_config.participants + || bootstrap.config.num_participants_per_round + != self.peer_config.num_participants_per_round + { + return Err(ReconciliationError::LocalConfigurationMismatch); + } + let info = bootstrap.config.round_info(&bootstrap.checkpoint)?; + let checkpoint_digest = Sha256::hash(&bootstrap.checkpoint.encode()); + if let Some(reconciliation) = storage.reconciliation() { + if reconciliation.phase == ReconciliationPhase::Importing + && (reconciliation.checkpoint_digest != checkpoint_digest + || reconciliation.target_epoch != bootstrap.checkpoint.epoch) + { + return Err(ReconciliationError::ImportInProgressConflict); + } + } + let importing = Reconciliation { + format_version: STATE_FORMAT_VERSION, + checkpoint_digest, + target_epoch: bootstrap.checkpoint.epoch, + phase: ReconciliationPhase::Importing, + }; + let complete = Reconciliation { + phase: ReconciliationPhase::Complete, + ..importing.clone() + }; + let mut logs = BTreeMap::new(); + for signed in bootstrap.logs { + let (dealer, log) = signed + .check(&info) + .ok_or(ReconciliationError::InvalidAuthenticatedLog)?; + if let Some(existing) = logs.insert(dealer, log.clone()) { + if existing != log { + return Err(ReconciliationError::ConflictingAuthenticatedLog); + } + } + } + + let target_epoch = bootstrap.checkpoint.epoch; + let mut rewrite_state = false; + let share = match storage.epoch() { + None if target_epoch == Epoch::zero() => { + if let Some(share) = bootstrap.initial_share { + validate_share(&bootstrap.checkpoint.output, self_pk, &share)?; + Some(share) + } else { + None + } + } + None => None, + Some((local_epoch, _)) if local_epoch > target_epoch => { + return Err(ReconciliationError::LocalStateAhead) + } + Some((local_epoch, _)) if local_epoch < target_epoch => { + rewrite_state = true; + None + } + Some((_, local)) => { + if local.round != bootstrap.checkpoint.successful_round + || local.output.as_ref() != Some(&bootstrap.checkpoint.output) + { + return Err(ReconciliationError::MatchingEpochConflict); + } + if let Some(share) = local.share.as_ref() { + validate_share(&bootstrap.checkpoint.output, self_pk, share)?; + } + for (dealer, log) in &logs { + if let Some(existing) = storage.logs(target_epoch).get(dealer) { + if existing != log { + return Err(ReconciliationError::ConflictingAuthenticatedLog); + } + } + } + storage.set_reconciliation(importing).await?; + for (dealer, log) in logs { + storage.append_log(target_epoch, dealer, log).await?; + } + storage.set_reconciliation(complete).await?; + return Ok(()); + } + }; + + storage.set_reconciliation(importing).await?; + for (dealer, log) in logs { + if let Some(existing) = storage.logs(target_epoch).get(&dealer) { + if existing != &log { + return Err(ReconciliationError::ConflictingAuthenticatedLog); + } + continue; + } + storage.append_log(target_epoch, dealer, log).await?; + } + if rewrite_state || storage.epoch().is_none() { + storage + .set_epoch( + target_epoch, + EpochState { + round: bootstrap.checkpoint.successful_round, + rng_seed: Summary::random(self.context.as_present_mut()), + output: Some(bootstrap.checkpoint.output), + share, + }, + ) + .await?; + } + storage.set_reconciliation(complete).await?; + Ok(()) + } + async fn distribute_shares>( self_pk: &ed25519::PublicKey, storage: &mut Storage, @@ -691,3 +1019,23 @@ where } } } + +#[derive(Debug, thiserror::Error)] +enum ReconciliationError { + #[error("authenticated public DKG state error: {0}")] + Public(#[from] crate::public::Error), + #[error("protected DKG storage error: {0}")] + Storage(#[from] crate::state::Error), + #[error("local DKG protocol configuration differs from authenticated state")] + LocalConfigurationMismatch, + #[error("authenticated dealer log is invalid")] + InvalidAuthenticatedLog, + #[error("authenticated dealer logs conflict")] + ConflictingAuthenticatedLog, + #[error("protected DKG state is ahead of authenticated QMDB state")] + LocalStateAhead, + #[error("protected DKG state conflicts with authenticated QMDB at the same epoch")] + MatchingEpochConflict, + #[error("a different authenticated DKG checkpoint import is already in progress")] + ImportInProgressConflict, +} diff --git a/dkg/src/lib.rs b/dkg/src/lib.rs index 4d92d01..1bfea95 100644 --- a/dkg/src/lib.rs +++ b/dkg/src/lib.rs @@ -9,6 +9,7 @@ mod actor; mod consensus; mod egress; mod ingress; +pub mod public; pub mod protector; pub mod orchestrator; mod setup; @@ -16,7 +17,7 @@ mod state; #[cfg(test)] mod tests; -pub use actor::{Actor, Config, Execution}; +pub use actor::{Actor, AuthenticatedBootstrap, Config, Execution}; pub use consensus::{ Activity, Context, EdScheme, EpochProvider, Finalization, Identity, Notarization, Provider, PublicKey, Scheme, Seed, Seedable, Signature, ThresholdScheme, @@ -24,8 +25,15 @@ pub use consensus::{ pub use egress::{ContinueOnUpdate, PostUpdate, Update, UpdateCallBack}; pub use ingress::{Mailbox, Message}; pub use protector::{StorageKey, StorageProtector}; +pub use public::{ + checked_threshold_scheme, transition as public_transition, transition_logs, validate_anchor, + validate_share, DkgProtocolConfig, PublicCheckpoint, PublicTransition, + ProtocolConfigReadCfg, STATE_FORMAT_VERSION, +}; pub use setup::PeerConfig; -pub use state::{Epoch as StoredEpoch, Storage}; +pub use state::{ + Epoch as StoredEpoch, Reconciliation, ReconciliationPhase, Storage, +}; pub type DealerLog = SignedDealerLog; diff --git a/dkg/src/orchestrator/actor.rs b/dkg/src/orchestrator/actor.rs index f815252..1f395f8 100644 --- a/dkg/src/orchestrator/actor.rs +++ b/dkg/src/orchestrator/actor.rs @@ -26,9 +26,14 @@ use commonware_parallel::Strategy; use commonware_runtime::{ buffer::paged::CacheRef, spawn_cell, - telemetry::metrics::{Gauge, GaugeExt, MetricsExt as _}, + telemetry::metrics::{ + histogram::Buckets, CounterFamily, EncodeLabelSet, Gauge, GaugeExt, Histogram, + MetricsExt as _, + }, BufferPooler, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage, }; +use commonware_storage::metadata::{self, Metadata}; +use commonware_utils::sequence::U64; use commonware_utils::{vec::NonEmptyVec, NZUsize, NZU16}; use rand::CryptoRng; use std::{ @@ -37,7 +42,14 @@ use std::{ num::{NonZeroU64, NonZeroUsize}, time::Duration, }; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; + +const CLEANUP_WATERMARK_KEY: U64 = U64::new(0); + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] +struct CleanupStatusLabel { + status: &'static str, +} /// Reporter that discards consensus activity. pub struct NoopReporter { @@ -101,10 +113,40 @@ where pub epoch_length: NonZeroU64, pub genesis_digest: Digest, pub recovered_floor: Option>, + /// A certificate-verified state-sync anchor for an in-progress epoch. + pub startup_finalization: Option>, + pub startup_floor: Option, pub _phantom: PhantomData, } +/// A certified startup boundary that can anchor the first entered epoch when +/// local finalized block history is absent. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StartupFloor { + pub height: Height, + pub digest: Digest, +} + +impl StartupFloor { + fn digest_for_epoch(&self, epocher: &FixedEpocher, epoch: Epoch) -> Option { + Self::floor_boundary(epocher, epoch) + .filter(|boundary| *boundary == self.height) + .map(|_| self.digest) + } + + // Epoch zero uses genesis as its floor; every later epoch is anchored by the + // last finalized block from the previous epoch. + fn floor_boundary(epocher: &FixedEpocher, epoch: Epoch) -> Option { + let previous_epoch = epoch.previous()?; + Some( + epocher + .last(previous_epoch) + .expect("previous epoch should be covered by epoch strategy"), + ) + } +} + pub struct Actor>> where E: BufferPooler + Spawner + Metrics + CryptoRng + Clock + Storage + Network, @@ -143,9 +185,15 @@ where epoch_length: NonZeroU64, genesis_digest: Digest, recovered_floor: Option>, + startup_finalization: Option>, + startup_floor: Option, page_cache_ref: CacheRef, latest_epoch: Gauge, + partition_cleanup_total: CounterFamily, + partition_cleanup_watermark: Gauge, + partitions_active: Gauge, + partition_cleanup_duration: Histogram, _phantom: PhantomData, } @@ -180,6 +228,23 @@ where // Register latest_epoch gauge for Grafana integration let latest_epoch = context.gauge("latest_epoch", "current epoch"); + let partition_cleanup_total = context.family( + "consensus_partition_cleanup", + "Total number of consensus partition cleanup outcomes", + ); + let partition_cleanup_watermark = context.gauge( + "consensus_partition_cleanup_watermark", + "Next consensus epoch partition requiring cleanup", + ); + let partitions_active = context.gauge( + "consensus_partitions_active", + "Number of consensus epoch partitions with active engines", + ); + let partition_cleanup_duration = context.histogram( + "consensus_partition_cleanup_duration_seconds", + "Duration of consensus epoch partition cleanup operations", + Buckets::LOCAL, + ); ( Self { @@ -198,14 +263,30 @@ where epoch_length: config.epoch_length, genesis_digest: config.genesis_digest, recovered_floor: config.recovered_floor, + startup_finalization: config.startup_finalization, + startup_floor: config.startup_floor, page_cache_ref, latest_epoch, + partition_cleanup_total, + partition_cleanup_watermark, + partitions_active, + partition_cleanup_duration, _phantom: PhantomData, }, Mailbox::new(sender), ) } + pub fn set_startup_floor(&mut self, startup_floor: StartupFloor) { + self.startup_floor = Some(startup_floor); + } + + /// Installs the certificate-verified state-sync anchor used to resume an + /// in-progress epoch when the local marshal archive is empty. + pub fn set_startup_finalization(&mut self, startup_finalization: Finalization) { + self.startup_finalization = Some(startup_finalization); + } + pub fn start( mut self, votes: ( @@ -268,6 +349,24 @@ where // Wait for instructions to transition epochs. let epocher = FixedEpocher::new(self.epoch_length); let mut engines: BTreeMap> = BTreeMap::new(); + let cleanup_partition = format!("{}_cleanup-v1", self.partition_prefix); + let mut cleanup_metadata = Metadata::::init( + self.context.child("partition_cleanup_metadata"), + metadata::Config { + partition: cleanup_partition, + codec_config: (), + }, + ) + .await + .expect("failed to initialize consensus partition cleanup metadata"); + let mut next_epoch_to_clean = cleanup_metadata + .get(&CLEANUP_WATERMARK_KEY) + .cloned() + .unwrap_or_else(|| U64::new(0)); + let _ = self + .partition_cleanup_watermark + .try_set(u64::from(&next_epoch_to_clean)); + let mut startup_cleanup_done = false; select_loop! { self.context, @@ -317,34 +416,30 @@ where continue; } - let floor = if self - .recovered_floor - .as_ref() - .is_some_and(|floor| floor.epoch() == transition.epoch) - { - Floor::Finalized( - self.recovered_floor - .take() - .expect("matching recovered floor must exist"), - ) - } else { - // DKG state does not persist the consensus floor; derive the epoch's - // genesis from marshal's finalized boundary block. - let digest = match Self::floor_boundary(&epocher, transition.epoch) { - Some(boundary_height) => self - .marshal - .get_block(boundary_height) - .await - .unwrap_or_else(|| { - panic!( - "missing finalized boundary block at height {} for epoch {}", - boundary_height, transition.epoch - ) - }) - .digest(), - None => self.genesis_digest, - }; - Floor::Genesis(digest) + if !startup_cleanup_done { + assert!( + u64::from(&next_epoch_to_clean) <= transition.epoch.get(), + "consensus partition cleanup watermark {} is ahead of first entering epoch {}", + u64::from(&next_epoch_to_clean), + transition.epoch, + ); + if let Some(previous) = transition.epoch.previous() { + assert!( + !engines.keys().any(|epoch| *epoch <= previous), + "refusing to clean a consensus partition with an active engine" + ); + self.cleanup_through( + &mut cleanup_metadata, + &mut next_epoch_to_clean, + previous, + ) + .await; + } + startup_cleanup_done = true; + } + + let Some(floor) = self.resolve_floor(&epocher, transition.epoch).await else { + break; }; // Register the new signing scheme with the scheme provider. @@ -364,6 +459,7 @@ where .await; engines.insert(transition.epoch, handle); let _ = self.latest_epoch.try_set(transition.epoch.get()); + let _ = self.partitions_active.try_set(engines.len()); info!(epoch = %transition.epoch, "entered epoch"); } @@ -374,25 +470,142 @@ where continue; }; handle.abort(); + // Spawned task handles close their completion channel when aborted. + match handle.await { + Ok(()) + | Err(commonware_runtime::Error::Aborted) + | Err(commonware_runtime::Error::Closed) => {} + Err(error) => { + panic!("consensus engine for epoch {epoch} failed while stopping: {error}") + } + } + let _ = self.partitions_active.try_set(engines.len()); // Unregister the signing scheme for the epoch. assert!(self.provider.unregister(&epoch)); + if u64::from(&next_epoch_to_clean) <= epoch.get() { + assert!( + !engines.keys().any(|active| *active <= epoch), + "refusing to clean a consensus partition with an active engine" + ); + self.cleanup_through( + &mut cleanup_metadata, + &mut next_epoch_to_clean, + epoch, + ) + .await; + } + info!(%epoch, "exited epoch"); } }, } } - // Epoch zero uses genesis as its floor; every later epoch is anchored by the - // last finalized block from the previous epoch. - fn floor_boundary(epocher: &FixedEpocher, epoch: Epoch) -> Option { - let previous_epoch = epoch.previous()?; - Some( - epocher - .last(previous_epoch) - .expect("previous epoch should be covered by epoch strategy"), - ) + async fn cleanup_through( + &mut self, + metadata: &mut Metadata, + next_epoch_to_clean: &mut U64, + through: Epoch, + ) { + while u64::from(&*next_epoch_to_clean) <= through.get() { + let epoch = Epoch::new(u64::from(&*next_epoch_to_clean)); + let partition = format!("{}_consensus_{}", self.partition_prefix, epoch); + let started = std::time::Instant::now(); + let status = match self.context.remove(&partition, None).await { + Ok(()) => { + info!(%epoch, %partition, "removed retired consensus partition"); + "removed" + } + Err(commonware_runtime::Error::PartitionMissing(_)) => { + info!(%epoch, %partition, "retired consensus partition already missing"); + "missing" + } + Err(error) => { + self.partition_cleanup_total + .get_or_create(&CleanupStatusLabel { status: "failed" }) + .inc(); + self.partition_cleanup_duration + .observe(started.elapsed().as_secs_f64()); + warn!(%epoch, %partition, %error, "failed to remove retired consensus partition"); + panic!("failed to remove retired consensus partition {partition}: {error}"); + } + }; + self.partition_cleanup_total + .get_or_create(&CleanupStatusLabel { status }) + .inc(); + self.partition_cleanup_duration + .observe(started.elapsed().as_secs_f64()); + + let next = epoch + .get() + .checked_add(1) + .expect("consensus cleanup watermark overflow"); + metadata + .put_sync(CLEANUP_WATERMARK_KEY, U64::new(next)) + .await + .unwrap_or_else(|error| { + panic!( + "failed to persist consensus cleanup watermark after removing {partition}: {error}" + ) + }); + *next_epoch_to_clean = U64::new(next); + let _ = self.partition_cleanup_watermark.try_set(next); + } + } + + async fn resolve_floor( + &mut self, + epocher: &FixedEpocher, + epoch: Epoch, + ) -> Option> { + if self + .recovered_floor + .as_ref() + .is_some_and(|floor| floor.epoch() == epoch) + { + return Some(Floor::Finalized( + self.recovered_floor + .take() + .expect("matching recovered floor must exist"), + )); + } + + if self + .startup_finalization + .as_ref() + .is_some_and(|finalization| finalization.epoch() == epoch) + { + return Some(Floor::Finalized( + self.startup_finalization + .take() + .expect("matching startup finalization must exist"), + )); + } + + let Some(boundary_height) = StartupFloor::floor_boundary(epocher, epoch) else { + return Some(Floor::Genesis(self.genesis_digest)); + }; + + if let Some(block) = self.marshal.get_block(boundary_height).await { + return Some(Floor::Genesis(block.digest())); + } + + if let Some(digest) = self + .startup_floor + .as_ref() + .and_then(|startup_floor| startup_floor.digest_for_epoch(epocher, epoch)) + { + return Some(Floor::Genesis(digest)); + } + + error!( + %boundary_height, + %epoch, + "refusing to enter epoch without recovered floor, certified startup boundary, or local finalized boundary block" + ); + None } async fn enter_epoch( @@ -454,3 +667,38 @@ where engine.start(vote, certificate, resolver) } } + +#[cfg(test)] +mod tests { + use super::*; + use commonware_utils::NZU64; + + #[test] + fn startup_floor_only_matches_previous_epoch_boundary() { + let epocher = FixedEpocher::new(NZU64!(10)); + let digest = Digest([9; 32]); + let floor = StartupFloor { + height: Height::new(19), + digest, + }; + + assert_eq!( + floor.digest_for_epoch(&epocher, Epoch::new(2)), + Some(digest) + ); + assert_eq!(floor.digest_for_epoch(&epocher, Epoch::new(1)), None); + assert_eq!(floor.digest_for_epoch(&epocher, Epoch::zero()), None); + } + + #[test] + fn non_boundary_startup_floor_is_not_inferred() { + let epocher = FixedEpocher::new(NZU64!(10)); + let floor = StartupFloor { + height: Height::new(12), + digest: Digest([7; 32]), + }; + + assert_eq!(floor.digest_for_epoch(&epocher, Epoch::new(2)), None); + } + +} diff --git a/dkg/src/orchestrator/mod.rs b/dkg/src/orchestrator/mod.rs index 36faace..f52bea3 100644 --- a/dkg/src/orchestrator/mod.rs +++ b/dkg/src/orchestrator/mod.rs @@ -1,7 +1,7 @@ //! Consensus engine orchestrator for epoch transitions. mod actor; -pub use actor::{Actor, Config, NoopReporter}; +pub use actor::{Actor, Config, NoopReporter, StartupFloor}; mod ingress; #[cfg(test)] diff --git a/dkg/src/public.rs b/dkg/src/public.rs new file mode 100644 index 0000000..5616176 --- /dev/null +++ b/dkg/src/public.rs @@ -0,0 +1,528 @@ +//! Authenticated public state for DKG resharing. +//! +//! This module contains the canonical protocol configuration, the public +//! checkpoint committed by an application, and the deterministic transition +//! used by both applications and DKG actors. + +use commonware_codec::{Encode, EncodeSize, Error as CodecError, RangeCfg, Read, ReadExt, Write}; +use commonware_consensus::types::{Epoch, Epocher, FixedEpocher, Height}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::{ + observe, DealerLog, Info, Logs, Output, SignedDealerLog, + }, + primitives::{ + group::Share, + sharing::{Mode, ModeVersion}, + variant::{MinSig, Variant}, + }, + }, + ed25519, + sha256::{Digest, Sha256}, + BatchVerifier, Hasher, PublicKey, Signer, +}; +use commonware_parallel::Strategy; +use commonware_utils::{ordered::Set, N3f1, Participant, TryCollect}; +use rand::{rngs::StdRng, seq::IteratorRandom, CryptoRng, SeedableRng}; +use std::{collections::BTreeMap, num::{NonZeroU32, NonZeroU64}}; + +/// Current authenticated DKG state format. +pub const STATE_FORMAT_VERSION: u16 = 1; + +/// Identifier committed for the `N3f1` fault model. +pub const N3F1_FAULT_MODEL: u8 = 1; + +const CONFIG_DOMAIN: &[u8] = b"NUNCHI_DKG_PROTOCOL_CONFIG"; + +/// Canonical protocol configuration committed by every public checkpoint. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DkgProtocolConfig { + pub state_format_version: u16, + pub namespace: Vec, + pub epoch_length: NonZeroU64, + pub participants: Set

, + pub num_participants_per_round: Vec, + pub mode: Mode, + pub mode_version: u8, + pub fault_model: u8, + pub trusted_initial_identity: V::Public, +} + +impl DkgProtocolConfig { + /// Validate semantic constraints that are not expressible through the codec. + pub fn validate(&self) -> Result<(), Error> { + if self.state_format_version != STATE_FORMAT_VERSION { + return Err(Error::UnsupportedFormat(self.state_format_version)); + } + if self.namespace.is_empty() { + return Err(Error::EmptyNamespace); + } + if self.participants.is_empty() { + return Err(Error::EmptyParticipants); + } + if self.num_participants_per_round.is_empty() { + return Err(Error::EmptyRoundSchedule); + } + if self.mode_version != 0 { + return Err(Error::UnsupportedModeVersion(self.mode_version)); + } + if self.fault_model != N3F1_FAULT_MODEL { + return Err(Error::UnsupportedFaultModel(self.fault_model)); + } + let total = self.participants.len(); + if self + .num_participants_per_round + .iter() + .any(|&count| count == 0 || count as usize > total) + { + return Err(Error::InvalidRoundSchedule); + } + Ok(()) + } + + /// Hash the canonical Commonware codec encoding. + pub fn digest(&self) -> Result { + self.validate()?; + Ok(Sha256::hash(&self.encode())) + } + + /// Return the maximum configured participant count. + pub fn max_participants_per_round(&self) -> u32 { + self.num_participants_per_round + .iter() + .copied() + .max() + .expect("validated schedule is non-empty") + } + + /// Select the participants for a successful-round number. + pub fn participants_for_round(&self, round: u64) -> Set

{ + let schedule_index = (round % self.num_participants_per_round.len() as u64) as usize; + let count = self.num_participants_per_round[schedule_index] as usize; + let participants = self.participants.iter().cloned(); + if round == 0 { + return participants.take(count).try_collect().unwrap(); + } + let mut rng = StdRng::seed_from_u64(round); + participants + .sample(&mut rng, count) + .into_iter() + .try_collect() + .unwrap() + } + + /// Reconstruct the exact public round information for a checkpoint. + pub fn round_info( + &self, + checkpoint: &PublicCheckpoint, + ) -> Result, Error> { + self.validate_checkpoint(checkpoint)?; + let dealers = self.participants_for_round(checkpoint.successful_round); + let players = self.participants_for_round(checkpoint.successful_round + 1); + Info::new::( + &self.namespace, + checkpoint.epoch.get(), + Some(checkpoint.output.clone()), + self.mode, + dealers, + players, + ) + .map_err(|error| Error::RoundInfo(error.to_string())) + } + + /// Validate that a checkpoint belongs to this protocol configuration. + pub fn validate_checkpoint( + &self, + checkpoint: &PublicCheckpoint, + ) -> Result<(), Error> { + let expected = self.digest()?; + if checkpoint.format_version != self.state_format_version { + return Err(Error::UnsupportedFormat(checkpoint.format_version)); + } + if checkpoint.protocol_config_digest != expected { + return Err(Error::ProtocolConfigMismatch); + } + if checkpoint.output.public().public() != &self.trusted_initial_identity { + return Err(Error::ThresholdIdentityMismatch); + } + Ok(()) + } +} + +impl EncodeSize for DkgProtocolConfig { + fn encode_size(&self) -> usize { + CONFIG_DOMAIN.len() + + self.state_format_version.encode_size() + + self.namespace.encode_size() + + self.epoch_length.encode_size() + + self.participants.encode_size() + + self.num_participants_per_round.encode_size() + + self.mode.encode_size() + + self.mode_version.encode_size() + + self.fault_model.encode_size() + + self.trusted_initial_identity.encode_size() + } +} + +impl Write for DkgProtocolConfig { + fn write(&self, buf: &mut impl bytes::BufMut) { + buf.put_slice(CONFIG_DOMAIN); + self.state_format_version.write(buf); + self.namespace.write(buf); + self.epoch_length.write(buf); + self.participants.write(buf); + self.num_participants_per_round.write(buf); + self.mode.write(buf); + self.mode_version.write(buf); + self.fault_model.write(buf); + self.trusted_initial_identity.write(buf); + } +} + +/// Codec limits for a protocol configuration. +#[derive(Clone, Copy)] +pub struct ProtocolConfigReadCfg { + pub max_namespace_len: usize, + pub max_participants: NonZeroU32, + pub max_round_schedule_len: usize, + pub max_supported_mode: ModeVersion, +} + +impl Read for DkgProtocolConfig { + type Cfg = ProtocolConfigReadCfg; + + fn read_cfg( + buf: &mut impl bytes::Buf, + cfg: &Self::Cfg, + ) -> Result { + let domain: [u8; CONFIG_DOMAIN.len()] = ReadExt::read(buf)?; + if domain.as_slice() != CONFIG_DOMAIN { + return Err(CodecError::Invalid("DkgProtocolConfig", "invalid domain")); + } + let value = Self { + state_format_version: ReadExt::read(buf)?, + namespace: Vec::::read_cfg( + buf, + &(RangeCfg::from(1..=cfg.max_namespace_len), ()), + )?, + epoch_length: ReadExt::read(buf)?, + participants: Set::

::read_cfg( + buf, + &(RangeCfg::from(1..=cfg.max_participants.get() as usize), ()), + )?, + num_participants_per_round: Vec::::read_cfg( + buf, + &(RangeCfg::from(1..=cfg.max_round_schedule_len), ()), + )?, + mode: Mode::read_cfg(buf, &cfg.max_supported_mode)?, + mode_version: ReadExt::read(buf)?, + fault_model: ReadExt::read(buf)?, + trusted_initial_identity: ReadExt::read(buf)?, + }; + value + .validate() + .map_err(|_| CodecError::Invalid("DkgProtocolConfig", "invalid configuration"))?; + Ok(value) + } +} + +/// Public DKG state authenticated by the application state root. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PublicCheckpoint { + pub format_version: u16, + pub protocol_config_digest: Digest, + pub epoch: Epoch, + pub successful_round: u64, + pub activation_height: Height, + pub output: Output, +} + +impl PublicCheckpoint { + /// Construct the epoch-zero checkpoint from trusted configuration. + pub fn genesis( + config: &DkgProtocolConfig, + output: Output, + ) -> Result { + let checkpoint = Self { + format_version: config.state_format_version, + protocol_config_digest: config.digest()?, + epoch: Epoch::zero(), + successful_round: 0, + activation_height: Height::zero(), + output, + }; + config.validate_checkpoint(&checkpoint)?; + Ok(checkpoint) + } +} + +impl EncodeSize for PublicCheckpoint { + fn encode_size(&self) -> usize { + self.format_version.encode_size() + + self.protocol_config_digest.encode_size() + + self.epoch.encode_size() + + self.successful_round.encode_size() + + self.activation_height.encode_size() + + self.output.encode_size() + } +} + +impl Write for PublicCheckpoint { + fn write(&self, buf: &mut impl bytes::BufMut) { + self.format_version.write(buf); + self.protocol_config_digest.write(buf); + self.epoch.write(buf); + self.successful_round.write(buf); + self.activation_height.write(buf); + self.output.write(buf); + } +} + +impl Read for PublicCheckpoint { + type Cfg = (NonZeroU32, ModeVersion); + + fn read_cfg(buf: &mut impl bytes::Buf, cfg: &Self::Cfg) -> Result { + let format_version = u16::read(buf)?; + if format_version != STATE_FORMAT_VERSION { + return Err(CodecError::Invalid( + "PublicCheckpoint", + "unsupported format version", + )); + } + Ok(Self { + format_version, + protocol_config_digest: ReadExt::read(buf)?, + epoch: ReadExt::read(buf)?, + successful_round: ReadExt::read(buf)?, + activation_height: ReadExt::read(buf)?, + output: Output::read_cfg(buf, cfg)?, + }) + } +} + +/// Result of a deterministic public transition. +pub struct PublicTransition { + pub checkpoint: PublicCheckpoint, + pub succeeded: bool, +} + +/// Execute the public transition committed at an epoch boundary. +pub fn transition( + config: &DkgProtocolConfig, + checkpoint: &PublicCheckpoint, + signed_logs: impl IntoIterator>, + boundary_height: Height, + rng: &mut impl CryptoRng, + strategy: &impl Strategy, +) -> Result, Error> +where + V: Variant, + P: PublicKey, + S: Signer, + B: BatchVerifier, +{ + config.validate_checkpoint(checkpoint)?; + let epocher = FixedEpocher::new(config.epoch_length); + if epocher.last(checkpoint.epoch) != Some(boundary_height) { + return Err(Error::WrongBoundaryHeight); + } + let info = config.round_info(checkpoint)?; + let eligible = config.participants_for_round(checkpoint.successful_round); + let mut unique = BTreeMap::>::new(); + for signed in signed_logs { + let checked = signed.clone().check(&info).ok_or(Error::InvalidSignedLog)?; + let dealer = checked.0; + if eligible.position(&dealer).is_none() { + return Err(Error::IneligibleDealer); + } + if let Some(existing) = unique.get(&dealer) { + if existing != &signed { + return Err(Error::ConflictingDealerLog); + } + continue; + } + unique.insert(dealer, signed); + } + + let mut checked = BTreeMap::new(); + for signed in unique.into_values() { + let (dealer, log) = signed.check(&info) + .ok_or(Error::InvalidSignedLog)?; + checked.insert(dealer, log); + } + transition_logs::( + config, + checkpoint, + checked, + boundary_height, + rng, + strategy, + ) +} + +/// Execute a transition from logs whose dealer signatures were already +/// authenticated while ingesting finalized blocks. +/// +/// This is the actor-facing half of [`transition`]. Applications should use +/// [`transition`] so signatures and duplicate submissions are checked in the +/// same call. +pub fn transition_logs( + config: &DkgProtocolConfig, + checkpoint: &PublicCheckpoint, + checked_logs: BTreeMap>, + boundary_height: Height, + rng: &mut impl CryptoRng, + strategy: &impl Strategy, +) -> Result, Error> +where + V: Variant, + P: PublicKey, + B: BatchVerifier, +{ + config.validate_checkpoint(checkpoint)?; + let epocher = FixedEpocher::new(config.epoch_length); + if epocher.last(checkpoint.epoch) != Some(boundary_height) { + return Err(Error::WrongBoundaryHeight); + } + let info = config.round_info(checkpoint)?; + let eligible = config.participants_for_round(checkpoint.successful_round); + let mut logs = Logs::::new(info); + for (dealer, log) in checked_logs { + if eligible.position(&dealer).is_none() { + return Err(Error::IneligibleDealer); + } + logs.record(dealer, log); + } + let observed = observe::(rng, logs, strategy); + let (succeeded, successful_round, output) = match observed { + Ok(output) => (true, checkpoint.successful_round + 1, output), + Err(_) => ( + false, + checkpoint.successful_round, + checkpoint.output.clone(), + ), + }; + Ok(PublicTransition { + checkpoint: PublicCheckpoint { + format_version: checkpoint.format_version, + protocol_config_digest: checkpoint.protocol_config_digest, + epoch: checkpoint.epoch.next(), + successful_round, + activation_height: boundary_height, + output, + }, + succeeded, + }) +} + +/// Validate exact checkpoint semantics at a certified startup anchor. +pub fn validate_anchor( + config: &DkgProtocolConfig, + checkpoint: &PublicCheckpoint, + anchor_height: Height, +) -> Result<(), Error> { + config.validate_checkpoint(checkpoint)?; + if anchor_height == Height::zero() { + return (checkpoint.epoch == Epoch::zero() + && checkpoint.activation_height == Height::zero()) + .then_some(()) + .ok_or(Error::CheckpointHeightMismatch); + } + let epocher = FixedEpocher::new(config.epoch_length); + let bounds = epocher + .containing(anchor_height) + .ok_or(Error::CheckpointHeightMismatch)?; + let expected = if anchor_height == bounds.last() { + (bounds.epoch().next(), anchor_height) + } else { + let epoch = bounds.epoch(); + let activation = epoch + .previous() + .and_then(|previous| epocher.last(previous)) + .unwrap_or(Height::zero()); + (epoch, activation) + }; + (checkpoint.epoch == expected.0 && checkpoint.activation_height == expected.1) + .then_some(()) + .ok_or(Error::CheckpointHeightMismatch) +} + +/// Validate a protected private share against authenticated public state. +pub fn validate_share( + output: &Output, + participant: &P, + share: &Share, +) -> Result<(), Error> { + let index = output + .players() + .position(participant) + .map(Participant::from_usize) + .ok_or(Error::ParticipantHasNoShare)?; + if share.index != index { + return Err(Error::InvalidLocalShare); + } + let expected = output + .public() + .partial_public(index) + .map_err(|_| Error::InvalidLocalShare)?; + if expected != share.public::() { + return Err(Error::InvalidLocalShare); + } + Ok(()) +} + +/// Construct a threshold scheme only after validating its private share. +pub fn checked_threshold_scheme( + namespace: &[u8], + output: &Output, + participant: &ed25519::PublicKey, + share: Share, +) -> Result { + validate_share(output, participant, &share)?; + crate::ThresholdScheme::signer( + namespace, + output.players().clone(), + output.public().clone(), + share, + ) + .ok_or(Error::InvalidLocalShare) +} + +/// Public-state validation failures. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("unsupported DKG state format version {0}")] + UnsupportedFormat(u16), + #[error("DKG namespace must not be empty")] + EmptyNamespace, + #[error("DKG participant set must not be empty")] + EmptyParticipants, + #[error("DKG round schedule must not be empty")] + EmptyRoundSchedule, + #[error("DKG round schedule contains an invalid participant count")] + InvalidRoundSchedule, + #[error("unsupported DKG mode version {0}")] + UnsupportedModeVersion(u8), + #[error("unsupported DKG fault model {0}")] + UnsupportedFaultModel(u8), + #[error("checkpoint protocol configuration digest does not match")] + ProtocolConfigMismatch, + #[error("checkpoint threshold identity does not match")] + ThresholdIdentityMismatch, + #[error("failed to construct DKG round info: {0}")] + RoundInfo(String), + #[error("transition height is not the current epoch boundary")] + WrongBoundaryHeight, + #[error("signed dealer log is invalid for the checkpoint")] + InvalidSignedLog, + #[error("dealer is not eligible in this DKG round")] + IneligibleDealer, + #[error("dealer submitted conflicting logs")] + ConflictingDealerLog, + #[error("checkpoint does not match the certified anchor height")] + CheckpointHeightMismatch, + #[error("validator is not a player in the checkpoint output")] + ParticipantHasNoShare, + #[error("protected local share does not match authenticated public state")] + InvalidLocalShare, +} diff --git a/dkg/src/state.rs b/dkg/src/state.rs index d0c309e..1c0474f 100644 --- a/dkg/src/state.rs +++ b/dkg/src/state.rs @@ -15,6 +15,7 @@ use commonware_cryptography::{ }, primitives::{group::Share, sharing::ModeVersion, variant::Variant}, }, + sha256::Digest, transcript::{Summary, Transcript}, BatchVerifier, PublicKey, Signer, }; @@ -23,7 +24,10 @@ use commonware_runtime::{ buffer::paged::CacheRef, Buf, BufMut, BufferPooler, Clock, Metrics, Storage as RuntimeStorage, }; use commonware_storage::{ - journal::{self, segmented::variable::{Config as SVConfig, Journal as SVJournal}}, + journal::{ + self, + segmented::variable::{Config as SVConfig, Journal as SVJournal}, + }, metadata::{self, Config as MetadataConfig, Metadata}, }; use commonware_utils::{Faults, NZUsize, NZU16}; @@ -45,6 +49,92 @@ const READ_BUFFER: NonZeroUsize = NZUsize!(1 << 20); const RECORD_AD_DOMAIN: &[u8] = b"nunchi-dkg-storage"; const RECORD_KIND_EPOCH: u8 = 0; const RECORD_KIND_EVENT: u8 = 1; +const RECONCILIATION_KEY: u8 = 0; + +/// Error returned when resuming a player from protected storage. +#[derive(Debug, thiserror::Error)] +pub enum CreatePlayerError { + /// Public dealer logs are present, but matching private dealer messages are not. + #[error("missing private player dealing")] + MissingPlayerDealing, + /// The crypto player could not be resumed from the persisted state. + #[error("failed to resume DKG player: {0}")] + Crypto(commonware_cryptography::bls12381::dkg::feldman_desmedt::Error), +} + +/// Durable authenticated-state import phase. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReconciliationPhase { + Importing, + Complete, +} + +impl EncodeSize for ReconciliationPhase { + fn encode_size(&self) -> usize { + 1 + } +} + +impl Write for ReconciliationPhase { + fn write(&self, buf: &mut impl BufMut) { + match self { + Self::Importing => 0u8.write(buf), + Self::Complete => 1u8.write(buf), + } + } +} + +impl Read for ReconciliationPhase { + type Cfg = (); + + fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { + match u8::read(buf)? { + 0 => Ok(Self::Importing), + 1 => Ok(Self::Complete), + other => Err(CodecError::InvalidEnum(other)), + } + } +} + +/// Crash-recovery marker for an authenticated checkpoint import. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Reconciliation { + pub format_version: u16, + pub checkpoint_digest: Digest, + pub target_epoch: EpochNum, + pub phase: ReconciliationPhase, +} + +impl EncodeSize for Reconciliation { + fn encode_size(&self) -> usize { + self.format_version.encode_size() + + self.checkpoint_digest.encode_size() + + self.target_epoch.encode_size() + + self.phase.encode_size() + } +} + +impl Write for Reconciliation { + fn write(&self, buf: &mut impl BufMut) { + self.format_version.write(buf); + self.checkpoint_digest.write(buf); + self.target_epoch.write(buf); + self.phase.write(buf); + } +} + +impl Read for Reconciliation { + type Cfg = (); + + fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { + Ok(Self { + format_version: ReadExt::read(buf)?, + checkpoint_digest: ReadExt::read(buf)?, + target_epoch: ReadExt::read(buf)?, + phase: ReadExt::read(buf)?, + }) + } +} /// Errors returned by DKG persistent storage. #[derive(Debug, thiserror::Error)] @@ -204,6 +294,7 @@ where public_key: P, states: Metadata, + reconciliation: Metadata, msgs: SVJournal, // In-memory state @@ -238,6 +329,14 @@ where }, ) .await?; + let reconciliation = Metadata::init( + context.child("reconciliation"), + MetadataConfig { + partition: format!("{partition_prefix}_reconciliation"), + codec_config: (), + }, + ) + .await?; let mut msgs = SVJournal::init( context.child("msgs"), @@ -311,6 +410,7 @@ where namespace, public_key, states, + reconciliation, msgs, current, epochs, @@ -459,6 +559,22 @@ where self.current.as_ref().map(|(e, s)| (*e, s.clone())) } + /// Return the durable authenticated-state reconciliation marker. + pub fn reconciliation(&self) -> Option { + self.reconciliation.get(&RECONCILIATION_KEY).cloned() + } + + /// Persist and sync an authenticated-state reconciliation phase. + pub async fn set_reconciliation( + &mut self, + reconciliation: Reconciliation, + ) -> Result<(), Error> { + self.reconciliation + .put(RECONCILIATION_KEY, reconciliation); + self.reconciliation.sync().await?; + Ok(()) + } + fn get_or_create_epoch(&mut self, epoch: EpochNum) -> &mut EpochCache { self.epochs.entry(epoch).or_default() } @@ -630,16 +746,25 @@ where epoch: EpochNum, signer: C, round_info: Info, - ) -> Option> { + ) -> Result, CreatePlayerError> { let logs = self.logs(epoch); let dealings = self.dealings(epoch); - let (crypto_player, acks) = CryptoPlayer::resume::(round_info, signer, &logs, dealings) - .expect("should be able to resume player"); + let (crypto_player, acks) = + CryptoPlayer::resume::(round_info, signer, &logs, dealings).map_err(|err| { + if matches!( + err, + commonware_cryptography::bls12381::dkg::feldman_desmedt::Error::MissingPlayerDealing + ) { + CreatePlayerError::MissingPlayerDealing + } else { + CreatePlayerError::Crypto(err) + } + })?; for dealer in acks.keys() { debug!(?epoch, ?dealer, "restored committed dealer message"); } - Some(Player { + Ok(Player { player: crypto_player, acks, }) diff --git a/dkg/src/tests/actor.rs b/dkg/src/tests/actor.rs index 869a84e..eef84cf 100644 --- a/dkg/src/tests/actor.rs +++ b/dkg/src/tests/actor.rs @@ -7,7 +7,10 @@ use commonware_codec::{EncodeSize, Error as CodecError, Read, ReadExt, Write}; use commonware_consensus::types::Epoch; use commonware_consensus::{types::Height, Heightable}; use commonware_cryptography::{ - bls12381::{dkg::feldman_desmedt::deal, primitives::variant::MinSig}, + bls12381::{ + dkg::feldman_desmedt::{deal, Dealer, DealerLog, Player, Verdict}, + primitives::{sharing::Mode, variant::MinSig}, + }, ed25519::{PrivateKey, PublicKey as Ed25519PublicKey}, sha256, transcript::Summary, @@ -16,7 +19,7 @@ use commonware_cryptography::{ use commonware_macros::test_traced; use commonware_math::algebra::Random; use commonware_p2p::{utils::mocks::inert_channel, PeerSetSubscription, Provider}; -use commonware_runtime::{deterministic, Runner, Supervisor as _}; +use commonware_runtime::{deterministic, Clock, Runner, Supervisor as _}; use commonware_utils::{channel::mpsc, N3f1, NZUsize, TryCollect, NZU32, NZU64}; use core::marker::PhantomData; use std::collections::BTreeMap; @@ -130,6 +133,56 @@ fn peer_config( (peer_config, participants) } +fn finalized_dkg_log( + namespace: &[u8], + epoch: Epoch, + dealers: commonware_utils::ordered::Set, + players: commonware_utils::ordered::Set, + participants: &BTreeMap, + dealer_pk: &Ed25519PublicKey, +) -> DealerLog { + let round = commonware_cryptography::bls12381::dkg::feldman_desmedt::Info::new::( + namespace, + epoch.get(), + None, + Mode::NonZeroCounter, + dealers, + players, + ) + .expect("round info should be valid"); + let dealer_signer = participants + .get(dealer_pk) + .cloned() + .expect("dealer signer should exist"); + let (mut dealer, public, private) = Dealer::::start::( + commonware_utils::test_rng(), + round.clone(), + dealer_signer, + None, + ) + .expect("dealer should start"); + + for (player_pk, private) in private { + let player_signer = participants + .get(&player_pk) + .cloned() + .expect("player signer should exist"); + let mut player = Player::new(round.clone(), player_signer).expect("player should start"); + let Verdict::Valid(ack) = + player.dealer_message::(dealer_pk.clone(), public.clone(), private) + else { + panic!("valid dealing should be acknowledged"); + }; + dealer.receive_player_ack(player_pk, ack).unwrap(); + } + + dealer + .finalize::() + .check(&round) + .expect("finalized log should check") + .1 +} + fn assert_recovered_storage_controls_dkg_mode_on_restart(execution: Execution, suffix: &str) { let executor = deterministic::Runner::seeded(8); executor.start(|mut context| async move { @@ -227,3 +280,105 @@ fn default_execution_recovered_storage_controls_dkg_mode_on_restart() { fn dedicated_execution_recovered_storage_controls_dkg_mode_on_restart() { assert_recovered_storage_controls_dkg_mode_on_restart(Execution::Dedicated, "dedicated"); } + +#[test_traced] +fn legacy_missing_player_dealing_exits_actor() { + let executor = deterministic::Runner::seeded(9); + executor.start(|mut context| async move { + let namespace = b"test_dkg".to_vec(); + let epoch = Epoch::zero(); + let (peer_config, participants) = peer_config(4, vec![4]); + let self_pk = peer_config + .participants + .iter() + .next() + .cloned() + .expect("participant exists"); + let signer = participants + .get(&self_pk) + .cloned() + .expect("signer should exist"); + let dealer_pk = peer_config + .participants + .iter() + .find(|candidate| **candidate != self_pk) + .cloned() + .expect("dealer exists"); + let log = finalized_dkg_log( + &namespace, + epoch, + peer_config.participants.clone(), + peer_config.dealers(0), + &participants, + &dealer_pk, + ); + let partition_prefix = format!("legacy_missing_player_dealing_{self_pk}"); + + let mut storage = Storage::<_, MinSig, Ed25519PublicKey>::init( + context.child("seed_storage"), + &partition_prefix, + StorageProtector::new(TEST_STORAGE_KEY), + namespace.clone(), + self_pk.clone(), + NZU32!(peer_config.max_participants_per_round()), + crate::MAX_SUPPORTED_MODE, + ) + .await + .expect("storage init should succeed"); + storage + .set_epoch( + epoch, + EpochState { + round: 0, + rng_seed: Summary::random(&mut context), + output: None, + share: None, + }, + ) + .await + .expect("set epoch should succeed"); + storage + .append_log(epoch, dealer_pk, log) + .await + .expect("append log should succeed"); + drop(storage); + + let (actor, _mailbox) = Actor::<_, _, TestBlock>::new( + context.child("actor"), + Config { + manager: NoopManager::::default(), + signer, + mailbox_size: NZUsize!(8), + execution: Execution::default(), + partition_prefix, + peer_config: peer_config.clone(), + max_supported_mode: crate::MAX_SUPPORTED_MODE, + namespace, + storage_protector: StorageProtector::new(TEST_STORAGE_KEY), + epoch_length: NZU64!(200), + }, + ); + let (sender, receiver) = inert_channel(&peer_config.participants); + let (orchestrator_sender, mut orchestrator_receiver) = + mailbox::new(context.child("orchestrator_mailbox"), NZUsize!(4)); + let handle = actor.start( + None, + None, + crate::orchestrator::Mailbox::new(orchestrator_sender), + (sender, receiver), + ContinueOnUpdate::boxed(), + ); + + let Some(Message::Enter(transition)) = orchestrator_receiver.recv().await else { + panic!("actor should emit an epoch transition before detecting bad player state"); + }; + assert_eq!(transition.epoch, epoch); + + commonware_macros::select! { + _ = handle => {}, + _ = context.sleep(std::time::Duration::from_secs(1)) => { + panic!("legacy actor should fail closed instead of continuing shareless"); + }, + } + }); +} diff --git a/dkg/src/tests/mod.rs b/dkg/src/tests/mod.rs index 8172f8b..ae41927 100644 --- a/dkg/src/tests/mod.rs +++ b/dkg/src/tests/mod.rs @@ -1,3 +1,4 @@ mod actor; mod protector; +mod public; mod state; diff --git a/dkg/src/tests/public.rs b/dkg/src/tests/public.rs new file mode 100644 index 0000000..a7ff59d --- /dev/null +++ b/dkg/src/tests/public.rs @@ -0,0 +1,187 @@ +use crate::{ + public_transition, transition_logs, validate_anchor, validate_share, + DkgProtocolConfig, PublicCheckpoint, STATE_FORMAT_VERSION, +}; +use commonware_codec::{Encode, Read}; +use commonware_consensus::types::{Epoch, Epocher, FixedEpocher, Height}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::{deal, Dealer, Player, Verdict}, + primitives::{ + group::{Private, Scalar, Share}, + sharing::Mode, + variant::MinSig, + }, + }, + ed25519::{self, Batch}, + Signer, +}; +use commonware_math::algebra::Ring; +use commonware_parallel::Sequential; +use commonware_utils::{ + ordered::Set, test_rng, N3f1, Participant, TestRng, NZU64, +}; + +fn setup() -> ( + DkgProtocolConfig, + Vec, + PublicCheckpoint, + commonware_utils::ordered::Map, +) { + let signers = (0..4) + .map(ed25519::PrivateKey::from_seed) + .collect::>(); + let participants = + Set::from_iter_dedup(signers.iter().map(Signer::public_key)); + let (output, shares) = deal::( + test_rng(), + Mode::NonZeroCounter, + participants.clone(), + ) + .unwrap(); + let config = DkgProtocolConfig { + state_format_version: STATE_FORMAT_VERSION, + namespace: b"nunchi-public-state-test".to_vec(), + epoch_length: NZU64!(10), + participants, + num_participants_per_round: vec![4], + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: crate::public::N3F1_FAULT_MODEL, + trusted_initial_identity: *output.public().public(), + }; + let checkpoint = PublicCheckpoint::genesis(&config, output).unwrap(); + (config, signers, checkpoint, shares) +} + +#[test] +fn protocol_digest_is_canonical_and_binds_configuration() { + let (config, _, _, _) = setup(); + let mut same = config.clone(); + assert_eq!(config.digest().unwrap(), same.digest().unwrap()); + same.epoch_length = NZU64!(11); + assert_ne!(config.digest().unwrap(), same.digest().unwrap()); + assert_eq!( + commonware_formatting::hex(&config.digest().unwrap()), + "cef10ca10a298de2fc304c09729e8e846ed682db984a66a8d5c09cd8a1be8b0d" + ); +} + +#[test] +fn checkpoint_codec_is_bounded_and_rejects_wrong_format() { + let (config, _, checkpoint, _) = setup(); + let encoded = checkpoint.encode(); + let mut input = encoded.as_ref(); + let decoded = PublicCheckpoint::read_cfg( + &mut input, + &( + commonware_utils::NZU32!(config.max_participants_per_round()), + crate::MAX_SUPPORTED_MODE, + ), + ) + .unwrap(); + assert_eq!(decoded, checkpoint); + assert!(input.is_empty()); + + let mut malformed = encoded.to_vec(); + malformed[0] ^= 1; + assert!(PublicCheckpoint::::read_cfg( + &mut malformed.as_slice(), + &( + commonware_utils::NZU32!(config.max_participants_per_round()), + crate::MAX_SUPPORTED_MODE, + ), + ) + .is_err()); +} + +#[test] +fn failed_public_transition_advances_epoch_only() { + let (config, _, checkpoint, _) = setup(); + let boundary = FixedEpocher::new(config.epoch_length) + .last(Epoch::zero()) + .unwrap(); + let transition = transition_logs::<_, _, Batch>( + &config, + &checkpoint, + Default::default(), + boundary, + &mut test_rng(), + &Sequential, + ) + .unwrap(); + assert!(!transition.succeeded); + assert_eq!(transition.checkpoint.epoch, Epoch::new(1)); + assert_eq!( + transition.checkpoint.successful_round, + checkpoint.successful_round + ); + assert_eq!(transition.checkpoint.output, checkpoint.output); + assert_eq!(transition.checkpoint.activation_height, boundary); + validate_anchor(&config, &transition.checkpoint, boundary).unwrap(); +} + +#[test] +fn successful_transition_and_identical_duplicate_are_deterministic() { + let (config, signers, checkpoint, shares) = setup(); + let info = config.round_info(&checkpoint).unwrap(); + let mut logs = Vec::new(); + for (dealer_index, signer) in signers.iter().enumerate() { + let dealer_pk = signer.public_key(); + let (mut dealer, public, private) = Dealer::::start::( + TestRng::new(100 + dealer_index as u64), + info.clone(), + signer.clone(), + shares.get_value(&dealer_pk).cloned(), + ) + .unwrap(); + for (player_pk, private) in private { + let player_signer = signers + .iter() + .find(|candidate| candidate.public_key() == player_pk) + .unwrap() + .clone(); + let mut player = Player::new(info.clone(), player_signer).unwrap(); + let Verdict::Valid(ack) = + player.dealer_message::(dealer_pk.clone(), public.clone(), private) + else { + panic!("valid dealing should be acknowledged"); + }; + dealer.receive_player_ack(player_pk, ack).unwrap(); + } + logs.push(dealer.finalize::()); + } + logs.push(logs[0].clone()); + let boundary = FixedEpocher::new(config.epoch_length) + .last(Epoch::zero()) + .unwrap(); + let transition = public_transition::<_, _, ed25519::PrivateKey, Batch>( + &config, + &checkpoint, + logs, + boundary, + &mut test_rng(), + &Sequential, + ) + .unwrap(); + assert!(transition.succeeded); + assert_eq!(transition.checkpoint.epoch, Epoch::new(1)); + assert_eq!(transition.checkpoint.successful_round, 1); +} + +#[test] +fn local_share_validation_is_checked_without_panicking() { + let (_, signers, checkpoint, shares) = setup(); + let participant = signers[0].public_key(); + let share = shares.get_value(&participant).unwrap(); + validate_share(&checkpoint.output, &participant, share).unwrap(); + + let invalid = Share::new( + Participant::new(share.index.get()), + Private::new(Scalar::one()), + ); + assert!(validate_share(&checkpoint.output, &participant, &invalid).is_err()); + let wrong_index = Share::new(Participant::new(1), share.private.clone()); + assert!(validate_share(&checkpoint.output, &participant, &wrong_index).is_err()); + assert!(validate_anchor(&setup().0, &checkpoint, Height::new(10)).is_err()); +} diff --git a/dkg/src/tests/state.rs b/dkg/src/tests/state.rs index 4a27bf8..2bdf01d 100644 --- a/dkg/src/tests/state.rs +++ b/dkg/src/tests/state.rs @@ -1,13 +1,16 @@ use crate::{ protector::{ProtectionError, SealedRecord, StorageProtector}, - state::{Dealer, Epoch as EpochState, Error as StorageError, Storage}, + state::{ + CreatePlayerError, Dealer, Epoch as EpochState, Error as StorageError, Reconciliation, + ReconciliationPhase, Storage, + }, }; use bytes::Bytes; use commonware_codec::{Encode, RangeCfg, ReadExt}; use commonware_consensus::types::Epoch; use commonware_cryptography::{ bls12381::{ - dkg::feldman_desmedt::{DealerPrivMsg, DealerPubMsg, Info, PlayerAck, Verdict}, + dkg::feldman_desmedt::{DealerPrivMsg, DealerPubMsg, Info, Player, PlayerAck, Verdict}, primitives::{ group::{Private, Scalar, Share}, sharing::Mode, @@ -15,6 +18,7 @@ use commonware_cryptography::{ }, }, ed25519::{self}, + sha256::Digest, transcript::Summary, Signer, }; @@ -117,6 +121,45 @@ fn test_dealing( (dealer_signer.public_key(), pub_msg, priv_msg) } +fn finalized_dealer_log( + signers: &[ed25519::PrivateKey], + dealer_index: usize, +) -> ( + ed25519::PublicKey, + commonware_cryptography::bls12381::dkg::feldman_desmedt::DealerLog< + MinPk, + ed25519::PublicKey, + >, +) { + let round_info = create_round_info(signers); + let dealer_signer = signers[dealer_index].clone(); + let dealer_pk = dealer_signer.public_key(); + let mut rng = test_rng(); + let (mut dealer, pub_msg, priv_msgs) = + commonware_cryptography::bls12381::dkg::feldman_desmedt::Dealer::::start::< + N3f1, + >(&mut rng, round_info.clone(), dealer_signer, None) + .expect("valid dealer"); + for (player_pk, priv_msg) in priv_msgs { + let player_signer = signers + .iter() + .find(|candidate| candidate.public_key() == player_pk) + .expect("player signer should exist") + .clone(); + let mut player = Player::new(round_info.clone(), player_signer).expect("valid player"); + let Verdict::Valid(ack) = + player.dealer_message::(dealer_pk.clone(), pub_msg.clone(), priv_msg) + else { + panic!("valid dealing should be acknowledged"); + }; + dealer.receive_player_ack(player_pk, ack).unwrap(); + } + let signed = dealer.finalize::(); + signed + .check(&round_info) + .expect("finalized log should check") +} + async fn corrupt_epoch_record(context: E, partition: &str, epoch: Epoch) where E: StorageContext, @@ -458,6 +501,107 @@ fn storage_recovers_no_share_observer_epoch() { }); } +#[test_traced] +fn storage_recovers_reconciliation_phase_from_separate_partition() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let signers = create_test_signers(4); + let public_key = signers[0].public_key(); + let partition = "reconciliation_marker"; + let marker = Reconciliation { + format_version: crate::STATE_FORMAT_VERSION, + checkpoint_digest: Digest([9; 32]), + target_epoch: Epoch::new(15), + phase: ReconciliationPhase::Importing, + }; + + let mut storage = init_storage( + context.child("storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + public_key.clone(), + ) + .await; + storage + .set_reconciliation(marker.clone()) + .await + .expect("reconciliation marker should persist"); + drop(storage); + + let mut recovered = init_storage( + context.child("recovered_storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + public_key, + ) + .await; + assert_eq!(recovered.reconciliation(), Some(marker.clone())); + + let complete = Reconciliation { + phase: ReconciliationPhase::Complete, + ..marker + }; + recovered + .set_reconciliation(complete.clone()) + .await + .expect("completed marker should persist"); + drop(recovered); + + let recovered = init_storage( + context.child("completed_storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + signers[0].public_key(), + ) + .await; + assert_eq!(recovered.reconciliation(), Some(complete)); + }); +} + +#[test_traced] +fn create_player_returns_missing_dealing_for_imported_public_logs() { + let executor = deterministic::Runner::default(); + executor.start(|mut context| async move { + let signers = create_test_signers(4); + let player_signer = signers[1].clone(); + let player_pk = player_signer.public_key(); + let partition = "missing_player_dealing"; + let epoch = Epoch::zero(); + let round_info = create_round_info(&signers); + let (dealer_pk, log) = finalized_dealer_log(&signers, 0); + + let mut storage = init_storage( + context.child("storage"), + partition, + TEST_STORAGE_KEY, + TEST_NAMESPACE.to_vec(), + player_pk, + ) + .await; + storage + .set_epoch(epoch, epoch_state(&mut context, 0, None)) + .await + .expect("set epoch should succeed"); + storage + .append_log(epoch, dealer_pk, log) + .await + .expect("append log should succeed"); + + match storage.create_player::( + epoch, + player_signer, + round_info, + ) { + Err(CreatePlayerError::MissingPlayerDealing) => {} + Err(err) => panic!("expected missing private dealing, got {err}"), + Ok(_) => panic!("missing private dealing should be reported"), + } + }); +} + #[test_traced] fn test_dealer_handle_returns_false_when_player_not_in_unsent() { let executor = deterministic::Runner::default(); diff --git a/examples/bridge-chain/src/engine.rs b/examples/bridge-chain/src/engine.rs index 3bda1e5..c61e60d 100644 --- a/examples/bridge-chain/src/engine.rs +++ b/examples/bridge-chain/src/engine.rs @@ -81,6 +81,19 @@ pub struct Config, P: Manager, } +#[derive(Debug, thiserror::Error)] +pub enum StartupError { + #[error("bridge-chain does not authenticate DKG progress in QMDB; state_sync = true is unsupported")] + UnsupportedDkgStateSync, +} + +pub fn validate_state_sync(enabled: bool) -> Result<(), StartupError> { + if enabled { + return Err(StartupError::UnsupportedDkgStateSync); + } + Ok(()) +} + type DkgActor = nunchi_chain::DkgActor; type DkgMailbox = nunchi_chain::DkgMailbox; type StatefulApp = @@ -168,7 +181,8 @@ where impl Sender, impl Receiver, ), - ) -> (Self, NodeHandle) { + ) -> Result<(Self, NodeHandle), StartupError> { + validate_state_sync(config.state_sync)?; let (mempool, submitter) = Mempool::::new(config.pool_config.clone()); let mempool = mempool.start(context.child("mempool")); @@ -469,6 +483,8 @@ where epoch_length: BLOCKS_PER_EPOCH, genesis_digest, recovered_floor, + startup_finalization: None, + startup_floor: None, _phantom: PhantomData, }, ); @@ -489,7 +505,7 @@ where stateful, stateful_mailbox, }; - (engine, node_handle) + Ok((engine, node_handle)) } #[allow(clippy::too_many_arguments)] diff --git a/examples/bridge-chain/src/execution.rs b/examples/bridge-chain/src/execution.rs index e6e6b5f..35aaa7b 100644 --- a/examples/bridge-chain/src/execution.rs +++ b/examples/bridge-chain/src/execution.rs @@ -13,7 +13,7 @@ pub use nunchi_chain::SharedAppliedHeight; #[derive(Clone)] pub struct NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub bridge: BridgeMailbox, pub stateful: StatefulMailbox, @@ -23,7 +23,7 @@ where impl NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub fn new( bridge: BridgeMailbox, diff --git a/examples/bridge-chain/src/rpc.rs b/examples/bridge-chain/src/rpc.rs index e127f05..7a34724 100644 --- a/examples/bridge-chain/src/rpc.rs +++ b/examples/bridge-chain/src/rpc.rs @@ -21,7 +21,8 @@ where + commonware_runtime::Spawner + commonware_runtime::Metrics + commonware_runtime::Clock - + rand::Rng, + + rand::Rng + + rand::CryptoRng, { node: NodeHandle, } @@ -32,7 +33,8 @@ where + commonware_runtime::Spawner + commonware_runtime::Metrics + commonware_runtime::Clock - + rand::Rng, + + rand::Rng + + rand::CryptoRng, { pub const fn new(node: NodeHandle) -> Self { Self { node } @@ -74,6 +76,7 @@ where + commonware_runtime::Metrics + commonware_runtime::Clock + rand::Rng + + rand::CryptoRng + Send + Sync + 'static, @@ -91,6 +94,7 @@ where + commonware_runtime::Metrics + commonware_runtime::Clock + rand::Rng + + rand::CryptoRng + Send + Sync + 'static, diff --git a/examples/bridge-chain/src/testnet.rs b/examples/bridge-chain/src/testnet.rs index 2c1107a..635d657 100644 --- a/examples/bridge-chain/src/testnet.rs +++ b/examples/bridge-chain/src/testnet.rs @@ -209,6 +209,8 @@ pub enum Error { TomlDeserialize(#[from] toml::de::Error), #[error("engine stopped unexpectedly: {0}")] Engine(commonware_runtime::Error), + #[error("engine startup failed: {0}")] + EngineStartup(#[from] crate::engine::StartupError), } struct Material { @@ -585,7 +587,7 @@ async fn start_node( probe, state_sync, ) - .await; + .await?; let engine_handle = engine.start( pending, recovered, diff --git a/examples/bridge-chain/src/tests/mod.rs b/examples/bridge-chain/src/tests/mod.rs index 34d6682..11dfd7c 100644 --- a/examples/bridge-chain/src/tests/mod.rs +++ b/examples/bridge-chain/src/tests/mod.rs @@ -27,6 +27,15 @@ use crate::{application, Application, Block, TxPool}; const FOREIGN_NAMESPACE: &[u8] = b"_NUNCHI_BRIDGE_CHAIN_FOREIGN"; const WRONG_NAMESPACE: &[u8] = b"_NUNCHI_BRIDGE_CHAIN_WRONG"; +#[test] +fn peer_state_sync_is_rejected_until_bridge_authenticates_dkg_state() { + assert!(matches!( + crate::engine::validate_state_sync(true), + Err(crate::engine::StartupError::UnsupportedDkgStateSync) + )); + assert!(crate::engine::validate_state_sync(false).is_ok()); +} + fn schemes(namespace: &[u8], seed: u64) -> Vec { let mut rng = TestRng::new(seed); vrf::fixture::(&mut rng, namespace, 4).schemes diff --git a/examples/coins/chain/src/engine.rs b/examples/coins/chain/src/engine.rs index 00a47dd..cb9d4c6 100644 --- a/examples/coins/chain/src/engine.rs +++ b/examples/coins/chain/src/engine.rs @@ -1,11 +1,9 @@ use crate::application::{self, Application}; use crate::execution::NodeHandle; -use crate::genesis::{genesis_target, state_commitment, ChainGenesis}; +use crate::genesis::{authenticated_genesis_target, state_commitment, ChainGenesis}; +use crate::history::{self, BlocksArchive, FinalizationsArchive}; use crate::indexer; -use crate::{ - Block, EpochProvider, Finalization, Provider, PublicKey, Scheme, Transaction, BLOCKS_PER_EPOCH, - NAMESPACE, -}; +use crate::{Block, EpochProvider, Provider, PublicKey, Scheme, Transaction, NAMESPACE}; use commonware_broadcast::buffered; use commonware_consensus::{ marshal::{ @@ -13,25 +11,25 @@ use commonware_consensus::{ core::Actor as MarshalActor, resolver, standard::{Inline, Standard}, - store::{Blocks as _, Certificates}, + store::Certificates, }, simplex::elector::Random, + simplex::types::Finalization, types::{Epoch, FixedEpocher, Height, ViewDelta}, - Reporters, + Epochable, Reporters, }; use commonware_cryptography::{ bls12381::{ dkg::feldman_desmedt::Output, - primitives::{group, variant::MinSig}, + primitives::{group, sharing::Mode, variant::MinSig}, }, - certificate::Verifier as _, ed25519::{self, Batch}, sha256::Digest, BatchVerifier, Digestible, Signer, }; use commonware_glue::stateful::{ Application as StatefulApplication, - db::ManagedDb as _, + db::{DatabaseSet as _, ManagedDb as _}, probe::{Config as ProbeConfig, Probe}, Config as StatefulConfig, Mailbox as StatefulMailbox, Stateful as StatefulActor, SyncPlan, }; @@ -41,12 +39,8 @@ use commonware_runtime::{ buffer::paged::CacheRef, spawn_cell, BufferPooler, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage, Strategizer, }; -use commonware_storage::{ - archive::{immutable, Identifier as ArchiveIdentifier}, - metadata::{self, Metadata}, - queue, -}; -use commonware_utils::{sequence::U64, union, NZDuration, NZU64}; +use commonware_storage::{archive::{Archive as ArchiveStore, Identifier as ArchiveIdentifier}, metadata::{self, Metadata}, queue}; +use commonware_utils::{sequence::U64, union, NZDuration}; use futures::lock::Mutex as AsyncMutex; use governor::clock::Clock as GClock; use nunchi_chain::engine::*; @@ -55,13 +49,14 @@ use nunchi_chain::state_sync::{ Mailbox as StateSyncMailbox, }; use nunchi_clob::{ClobActor, ClobConfig, ClobExtension}; -use nunchi_common::{QmdbBackend, QmdbState}; +use nunchi_common::{QmdbBackend, QmdbReader, QmdbState}; use nunchi_dkg::{self as dkg, orchestrator, PeerConfig, UpdateCallBack, MAX_SUPPORTED_MODE}; use nunchi_mempool::{Mempool, PoolConfig}; use rand::{CryptoRng, Rng}; use std::{ marker::PhantomData, - sync::Arc, + num::NonZeroU64, + sync::{Arc, Mutex}, time::{Duration, Instant}, }; use tracing::{info, warn}; @@ -84,13 +79,12 @@ pub struct Config, P: Manager, pub share: Option, pub peer_config: PeerConfig, + pub epoch_length: NonZeroU64, pub leader_timeout: Duration, pub certification_timeout: Duration, pub strategy: S, @@ -100,6 +94,7 @@ pub struct Config, P: Manager, pub indexer: Option, + pub indexer_spool_limits: indexer::SpoolLimits, } type DkgActor = nunchi_chain::DkgActor; @@ -110,8 +105,6 @@ type LimitedStatefulAppMailbox = VerifyLimiter>; type InlineApp = Inline, Block, FixedEpocher>; type Marshaled = BoxedAutomaton>; type SchemeProvider = Provider; -type FinalizationsArchive = immutable::Archive; -type BlocksArchive = immutable::Archive; type Marshal = MarshalActor< E, Standard, @@ -132,6 +125,7 @@ type Orchestrator = orchestrator::Actor< Option>, >; type IndexerConsumer = indexer::Consumer; +type StartupReporter = nunchi_chain::startup::StartupReporter; /// The engine that drives the coins-chain [Application]. #[allow(clippy::type_complexity)] @@ -155,6 +149,10 @@ where config: Config, dkg: DkgActor, dkg_mailbox: DkgMailbox, + dkg_state: nunchi_chain::DkgState, + startup_coordinator: Arc>, + startup_reporter: StartupReporter, + startup_finalization: Option>, buffer: buffered::Engine, buffered_mailbox: buffered::Mailbox, marshal: Marshal, @@ -167,6 +165,7 @@ where stateful: StatefulApp, stateful_mailbox: StatefulAppMailbox, indexer_producer: Option, + indexer_producer_handle: Option>, indexer_consumer: Option>, } @@ -221,6 +220,19 @@ where let num_participants = commonware_utils::NZU32!(config.peer_config.max_participants_per_round()); let block_codec_config = (num_participants, ()); + let protocol_config = dkg::DkgProtocolConfig { + state_format_version: dkg::STATE_FORMAT_VERSION, + namespace: NAMESPACE.to_vec(), + epoch_length: config.epoch_length, + participants: config.peer_config.participants.clone(), + num_participants_per_round: config.peer_config.num_participants_per_round.clone(), + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: dkg::public::N3F1_FAULT_MODEL, + trusted_initial_identity: *config.output.public().public(), + }; + let dkg_state = nunchi_chain::DkgState::new(protocol_config) + .expect("invalid authenticated DKG protocol configuration"); let (dkg, dkg_mailbox) = dkg::Actor::new( context.child("dkg"), @@ -234,7 +246,7 @@ where max_supported_mode: MAX_SUPPORTED_MODE, namespace: NAMESPACE.to_vec(), storage_protector: dkg::StorageProtector::new(config.dkg_storage_key), - epoch_length: BLOCKS_PER_EPOCH, + epoch_length: config.epoch_length, }, ); @@ -251,88 +263,36 @@ where ); let start = Instant::now(); - let finalizations_by_height = immutable::Archive::init( - context.child("finalizations_by_height"), - immutable::Config { - metadata_partition: format!( - "{}-finalizations-by-height-metadata", - config.partition_prefix - ), - freezer_table_partition: format!( - "{}-finalizations-by-height-freezer-table", - config.partition_prefix - ), - freezer_table_initial_size: config.finalized_freezer_table_initial_size, - freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY, - freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE, - freezer_key_partition: format!( - "{}-finalizations-by-height-freezer-key", - config.partition_prefix - ), - freezer_key_page_cache: page_cache.clone(), - freezer_key_write_buffer: WRITE_BUFFER, - freezer_value_partition: format!( - "{}-finalizations-by-height-freezer-value", - config.partition_prefix - ), - freezer_value_write_buffer: WRITE_BUFFER, - freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE, - freezer_value_compression: FREEZER_VALUE_COMPRESSION, - ordinal_partition: format!( - "{}-finalizations-by-height-ordinal", - config.partition_prefix - ), - ordinal_write_buffer: WRITE_BUFFER, - items_per_section: IMMUTABLE_ITEMS_PER_SECTION, - codec_config: Scheme::certificate_codec_config_unbounded(), - replay_buffer: REPLAY_BUFFER, - }, - ) - .await - .expect("failed to initialize finalizations by height archive"); - info!(elapsed = ?start.elapsed(), "restored finalizations by height archive"); - - let start = Instant::now(); - let finalized_blocks = immutable::Archive::init( - context.child("finalized_blocks"), - immutable::Config { - metadata_partition: format!( - "{}-finalized_blocks-metadata", - config.partition_prefix - ), - freezer_table_partition: format!( - "{}-finalized_blocks-freezer-table", - config.partition_prefix - ), - freezer_table_initial_size: config.blocks_freezer_table_initial_size, - freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY, - freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE, - freezer_key_partition: format!( - "{}-finalized_blocks-freezer-key", - config.partition_prefix - ), - freezer_key_page_cache: page_cache.clone(), - freezer_key_write_buffer: WRITE_BUFFER, - freezer_value_partition: format!( - "{}-finalized_blocks-freezer-value", - config.partition_prefix - ), - freezer_value_write_buffer: WRITE_BUFFER, - freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE, - freezer_value_compression: FREEZER_VALUE_COMPRESSION, - ordinal_partition: format!("{}-finalized_blocks-ordinal", config.partition_prefix), - ordinal_write_buffer: WRITE_BUFFER, - items_per_section: IMMUTABLE_ITEMS_PER_SECTION, - codec_config: block_codec_config, - replay_buffer: REPLAY_BUFFER, - }, + let opened_history = history::open( + &context, + &config.partition_prefix, + page_cache.clone(), + block_codec_config, ) .await - .expect("failed to initialize finalized blocks archive"); - info!(elapsed = ?start.elapsed(), "restored finalized blocks archive"); + .unwrap_or_else(|error| panic!("failed to initialize bounded finalized history: {error}")); + let history::Opened { + finalizations: mut finalizations_by_height, + blocks: mut finalized_blocks, + partitions: history_partitions, + marker_status, + } = opened_history; + info!( + elapsed = ?start.elapsed(), + format_version = history::FORMAT_VERSION, + finalizations_key = %history_partitions.finalizations_key, + finalizations_value = %history_partitions.finalizations_value, + blocks_key = %history_partitions.blocks_key, + blocks_value = %history_partitions.blocks_value, + logical_retention = history::LOGICAL_RETENTION, + items_per_section = history::PRUNABLE_ITEMS_PER_SECTION.get(), + maximum_section_and_cadence_slack = history::MAX_RETAINED_HEIGHTS - history::LOGICAL_RETENTION, + ?marker_status, + "restored prunable finalized history" + ); let recovered_height = Certificates::last_index(&finalizations_by_height); - let mut recovered_floor = if let Some(height) = recovered_height { + let recovered_floor = if let Some(height) = recovered_height { Certificates::get( &finalizations_by_height, ArchiveIdentifier::Index(height.get()), @@ -381,41 +341,65 @@ where .expect("failed to initialize empty state database for genesis commitment"); state_commitment(empty.sync_target()) }; - let genesis_state = if let Some(genesis) = &config.genesis { - let fingerprint = commonware_formatting::hex( - &genesis - .fingerprint() - .expect("failed to fingerprint statically configured genesis"), - ); - let compute_partition = format!("{}-genesis-{}", config.partition_prefix, fingerprint); - let compute_config = - QmdbState::::config_with_page_cache(&compute_partition, page_cache.clone()); - let expected = genesis_target( - context.child("genesis_commitment"), - compute_config, - genesis, - &empty_state, + let dkg_fingerprint = commonware_formatting::hex( + &dkg_state + .config() + .digest() + .expect("failed to fingerprint authenticated DKG configuration"), + ); + let app_fingerprint = config + .genesis + .as_ref() + .map(|genesis| { + commonware_formatting::hex( + &genesis + .fingerprint() + .expect("failed to fingerprint statically configured genesis"), + ) + }) + .unwrap_or_else(|| "none".to_owned()); + let compute_partition = format!( + "{}-genesis-{}-{}", + config.partition_prefix, dkg_fingerprint, app_fingerprint + ); + let compute_config = + QmdbState::::config_with_page_cache(&compute_partition, page_cache.clone()); + let expected = authenticated_genesis_target( + context.child("genesis_commitment"), + compute_config, + &dkg_state, + config.output.clone(), + config.genesis.as_ref(), + &empty_state, + ) + .await + .expect("failed to materialize authenticated genesis commitment"); + let state_was_empty = { + let state = QmdbState::init_with_config( + context.child("genesis_existing_state_probe"), + db_config.clone(), ) .await - .expect("failed to materialize genesis commitment"); - - let mut state = - QmdbState::init_with_config(context.child("genesis_seed"), db_config.clone()) - .await - .expect("failed to initialize state database for genesis seeding"); - genesis - .apply_to_state(&mut state, &empty_state) - .await - .expect("failed to seed state database with genesis"); - let actual = state_commitment(state.sync_target()); + .expect("failed to inspect state database before genesis seeding"); + state.sync_target().root == empty_state.root + }; + let actual = authenticated_genesis_target( + context.child("genesis_seed"), + db_config.clone(), + &dkg_state, + config.output.clone(), + config.genesis.as_ref(), + &empty_state, + ) + .await + .expect("failed to seed authenticated state database"); + if state_was_empty { assert_eq!( expected, actual, - "state database genesis commitment must match the genesis block commitment" + "fresh state database genesis commitment must match the genesis block commitment" ); - expected - } else { - empty_state - }; + } + let genesis_state = expected; let current_state_target = { let state = QmdbState::init_with_config( context.child("startup_state_probe"), @@ -425,22 +409,49 @@ where .expect("failed to initialize state database for startup probe"); state.sync_target() }; - if repair_marshal_progress_if_state_is_behind( + let local_startup_candidate = if let Some((processed_height, candidate)) = + validate_marshal_progress_against_state( &context, &config.partition_prefix, + &finalizations_by_height, &finalized_blocks, ¤t_state_target, ) .await { - recovered_floor = None; - } + validate_history_through_processed( + &finalizations_by_height, + &finalized_blocks, + processed_height, + ) + .await; + if let Some(tip) = ArchiveStore::last_index(&finalized_blocks) { + let policy_floor = history::policy_floor(tip, history::LOGICAL_RETENTION); + let safe_floor = policy_floor.min(processed_height.get()); + finalizations_by_height + .prune(safe_floor) + .await + .expect("failed to startup-prune finalized certificates"); + finalized_blocks + .prune(safe_floor) + .await + .expect("failed to startup-prune finalized blocks"); + } + Some(candidate) + } else if ArchiveStore::last_index(&finalized_blocks).is_some() + || ArchiveStore::last_index(&finalizations_by_height).is_some() + { + panic!("finalized history exists without persisted marshal processed height; rebuild or perform verified peer state sync"); + } else { + None + }; let applied_height = Arc::new(AsyncMutex::new(Height::zero())); - let app = Application::with_consensus( + let app = Application::with_authenticated_dkg( submitter.clone(), config.max_block_transactions, ClobExtension::new(clob_mailbox.clone()), - Some(dkg_mailbox.clone()), + dkg_mailbox.clone(), + dkg_state.clone(), applied_height.clone(), genesis_state, application::genesis_payload(), @@ -488,6 +499,44 @@ where .expect("state-sync floor probe stopped"); plan = plan.with_floor(floor); } + // Preserve the certificate-verified anchor while the plan is consumed + // by the stateful actor. It is needed to resume Simplex from a QMDB + // snapshot taken within, rather than at the start of, an epoch. + let startup_finalization = plan.floor().cloned(); + let startup_coordinator = Arc::new(Mutex::new( + nunchi_chain::startup::StartupCoordinator::new(MAX_PENDING_ACKS), + )); + startup_coordinator + .lock() + .expect("startup coordinator lock poisoned") + .record(nunchi_chain::startup::StartupCandidate { + height: Height::zero(), + digest: genesis_digest, + state_target: >::sync_targets(&genesis), + certificate_payload: None, + genesis: true, + }) + .expect("genesis startup candidate should fit"); + if let Some(candidate) = local_startup_candidate { + startup_coordinator + .lock() + .expect("startup coordinator lock poisoned") + .record(candidate) + .expect("local startup candidate should fit"); + } + let certified_payloads = recovered_floor + .iter() + .map(|certificate| certificate.proposal.payload) + .chain( + plan.floor() + .into_iter() + .map(|certificate| certificate.proposal.payload), + ); + let startup_reporter = nunchi_chain::startup::StartupReporter::new( + startup_coordinator.clone(), + certified_payloads, + block_state_target, + ); let marshal_start = recovered_floor .clone() .map_or_else(|| plan.marshal_start(genesis), marshal::Start::Floor); @@ -497,7 +546,7 @@ where finalized_blocks, marshal::Config { provider: provider.clone(), - epocher: FixedEpocher::new(BLOCKS_PER_EPOCH), + epocher: FixedEpocher::new(config.epoch_length), start: marshal_start, partition_prefix: format!("{}_marshal", config.partition_prefix), mailbox_size: MAILBOX_SIZE, @@ -551,10 +600,10 @@ where APPLICATION_VERIFY_CONCURRENCY, ), marshal_mailbox.clone(), - FixedEpocher::new(BLOCKS_PER_EPOCH), + FixedEpocher::new(config.epoch_length), )); - let (indexer_producer, indexer_pusher, indexer_consumer) = + let (indexer_producer, indexer_producer_handle, indexer_pusher, indexer_consumer) = if let Some(client) = config.indexer.clone() { let indexer_context = context.child("indexer"); let indexer_metrics = indexer::IndexerMetrics::register(&indexer_context); @@ -562,10 +611,13 @@ where let queue = queue::shared::init( context.child("indexer_queue"), queue::Config { - partition: format!("{}-indexer-finalized-queue", config.partition_prefix), - items_per_section: NZU64!(128), + partition: format!( + "{}-indexer-finalized-payload-queue-v1", + config.partition_prefix + ), + items_per_section: indexer::SPOOL_ITEMS_PER_SECTION, compression: None, - codec_config: (), + codec_config: block_codec_config, page_cache: page_cache.clone(), write_buffer: WRITE_BUFFER, }, @@ -581,14 +633,20 @@ where mailbox_size: MAILBOX_SIZE, backfiller_max_active: commonware_utils::NZUsize!(16), backfiller_retry: Duration::from_millis(500), + spool_limits: config.indexer_spool_limits, metrics: indexer_metrics, }, ) .await; - let (producer, pusher, consumer) = indexer.split(); - (Some(producer), Some(pusher), Some(consumer)) + let (producer, producer_handle, pusher, consumer) = indexer.split(); + ( + Some(producer), + Some(producer_handle), + Some(pusher), + Some(consumer), + ) } else { - (None, None, None) + (None, None, None, None) }; let (orchestrator, orchestrator_mailbox) = orchestrator::Actor::new( @@ -605,9 +663,11 @@ where muxer_size: MAILBOX_SIZE.get(), mailbox_size: MAILBOX_SIZE, partition_prefix: format!("{}_consensus", config.partition_prefix), - epoch_length: BLOCKS_PER_EPOCH, + epoch_length: config.epoch_length, genesis_digest, recovered_floor, + startup_finalization: None, + startup_floor: None, _phantom: PhantomData, }, ); @@ -617,6 +677,10 @@ where config, dkg, dkg_mailbox, + dkg_state, + startup_coordinator, + startup_reporter, + startup_finalization, buffer, buffered_mailbox, marshal, @@ -629,6 +693,7 @@ where stateful, stateful_mailbox, indexer_producer, + indexer_producer_handle, indexer_consumer, }; (engine, node_handle) @@ -689,7 +754,7 @@ where #[allow(clippy::too_many_arguments)] async fn run( - self, + mut self, votes: ( impl Sender, impl Receiver, @@ -724,16 +789,15 @@ where ), callback: Box>, ) -> Result<(), EngineError> { - let dkg_handle = self.dkg.start( - Some(self.config.output), - self.config.share, - self.orchestrator_mailbox, - dkg, - callback, - ); let buffer_handle = self.buffer.start(broadcast); let reporters: Reporters<_, _, indexer::Producer> = Reporters::from(( - nunchi_chain::dkg_reporters(self.stateful_mailbox, self.dkg_mailbox), + Reporters::from(( + nunchi_chain::dkg_reporters( + self.stateful_mailbox.clone(), + self.dkg_mailbox, + ), + self.startup_reporter, + )), self.indexer_producer, )); let marshal_handle = self @@ -742,15 +806,74 @@ where let probe_handle = self.probe_handle; let state_sync_handle = self.state_sync_handle; let stateful_handle = self.stateful.start(); + // Marshal may queue one finalized block for the not-yet-started DKG + // mailbox while stateful attaches or reconstructs QMDB. Once the + // database subscription resolves, reconcile protected DKG storage + // against the authenticated checkpoint before acknowledging that block. + let databases = self.stateful_mailbox.subscribe_databases().await; + let attached_target = databases.committed_targets().await; + let artifact = self + .startup_coordinator + .lock() + .expect("startup coordinator lock poisoned") + .resolve(&attached_target) + .expect("attached QMDB does not exactly match one certified startup block"); + let reader = QmdbReader::new(databases); + let checkpoint = self + .dkg_state + .load_checkpoint(&reader) + .await + .expect("attached QMDB has invalid authenticated DKG checkpoint"); + dkg::validate_anchor( + self.dkg_state.config(), + &checkpoint, + artifact.anchor_height, + ) + .expect("authenticated DKG checkpoint does not match startup anchor height"); + if let Some(finalization) = self.startup_finalization.take() { + assert_eq!( + finalization.proposal.payload, artifact.anchor_digest, + "state-sync finalization does not certify the attached QMDB anchor" + ); + assert_eq!( + finalization.epoch(), checkpoint.epoch, + "state-sync finalization epoch does not match authenticated DKG checkpoint" + ); + self.orchestrator.set_startup_finalization(finalization); + } + self.orchestrator + .set_startup_floor(orchestrator::StartupFloor { + height: artifact.anchor_height, + digest: artifact.anchor_digest, + }); + let logs = self + .dkg_state + .load_logs(&reader) + .await + .expect("attached QMDB has invalid authenticated DKG logs"); let orchestrator_handle = self.orchestrator.start(votes, certificates, resolver); + let dkg_handle = self.dkg.start_authenticated( + dkg::AuthenticatedBootstrap { + config: self.dkg_state.config().clone(), + checkpoint, + logs, + initial_share: self.config.share, + }, + self.orchestrator_mailbox, + dkg, + callback, + ); let mempool_handle = self .mempool .start_p2p(self.context.child("mempool"), mempool); let indexer_consumer_handle = self.indexer_consumer.map(indexer::Consumer::start); + let indexer_producer_handle = self.indexer_producer_handle; let clob_handle = self.clob.start_p2p(self.context.child("clob"), clob); let mut shutdown = self.context.stopped(); - if let Some(indexer_consumer_handle) = indexer_consumer_handle { + if let (Some(indexer_producer_handle), Some(indexer_consumer_handle)) = + (indexer_producer_handle, indexer_consumer_handle) + { commonware_macros::select! { stopped = &mut shutdown => match stopped { Ok(0) => { @@ -769,6 +892,7 @@ where result = orchestrator_handle => unexpected_exit("orchestrator", result), result = mempool_handle => unexpected_exit("mempool", result), result = clob_handle => unexpected_exit("clob", result), + result = indexer_producer_handle => unexpected_exit("indexer_producer", result), result = indexer_consumer_handle => unexpected_exit("indexer_consumer", result), } } else { @@ -805,21 +929,34 @@ fn unexpected_exit( } } +fn block_state_target( + block: &Block, +) -> commonware_storage::qmdb::sync::Target< + commonware_storage::mmr::Family, + Digest, +> { + commonware_storage::qmdb::sync::Target::new( + block.state_root, + block.state_range.clone(), + ) +} + const MARSHAL_PROCESSED_KEY: U64 = U64::new(0xFF); -const STATE_SYNC_METADATA_SUFFIX: &str = "state_sync_metadata"; -async fn repair_marshal_progress_if_state_is_behind( +async fn validate_marshal_progress_against_state( context: &E, partition_prefix: &str, + finalizations: &FinalizationsArchive, finalized_blocks: &BlocksArchive, current_target: &commonware_storage::qmdb::sync::Target, -) -> bool +) -> Option<(Height, nunchi_chain::startup::StartupCandidate)> where E: BufferPooler + Clock + Metrics + Network + Rng + + CryptoRng + Spawner + Storage + Strategizer @@ -837,63 +974,108 @@ where ) .await .expect("failed to initialize marshal progress metadata probe"); - let Some(processed_height) = metadata.get(&MARSHAL_PROCESSED_KEY).copied() else { - return false; - }; + let processed_height = metadata.get(&MARSHAL_PROCESSED_KEY).copied()?; drop(metadata); - let Some(processed_block) = finalized_blocks - .get(ArchiveIdentifier::Index(processed_height.get())) - .await - .expect("failed to read processed block for state repair") - else { - warn!( - %processed_height, - "marshal progress references a missing finalized block; clearing startup metadata to replay" - ); - remove_partition_if_exists(context, &marshal_metadata_partition).await; - clear_disabled_state_sync_metadata(context, partition_prefix).await; - return true; + let Some(tip) = ArchiveStore::last_index(finalized_blocks) else { + panic!("marshal progress references height {processed_height}, but finalized block history is empty; rebuild or perform verified peer state sync"); }; - let processed_target = >::sync_targets(&processed_block); - let current_size = current_target.range.end().as_u64(); - let processed_size = processed_target.range.end().as_u64(); - if current_size >= processed_size { - return false; + let search_end = tip.min( + processed_height + .get() + .saturating_add(MAX_PENDING_ACKS.get() as u64), + ); + for height in processed_height.get()..=search_end { + let Some(block) = ArchiveStore::get( + finalized_blocks, + ArchiveIdentifier::Index(height), + ) + .await + .expect("failed to read finalized block while validating QMDB startup target") + else { + continue; + }; + let target = >::sync_targets(&block); + if &target == current_target { + let certificate = Certificates::get( + finalizations, + ArchiveIdentifier::Index(height), + ) + .await + .expect("failed to read startup candidate finalization") + .expect("startup candidate block has no finalization"); + assert_eq!( + certificate.proposal.payload, + block.digest(), + "startup candidate certificate payload must equal block digest" + ); + return Some(( + processed_height, + nunchi_chain::startup::StartupCandidate { + height: block.height, + digest: block.digest(), + state_target: target, + certificate_payload: Some(certificate.proposal.payload), + genesis: false, + }, + )); + } } - warn!( - %processed_height, - current_size, - processed_size, - "state database is behind marshal progress; clearing startup metadata to replay finalized blocks" + panic!( + "QMDB committed target does not exactly match canonical finalized history from processed height {} through {}; rebuild or perform verified peer state sync", + processed_height, + search_end, ); - remove_partition_if_exists(context, &marshal_metadata_partition).await; - remove_partition_if_exists( - context, - &format!("{partition_prefix}{STATE_SYNC_METADATA_SUFFIX}"), - ) - .await; - true } -async fn clear_disabled_state_sync_metadata(context: &E, partition_prefix: &str) +async fn validate_history_through_processed( + finalizations: &FinalizationsArchive, + blocks: &BlocksArchive, + processed_height: Height, +) where - E: Storage, + E: BufferPooler + Metrics + Storage, { - remove_partition_if_exists( - context, - &format!("{partition_prefix}{STATE_SYNC_METADATA_SUFFIX}"), - ) - .await; -} + let processed = processed_height.get(); + let first = match (finalizations.first_index(), blocks.first_index()) { + (Some(finalization), Some(block)) => finalization.max(block), + _ => panic!( + "marshal processed height {processed_height} has incomplete finalized history; rebuild or perform verified peer state sync" + ), + }; + if first > processed { + panic!( + "marshal processed height {processed_height} is below retained finalized history floor {first}; rebuild or perform verified peer state sync" + ); + } -async fn remove_partition_if_exists(context: &E, partition: &str) -where - E: Storage, -{ - match context.remove(partition, None).await { - Ok(()) | Err(commonware_runtime::Error::PartitionMissing(_)) => {} - Err(error) => panic!("failed to remove partition {partition}: {error}"), + for height in first..=processed { + let certificate = finalizations + .get(ArchiveIdentifier::Index(height)) + .await + .expect("failed to validate finalized certificate archive"); + let block = blocks + .get(ArchiveIdentifier::Index(height)) + .await + .expect("failed to validate finalized block archive"); + if height == processed { + assert!( + block.is_some(), + "marshal processed height {processed_height} references a missing finalized block; rebuild or perform verified peer state sync" + ); + } + if let (Some(certificate), Some(block)) = (certificate, block) { + assert_eq!( + certificate.proposal.payload, + block.digest(), + "finalized block and certificate conflict at height {height}" + ); + assert_eq!( + certificate.proposal.round.epoch(), + block.context.round.epoch(), + "finalized block and certificate epochs conflict at height {height}" + ); + } } } diff --git a/examples/coins/chain/src/execution.rs b/examples/coins/chain/src/execution.rs index 752eee9..f6dde33 100644 --- a/examples/coins/chain/src/execution.rs +++ b/examples/coins/chain/src/execution.rs @@ -17,7 +17,7 @@ pub use nunchi_chain::SharedAppliedHeight; #[derive(Clone)] pub struct NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub submitter: MempoolHandle, pub clob: ClobMailbox, @@ -27,7 +27,7 @@ where impl NodeHandle where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub fn new( submitter: MempoolHandle, @@ -53,14 +53,14 @@ where /// Read-only coin queries answered from the stateful actor's committed databases. pub struct StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { stateful: StatefulMailbox, } impl Clone for StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { fn clone(&self) -> Self { Self { @@ -71,7 +71,7 @@ where impl StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng, + E: Context + Spawner + Metrics + Clock + rand::Rng + rand::CryptoRng, { pub fn new(stateful: StatefulMailbox) -> Self { Self { stateful } @@ -85,7 +85,15 @@ where #[async_trait] impl CoinQuery for StatefulQuery where - E: Context + Spawner + Metrics + Clock + rand::Rng + Send + Sync + 'static, + E: Context + + Spawner + + Metrics + + Clock + + rand::Rng + + rand::CryptoRng + + Send + + Sync + + 'static, { async fn nonce(&self, account: Address) -> Result { self.ledger().await.nonce(&account).await diff --git a/examples/coins/chain/src/genesis.rs b/examples/coins/chain/src/genesis.rs index 06b8823..6a86f51 100644 --- a/examples/coins/chain/src/genesis.rs +++ b/examples/coins/chain/src/genesis.rs @@ -1,6 +1,13 @@ use crate::StateCommitment; use commonware_codec::{DecodeExt, Encode}; use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::Output, + primitives::variant::MinSig, + }, + ed25519, +}; use commonware_storage::{mmr::Family, qmdb::sync::Target, Context}; use nunchi_authority::{AuthorityGenesis, AuthorityLedger}; use nunchi_clob::{ClobGenesis, ClobLedger}; @@ -8,6 +15,7 @@ use nunchi_coins::{CoinsGenesis, Ledger}; use nunchi_common::{ CommitState, Namespace, Overlay, QmdbConfig, QmdbState, StateError, StateStore, }; +use nunchi_chain::DkgState; use nunchi_oracle::{OracleGenesis, OracleLedger}; use serde::{Deserialize, Serialize}; use std::{fs, path::Path}; @@ -59,6 +67,8 @@ pub enum GenesisError { MismatchedGenesis, #[error("existing chain state is non-empty but has no genesis marker")] UnmarkedState, + #[error("authenticated DKG genesis error: {0}")] + Dkg(#[from] nunchi_chain::DkgStateError), } impl ChainGenesis { @@ -136,6 +146,29 @@ where Ok(state_commitment(state.sync_target())) } +/// Initialize the mandatory authenticated DKG checkpoint and optional +/// application genesis in one canonical order. +pub async fn authenticated_genesis_target( + context: E, + config: QmdbConfig, + dkg: &DkgState, + initial_output: Output, + genesis: Option<&ChainGenesis>, + empty: &StateCommitment, +) -> Result +where + E: Context + commonware_runtime::BufferPooler, +{ + let mut state = QmdbState::init_with_config(context, config).await?; + dkg.seed(&mut state, empty.root, initial_output).await?; + state.commit().await?; + let dkg_baseline = state_commitment(state.sync_target()); + if let Some(genesis) = genesis { + genesis.apply_to_state(&mut state, &dkg_baseline).await?; + } + Ok(state_commitment(state.sync_target())) +} + pub fn state_commitment(target: Target) -> StateCommitment { StateCommitment { root: target.root, diff --git a/examples/coins/chain/src/history.rs b/examples/coins/chain/src/history.rs new file mode 100644 index 0000000..6d39607 --- /dev/null +++ b/examples/coins/chain/src/history.rs @@ -0,0 +1,466 @@ +//! Bounded finalized-history storage for coins-chain validators. + +use crate::{Block, Finalization}; +use commonware_codec::Read; +use commonware_cryptography::sha256::Digest; +use commonware_runtime::{buffer::paged::CacheRef, BufferPooler, Clock, Metrics, Spawner, Storage}; +use commonware_storage::{ + archive::{prunable, Archive as _}, + metadata::{self, Metadata}, + translator::EightCap, +}; +use commonware_utils::{sequence::U64, NZU64}; +use nunchi_chain::engine::{ + MAX_PENDING_ACKS, PRUNE_MAINTENANCE_INTERVAL, PRUNE_RETAINED_MARSHAL_BLOCKS, REPLAY_BUFFER, + WRITE_BUFFER, +}; +use std::num::NonZeroU64; + +pub(crate) const FORMAT_VERSION: u64 = 1; +pub(crate) const PRUNABLE_ITEMS_PER_SECTION: NonZeroU64 = NZU64!(4_096); +pub(crate) const LOGICAL_RETENTION: u64 = + PRUNE_RETAINED_MARSHAL_BLOCKS as u64 + MAX_PENDING_ACKS.get() as u64 + 1; +pub(crate) const MAX_RETAINED_HEIGHTS: u64 = LOGICAL_RETENTION + + PRUNABLE_ITEMS_PER_SECTION.get() + - 1 + + PRUNE_MAINTENANCE_INTERVAL.get() as u64 + - 1; + +const MARKER_KEY: U64 = U64::new(0); + +pub(crate) type FinalizationsArchive = + prunable::Archive; +pub(crate) type BlocksArchive = prunable::Archive; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MarkerStatus { + Created, + RecoveredPartialInitialization, + Present, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum HistoryError { + #[error("legacy finalized-history partition exists: {partition}; stop the devnet and clear the complete validator storage directory before relaunch")] + LegacyPartition { partition: String }, + #[error("history format marker exists but required partition is missing: {partition}; clear the complete devnet validator storage directory before relaunch")] + MarkedPartitionMissing { partition: String }, + #[error("history format marker is missing but {partition} contains finalized data at height {height}; refusing to modify unmarked storage")] + NonEmptyUnmarked { partition: String, height: u64 }, + #[error("invalid history format marker in {partition}: expected version {expected}, found {found:?}")] + InvalidMarker { + partition: String, + expected: u64, + found: Option, + }, + #[error("failed to inspect history partition {partition}: {source}")] + Inspect { + partition: String, + #[source] + source: commonware_runtime::Error, + }, + #[error("failed to open history archive partition {partition}: {source}")] + Archive { + partition: String, + #[source] + source: commonware_storage::archive::Error, + }, + #[error("failed to access history marker partition {partition}: {source}")] + Marker { + partition: String, + #[source] + source: commonware_storage::metadata::Error, + }, + #[error("invalid history retention configuration: {0}")] + Retention(&'static str), +} + +#[derive(Clone, Debug)] +pub(crate) struct Partitions { + pub(crate) finalizations_key: String, + pub(crate) finalizations_value: String, + pub(crate) blocks_key: String, + pub(crate) blocks_value: String, + pub(crate) metadata: String, +} + +impl Partitions { + pub(crate) fn new(prefix: &str) -> Self { + Self { + finalizations_key: format!( + "{prefix}-finalizations-by-height-prunable-v1-key" + ), + finalizations_value: format!( + "{prefix}-finalizations-by-height-prunable-v1-value" + ), + blocks_key: format!("{prefix}-finalized-blocks-prunable-v1-key"), + blocks_value: format!("{prefix}-finalized-blocks-prunable-v1-value"), + metadata: format!("{prefix}-history-prunable-v1-metadata"), + } + } + + fn archives(&self) -> [&str; 4] { + [ + &self.finalizations_key, + &self.finalizations_value, + &self.blocks_key, + &self.blocks_value, + ] + } +} + +pub(crate) struct Opened +where + E: BufferPooler + Clock + Storage + Metrics, +{ + pub(crate) finalizations: FinalizationsArchive, + pub(crate) blocks: BlocksArchive, + pub(crate) partitions: Partitions, + pub(crate) marker_status: MarkerStatus, +} + +pub(crate) const fn policy_floor(tip: u64, retained: u64) -> u64 { + tip.saturating_add(1).saturating_sub(retained) +} + +pub(crate) fn validate_production_retention() -> Result<(), HistoryError> { + if MAX_PENDING_ACKS.get() != 16 { + return Err(HistoryError::Retention( + "MAX_PENDING_ACKS changed; update the mandatory rewind assertion", + )); + } + if LOGICAL_RETENTION != 217 { + return Err(HistoryError::Retention( + "logical finalized-history retention must be 217", + )); + } + if MAX_RETAINED_HEIGHTS != 4_343 { + return Err(HistoryError::Retention( + "section-granular finalized-history bound must be 4,343", + )); + } + Ok(()) +} + +pub(crate) async fn open( + context: &E, + prefix: &str, + page_cache: CacheRef, + block_codec_config: ::Cfg, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + validate_production_retention()?; + reject_legacy(context, prefix).await?; + + let partitions = Partitions::new(prefix); + let marker_existed = partition_exists(context, &partitions.metadata).await?; + let mut archive_existed = [false; 4]; + for (exists, partition) in archive_existed + .iter_mut() + .zip(partitions.archives()) + { + *exists = partition_exists(context, partition).await?; + } + let mut finalizations = + init_finalizations(context, &partitions, page_cache.clone()).await?; + let mut blocks = init_blocks( + context, + &partitions, + page_cache.clone(), + block_codec_config, + ) + .await?; + + let marker_status = if marker_existed { + validate_marker(context, &partitions.metadata).await?; + let archives_empty = finalizations.first_index().is_none() && blocks.first_index().is_none(); + let all_lazy = archive_existed.iter().all(|exists| !exists); + if !(archive_existed.iter().all(|exists| *exists) || archives_empty && all_lazy) { + let partition = partitions + .archives() + .into_iter() + .zip(archive_existed) + .find_map(|(partition, exists)| (!exists).then_some(partition)) + .expect("at least one marked archive partition is missing"); + return Err(HistoryError::MarkedPartitionMissing { + partition: partition.to_string(), + }); + } + MarkerStatus::Present + } else { + if let Some(height) = finalizations.first_index() { + return Err(HistoryError::NonEmptyUnmarked { + partition: partitions.finalizations_key.clone(), + height, + }); + } + if let Some(height) = blocks.first_index() { + return Err(HistoryError::NonEmptyUnmarked { + partition: partitions.blocks_key.clone(), + height, + }); + } + + // Ensure all newly-created archive blobs are durable and structurally reopenable before + // committing the format marker. + finalizations.sync().await.map_err(|source| HistoryError::Archive { + partition: partitions.finalizations_key.clone(), + source, + })?; + blocks.sync().await.map_err(|source| HistoryError::Archive { + partition: partitions.blocks_key.clone(), + source, + })?; + drop(finalizations); + drop(blocks); + finalizations = init_finalizations(context, &partitions, page_cache.clone()).await?; + blocks = init_blocks(context, &partitions, page_cache, block_codec_config).await?; + write_marker(context, &partitions.metadata).await?; + + if archive_existed.into_iter().any(|exists| exists) { + MarkerStatus::RecoveredPartialInitialization + } else { + MarkerStatus::Created + } + }; + + Ok(Opened { + finalizations, + blocks, + partitions, + marker_status, + }) +} + +async fn init_finalizations( + context: &E, + partitions: &Partitions, + page_cache: CacheRef, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + prunable::Archive::init( + context.child("finalizations_by_height"), + prunable::Config { + translator: EightCap, + key_partition: partitions.finalizations_key.clone(), + key_page_cache: page_cache, + value_partition: partitions.finalizations_value.clone(), + compression: Some(3), + codec_config: (), + items_per_section: PRUNABLE_ITEMS_PER_SECTION, + key_write_buffer: WRITE_BUFFER, + value_write_buffer: WRITE_BUFFER, + replay_buffer: REPLAY_BUFFER, + }, + ) + .await + .map_err(|source| HistoryError::Archive { + partition: partitions.finalizations_key.clone(), + source, + }) +} + +async fn init_blocks( + context: &E, + partitions: &Partitions, + page_cache: CacheRef, + codec_config: ::Cfg, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + prunable::Archive::init( + context.child("finalized_blocks"), + prunable::Config { + translator: EightCap, + key_partition: partitions.blocks_key.clone(), + key_page_cache: page_cache, + value_partition: partitions.blocks_value.clone(), + compression: Some(3), + codec_config, + items_per_section: PRUNABLE_ITEMS_PER_SECTION, + key_write_buffer: WRITE_BUFFER, + value_write_buffer: WRITE_BUFFER, + replay_buffer: REPLAY_BUFFER, + }, + ) + .await + .map_err(|source| HistoryError::Archive { + partition: partitions.blocks_key.clone(), + source, + }) +} + +async fn partition_exists(context: &E, partition: &str) -> Result { + match context.scan(partition).await { + Ok(_) => Ok(true), + Err(commonware_runtime::Error::PartitionMissing(_)) => Ok(false), + Err(source) => Err(HistoryError::Inspect { + partition: partition.to_string(), + source, + }), + } +} + +async fn validate_marker(context: &E, partition: &str) -> Result<(), HistoryError> +where + E: BufferPooler + Clock + Storage + Metrics + Spawner, +{ + let marker = Metadata::::init( + context.child("history_marker"), + metadata::Config { + partition: partition.to_string(), + codec_config: (), + }, + ) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + })?; + let found = marker.get(&MARKER_KEY).copied(); + if found != Some(FORMAT_VERSION) { + return Err(HistoryError::InvalidMarker { + partition: partition.to_string(), + expected: FORMAT_VERSION, + found, + }); + } + Ok(()) +} + +async fn write_marker(context: &E, partition: &str) -> Result<(), HistoryError> +where + E: BufferPooler + Clock + Storage + Metrics + Spawner, +{ + let mut marker = Metadata::::init( + context.child("history_marker"), + metadata::Config { + partition: partition.to_string(), + codec_config: (), + }, + ) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + })?; + marker + .put_sync(MARKER_KEY, FORMAT_VERSION) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + }) +} + +async fn reject_legacy(context: &E, prefix: &str) -> Result<(), HistoryError> { + const FINALIZATION_SUFFIXES: [&str; 5] = [ + "finalizations-by-height-metadata", + "finalizations-by-height-freezer-table", + "finalizations-by-height-freezer-key", + "finalizations-by-height-freezer-value", + "finalizations-by-height-ordinal", + ]; + const BLOCK_SUFFIXES: [&str; 5] = [ + "finalized_blocks-metadata", + "finalized_blocks-freezer-table", + "finalized_blocks-freezer-key", + "finalized_blocks-freezer-value", + "finalized_blocks-ordinal", + ]; + + for suffix in FINALIZATION_SUFFIXES.into_iter().chain(BLOCK_SUFFIXES) { + let partition = format!("{prefix}-{suffix}"); + if partition_exists(context, &partition).await? { + return Err(HistoryError::LegacyPartition { partition }); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use commonware_runtime::{buffer::paged::CacheRef, deterministic, Runner as _}; + use commonware_utils::{NZU16, NZU32, NZUsize}; + + #[test] + fn production_retention_arithmetic() { + validate_production_retention().unwrap(); + assert_eq!(policy_floor(216, LOGICAL_RETENTION), 0); + assert_eq!(policy_floor(217, LOGICAL_RETENTION), 1); + assert_eq!(LOGICAL_RETENTION, 217); + assert_eq!(MAX_RETAINED_HEIGHTS, 4_343); + } + + #[test] + fn fresh_format_marker_is_durable_and_legacy_is_not_created() { + deterministic::Runner::default().start(|context| async move { + let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(32)); + let first = open( + &context, + "validator", + page_cache.clone(), + (NZU32!(1), ()), + ) + .await + .expect("initialize fresh history"); + assert_eq!(first.marker_status, MarkerStatus::Created); + assert_eq!(first.finalizations.first_index(), None); + assert_eq!(first.blocks.first_index(), None); + drop(first); + + let reopened = open( + &context, + "validator", + page_cache, + (NZU32!(1), ()), + ) + .await + .expect("reopen marked history"); + assert_eq!(reopened.marker_status, MarkerStatus::Present); + + for suffix in [ + "finalizations-by-height-metadata", + "finalizations-by-height-freezer-table", + "finalizations-by-height-freezer-key", + "finalizations-by-height-freezer-value", + "finalizations-by-height-ordinal", + "finalized_blocks-metadata", + "finalized_blocks-freezer-table", + "finalized_blocks-freezer-key", + "finalized_blocks-freezer-value", + "finalized_blocks-ordinal", + ] { + assert!(matches!( + context.scan(&format!("validator-{suffix}")).await, + Err(commonware_runtime::Error::PartitionMissing(_)) + )); + } + }); + } + + #[test] + fn refuses_known_legacy_partition() { + deterministic::Runner::default().start(|context| async move { + context + .open("validator-finalized_blocks-ordinal", b"legacy") + .await + .expect("create legacy partition"); + let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(32)); + let result = open( + &context, + "validator", + page_cache, + (NZU32!(1), ()), + ) + .await; + assert!(matches!(result, Err(HistoryError::LegacyPartition { .. }))); + }); + } +} diff --git a/examples/coins/chain/src/indexer/backfiller/consumer.rs b/examples/coins/chain/src/indexer/backfiller/consumer.rs index e714eb2..0bd66e8 100644 --- a/examples/coins/chain/src/indexer/backfiller/consumer.rs +++ b/examples/coins/chain/src/indexer/backfiller/consumer.rs @@ -1,17 +1,14 @@ -use super::{Decision, Entry, SharedState}; +use super::{ + producer::{Admission, AdmissionReceiver}, + Decision, Entry, SharedState, +}; use crate::indexer::{ metrics::{ - estimated_finalized_bytes, BackfillDecision, BackfillPhase, BackfillResetReason, - BackfillUploadGuard, BackfillWaitReason, BlockMetricSource, QueueReadSource, QueueStatus, - SharedCacheSource, + estimated_finalized_bytes, BackfillDecision, BackfillPhase, BackfillWaitReason, + BlockMetricSource, ProducerStatus, QueueReadSource, QueueStatus, }, - Client, IndexerMetrics, -}; -use crate::{Block, Finalized, Scheme}; -use commonware_consensus::marshal::{ - core::Mailbox as MarshalMailbox, standard::Standard, Identifier, + Client, IndexerMetrics, SpoolLimits, }; -use commonware_consensus::types::Height; use commonware_cryptography::sha256::Digest; use commonware_macros::select_loop; use commonware_runtime::{ @@ -22,7 +19,7 @@ use commonware_storage::queue; use commonware_utils::futures::{OptionFuture, Pool}; use std::{ num::NonZeroUsize, - time::{Duration, Instant}, + time::{Duration, Instant, UNIX_EPOCH}, }; use tracing::{debug, warn}; @@ -45,47 +42,39 @@ enum Completion { Skipped { position: u64, height: u64, - }, - Stale { - reason: BackfillResetReason, - first_height: u64, - first_digest: Digest, - attempts: u64, - elapsed: Duration, + digest: Digest, }, } pub(crate) struct Config { pub(crate) max_active: NonZeroUsize, pub(crate) retry: Duration, - pub(crate) missing_finalization_grace: Duration, - pub(crate) mismatched_finalization_grace: Duration, + pub(crate) spool_limits: SpoolLimits, } pub struct Consumer { context: ContextCell, client: C, - marshal: MarshalMailbox>, metrics: IndexerMetrics, upload_results: status::Counter, uploads: SharedState, writer: queue::Writer, reader: queue::Reader, + admission: AdmissionReceiver, active: Pool, max_active: NonZeroUsize, retry: Duration, - missing_finalization_grace: Duration, - mismatched_finalization_grace: Duration, + spool_limits: SpoolLimits, } impl Consumer { pub fn new( context: E, client: C, - marshal: MarshalMailbox>, metrics: IndexerMetrics, uploads: SharedState, backfiller: (queue::Writer, queue::Reader), + admission: AdmissionReceiver, config: Config, ) -> Self { let upload_results = context.register( @@ -96,25 +85,23 @@ impl Consumer< let Config { max_active, retry, - missing_finalization_grace, - mismatched_finalization_grace, + spool_limits, } = config; metrics.backfill_configured(max_active.get()); let (writer, reader) = backfiller; Self { context: ContextCell::new(context), client, - marshal, metrics, upload_results, uploads, writer, reader, + admission, active: Pool::default(), max_active, retry, - missing_finalization_grace, - mismatched_finalization_grace, + spool_limits, } } @@ -127,25 +114,24 @@ impl Consumer< self.context, on_start => { self.fill_slots().await; - if self.active.is_empty() { - let item = Self::recv_queue(&mut self.reader, self.metrics.clone()).await; - let Some((position, entry)) = item else { - warn!("consumer queue closed"); - break; - }; - self.start_upload(position, entry).await; - continue; - } let item = OptionFuture::from( (self.active.len() < self.max_active.get()).then(|| { Self::recv_queue(&mut self.reader, self.metrics.clone()) }), ); + let admission = self.admission.recv(); }, on_stopped => {}, completion = self.active.next_completed() => { self.complete(completion).await; }, + request = admission => { + let Some(request) = request else { + warn!("indexer spool admission coordinator closed"); + break; + }; + self.admit(request).await; + }, item = item => { match item { Some((position, entry)) => { @@ -160,6 +146,87 @@ impl Consumer< } } + async fn admit(&mut self, admission: Admission) { + let Admission { entry, response } = admission; + loop { + let now_millis: u64 = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + let next = { + let uploads = self.uploads.lock(); + let (entries, bytes) = uploads.spool_usage(); + uploads.oldest_queued().and_then( + |(digest, position, height, old_bytes, enqueued_at)| { + let age = Duration::from_millis( + now_millis.saturating_sub(enqueued_at), + ); + let status = if age >= self.spool_limits.max_age { + Some(ProducerStatus::ExpiredAge) + } else if entries >= self.spool_limits.max_entries { + Some(ProducerStatus::ExpiredEntries) + } else if bytes.saturating_add(entry.encoded_len) + > self.spool_limits.max_bytes + { + Some(ProducerStatus::ExpiredBytes) + } else { + None + }; + status.map(|status| { + (digest, position, height, old_bytes, age, status) + }) + }, + ) + }; + let Some((digest, position, height, encoded_len, age, status)) = next else { + break; + }; + + // Advancing the floor invalidates positions held by upload tasks. Cancel all of them + // and replay the still-live suffix so a stale completion cannot be applied twice. + self.active.cancel_all(); + self.reader + .ack_up_to(position.saturating_add(1)) + .await + .unwrap_or_else(|error| panic!("failed to expire indexer spool floor: {error:?}")); + let sync_started = Instant::now(); + self.writer.sync().await.unwrap_or_else(|error| { + panic!("failed to durably sync expired indexer spool floor: {error:?}") + }); + self.metrics + .queue_synced(QueueStatus::Success, sync_started.elapsed()); + self.reader.reset().await; + let expired = self.uploads.lock().expire_through(position); + for (_, _, _) in &expired { + self.metrics.producer_recorded(status, Duration::ZERO); + } + warn!( + height, + ?digest, + encoded_len, + ?age, + ?status, + expired_entries = expired.len(), + "terminally expired oldest finalized indexer payload before admission" + ); + } + + self.metrics.queue_entry(entry.height()); + let position = self.writer.enqueue(entry.clone()).await.unwrap_or_else(|error| { + self.metrics.queue_enqueued(QueueStatus::Failure); + panic!("failed to enqueue finalized indexer payload: {error:?}") + }); + self.uploads.lock().mark_queued(position, &entry); + self.metrics.queue_enqueued(QueueStatus::Success); + response + .send(()) + .expect("indexer spool producer stopped before admission completed"); + } + async fn fill_slots(&mut self) { while self.active.len() < self.max_active.get() { let item = Self::try_recv_queue(&mut self.reader, self.metrics.clone()).await; @@ -177,7 +244,7 @@ impl Consumer< match reader.recv().await { Ok(Some((position, entry))) => { metrics.queue_read(QueueReadSource::Recv, QueueStatus::Success); - metrics.queue_entry(entry.height); + metrics.queue_entry(entry.height()); Some((position, entry)) } Ok(None) => { @@ -198,7 +265,7 @@ impl Consumer< match reader.try_recv().await { Ok(Some((position, entry))) => { metrics.queue_read(QueueReadSource::TryRecv, QueueStatus::Success); - metrics.queue_entry(entry.height); + metrics.queue_entry(entry.height()); Some((position, entry)) } Ok(None) => { @@ -213,7 +280,26 @@ impl Consumer< } async fn start_upload(&mut self, position: u64, entry: Entry) { - let Entry { height, digest } = entry; + let height = entry.height(); + let digest = entry.digest(); + let enqueued_at_millis = entry.enqueued_at_millis; + let finalized = entry.finalized; + if self.payload_age(enqueued_at_millis) >= self.spool_limits.max_age { + self.metrics + .producer_recorded(ProducerStatus::ExpiredAge, Duration::ZERO); + warn!( + height, + ?digest, + "terminally expiring finalized indexer payload at configured age bound" + ); + self.complete(Completion::Skipped { + position, + height, + digest, + }) + .await; + return; + } let decision = self.uploads.lock().should_upload(&digest); self.metrics .backfill_decision((&decision).into(), BackfillPhase::Start); @@ -222,7 +308,11 @@ impl Consumer< let mut upload = self.metrics.start_backfill_upload(); upload.skipped(); } - self.complete(Completion::Skipped { position, height }) + self.complete(Completion::Skipped { + position, + height, + digest, + }) .await; debug!(?digest, "consumer skipping already-uploaded block"); return; @@ -235,82 +325,35 @@ impl Consumer< .with_attribute("digest", digest) .with_attribute("height", height); let client = self.client.clone(); - let marshal = self.marshal.clone(); let metrics = self.metrics.clone(); let upload_results = self.upload_results.clone(); let uploads = self.uploads.clone(); let retry = self.retry; - let missing_finalization_grace = self.missing_finalization_grace; - let mismatched_finalization_grace = self.mismatched_finalization_grace; + let max_age = self.spool_limits.max_age; async move { let mut upload = metrics.start_backfill_upload(); - let Some(block) = - Self::wait_for_uploadable_block( - &context, &marshal, &metrics, &uploads, &mut upload, digest, retry, - ) - .await - else { - upload.skipped(); - debug!(?digest, "skipping previously uploaded block"); - return Completion::Skipped { position, height }; - }; + metrics.observe_block(BlockMetricSource::ConsumerCached, &finalized.block); + upload.hold_block(&finalized.block); - let mut proof_failure = ProofFailure::default(); - let finalized = loop { - let Some(proof) = marshal.get_finalization(Height::new(height)).await else { - if let Some(stale) = proof_failure.observe( - &context, - BackfillResetReason::MissingFinalization, - missing_finalization_grace, - height, - digest, - ) { - upload.abandoned(); - return stale; - } - warn!( - height, - ?digest, - "consumer could not find finalization, retrying" - ); - Self::wait( - &context, - &metrics, - BackfillWaitReason::MissingFinalization, - retry, - ) - .await; - continue; - }; - if proof.proposal.payload != digest { - if let Some(stale) = proof_failure.observe( - &context, - BackfillResetReason::MismatchedFinalization, - mismatched_finalization_grace, + loop { + let now_millis: u64 = context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + // Backward wall-clock movement produces age zero until the clock catches up. + let age = Duration::from_millis(now_millis.saturating_sub(enqueued_at_millis)); + if age >= max_age { + metrics.producer_recorded(ProducerStatus::ExpiredAge, Duration::ZERO); + warn!(height, ?digest, ?age, "terminally expiring active finalized indexer upload at configured age bound"); + return Completion::Skipped { + position, height, digest, - ) { - upload.abandoned(); - return stale; - } - warn!( - height, - ?digest, - "consumer found mismatched finalization, retrying" - ); - Self::wait( - &context, - &metrics, - BackfillWaitReason::MismatchedFinalization, - retry, - ) - .await; - continue; + }; } - break Finalized::new(proof, block.clone()); - }; - - loop { let decision = { let uploads = uploads.lock(); uploads.should_upload(&digest) @@ -320,7 +363,11 @@ impl Consumer< Decision::Skip => { upload.skipped(); debug!(?digest, "skipping previously uploaded block"); - return Completion::Skipped { position, height }; + return Completion::Skipped { + position, + height, + digest, + }; } Decision::Wait => { Self::wait( @@ -363,86 +410,17 @@ impl Consumer< }); } - async fn wait_for_uploadable_block( - context: &E, - marshal: &MarshalMailbox>, - metrics: &IndexerMetrics, - uploads: &SharedState, - upload: &mut BackfillUploadGuard, - digest: Digest, - retry: Duration, - ) -> Option { - enum NextBlock { - AlreadyUploaded, - WaitForCertificate, - Ready(Box), - FetchFromMarshal, - } - - loop { - let next = { - let uploads = uploads.lock(); - match uploads.should_upload(&digest) { - Decision::Skip => { - metrics.backfill_decision( - BackfillDecision::Skip, - BackfillPhase::BeforeBlock, - ); - NextBlock::AlreadyUploaded - } - Decision::Wait => { - metrics.backfill_decision( - BackfillDecision::Wait, - BackfillPhase::BeforeBlock, - ); - NextBlock::WaitForCertificate - } - Decision::Proceed => { - metrics.backfill_decision( - BackfillDecision::Proceed, - BackfillPhase::BeforeBlock, - ); - uploads - .cached_block(&digest) - .map(|block| NextBlock::Ready(Box::new(block))) - .unwrap_or(NextBlock::FetchFromMarshal) - } - } - }; - - match next { - NextBlock::AlreadyUploaded => return None, - NextBlock::WaitForCertificate => { - Self::wait( - context, - metrics, - BackfillWaitReason::CertificateUpload, - retry, - ) - .await; - } - NextBlock::Ready(block) => { - metrics.observe_block(BlockMetricSource::ConsumerCached, &block); - upload.hold_block(&block); - return Some(*block); - } - NextBlock::FetchFromMarshal => { - if let Some(block) = marshal.get_block(Identifier::Digest(digest)).await { - metrics.observe_block(BlockMetricSource::ConsumerMarshal, &block); - uploads - .lock() - .cache_block(block.clone(), SharedCacheSource::ConsumerMarshal); - upload.hold_block(&block); - return Some(block); - } - warn!( - ?digest, - "consumer could not find block in marshal, retrying" - ); - Self::wait(context, metrics, BackfillWaitReason::MissingBlock, retry).await; - } - } - } + fn payload_age(&self, enqueued_at_millis: u64) -> Duration { + let now_millis: u64 = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + // Backward wall-clock movement produces age zero until the clock catches up. + Duration::from_millis(now_millis.saturating_sub(enqueued_at_millis)) } async fn wait( @@ -462,28 +440,22 @@ impl Consumer< } async fn complete(&mut self, completion: Completion) { - let (position, height) = match completion { + let (position, height, digest) = match completion { Completion::Uploaded { position, height, digest, } => { self.uploads.lock().mark_uploaded(digest, height); - (position, height) - } - Completion::Skipped { position, height } => (position, height), - Completion::Stale { - reason, - first_height, - first_digest, - attempts, - elapsed, - } => { - self.reset_stale_queue(reason, first_height, first_digest, attempts, elapsed) - .await; - return; + (position, height, digest) } + Completion::Skipped { + position, + height, + digest, + } => (position, height, digest), }; + self.uploads.lock().finish_queued(&digest); let floor = self.reader.ack_floor().await; match self.reader.ack(position).await { @@ -511,109 +483,4 @@ impl Consumer< } } - async fn reset_stale_queue( - &mut self, - reason: BackfillResetReason, - first_height: u64, - first_digest: Digest, - attempts: u64, - elapsed: Duration, - ) { - let queue_size = self.writer.size().await; - let ack_floor = self.reader.ack_floor().await; - let abandoned_entries = queue_size.saturating_sub(ack_floor); - let tip = Self::latest_proof_bearing_tip(self.marshal.clone()).await; - let (tip_height, tip_digest) = tip.unwrap_or((0, first_digest)); - let abandoned_height_span = tip_height.saturating_sub(first_height); - - warn!( - ?reason, - first_height, - ?first_digest, - tip_height, - ?tip_digest, - attempts, - ?elapsed, - abandoned_entries, - abandoned_height_span, - "resetting stale durable backfill queue" - ); - - self.active.cancel_all(); - match self.reader.ack_up_to(queue_size).await { - Ok(()) => self.metrics.queue_acked(QueueStatus::Success), - Err(err) => { - self.metrics.queue_acked(QueueStatus::Failure); - panic!("failed to ack stale finalized queue: {err:?}"); - } - } - let sync_started = Instant::now(); - match self.writer.sync().await { - Ok(()) => self - .metrics - .queue_synced(QueueStatus::Success, sync_started.elapsed()), - Err(err) => { - self.metrics - .queue_synced(QueueStatus::Failure, sync_started.elapsed()); - panic!("failed to sync stale finalized queue reset: {err:?}"); - } - } - self.reader.reset().await; - self.metrics.queue_ack_floor(tip_height); - self.metrics.backfill_queue_reset( - reason, - abandoned_entries, - abandoned_height_span, - ); - self.uploads.lock().advance_queue_floor(tip_height); - self.uploads.lock().restart_above(tip_height); - } - - async fn latest_proof_bearing_tip( - marshal: MarshalMailbox>, - ) -> Option<(u64, Digest)> { - let (height, _) = marshal.get_info(Identifier::Latest).await?; - for height in (0..=height.get()).rev() { - if let Some(proof) = marshal.get_finalization(Height::new(height)).await { - return Some((height, proof.proposal.payload)); - } - } - None - } -} - -#[derive(Default)] -struct ProofFailure { - reason: Option, - first_seen: Option, - attempts: u64, -} - -impl ProofFailure { - fn observe( - &mut self, - context: &E, - reason: BackfillResetReason, - grace: Duration, - height: u64, - digest: Digest, - ) -> Option { - if self.reason != Some(reason) { - self.reason = Some(reason); - self.first_seen = Some(context.current()); - self.attempts = 0; - } - self.attempts = self.attempts.saturating_add(1); - let elapsed = context - .current() - .duration_since(self.first_seen.expect("proof failure start")) - .unwrap_or_default(); - (elapsed >= grace).then_some(Completion::Stale { - reason, - first_height: height, - first_digest: digest, - attempts: self.attempts, - elapsed, - }) - } } diff --git a/examples/coins/chain/src/indexer/backfiller/producer.rs b/examples/coins/chain/src/indexer/backfiller/producer.rs index 88b01e4..fccdb5d 100644 --- a/examples/coins/chain/src/indexer/backfiller/producer.rs +++ b/examples/coins/chain/src/indexer/backfiller/producer.rs @@ -3,23 +3,32 @@ use crate::{ indexer::{ metrics::{ estimated_block_bytes, BlockMetricSource, ProducerActivity, ProducerStatus, - QueueStatus, }, - IndexerMetrics, + IndexerMetrics, SpoolLimits, }, - Block, + Block, Finalized, Scheme, }; use commonware_actor::{ mailbox::{self, Overflow, Policy}, Feedback, }; -use commonware_consensus::{marshal::Update, Reporter}; +use commonware_consensus::{ + marshal::{core::Mailbox as MarshalMailbox, standard::Standard, Update}, + types::Height, + Reporter, +}; use commonware_runtime::{ spawn_cell, BufferPooler, Clock, ContextCell, Handle, Metrics, Spawner, Storage, }; -use commonware_storage::queue; use commonware_utils::{acknowledgement::Exact, Acknowledgement}; -use std::{collections::VecDeque, num::NonZeroUsize, sync::Arc, time::Instant}; +use commonware_utils::channel::oneshot; +use std::{ + collections::VecDeque, + num::NonZeroUsize, + sync::Arc, + time::{Duration, Instant, UNIX_EPOCH}, +}; +use tracing::warn; #[derive(Clone)] pub struct Producer { @@ -27,6 +36,22 @@ pub struct Producer { metrics: IndexerMetrics, } +pub(crate) struct Admission { + pub(crate) entry: Entry, + pub(crate) response: oneshot::Sender<()>, +} + +impl Policy for Admission { + type Overflow = VecDeque; + + fn handle(overflow: &mut Self::Overflow, message: Self) { + overflow.push_back(message); + } +} + +pub(crate) type AdmissionSender = mailbox::Sender; +pub(crate) type AdmissionReceiver = mailbox::Receiver; + struct Message { block: Arc, block_estimated_bytes: u64, @@ -87,8 +112,21 @@ struct Actor { context: ContextCell, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, + marshal: MarshalMailbox>, + admission: AdmissionSender, receiver: mailbox::Receiver, + retry: Duration, + missing_finalization_grace: Duration, + mismatched_finalization_grace: Duration, + spool_limits: SpoolLimits, +} + +pub(crate) struct Config { + pub(crate) mailbox_size: NonZeroUsize, + pub(crate) retry: Duration, + pub(crate) missing_finalization_grace: Duration, + pub(crate) mismatched_finalization_grace: Duration, + pub(crate) spool_limits: SpoolLimits, } impl Reporter for Producer { @@ -123,16 +161,29 @@ impl Actor { context: E, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, - mailbox_size: NonZeroUsize, + marshal: MarshalMailbox>, + admission: AdmissionSender, + config: Config, ) -> (Self, Producer) { + let Config { + mailbox_size, + retry, + missing_finalization_grace, + mismatched_finalization_grace, + spool_limits, + } = config; let (sender, receiver) = mailbox::new(context.child("mailbox"), mailbox_size); let actor = Self { context: ContextCell::new(context), uploads, metrics: metrics.clone(), - writer, + marshal, + admission, receiver, + retry, + missing_finalization_grace, + mismatched_finalization_grace, + spool_limits, }; ( actor, @@ -158,25 +209,91 @@ impl Actor { let started = Instant::now(); self.metrics .observe_block(BlockMetricSource::ProducerRecord, block); - let Some(entry) = self.uploads.lock().record(block) else { + let Some(candidate) = self.uploads.lock().record(block) else { self.metrics.producer_recorded( ProducerStatus::AlreadyUploaded, started.elapsed(), ); return; }; - self.metrics.queue_entry(entry.height); - match self.writer.enqueue(entry).await { - Ok(_) => { - self.metrics.queue_enqueued(QueueStatus::Success); - self.metrics - .producer_recorded(ProducerStatus::Recorded, started.elapsed()); - } - Err(err) => { - self.metrics.queue_enqueued(QueueStatus::Failure); - panic!("failed to enqueue finalized digest: {err:?}"); + let first_seen = self.context.current(); + self.metrics.certificate_request_started(); + let proof = loop { + let Some(proof) = self + .marshal + .get_finalization(Height::new(candidate.height)) + .await + else { + let elapsed = self + .context + .current() + .duration_since(first_seen) + .unwrap_or_default(); + assert!( + elapsed < self.missing_finalization_grace, + "marshal has no finalization certificate for durable indexer payload at height {} after {:?}", + candidate.height, + elapsed, + ); + warn!(height = candidate.height, ?elapsed, "waiting for finalized certificate before spooling indexer payload"); + self.context.sleep(self.retry).await; + continue; + }; + if proof.proposal.payload != candidate.digest + || proof.proposal.round.epoch() != block.context.round.epoch() + { + let elapsed = self + .context + .current() + .duration_since(first_seen) + .unwrap_or_default(); + assert!( + elapsed < self.mismatched_finalization_grace, + "marshal finalization certificate conflicts with block at height {} after {:?}", + candidate.height, + elapsed, + ); + warn!(height = candidate.height, ?elapsed, "waiting for matching finalized certificate before spooling indexer payload"); + self.context.sleep(self.retry).await; + continue; } + break proof; + }; + self.metrics.certificate_request_finished(); + let enqueued_at_millis = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + let entry = Entry::new(enqueued_at_millis, Finalized::new(proof, block.clone())); + let expiry = if entry.encoded_len > self.spool_limits.max_payload_bytes + || entry.encoded_len > self.spool_limits.max_bytes + { + Some(ProducerStatus::ExpiredOversized) + } else { + None + }; + if let Some(status) = expiry { + self.metrics.producer_recorded(status, started.elapsed()); + warn!( + height = entry.height(), + digest = ?entry.digest(), + encoded_len = entry.encoded_len, + ?status, + "terminally expiring finalized indexer payload to preserve spool availability bound" + ); + return; } + let (response, completed) = oneshot::channel(); + let _ = self.admission.enqueue(Admission { entry, response }); + completed + .await + .expect("indexer spool admission coordinator stopped"); + self.metrics + .producer_recorded(ProducerStatus::Recorded, started.elapsed()); } } @@ -184,15 +301,23 @@ pub fn init( context: E, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, - mailbox_size: NonZeroUsize, -) -> Producer + marshal: MarshalMailbox>, + admission: AdmissionSender, + config: Config, +) -> (Producer, Handle<()>) where E: BufferPooler + Clock + Storage + Metrics + Spawner, { - let (actor, producer) = Actor::new(context, uploads, metrics, writer, mailbox_size); - actor.start(); - producer + let (actor, producer) = Actor::new( + context, + uploads, + metrics, + marshal, + admission, + config, + ); + let handle = actor.start(); + (producer, handle) } #[cfg(test)] @@ -200,16 +325,12 @@ mod tests { use super::*; use crate::{StateCommitment, Transaction, EPOCH}; use commonware_consensus::types::{Height, Round, View}; - use commonware_cryptography::{ed25519, Digestible, Hasher, Sha256, Signer}; - use commonware_runtime::{ - buffer::paged::CacheRef, deterministic, Runner as _, Supervisor as _, - }; + use commonware_cryptography::{ed25519, Hasher, Sha256, Signer}; + use commonware_runtime::{deterministic, Runner as _, Supervisor as _}; use commonware_storage::mmr::Location; use commonware_utils::{ - acknowledgement::Exact, range::NonEmptyRange, sync::Mutex, NZUsize, NZU16, NZU64, + acknowledgement::Exact, range::NonEmptyRange, NZUsize, }; - use futures::FutureExt; - use std::{sync::Arc, time::Duration}; fn state(height: u64) -> StateCommitment { StateCommitment { @@ -239,60 +360,6 @@ mod tests { ) } - #[test] - fn queues_finalized_block_before_acknowledging() { - deterministic::Runner::default().start(|context| async move { - let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(128)); - let (writer, mut reader) = queue::shared::init( - context.child("queue"), - queue::Config { - partition: "indexer-producer-test".to_string(), - items_per_section: NZU64!(16), - compression: None, - codec_config: (), - page_cache, - write_buffer: NZUsize!(1024), - }, - ) - .await - .expect("init queue"); - let uploads = Arc::new(Mutex::new(crate::indexer::backfiller::State::new())); - let metrics = IndexerMetrics::register(&context.child("indexer")); - let mut producer = init(context.child("producer"), uploads, metrics, writer, NZUsize!(4)); - let block = block(2, 2, b"block"); - let digest = block.digest(); - let (ack, waiter) = Exact::handle(); - - assert!(producer - .report(commonware_consensus::marshal::Update::Block( - block.into(), - ack, - )) - .accepted()); - commonware_macros::select! { - result = waiter.fuse() => result.expect("acknowledged"), - _ = context.sleep(Duration::from_secs(1)) => panic!("ack timed out"), - } - - let (_, entry) = reader - .recv() - .await - .expect("read queue") - .expect("queued entry"); - assert_eq!(entry.height, 2); - assert_eq!(entry.digest, digest); - - let encoded = context.encode(); - assert!(encoded.contains( - "indexer_producer_report_total{activity=\"block\",status=\"enqueued\"} 1", - )); - assert!(encoded.contains( - "indexer_producer_report_total{activity=\"block\",status=\"recorded\"} 1", - )); - assert!(encoded.contains("indexer_producer_record_duration_seconds_bucket")); - }); - } - #[test] fn mailbox_overflow_tracks_retained_block_pressure() { deterministic::Runner::default().start(|context| async move { diff --git a/examples/coins/chain/src/indexer/backfiller/state.rs b/examples/coins/chain/src/indexer/backfiller/state.rs index 217b70f..e52c24c 100644 --- a/examples/coins/chain/src/indexer/backfiller/state.rs +++ b/examples/coins/chain/src/indexer/backfiller/state.rs @@ -5,10 +5,10 @@ use crate::{ }, IndexerMetrics, }, - Block, + Block, Finalized, }; use bytes::{Buf, BufMut}; -use commonware_codec::{self, FixedSize, Read, Write}; +use commonware_codec::{self, EncodeSize, FixedSize, Read, Write}; use commonware_cryptography::{sha256::Digest, Digestible}; use commonware_utils::{sync::Mutex, PrioritySet}; use std::{collections::BTreeMap, sync::Arc}; @@ -19,30 +19,84 @@ pub enum Decision { Proceed, } -#[derive(Clone, Copy, PartialEq, Eq)] -pub struct Entry { +pub struct Candidate { pub height: u64, pub digest: Digest, } -impl FixedSize for Entry { - const SIZE: usize = u64::SIZE + Digest::SIZE; +const ENTRY_VERSION: u8 = 1; + +#[derive(Clone)] +pub struct Entry { + pub version: u8, + pub enqueued_at_millis: u64, + pub encoded_len: u64, + pub finalized: Finalized, +} + +impl Entry { + pub fn new(enqueued_at_millis: u64, finalized: Finalized) -> Self { + let mut entry = Self { + version: ENTRY_VERSION, + enqueued_at_millis, + encoded_len: 0, + finalized, + }; + entry.encoded_len = entry.encode_size() as u64; + entry + } + + pub fn height(&self) -> u64 { + self.finalized.block.height.get() + } + + pub fn digest(&self) -> Digest { + self.finalized.block.digest() + } } impl Write for Entry { fn write(&self, buf: &mut impl BufMut) { - self.height.write(buf); - self.digest.write(buf); + self.version.write(buf); + self.enqueued_at_millis.write(buf); + self.encoded_len.write(buf); + self.finalized.write(buf); } } impl Read for Entry { - type Cfg = (); + type Cfg = ::Cfg; + + fn read_cfg(buf: &mut impl Buf, cfg: &Self::Cfg) -> Result { + let version = u8::read_cfg(buf, &())?; + if version != ENTRY_VERSION { + return Err(commonware_codec::Error::Invalid( + "indexer spool entry", + "unsupported version", + )); + } + let enqueued_at_millis = u64::read_cfg(buf, &())?; + let encoded_len = u64::read_cfg(buf, &())?; + let finalized = Finalized::read_cfg(buf, cfg)?; + let entry = Self { + version, + enqueued_at_millis, + encoded_len, + finalized, + }; + if entry.encode_size() as u64 != encoded_len { + return Err(commonware_codec::Error::Invalid( + "indexer spool entry", + "encoded length mismatch", + )); + } + Ok(entry) + } +} - fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { - let height = u64::read_cfg(buf, &())?; - let digest = Digest::read_cfg(buf, &())?; - Ok(Self { height, digest }) +impl EncodeSize for Entry { + fn encode_size(&self) -> usize { + u8::SIZE + u64::SIZE + u64::SIZE + self.finalized.encode_size() } } @@ -55,6 +109,8 @@ pub struct State { cached_block_estimated_bytes: u64, certificate_uploads: BTreeMap, certificate_upload_refs: usize, + queued: BTreeMap, + queued_bytes: u64, metrics: Option, } @@ -74,6 +130,8 @@ impl State { cached_block_estimated_bytes: 0, certificate_uploads: BTreeMap::new(), certificate_upload_refs: 0, + queued: BTreeMap::new(), + queued_bytes: 0, metrics: None, } } @@ -89,8 +147,8 @@ impl State { self.uploaded.contains(digest) } - pub fn record(&mut self, block: &Block) -> Option { - let entry = Entry { + pub fn record(&mut self, block: &Block) -> Option { + let entry = Candidate { height: block.height.get(), digest: block.digest(), }; @@ -99,7 +157,7 @@ impl State { self.sync_metrics(); return None; } - if self.is_uploaded(&entry.digest) { + if self.is_uploaded(&entry.digest) || self.queued(&entry.digest) { self.sync_metrics(); return None; } @@ -108,12 +166,74 @@ impl State { Some(entry) } + pub fn queued(&self, digest: &Digest) -> bool { + self.queued.contains_key(digest) + } + + pub fn mark_queued(&mut self, position: u64, entry: &Entry) { + if let Some((_, _, old_bytes, _)) = + self.queued + .insert( + entry.digest(), + ( + position, + entry.height(), + entry.encoded_len, + entry.enqueued_at_millis, + ), + ) + { + self.queued_bytes = self.queued_bytes.saturating_sub(old_bytes); + } + self.queued_bytes = self.queued_bytes.saturating_add(entry.encoded_len); + self.sync_metrics(); + } + + pub fn recover_queued(&mut self, position: u64, entry: &Entry) { + self.observe_finalization(entry.height()); + self.mark_queued(position, entry); + } + + pub fn finish_queued(&mut self, digest: &Digest) { + if let Some((_, _, bytes, _)) = self.queued.remove(digest) { + self.queued_bytes = self.queued_bytes.saturating_sub(bytes); + } + self.sync_metrics(); + } + + pub fn spool_usage(&self) -> (u64, u64) { + (self.queued.len() as u64, self.queued_bytes) + } + + pub fn oldest_queued(&self) -> Option<(Digest, u64, u64, u64, u64)> { + self.queued + .iter() + .min_by_key(|(_, (position, _, _, _))| *position) + .map(|(digest, (position, height, bytes, enqueued_at))| { + (*digest, *position, *height, *bytes, *enqueued_at) + }) + } + + pub fn expire_through(&mut self, position: u64) -> Vec<(Digest, u64, u64)> { + let expired: Vec<_> = self + .queued + .iter() + .filter(|(_, (queued_position, _, _, _))| *queued_position <= position) + .map(|(digest, (_, height, bytes, _))| (*digest, *height, *bytes)) + .collect(); + for (digest, _, _) in &expired { + self.finish_queued(digest); + } + expired + } + pub fn advance_queue_floor(&mut self, height: u64) { self.acked_through = self.acked_through.max(height); self.prune(); self.sync_metrics(); } + #[cfg(test)] pub fn restart_above(&mut self, height: u64) { self.restart_watermark = self.restart_watermark.max(height); let pruned = self @@ -145,6 +265,7 @@ impl State { self.sync_metrics(); } + #[cfg(test)] pub fn cached_block(&self, digest: &Digest) -> Option { self.cached_blocks .get(digest) @@ -279,6 +400,20 @@ impl State { uploaded_digests: self.uploaded.len(), latest_finalized_height: self.latest_finalized, acked_through_height: self.acked_through, + spool_entries: self.queued.len(), + spool_logical_bytes: self.queued_bytes, + spool_oldest_height: self + .queued + .values() + .map(|(_, height, _, _)| *height) + .min() + .unwrap_or(0), + spool_oldest_enqueued_at_millis: self + .queued + .values() + .map(|(_, _, _, enqueued_at)| *enqueued_at) + .min() + .unwrap_or(0), }); } } @@ -290,7 +425,6 @@ pub type SharedState = Arc>; mod tests { use super::*; use crate::{StateCommitment, Transaction, EPOCH}; - use commonware_codec::{DecodeExt, Encode}; use commonware_consensus::types::{Height, Round, View}; use commonware_cryptography::{ed25519, Hasher, Sha256, Signer}; use commonware_runtime::{deterministic, Metrics as _, Runner as _, Supervisor as _}; @@ -325,20 +459,6 @@ mod tests { ) } - #[test] - fn entry_codec_roundtrips() { - let entry = Entry { - height: 7, - digest: Sha256::hash(b"block"), - }; - - let encoded = entry.encode(); - let decoded = Entry::decode(encoded).expect("decode entry"); - - assert_eq!(decoded.height, entry.height); - assert_eq!(decoded.digest, entry.digest); - } - #[test] fn record_caches_block_until_uploaded() { let mut state = State::new(); diff --git a/examples/coins/chain/src/indexer/metrics.rs b/examples/coins/chain/src/indexer/metrics.rs index 18d2f08..ca211ea 100644 --- a/examples/coins/chain/src/indexer/metrics.rs +++ b/examples/coins/chain/src/indexer/metrics.rs @@ -163,7 +163,6 @@ pub(crate) enum BlockMetricSource { ProducerRecord, LiveCertificate, ConsumerCached, - ConsumerMarshal, } impl BlockMetricSource { @@ -172,7 +171,6 @@ impl BlockMetricSource { Self::ProducerRecord => "producer_record", Self::LiveCertificate => "live_certificate", Self::ConsumerCached => "consumer_cached", - Self::ConsumerMarshal => "consumer_marshal", } } } @@ -192,7 +190,6 @@ impl EncodeLabelValueTrait for BlockMetricSource { pub(crate) enum SharedCacheSource { ProducerRecord, LiveCertificate, - ConsumerMarshal, } impl SharedCacheSource { @@ -200,7 +197,6 @@ impl SharedCacheSource { match self { Self::ProducerRecord => "producer_record", Self::LiveCertificate => "live_certificate", - Self::ConsumerMarshal => "consumer_marshal", } } } @@ -248,7 +244,6 @@ impl EncodeLabelValueTrait for SharedRetentionReason { pub(crate) enum BackfillStatus { Uploaded, Skipped, - Abandoned, } impl BackfillStatus { @@ -256,7 +251,6 @@ impl BackfillStatus { match self { Self::Uploaded => "uploaded", Self::Skipped => "skipped", - Self::Abandoned => "abandoned", } } } @@ -303,7 +297,6 @@ impl EncodeLabelValueTrait for BackfillDecision { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum BackfillPhase { Start, - BeforeBlock, BeforeAttempt, } @@ -311,7 +304,6 @@ impl BackfillPhase { const fn as_str(self) -> &'static str { match self { Self::Start => "start", - Self::BeforeBlock => "before_block", Self::BeforeAttempt => "before_attempt", } } @@ -331,9 +323,6 @@ impl EncodeLabelValueTrait for BackfillPhase { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum BackfillWaitReason { CertificateUpload, - MissingBlock, - MissingFinalization, - MismatchedFinalization, HttpError, } @@ -341,40 +330,11 @@ impl BackfillWaitReason { const fn as_str(self) -> &'static str { match self { Self::CertificateUpload => "certificate_upload", - Self::MissingBlock => "missing_block", - Self::MissingFinalization => "missing_finalization", - Self::MismatchedFinalization => "mismatched_finalization", Self::HttpError => "http_error", } } } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub(crate) enum BackfillResetReason { - MissingFinalization, - MismatchedFinalization, -} - -impl BackfillResetReason { - const fn as_str(self) -> &'static str { - match self { - Self::MissingFinalization => "missing_finalization", - Self::MismatchedFinalization => "mismatched_finalization", - } - } -} - -impl EncodeLabelValueTrait for BackfillResetReason { - fn encode( - &self, - encoder: &mut commonware_runtime::telemetry::metrics::LabelValueEncoder, - ) -> Result<(), fmt::Error> { - use fmt::Write as _; - - encoder.write_str(self.as_str()) - } -} - impl EncodeLabelValueTrait for BackfillWaitReason { fn encode( &self, @@ -475,6 +435,10 @@ pub(crate) enum ProducerStatus { Ignored, Recorded, AlreadyUploaded, + ExpiredOversized, + ExpiredEntries, + ExpiredBytes, + ExpiredAge, } impl ProducerStatus { @@ -485,6 +449,10 @@ impl ProducerStatus { Self::Ignored => "ignored", Self::Recorded => "recorded", Self::AlreadyUploaded => "already_uploaded", + Self::ExpiredOversized => "expired_oversized", + Self::ExpiredEntries => "expired_entries", + Self::ExpiredBytes => "expired_bytes", + Self::ExpiredAge => "expired_age", } } } @@ -538,6 +506,10 @@ pub(crate) struct SharedStateSnapshot { pub(crate) uploaded_digests: usize, pub(crate) latest_finalized_height: u64, pub(crate) acked_through_height: u64, + pub(crate) spool_entries: usize, + pub(crate) spool_logical_bytes: u64, + pub(crate) spool_oldest_height: u64, + pub(crate) spool_oldest_enqueued_at_millis: u64, } #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] @@ -599,11 +571,6 @@ struct BackfillWaitReasonLabel { reason: BackfillWaitReason, } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] -struct BackfillResetReasonLabel { - reason: BackfillResetReason, -} - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] struct QueueStatusLabel { status: QueueStatus, @@ -666,6 +633,11 @@ pub(crate) struct Inner { shared_uploaded_digests: Gauge, shared_latest_finalized_height: Gauge, shared_acked_through_height: Gauge, + spool_entries: Gauge, + spool_logical_bytes: Gauge, + spool_oldest_height: Gauge, + spool_oldest_enqueued_at_millis: Gauge, + producer_certificate_requests: Gauge, shared_pruned: CounterFamily, shared_cache_inserted: CounterFamily, shared_cache_removed: CounterFamily, @@ -677,9 +649,6 @@ pub(crate) struct Inner { backfill_decision: CounterFamily, backfill_wait_duration: HistogramFamily, backfill_retry: CounterFamily, - backfill_queue_reset: CounterFamily, - backfill_queue_reset_abandoned_entries: HistogramFamily, - backfill_queue_reset_abandoned_height_span: HistogramFamily, backfill_active_block_estimated_bytes: Gauge, backfill_active_body_estimated_bytes: Gauge, queue_enqueue: CounterFamily, @@ -817,6 +786,26 @@ impl IndexerMetrics { "shared_acked_through_height", "Queue acknowledgement floor tracked by shared indexer state", ), + spool_entries: context.gauge( + "spool_entries", + "Current number of self-contained finalized payloads in the durable spool", + ), + spool_logical_bytes: context.gauge( + "spool_logical_bytes", + "Current encoded bytes in self-contained finalized spool payloads", + ), + spool_oldest_height: context.gauge( + "spool_oldest_height", + "Oldest finalized height retained in the durable spool", + ), + spool_oldest_enqueued_at_millis: context.gauge( + "spool_oldest_enqueued_at_millis", + "Wall-clock Unix millisecond timestamp of the oldest durable spool payload", + ), + producer_certificate_requests: context.gauge( + "spool_certificate_requests", + "Current finalized blocks awaiting a marshal certificate before spool admission", + ), shared_pruned: context.family( "shared_prune", "Total shared indexer state retention removals by reason", @@ -867,24 +856,6 @@ impl IndexerMetrics { "backfill_retry", "Total durable backfill retries by reason", ), - backfill_queue_reset: context.family( - "backfill_queue_reset", - "Total durable backfill queue resets by reason", - ), - backfill_queue_reset_abandoned_entries: context.register( - "backfill_queue_reset_abandoned_entries", - "Durable backfill entries abandoned by queue reset", - raw::Family:: raw::Histogram>::new_with_constructor( - local_histogram, - ), - ), - backfill_queue_reset_abandoned_height_span: context.register( - "backfill_queue_reset_abandoned_height_span", - "Durable backfill height span abandoned by queue reset", - raw::Family:: raw::Histogram>::new_with_constructor( - local_histogram, - ), - ), backfill_active_block_estimated_bytes: context.gauge( "backfill_active_block_estimated_bytes", "Current estimated encoded block bytes held by durable backfill upload tasks", @@ -1074,6 +1045,16 @@ impl IndexerMetrics { let _ = self .shared_acked_through_height .try_set(snapshot.acked_through_height); + let _ = self.spool_entries.try_set(snapshot.spool_entries); + let _ = self + .spool_logical_bytes + .try_set(snapshot.spool_logical_bytes); + let _ = self + .spool_oldest_height + .try_set(snapshot.spool_oldest_height); + let _ = self + .spool_oldest_enqueued_at_millis + .try_set(snapshot.spool_oldest_enqueued_at_millis); self.queue_ack_floor(snapshot.acked_through_height); let lag = snapshot .latest_finalized_height @@ -1081,6 +1062,14 @@ impl IndexerMetrics { let _ = self.queue_lag_height.try_set(lag); } + pub(crate) fn certificate_request_started(&self) { + self.producer_certificate_requests.inc(); + } + + pub(crate) fn certificate_request_finished(&self) { + self.producer_certificate_requests.dec(); + } + pub(crate) fn shared_cache_inserted(&self, source: SharedCacheSource) { self.shared_cache_inserted .get_or_create(&SharedCacheSourceLabel { source }) @@ -1132,22 +1121,6 @@ impl IndexerMetrics { .observe(duration.as_secs_f64()); } - pub(crate) fn backfill_queue_reset( - &self, - reason: BackfillResetReason, - abandoned_entries: u64, - abandoned_height_span: u64, - ) { - let label = BackfillResetReasonLabel { reason }; - self.backfill_queue_reset.get_or_create(&label).inc(); - self.backfill_queue_reset_abandoned_entries - .get_or_create(&label) - .observe(abandoned_entries as f64); - self.backfill_queue_reset_abandoned_height_span - .get_or_create(&label) - .observe(abandoned_height_span as f64); - } - pub(crate) fn start_backfill_body(&self, bytes: u64) -> BackfillBodyGuard { let bytes = gauge_u64(bytes); self.backfill_active_body_estimated_bytes.inc_by(bytes); @@ -1403,9 +1376,6 @@ impl BackfillUploadGuard { self.status = Some(BackfillStatus::Skipped); } - pub(crate) const fn abandoned(&mut self) { - self.status = Some(BackfillStatus::Abandoned); - } } impl Drop for BackfillUploadGuard { diff --git a/examples/coins/chain/src/indexer/mod.rs b/examples/coins/chain/src/indexer/mod.rs index 2604221..fac82a2 100644 --- a/examples/coins/chain/src/indexer/mod.rs +++ b/examples/coins/chain/src/indexer/mod.rs @@ -6,9 +6,9 @@ use commonware_consensus::types::Epoch; use commonware_cryptography::bls12381::{ dkg::feldman_desmedt::Output as DkgOutput, primitives::variant::MinSig, }; -use commonware_runtime::{BufferPooler, Clock, Metrics, Spawner, Storage}; +use commonware_runtime::{BufferPooler, Clock, Handle, Metrics, Spawner, Storage}; use commonware_storage::queue; -use commonware_utils::sync::Mutex; +use commonware_utils::{sync::Mutex, NZU64}; use std::{future::Future, num::NonZeroUsize, sync::Arc, time::Duration}; use thiserror::Error; @@ -21,7 +21,7 @@ use backfiller::{SharedState, State}; pub(crate) use metrics::{DkgUploadStatus, IndexerMetrics}; #[cfg(test)] pub(crate) use metrics::{ - BackfillDecision, BackfillPhase, BackfillResetReason, BackfillWaitReason, BlockMetricSource, + BackfillDecision, BackfillPhase, BackfillWaitReason, BlockMetricSource, HttpArtifact, LiveUploadArtifact, ProducerActivity, ProducerStatus, QueueReadSource, QueueStatus, SharedCacheSource, SharedRetentionReason, SharedStateSnapshot, }; @@ -32,6 +32,64 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const MISSING_FINALIZATION_GRACE: Duration = Duration::from_secs(120); const MISMATCHED_FINALIZATION_GRACE: Duration = Duration::from_secs(15); +pub(crate) const SPOOL_ITEMS_PER_SECTION: std::num::NonZeroU64 = NZU64!(128); + +/// Availability window for durable finalized uploads while the external indexer is unavailable. +#[derive(Clone, Copy, Debug)] +pub struct SpoolLimits { + pub max_entries: u64, + pub max_bytes: u64, + pub max_payload_bytes: u64, + pub max_age: Duration, +} + +impl Default for SpoolLimits { + fn default() -> Self { + Self { + max_entries: crate::BLOCKS_PER_EPOCH.get(), + max_bytes: 2 * 1024 * 1024 * 1024, + max_payload_bytes: 16 * 1024 * 1024, + max_age: Duration::from_secs(24 * 60 * 60), + } + } +} + +impl SpoolLimits { + fn validate(self) { + assert!(self.max_entries > 0, "indexer spool max_entries must be non-zero"); + assert!(self.max_bytes > 0, "indexer spool max_bytes must be non-zero"); + assert!( + self.max_payload_bytes > 0 && self.max_payload_bytes <= self.max_bytes, + "indexer spool max_payload_bytes must be in 1..=max_bytes" + ); + assert!(!self.max_age.is_zero(), "indexer spool max_age must be non-zero"); + self.max_encoded_payload_bytes() + .expect("indexer spool encoded payload bound overflows u64"); + } + + pub fn max_encoded_payload_bytes(self) -> Option { + (SPOOL_ITEMS_PER_SECTION.get() - 1) + .checked_mul(self.max_payload_bytes)? + .checked_add(self.max_bytes) + } +} + +#[cfg(test)] +mod spool_tests { + use super::*; + + #[test] + fn default_spool_bound_includes_section_slack() { + let limits = SpoolLimits::default(); + assert_eq!(limits.max_entries, crate::BLOCKS_PER_EPOCH.get()); + assert_eq!(limits.max_bytes, 2 * 1024 * 1024 * 1024); + assert_eq!(limits.max_age, Duration::from_secs(24 * 60 * 60)); + assert_eq!( + limits.max_encoded_payload_bytes(), + Some(limits.max_bytes + 127 * limits.max_payload_bytes) + ); + } +} /// Errors returned by the HTTP indexer client. #[derive(Debug, Error)] @@ -184,11 +242,13 @@ pub(crate) struct Config { pub(crate) mailbox_size: NonZeroUsize, pub(crate) backfiller_max_active: NonZeroUsize, pub(crate) backfiller_retry: Duration, + pub(crate) spool_limits: SpoolLimits, pub(crate) metrics: IndexerMetrics, } pub(crate) struct Indexer { producer: Producer, + producer_handle: Handle<()>, pusher: Pusher, consumer: Consumer, } @@ -205,8 +265,10 @@ impl Indexer Indexer (Producer, Pusher, Consumer) { + pub(crate) fn split(self) -> (Producer, Handle<()>, Pusher, Consumer) { let Self { producer, + producer_handle, pusher, consumer, } = self; - (producer, pusher, consumer) + (producer, producer_handle, pusher, consumer) } } diff --git a/examples/coins/chain/src/lib.rs b/examples/coins/chain/src/lib.rs index b034460..359f7f8 100644 --- a/examples/coins/chain/src/lib.rs +++ b/examples/coins/chain/src/lib.rs @@ -19,6 +19,7 @@ pub mod application; pub mod engine; pub mod execution; pub mod genesis; +pub(crate) mod history; pub mod indexer; pub mod rpc; pub mod runtime; diff --git a/examples/coins/chain/src/testnet.rs b/examples/coins/chain/src/testnet.rs index a1fc1ea..556b31d 100644 --- a/examples/coins/chain/src/testnet.rs +++ b/examples/coins/chain/src/testnet.rs @@ -10,7 +10,7 @@ use crate::{ channels, engine::{Config as EngineConfig, Engine}, genesis::{ChainGenesis, GenesisError}, - indexer, rpc, PublicKey, NAMESPACE, + indexer, rpc, PublicKey, BLOCKS_PER_EPOCH, NAMESPACE, }; use commonware_codec::{Decode, DecodeExt, Encode, EncodeSize}; use commonware_consensus::marshal; @@ -29,7 +29,7 @@ use commonware_p2p::{ use commonware_runtime::{ tokio, Clock as _, Handle, Runner as _, Spawner as _, Strategizer as _, Supervisor as _, }; -use commonware_utils::{ordered::Set, N3f1, NZUsize, NZU32}; +use commonware_utils::{ordered::Set, Hostname, N3f1, NZUsize, NZU32}; use governor::Quota; use nunchi_dkg::{ ContinueOnUpdate, PeerConfig, Storage as DkgStorage, StorageKey, StorageProtector, @@ -41,14 +41,13 @@ use serde::{Deserialize, Serialize}; use std::{ fs, net::{IpAddr, SocketAddr}, - num::{NonZeroU32, TryFromIntError}, + num::{NonZeroU32, NonZeroU64, TryFromIntError}, path::{Path, PathBuf}, str::FromStr, time::{Duration, Instant}, }; use tracing::{info, warn, Level}; -const FREEZER_TABLE_INITIAL_SIZE: u32 = 2u32.pow(14); const DEFAULT_MAX_BLOCK_TRANSACTIONS: usize = 4_096; const DEFAULT_MAX_MESSAGE_SIZE: u32 = 1024 * 1024; const DEFAULT_CHANNEL_BACKLOG: usize = 1024; @@ -124,7 +123,12 @@ pub struct NodeConfig { pub share: String, pub peer_config: PeerConfig, pub listen_address: SocketAddr, - pub dialable_address: SocketAddr, + /// Address advertised to peers. IP literals use socket syntax (with brackets around IPv6), + /// while DNS names use `hostname:port` without a URL scheme or path. DNS is resolved again + /// whenever a disconnected peer retries this node, so operators should use a stable hostname + /// and an appropriate TTL. + #[serde(with = "ingress_serde")] + pub dialable_address: Ingress, pub rpc_address: SocketAddr, pub metrics_address: SocketAddr, pub bootstrappers: Vec, @@ -132,6 +136,21 @@ pub struct NodeConfig { pub genesis_path: Option, #[serde(default)] pub indexer_url: Option, + /// Number of finalized blocks in each consensus epoch. + #[serde(default = "default_epoch_length")] + pub epoch_length: NonZeroU64, + /// Maximum self-contained finalized payloads retained while the indexer is unavailable. + #[serde(default = "default_indexer_spool_max_entries")] + pub indexer_spool_max_entries: u64, + /// Maximum logical encoded payload bytes retained by the indexer spool. + #[serde(default = "default_indexer_spool_max_bytes")] + pub indexer_spool_max_bytes: u64, + /// Maximum encoded size accepted for one finalized payload. + #[serde(default = "default_indexer_spool_max_payload_bytes")] + pub indexer_spool_max_payload_bytes: u64, + /// Maximum payload age before visible terminal expiry. + #[serde(default = "default_indexer_spool_max_age_seconds")] + pub indexer_spool_max_age_seconds: u64, pub consensus: ConsensusConfig, pub networking: NetworkConfig, /// Enable one-time peer QMDB state sync for a fresh joining node. @@ -152,10 +171,87 @@ impl NodeConfig { } } +fn default_epoch_length() -> NonZeroU64 { + BLOCKS_PER_EPOCH +} + +fn default_indexer_spool_max_entries() -> u64 { + indexer::SpoolLimits::default().max_entries +} + +fn default_indexer_spool_max_bytes() -> u64 { + indexer::SpoolLimits::default().max_bytes +} + +fn default_indexer_spool_max_payload_bytes() -> u64 { + indexer::SpoolLimits::default().max_payload_bytes +} + +fn default_indexer_spool_max_age_seconds() -> u64 { + indexer::SpoolLimits::default().max_age.as_secs() +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct BootstrapperConfig { pub public_key: String, - pub address: SocketAddr, + /// Address used to dial the bootstrapper. IP literals use socket syntax (with brackets around + /// IPv6), while DNS names use `hostname:port` without a URL scheme or path. DNS is resolved on + /// each reconnect attempt rather than continuously while a connection is healthy. + #[serde(with = "ingress_serde")] + pub address: Ingress, +} + +mod ingress_serde { + use super::*; + use serde::{de::Error as _, Deserializer, Serializer}; + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + if let Ok(address) = SocketAddr::from_str(&value) { + return Ok(Ingress::Socket(address)); + } + if value.contains("://") { + return Err(D::Error::custom( + "peer address must not contain a URL scheme or path", + )); + } + + let (host, port) = value + .rsplit_once(':') + .ok_or_else(|| D::Error::custom("peer address is missing a port"))?; + if host.is_empty() { + return Err(D::Error::custom("peer address is missing a host")); + } + if port.is_empty() { + return Err(D::Error::custom("peer address is missing a port")); + } + if host.contains(':') { + return Err(D::Error::custom( + "IPv6 peer addresses must use bracketed socket syntax", + )); + } + let port = port + .parse::() + .map_err(|error| D::Error::custom(format!("invalid peer address port: {error}")))?; + let host = Hostname::new(host) + .map_err(|error| D::Error::custom(format!("invalid peer address hostname: {error}")))?; + Ok(Ingress::Dns { host, port }) + } + + pub fn serialize(ingress: &Ingress, serializer: S) -> Result + where + S: Serializer, + { + match ingress { + Ingress::Socket(address) => serializer.serialize_str(&address.to_string()), + Ingress::Dns { host, port } => { + serializer.serialize_str(&format!("{host}:{port}")) + } + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -284,14 +380,14 @@ pub fn generate_local_testnet(config: LocalTestnetConfig) -> Result Result, Error>>()?; @@ -328,6 +424,11 @@ pub fn generate_local_testnet(config: LocalTestnetConfig) -> Result(&bootstrapper.public_key, "bootstrapper.public_key")?, - Ingress::from(bootstrapper.address), + bootstrapper.address.clone(), )) }) .collect::, Error>>()?; @@ -486,6 +587,7 @@ async fn start_node( node = %config.name, public_key = %public_key, listen = %config.listen_address, + dialable = ?config.dialable_address, rpc = %config.rpc_address, metrics = %config.metrics_address, "starting coins-chain validator" @@ -495,7 +597,7 @@ async fn start_node( private_key.clone(), NAMESPACE, config.listen_address, - config.dialable_address, + config.dialable_address.clone(), bootstrappers, config.networking.max_message_size, ); @@ -541,13 +643,12 @@ async fn start_node( blocker: oracle.clone(), manager: oracle.clone(), partition_prefix: config.name.clone(), - blocks_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, - finalized_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, signer: private_key, dkg_storage_key, output, share: Some(share), peer_config: config.peer_config.clone(), + epoch_length: config.epoch_length, leader_timeout: Duration::from_millis(config.consensus.leader_timeout_ms), certification_timeout: Duration::from_millis(config.consensus.certification_timeout_ms), strategy: context @@ -557,6 +658,12 @@ async fn start_node( pool_config: PoolConfig::default(), genesis: read_genesis(config.genesis_path.as_ref())?, indexer: indexer_client.map(|(client, _metrics)| client), + indexer_spool_limits: indexer::SpoolLimits { + max_entries: config.indexer_spool_max_entries, + max_bytes: config.indexer_spool_max_bytes, + max_payload_bytes: config.indexer_spool_max_payload_bytes, + max_age: Duration::from_secs(config.indexer_spool_max_age_seconds), + }, }; let dkg_callback: Box> = ContinueOnUpdate::boxed(); diff --git a/examples/coins/chain/src/tests/genesis.rs b/examples/coins/chain/src/tests/genesis.rs index 9c48f6e..b28773a 100644 --- a/examples/coins/chain/src/tests/genesis.rs +++ b/examples/coins/chain/src/tests/genesis.rs @@ -3,12 +3,19 @@ use nunchi_authority::{AuthorityGenesis, AuthorityLedger}; use nunchi_coins::{CoinsGenesis, Ledger}; use nunchi_common::{CommitState, QmdbState}; -use commonware_cryptography::{ed25519, Signer as _}; +use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::deal, + primitives::{sharing::Mode, variant::MinSig}, + }, + ed25519, Signer as _, +}; use commonware_runtime::{deterministic, Runner as _, Supervisor as _}; use nunchi_authority::{AuthorityDB, AuthorityOperation, Transaction as AuthorityTransaction}; use nunchi_coins::{Address, CoinDB, CoinSpec, TokenFactory, TokenName, TokenSymbol}; use nunchi_crypto::PrivateKey; use nunchi_oracle::{IntervalKey, OracleGenesis, OracleLedger}; +use commonware_utils::{ordered::Set, test_rng, N3f1, NZU64}; use crate::genesis::*; @@ -79,6 +86,35 @@ async fn empty_commitment(context: deterministic::Context, partition: &str) -> S state_commitment(state.sync_target()) } +fn authenticated_dkg() -> ( + nunchi_chain::DkgState, + commonware_cryptography::bls12381::dkg::feldman_desmedt::Output< + MinSig, + ed25519::PublicKey, + >, +) { + let participants = Set::from_iter_dedup( + (0..4) + .map(ed25519::PrivateKey::from_seed) + .map(|signer| signer.public_key()), + ); + let (output, _) = + deal::(test_rng(), Mode::NonZeroCounter, participants.clone()) + .unwrap(); + let config = nunchi_dkg::DkgProtocolConfig { + state_format_version: nunchi_dkg::STATE_FORMAT_VERSION, + namespace: crate::NAMESPACE.to_vec(), + epoch_length: NZU64!(10), + participants, + num_participants_per_round: vec![4], + mode: Mode::NonZeroCounter, + mode_version: 0, + fault_model: nunchi_dkg::public::N3F1_FAULT_MODEL, + trusted_initial_identity: *output.public().public(), + }; + (nunchi_chain::DkgState::new(config).unwrap(), output) +} + #[test] fn genesis_json_roundtrips() { let genesis = sample_genesis(); @@ -87,6 +123,68 @@ fn genesis_json_roundtrips() { assert_eq!(decoded, genesis); } +#[test] +fn authenticated_checkpoint_is_seeded_with_and_without_application_genesis() { + deterministic::Runner::default().start(|context| async move { + let (dkg, output) = authenticated_dkg(); + for (suffix, empty_label, config_label, state_label, reopen_label, genesis) in [ + ( + "none", + "empty_none", + "config_none", + "state_none", + "reopen_none", + None, + ), + ( + "some", + "empty_some", + "config_some", + "state_some", + "reopen_some", + Some(sample_genesis()), + ), + ] { + let partition = format!("authenticated-genesis-{suffix}"); + let empty = empty_commitment( + context.child(empty_label), + &format!("{partition}-empty"), + ) + .await; + let config_context = context.child(config_label); + let config = QmdbState::::config( + &config_context, + &partition, + ); + let commitment = authenticated_genesis_target( + context.child(state_label), + config.clone(), + &dkg, + output.clone(), + genesis.as_ref(), + &empty, + ) + .await + .unwrap(); + assert_ne!(commitment.root, empty.root); + + let state = QmdbState::init_with_config( + context.child(reopen_label), + config, + ) + .await + .unwrap(); + let checkpoint = dkg.load_checkpoint(&state).await.unwrap(); + assert_eq!(checkpoint.epoch, commonware_consensus::types::Epoch::zero()); + assert_eq!(checkpoint.activation_height.get(), 0); + assert_eq!( + checkpoint.protocol_config_digest, + dkg.config().digest().unwrap() + ); + } + }); +} + #[test] fn genesis_json_loads_from_disk() { let genesis = ChainGenesis::from_slice(GENESIS_FIXTURE).unwrap(); diff --git a/examples/coins/chain/src/tests/indexer.rs b/examples/coins/chain/src/tests/indexer.rs index b26e49e..d26399b 100644 --- a/examples/coins/chain/src/tests/indexer.rs +++ b/examples/coins/chain/src/tests/indexer.rs @@ -1,6 +1,6 @@ use crate::{ indexer::{ - BackfillDecision, BackfillPhase, BackfillResetReason, BackfillWaitReason, + BackfillDecision, BackfillPhase, BackfillWaitReason, BlockMetricSource, DkgUploadStatus, HttpArtifact, IndexerMetrics, LiveUploadArtifact, ProducerActivity, ProducerStatus, QueueReadSource, QueueStatus, SharedCacheSource, SharedRetentionReason, @@ -346,10 +346,13 @@ fn shared_state_metrics_use_expected_labels() { uploaded_digests: 4, latest_finalized_height: 8, acked_through_height: 6, + spool_entries: 5, + spool_logical_bytes: 512, + spool_oldest_height: 2, + spool_oldest_enqueued_at_millis: 1_000, }); metrics.shared_cache_inserted(SharedCacheSource::ProducerRecord); metrics.shared_cache_inserted(SharedCacheSource::LiveCertificate); - metrics.shared_cache_inserted(SharedCacheSource::ConsumerMarshal); metrics.shared_cache_removed(SharedRetentionReason::Uploaded); metrics.shared_cache_removed(SharedRetentionReason::CertificateFinished); metrics.shared_cache_removed(SharedRetentionReason::Pruned); @@ -373,9 +376,6 @@ fn shared_state_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_shared_cache_insert_total{source=\"live_certificate\"} 1", )); - assert!(encoded.contains( - "indexer_shared_cache_insert_total{source=\"consumer_marshal\"} 1", - )); assert!(encoded.contains( "indexer_shared_cache_remove_total{reason=\"uploaded\"} 1", )); @@ -402,7 +402,6 @@ fn block_metrics_use_expected_sources() { metrics.observe_block(BlockMetricSource::ProducerRecord, &block); metrics.observe_block(BlockMetricSource::LiveCertificate, &block); metrics.observe_block(BlockMetricSource::ConsumerCached, &block); - metrics.observe_block(BlockMetricSource::ConsumerMarshal, &block); let encoded = context.encode(); assert!(encoded.contains("indexer_block_estimated_bytes_bucket")); @@ -410,7 +409,6 @@ fn block_metrics_use_expected_sources() { assert!(encoded.contains("source=\"producer_record\"")); assert!(encoded.contains("source=\"live_certificate\"")); assert!(encoded.contains("source=\"consumer_cached\"")); - assert!(encoded.contains("source=\"consumer_marshal\"")); }); } @@ -433,20 +431,11 @@ fn backfill_metrics_use_expected_labels() { upload.skipped(); } metrics.backfill_decision(BackfillDecision::Skip, BackfillPhase::Start); - metrics.backfill_decision(BackfillDecision::Wait, BackfillPhase::BeforeBlock); metrics.backfill_decision(BackfillDecision::Proceed, BackfillPhase::BeforeAttempt); metrics.backfill_retry(BackfillWaitReason::CertificateUpload); - metrics.backfill_retry(BackfillWaitReason::MissingBlock); - metrics.backfill_retry(BackfillWaitReason::MissingFinalization); - metrics.backfill_retry(BackfillWaitReason::MismatchedFinalization); metrics.backfill_retry(BackfillWaitReason::HttpError); metrics.backfill_waited(BackfillWaitReason::CertificateUpload, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MissingBlock, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MissingFinalization, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MismatchedFinalization, Duration::from_millis(1)); metrics.backfill_waited(BackfillWaitReason::HttpError, Duration::from_millis(1)); - metrics.backfill_queue_reset(BackfillResetReason::MissingFinalization, 32, 31); - metrics.backfill_queue_reset(BackfillResetReason::MismatchedFinalization, 2, 1); metrics.queue_enqueued(QueueStatus::Success); metrics.queue_enqueued(QueueStatus::Failure); metrics.queue_acked(QueueStatus::Success); @@ -486,9 +475,6 @@ fn backfill_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_backfill_decision_total{decision=\"skip\",phase=\"start\"} 1", )); - assert!(encoded.contains( - "indexer_backfill_decision_total{decision=\"wait\",phase=\"before_block\"} 1", - )); assert!(encoded.contains( "indexer_backfill_decision_total{decision=\"proceed\",phase=\"before_attempt\"} 1", )); @@ -496,20 +482,7 @@ fn backfill_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_backfill_retry_total{reason=\"certificate_upload\"} 1", )); - assert!(encoded.contains("indexer_backfill_retry_total{reason=\"missing_block\"} 1")); - assert!(encoded - .contains("indexer_backfill_retry_total{reason=\"missing_finalization\"} 1")); - assert!(encoded - .contains("indexer_backfill_retry_total{reason=\"mismatched_finalization\"} 1")); assert!(encoded.contains("indexer_backfill_retry_total{reason=\"http_error\"} 1")); - assert!(encoded.contains( - "indexer_backfill_queue_reset_total{reason=\"missing_finalization\"} 1", - )); - assert!(encoded.contains( - "indexer_backfill_queue_reset_total{reason=\"mismatched_finalization\"} 1", - )); - assert!(encoded.contains("indexer_backfill_queue_reset_abandoned_entries_bucket")); - assert!(encoded.contains("indexer_backfill_queue_reset_abandoned_height_span_bucket")); assert!(encoded.contains("indexer_backfill_active_block_estimated_bytes 0")); assert!(encoded.contains("indexer_backfill_active_body_estimated_bytes 0")); assert!(encoded.contains("indexer_queue_enqueue_total{status=\"success\"} 1")); diff --git a/examples/coins/chain/src/tests/testnet.rs b/examples/coins/chain/src/tests/testnet.rs index ac240e4..329789c 100644 --- a/examples/coins/chain/src/tests/testnet.rs +++ b/examples/coins/chain/src/tests/testnet.rs @@ -5,16 +5,24 @@ //! as `narae` consume. [`run_node`] boots a single validator from one of those configs on the //! tokio runtime with authenticated peer discovery, and serves the aggregated JSON-RPC module. -use commonware_cryptography::{bls12381::primitives::group, ed25519}; +use commonware_cryptography::{bls12381::primitives::group, ed25519, Signer as _}; +use commonware_macros::select; +use commonware_p2p::{ + authenticated::discovery::{self, Network}, + Ingress, Manager, Receiver as _, Recipients, Sender as _, +}; +use commonware_runtime::{deterministic, Clock as _, Quota, Runner as _, Supervisor as _}; +use commonware_utils::{ordered::Set, Hostname, NZU32}; use std::collections::HashSet; use std::{ fs, - net::{IpAddr, Ipv4Addr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, num::NonZeroU32, path::PathBuf, + time::Duration, }; -use crate::testnet::*; +use crate::{testnet::*, NAMESPACE, BLOCKS_PER_EPOCH}; #[test] fn generated_testnet_has_unique_ports_dirs_and_complete_peer_sets() { @@ -75,6 +83,7 @@ fn generated_testnet_has_unique_ports_dirs_and_complete_peer_sets() { .any(|bootstrapper| bootstrapper.address.port() == node.port)); assert_eq!(config.rpc_address.port(), node.rpc_port); assert_eq!(config.metrics_address.port(), node.metrics_port); + assert_eq!(config.epoch_length, BLOCKS_PER_EPOCH); // The threshold material must round-trip from the written config. let max_participants = @@ -93,6 +102,52 @@ fn generated_testnet_has_unique_ports_dirs_and_complete_peer_sets() { let _ = fs::remove_dir_all(dir); } +#[test] +fn node_config_uses_default_epoch_length_when_omitted_and_reads_overrides() { + let dir = std::env::temp_dir().join(format!( + "coins-chain-epoch-config-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 1, + base_port: 43_000, + base_rpc_port: 44_000, + base_metrics_port: 45_000, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed: 11, + }) + .expect("generate testnet"); + let path = &manifest.nodes[0].config_path; + let raw = fs::read_to_string(path).expect("read generated config"); + + let legacy = raw.replace("epoch_length = 200000\n", ""); + fs::write(path, legacy).expect("write legacy config"); + assert_eq!( + NodeConfig::read(path) + .expect("read legacy config") + .epoch_length, + BLOCKS_PER_EPOCH + ); + + let configured = raw.replace("epoch_length = 200000", "epoch_length = 100"); + fs::write(path, configured).expect("write configured config"); + assert_eq!( + NodeConfig::read(path) + .expect("read configured config") + .epoch_length + .get(), + 100 + ); + + let _ = fs::remove_dir_all(dir); +} + #[test] fn generated_testnet_can_advertise_remote_hosts() { let dir = @@ -124,17 +179,20 @@ fn generated_testnet_can_advertise_remote_hosts() { let first = NodeConfig::read(&manifest.nodes[0].config_path).expect("read first config"); let second = NodeConfig::read(&manifest.nodes[1].config_path).expect("read second config"); + let first_raw = fs::read_to_string(&manifest.nodes[0].config_path).expect("read first config"); assert_eq!(first.listen_address.ip(), IpAddr::V4(Ipv4Addr::UNSPECIFIED)); - assert_eq!(first.dialable_address.ip(), public_ips[0]); - assert_eq!(first.bootstrappers[0].address.ip(), public_ips[1]); + assert_eq!(first.dialable_address.ip(), Some(public_ips[0])); + assert_eq!(first.bootstrappers[0].address.ip(), Some(public_ips[1])); + assert!(first_raw.contains("dialable_address = \"192.0.2.10:30000\"")); + assert!(first_raw.contains("address = \"192.0.2.11:30001\"")); assert_eq!(first.storage_dir, storage_dir); assert_eq!( first.indexer_url.as_deref(), Some("https://indexer.example.com/coins-chain") ); - assert_eq!(second.dialable_address.ip(), public_ips[1]); - assert_eq!(second.bootstrappers[0].address.ip(), public_ips[0]); + assert_eq!(second.dialable_address.ip(), Some(public_ips[1])); + assert_eq!(second.bootstrappers[0].address.ip(), Some(public_ips[0])); assert_eq!( second.indexer_url.as_deref(), Some("https://indexer.example.com/coins-chain") @@ -142,3 +200,454 @@ fn generated_testnet_can_advertise_remote_hosts() { let _ = fs::remove_dir_all(dir); } + +#[test] +fn peer_addresses_round_trip_dns_and_ipv6_as_strings() { + let dir = std::env::temp_dir().join(format!( + "coins-chain-address-round-trip-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 2, + base_port: 32_000, + base_rpc_port: 33_000, + base_metrics_port: 34_000, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed: 9, + }) + .expect("generate testnet"); + let path = &manifest.nodes[0].config_path; + let mut config = NodeConfig::read(path).expect("read generated config"); + config.dialable_address = Ingress::Dns { + host: Hostname::new("validator-0.example.com").unwrap(), + port: 30_000, + }; + config.bootstrappers[0].address = Ingress::Dns { + host: Hostname::new("validator-1.example.com").unwrap(), + port: 30_001, + }; + config.write(path).expect("write DNS config"); + + let raw = fs::read_to_string(path).expect("read DNS config"); + assert!(raw.contains("dialable_address = \"validator-0.example.com:30000\"")); + assert!(raw.contains("address = \"validator-1.example.com:30001\"")); + let decoded = NodeConfig::read(path).expect("read DNS config"); + assert_eq!(decoded.dialable_address, config.dialable_address); + assert_eq!( + decoded.bootstrappers[0].address, + config.bootstrappers[0].address + ); + + config.dialable_address = Ingress::Socket(SocketAddr::new( + IpAddr::V6("2001:db8::10".parse::().unwrap()), + 30_000, + )); + config.write(path).expect("write IPv6 config"); + let raw = fs::read_to_string(path).expect("read IPv6 config"); + assert!(raw.contains("dialable_address = \"[2001:db8::10]:30000\"")); + assert_eq!( + NodeConfig::read(path) + .expect("read IPv6 config") + .dialable_address, + config.dialable_address + ); + + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn invalid_peer_addresses_are_rejected_through_node_config_read() { + let dir = std::env::temp_dir().join(format!( + "coins-chain-invalid-addresses-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 1, + base_port: 35_000, + base_rpc_port: 36_000, + base_metrics_port: 37_000, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed: 10, + }) + .expect("generate testnet"); + let valid_path = &manifest.nodes[0].config_path; + let valid = fs::read_to_string(valid_path).expect("read generated config"); + let valid_address = "dialable_address = \"127.0.0.1:35000\""; + assert!(valid.contains(valid_address)); + + let cases = [ + ("validator.example.com", "missing a port"), + ("validator.example.com:not-a-port", "invalid peer address port"), + ("validator.example.com:65536", "invalid peer address port"), + ("-validator.example.com:30000", "invalid peer address hostname"), + ( + "https://validator.example.com:30000", + "must not contain a URL scheme or path", + ), + ("2001:db8::10:30000", "must use bracketed socket syntax"), + ]; + for (index, (address, expected)) in cases.into_iter().enumerate() { + let path = dir.join(format!("invalid-{index}.toml")); + let invalid = valid.replace( + valid_address, + &format!("dialable_address = \"{address}\""), + ); + fs::write(&path, invalid).expect("write invalid config"); + let error = NodeConfig::read(&path).expect_err("invalid address should fail"); + assert!( + error.to_string().contains(expected), + "unexpected error for {address}: {error}" + ); + } + + let _ = fs::remove_dir_all(dir); +} + +fn parse_dns_address_through_node_config(address: &str, seed: u64) -> Ingress { + let dir = std::env::temp_dir().join(format!( + "coins-chain-parse-dns-{seed}-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 1, + base_port: 38_000, + base_rpc_port: 38_100, + base_metrics_port: 38_200, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed, + }) + .expect("generate parser fixture"); + let path = &manifest.nodes[0].config_path; + let valid = fs::read_to_string(path).expect("read parser fixture"); + let dns = valid.replace( + "dialable_address = \"127.0.0.1:38000\"", + &format!("dialable_address = \"{address}\""), + ); + fs::write(path, dns).expect("write parser fixture"); + let ingress = NodeConfig::read(path) + .expect("parse DNS address") + .dialable_address; + let _ = fs::remove_dir_all(dir); + ingress +} + +fn reconnect_config( + key: ed25519::PrivateKey, + listen: SocketAddr, + dialable: Ingress, + bootstrappers: Vec<(ed25519::PublicKey, Ingress)>, +) -> discovery::Config { + let mut config = discovery::Config::local( + key, + NAMESPACE, + listen, + dialable, + bootstrappers, + 1024 * 1024, + ); + config.peer_connection_cooldown = Duration::from_millis(100); + config.dial_frequency = Duration::from_millis(50); + config.gossip_bit_vec_frequency = Duration::from_millis(100); + config +} + +fn run_dns_bootstrapper_redeployment(seed: u64, dns: Ingress) -> String { + let runner = deterministic::Runner::new( + deterministic::Config::new() + .with_seed(seed) + .with_timeout(Some(Duration::from_secs(20))), + ); + runner.start(|context| async move { + let key_a = ed25519::PrivateKey::from_seed(100); + let key_b = ed25519::PrivateKey::from_seed(101); + let public_a = key_a.public_key(); + let public_b = key_b.public_key(); + let peers = Set::try_from(vec![public_a.clone(), public_b.clone()]).unwrap(); + let socket_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 10)), 39_000); + let socket_b1 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 11)), 39_000); + let socket_b2 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 12)), 39_000); + let unreachable_a = Ingress::Socket(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 0, 250)), + 49_000, + )); + context.resolver_register("validator-b-bootstrap.test", Some(vec![socket_b1.ip()])); + + let config_b = reconnect_config(key_b.clone(), socket_b1, dns.clone(), vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_initial").child("network"), config_b); + oracle_b.track(0, peers.clone()); + let (mut sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let handle_b = network_b.start(); + + let config_a = reconnect_config( + key_a, + socket_a, + unreachable_a, + vec![(public_b.clone(), dns.clone())], + ); + assert!(config_a.allow_dns); + let (mut network_a, mut oracle_a) = + Network::new(context.child("a").child("network"), config_a); + oracle_a.track(0, peers.clone()); + let (mut sender_a, mut receiver_a) = + network_a.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_a = network_a.start(); + + let initial_exchange = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-before".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + loop { + sender_b.send( + Recipients::One(public_a.clone()), + b"b-before".to_vec(), + true, + ); + select! { + result = receiver_a.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_b); + assert_eq!(message.as_ref(), b"b-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = initial_exchange => {}, + _ = context.sleep(Duration::from_secs(5)) => panic!("initial DNS connection timed out"), + } + + handle_b.abort(); + drop(sender_b); + drop(receiver_b); + context.resolver_register("validator-b-bootstrap.test", Some(vec![socket_b2.ip()])); + + // The restarted peer has no bootstrappers and A advertises an unreachable address, so B + // cannot initiate the replacement connection. + let config_b = reconnect_config(key_b, socket_b2, dns, vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_restarted").child("network"), config_b); + oracle_b.track(0, peers); + let (_sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_b = network_b.start(); + + let reconnected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-after".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-after"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = reconnected => {}, + _ = context.sleep(Duration::from_secs(5)) => panic!("DNS bootstrapper did not reconnect"), + } + context.auditor().state() + }) +} + +#[test] +fn dns_bootstrapper_is_resolved_again_after_redeployment() { + let dns = parse_dns_address_through_node_config("validator-b-bootstrap.test:39000", 11); + let first = run_dns_bootstrapper_redeployment(42, dns.clone()); + let second = run_dns_bootstrapper_redeployment(42, dns); + assert_eq!(first, second); +} + +fn run_discovered_dns_redeployment(seed: u64, dns: Ingress) -> String { + let runner = deterministic::Runner::new( + deterministic::Config::new() + .with_seed(seed) + .with_timeout(Some(Duration::from_secs(20))), + ); + runner.start(|context| async move { + let key_a = ed25519::PrivateKey::from_seed(200); + let key_b = ed25519::PrivateKey::from_seed(201); + let key_c = ed25519::PrivateKey::from_seed(202); + let public_a = key_a.public_key(); + let public_b = key_b.public_key(); + let public_c = key_c.public_key(); + let peers = Set::try_from(vec![ + public_a.clone(), + public_b.clone(), + public_c.clone(), + ]) + .unwrap(); + let socket_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 10)), 40_000); + let socket_b1 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 11)), 40_000); + let socket_b2 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 12)), 40_000); + let socket_c = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 13)), 40_000); + let unreachable_a = Ingress::Socket(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 1, 250)), + 50_000, + )); + context.resolver_register("validator-b-discovered.test", Some(vec![socket_b1.ip()])); + + let config_c = reconnect_config(key_c, socket_c, socket_c.into(), vec![]); + let (mut network_c, mut oracle_c) = + Network::new(context.child("c").child("network"), config_c); + oracle_c.track(0, peers.clone()); + let (_sender_c, _receiver_c) = + network_c.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_c = network_c.start(); + + let config_b = reconnect_config( + key_b.clone(), + socket_b1, + dns.clone(), + vec![(public_c.clone(), socket_c.into())], + ); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_initial").child("network"), config_b); + oracle_b.track(0, peers.clone()); + let (mut sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let handle_b = network_b.start(); + + // A only knows C initially. It must learn B's DNS ingress through discovery. + let config_a = reconnect_config( + key_a, + socket_a, + unreachable_a, + vec![(public_c, socket_c.into())], + ); + let (mut network_a, mut oracle_a) = + Network::new(context.child("a").child("network"), config_a); + oracle_a.track(0, peers.clone()); + let (mut sender_a, mut receiver_a) = + network_a.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_a = network_a.start(); + + let learned_and_connected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-before".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + loop { + sender_b.send( + Recipients::One(public_a.clone()), + b"b-before".to_vec(), + true, + ); + select! { + result = receiver_a.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_b); + assert_eq!(message.as_ref(), b"b-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = learned_and_connected => {}, + _ = context.sleep(Duration::from_secs(7)) => panic!("A did not discover B's DNS ingress"), + } + + handle_b.abort(); + drop(sender_b); + drop(receiver_b); + context.resolver_register("validator-b-discovered.test", Some(vec![socket_b2.ip()])); + + // B has no bootstrapper after restart and A's advertised address is unreachable. A must + // retain the discovered hostname, resolve it again, and dial B at its replacement IP. + let config_b = reconnect_config(key_b, socket_b2, dns, vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_restarted").child("network"), config_b); + oracle_b.track(0, peers); + let (_sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_b = network_b.start(); + + let reconnected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-after".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-after"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = reconnected => {}, + _ = context.sleep(Duration::from_secs(7)) => panic!("discovered DNS peer did not reconnect"), + } + context.auditor().state() + }) +} + +#[test] +fn discovered_dns_address_is_resolved_again_after_redeployment() { + let dns = parse_dns_address_through_node_config("validator-b-discovered.test:40000", 12); + let first = run_discovered_dns_redeployment(84, dns.clone()); + let second = run_discovered_dns_redeployment(84, dns); + assert_eq!(first, second); +} diff --git a/examples/coins/chain/tests/coins.rs b/examples/coins/chain/tests/coins.rs index 9051289..22dabb6 100644 --- a/examples/coins/chain/tests/coins.rs +++ b/examples/coins/chain/tests/coins.rs @@ -8,7 +8,8 @@ use commonware_cryptography::Signer as _; use commonware_cryptography::{Hasher, Sha256}; use commonware_macros::{select, test_traced}; use commonware_p2p::simulated::Link; -use commonware_runtime::{deterministic, Clock, Runner as _}; +use commonware_runtime::{deterministic, Clock, Runner as _, Spawner as _, Supervisor as _}; +use commonware_utils::NZU64; use nunchi_authority::{ proposal_id, AuthorityOperation, MultisigPolicy, RegistryChange, Transaction as AuthorityTransaction, @@ -104,13 +105,68 @@ fn reaches_height_100() { }); } +#[test_traced] +fn crosses_epoch_boundary_and_reclaims_retired_partition() { + with_large_stack(|| { + let executor = deterministic::Runner::timed(Duration::from_secs(60)); + executor.start(|mut context| async move { + let cfg = ValidatorConfig { + epoch_length: NZU64!(20), + ..ValidatorConfig::default() + }; + let mut network = TestNetworkBuilder::new(VALIDATORS) + .with_initial_link(reliable_link()) + .with_validator_config(cfg) + .build(&mut context) + .await; + network.start_all().await; + + let validator_task_prefix = "validator"; + assert!(network.running_tasks(validator_task_prefix) > 0); + + network.run_until_height(25).await; + + network.assert_validator_metric("engine_orchestrator_latest_epoch", &[], 1); + network.assert_validator_metric( + "engine_orchestrator_consensus_partitions_active", + &[], + 1, + ); + network.assert_validator_metric( + "engine_orchestrator_consensus_partition_cleanup_watermark", + &[], + 1, + ); + network.assert_validator_metric( + "engine_orchestrator_consensus_partition_cleanup_total", + &[("status", "removed")], + 1, + ); + network.assert_consensus_partition_missing(0).await; + assert!(network.running_tasks(validator_task_prefix) > 0); + + let shutdown = network.context().child("shutdown"); + shutdown + .stop(0, Some(Duration::from_secs(10))) + .await + .expect("validator tasks should stop cleanly"); + assert_eq!(network.running_tasks(validator_task_prefix), 0); + }); + }); +} + #[test_traced] fn state_syncs_late_validator() { with_large_stack(|| { let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { + let cfg = ValidatorConfig { + epoch_length: NZU64!(20), + ..ValidatorConfig::default() + }; let mut network = TestNetworkBuilder::new(5) .without_initial_links() + .with_validator_config(cfg) .build(&mut context) .await; @@ -122,7 +178,11 @@ fn state_syncs_late_validator() { for index in 1..5 { network.start_validator(index).await; } - network.run_until_height(10).await; + // Select a state-sync anchor inside epoch 1. The joining validator + // has no marshal archive, so its orchestrator must use the + // certificate-verified in-epoch finalization rather than infer an + // epoch-0 boundary digest from the QMDB anchor. + network.run_until_height(25).await; network .link_where(link, |from, to| { @@ -130,7 +190,7 @@ fn state_syncs_late_validator() { }) .await; network.start_validator_with_state_sync(0).await; - network.run_until_height(20).await; + network.run_until_height(30).await; }); }); } @@ -154,6 +214,7 @@ fn recovers_unclean_shutdown() { let cfg = ValidatorConfig { leader_timeout: Duration::from_millis(250), certification_timeout: Duration::from_millis(500), + ..ValidatorConfig::default() }; let wait = diff --git a/examples/coins/chain/tests/common/network.rs b/examples/coins/chain/tests/common/network.rs index 04495cb..21d473f 100644 --- a/examples/coins/chain/tests/common/network.rs +++ b/examples/coins/chain/tests/common/network.rs @@ -15,7 +15,7 @@ use commonware_p2p::{ use commonware_parallel::Sequential; use commonware_runtime::{ deterministic::{self, Runner}, - Clock, Metrics, Runner as _, Supervisor, + Clock, Error as RuntimeError, Metrics, Runner as _, Storage, Supervisor, }; use commonware_utils::{ ordered::{Map, Set}, @@ -27,7 +27,7 @@ use nunchi_coins::{Address, Ledger}; use nunchi_coins_chain::{ engine::{Config, Engine}, execution::NodeHandle, - PublicKey, Transaction, + PublicKey, Transaction, BLOCKS_PER_EPOCH, }; use nunchi_common::QmdbReader; use nunchi_dkg::{ContinueOnUpdate, PeerConfig}; @@ -35,10 +35,10 @@ use nunchi_mempool::{MempoolHandle, PoolConfig}; use nunchi_oracle::OracleLedger; use std::{ collections::{HashMap, HashSet}, + num::NonZeroU64, time::Duration, }; -const FREEZER_TABLE_INITIAL_SIZE: u32 = 2u32.pow(14); // 1MB const TEST_QUOTA: Quota = Quota::per_second(NZU32!(u32::MAX)); const MAX_BLOCK_TRANSACTIONS: usize = 256; @@ -109,6 +109,7 @@ pub(crate) fn lossy_link() -> Link { #[derive(Clone)] pub(crate) struct ValidatorConfig { + pub(crate) epoch_length: NonZeroU64, pub(crate) leader_timeout: Duration, pub(crate) certification_timeout: Duration, } @@ -116,6 +117,7 @@ pub(crate) struct ValidatorConfig { impl Default for ValidatorConfig { fn default() -> Self { Self { + epoch_length: BLOCKS_PER_EPOCH, leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(2), } @@ -356,6 +358,83 @@ impl TestNetwork<'_> { } } + pub(crate) fn assert_validator_metric( + &self, + suffix: &str, + required_labels: &[(&str, &str)], + expected_value: u64, + ) { + let expected = self.started_validator_ids(); + let mut observed = HashSet::new(); + let metrics = self.context.encode(); + for line in metrics.lines() { + let Some((metric, labels, value)) = validator_metric_sample(line) else { + continue; + }; + if !metric.ends_with(suffix) + || !required_labels + .iter() + .all(|(name, expected)| metric_label(labels, name) == Some(*expected)) + { + continue; + } + let Some(id) = metric_label(labels, "id") else { + continue; + }; + if !expected.contains(id) { + continue; + } + assert_eq!( + value.parse::().unwrap(), + expected_value, + "unexpected {suffix} value for {id}: {line}", + ); + assert!( + observed.insert(id.to_string()), + "duplicate {suffix} for {id}" + ); + } + assert_eq!( + observed, expected, + "missing {suffix} samples in metrics: {metrics}", + ); + } + + pub(crate) async fn assert_consensus_partition_missing(&self, epoch: u64) { + for signer in &self.private_keys { + let public_key = signer.public_key(); + if self.registrations.contains_key(&public_key) { + continue; + } + let partition = format!("validator_{public_key}_consensus_consensus_{epoch}"); + let result = self.context.scan(&partition).await; + assert!( + matches!( + result, + Err(RuntimeError::PartitionMissing(ref missing)) if missing == &partition + ), + "retired consensus partition {partition} still exists: {result:?}", + ); + } + } + + pub(crate) fn running_tasks(&self, prefix: &str) -> usize { + self.context + .encode() + .lines() + .filter_map(|line| { + if !line.starts_with("runtime_tasks_running{") { + return None; + } + let name = line.split("name=\"").nth(1)?.split('"').next()?; + if !name.starts_with(prefix) { + return None; + } + line.rsplit(' ').next()?.parse::().ok() + }) + .sum() + } + fn started_validator_ids(&self) -> HashSet { self.private_keys .iter() @@ -512,13 +591,12 @@ async fn start_validator( blocker: oracle.control(public_key.clone()), manager: oracle.manager(), partition_prefix: uid.clone(), - blocks_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, - finalized_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, signer: signer.clone(), dkg_storage_key: [9u8; 32], output, share: Some(share), peer_config, + epoch_length: cfg.epoch_length, leader_timeout: cfg.leader_timeout, certification_timeout: cfg.certification_timeout, strategy: Sequential, @@ -527,6 +605,7 @@ async fn start_validator( pool_config: PoolConfig::default(), genesis: None, indexer: None, + indexer_spool_limits: Default::default(), }; let validator_context = context.child("validator").with_attribute("id", &uid);