diff --git a/dkg/src/orchestrator/actor.rs b/dkg/src/orchestrator/actor.rs index f815252..060b701 100644 --- a/dkg/src/orchestrator/actor.rs +++ b/dkg/src/orchestrator/actor.rs @@ -26,9 +26,14 @@ use commonware_parallel::Strategy; use commonware_runtime::{ buffer::paged::CacheRef, spawn_cell, - telemetry::metrics::{Gauge, GaugeExt, MetricsExt as _}, + telemetry::metrics::{ + histogram::Buckets, CounterFamily, EncodeLabelSet, Gauge, GaugeExt, Histogram, + MetricsExt as _, + }, BufferPooler, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage, }; +use commonware_storage::metadata::{self, Metadata}; +use commonware_utils::sequence::U64; use commonware_utils::{vec::NonEmptyVec, NZUsize, NZU16}; use rand::CryptoRng; use std::{ @@ -39,6 +44,13 @@ use std::{ }; use tracing::{debug, info, warn}; +const CLEANUP_WATERMARK_KEY: U64 = U64::new(0); + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] +struct CleanupStatusLabel { + status: &'static str, +} + /// Reporter that discards consensus activity. pub struct NoopReporter { _phantom: PhantomData, @@ -146,6 +158,10 @@ where page_cache_ref: CacheRef, latest_epoch: Gauge, + partition_cleanup_total: CounterFamily, + partition_cleanup_watermark: Gauge, + partitions_active: Gauge, + partition_cleanup_duration: Histogram, _phantom: PhantomData, } @@ -180,6 +196,23 @@ where // Register latest_epoch gauge for Grafana integration let latest_epoch = context.gauge("latest_epoch", "current epoch"); + let partition_cleanup_total = context.family( + "consensus_partition_cleanup", + "Total number of consensus partition cleanup outcomes", + ); + let partition_cleanup_watermark = context.gauge( + "consensus_partition_cleanup_watermark", + "Next consensus epoch partition requiring cleanup", + ); + let partitions_active = context.gauge( + "consensus_partitions_active", + "Number of consensus epoch partitions with active engines", + ); + let partition_cleanup_duration = context.histogram( + "consensus_partition_cleanup_duration_seconds", + "Duration of consensus epoch partition cleanup operations", + Buckets::LOCAL, + ); ( Self { @@ -200,6 +233,10 @@ where recovered_floor: config.recovered_floor, page_cache_ref, latest_epoch, + partition_cleanup_total, + partition_cleanup_watermark, + partitions_active, + partition_cleanup_duration, _phantom: PhantomData, }, Mailbox::new(sender), @@ -268,6 +305,24 @@ where // Wait for instructions to transition epochs. let epocher = FixedEpocher::new(self.epoch_length); let mut engines: BTreeMap> = BTreeMap::new(); + let cleanup_partition = format!("{}_cleanup-v1", self.partition_prefix); + let mut cleanup_metadata = Metadata::::init( + self.context.child("partition_cleanup_metadata"), + metadata::Config { + partition: cleanup_partition, + codec_config: (), + }, + ) + .await + .expect("failed to initialize consensus partition cleanup metadata"); + let mut next_epoch_to_clean = cleanup_metadata + .get(&CLEANUP_WATERMARK_KEY) + .cloned() + .unwrap_or_else(|| U64::new(0)); + let _ = self + .partition_cleanup_watermark + .try_set(u64::from(&next_epoch_to_clean)); + let mut startup_cleanup_done = false; select_loop! { self.context, @@ -317,6 +372,28 @@ where continue; } + if !startup_cleanup_done { + assert!( + u64::from(&next_epoch_to_clean) <= transition.epoch.get(), + "consensus partition cleanup watermark {} is ahead of first entering epoch {}", + u64::from(&next_epoch_to_clean), + transition.epoch, + ); + if let Some(previous) = transition.epoch.previous() { + assert!( + !engines.keys().any(|epoch| *epoch <= previous), + "refusing to clean a consensus partition with an active engine" + ); + self.cleanup_through( + &mut cleanup_metadata, + &mut next_epoch_to_clean, + previous, + ) + .await; + } + startup_cleanup_done = true; + } + let floor = if self .recovered_floor .as_ref() @@ -364,6 +441,7 @@ where .await; engines.insert(transition.epoch, handle); let _ = self.latest_epoch.try_set(transition.epoch.get()); + let _ = self.partitions_active.try_set(engines.len()); info!(epoch = %transition.epoch, "entered epoch"); } @@ -374,16 +452,91 @@ where continue; }; handle.abort(); + // Spawned task handles close their completion channel when aborted. + match handle.await { + Ok(()) + | Err(commonware_runtime::Error::Aborted) + | Err(commonware_runtime::Error::Closed) => {} + Err(error) => { + panic!("consensus engine for epoch {epoch} failed while stopping: {error}") + } + } + let _ = self.partitions_active.try_set(engines.len()); // Unregister the signing scheme for the epoch. assert!(self.provider.unregister(&epoch)); + if u64::from(&next_epoch_to_clean) <= epoch.get() { + assert!( + !engines.keys().any(|active| *active <= epoch), + "refusing to clean a consensus partition with an active engine" + ); + self.cleanup_through( + &mut cleanup_metadata, + &mut next_epoch_to_clean, + epoch, + ) + .await; + } + info!(%epoch, "exited epoch"); } }, } } + async fn cleanup_through( + &mut self, + metadata: &mut Metadata, + next_epoch_to_clean: &mut U64, + through: Epoch, + ) { + while u64::from(&*next_epoch_to_clean) <= through.get() { + let epoch = Epoch::new(u64::from(&*next_epoch_to_clean)); + let partition = format!("{}_consensus_{}", self.partition_prefix, epoch); + let started = std::time::Instant::now(); + let status = match self.context.remove(&partition, None).await { + Ok(()) => { + info!(%epoch, %partition, "removed retired consensus partition"); + "removed" + } + Err(commonware_runtime::Error::PartitionMissing(_)) => { + info!(%epoch, %partition, "retired consensus partition already missing"); + "missing" + } + Err(error) => { + self.partition_cleanup_total + .get_or_create(&CleanupStatusLabel { status: "failed" }) + .inc(); + self.partition_cleanup_duration + .observe(started.elapsed().as_secs_f64()); + warn!(%epoch, %partition, %error, "failed to remove retired consensus partition"); + panic!("failed to remove retired consensus partition {partition}: {error}"); + } + }; + self.partition_cleanup_total + .get_or_create(&CleanupStatusLabel { status }) + .inc(); + self.partition_cleanup_duration + .observe(started.elapsed().as_secs_f64()); + + let next = epoch + .get() + .checked_add(1) + .expect("consensus cleanup watermark overflow"); + metadata + .put_sync(CLEANUP_WATERMARK_KEY, U64::new(next)) + .await + .unwrap_or_else(|error| { + panic!( + "failed to persist consensus cleanup watermark after removing {partition}: {error}" + ) + }); + *next_epoch_to_clean = U64::new(next); + let _ = self.partition_cleanup_watermark.try_set(next); + } + } + // Epoch zero uses genesis as its floor; every later epoch is anchored by the // last finalized block from the previous epoch. fn floor_boundary(epocher: &FixedEpocher, epoch: Epoch) -> Option { diff --git a/examples/coins/chain/src/engine.rs b/examples/coins/chain/src/engine.rs index 00a47dd..2930cbf 100644 --- a/examples/coins/chain/src/engine.rs +++ b/examples/coins/chain/src/engine.rs @@ -1,11 +1,9 @@ use crate::application::{self, Application}; use crate::execution::NodeHandle; use crate::genesis::{genesis_target, state_commitment, ChainGenesis}; +use crate::history::{self, BlocksArchive, FinalizationsArchive}; use crate::indexer; -use crate::{ - Block, EpochProvider, Finalization, Provider, PublicKey, Scheme, Transaction, BLOCKS_PER_EPOCH, - NAMESPACE, -}; +use crate::{Block, EpochProvider, Provider, PublicKey, Scheme, Transaction, NAMESPACE}; use commonware_broadcast::buffered; use commonware_consensus::{ marshal::{ @@ -13,7 +11,7 @@ use commonware_consensus::{ core::Actor as MarshalActor, resolver, standard::{Inline, Standard}, - store::{Blocks as _, Certificates}, + store::Certificates, }, simplex::elector::Random, types::{Epoch, FixedEpocher, Height, ViewDelta}, @@ -24,7 +22,6 @@ use commonware_cryptography::{ dkg::feldman_desmedt::Output, primitives::{group, variant::MinSig}, }, - certificate::Verifier as _, ed25519::{self, Batch}, sha256::Digest, BatchVerifier, Digestible, Signer, @@ -41,12 +38,8 @@ use commonware_runtime::{ buffer::paged::CacheRef, spawn_cell, BufferPooler, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage, Strategizer, }; -use commonware_storage::{ - archive::{immutable, Identifier as ArchiveIdentifier}, - metadata::{self, Metadata}, - queue, -}; -use commonware_utils::{sequence::U64, union, NZDuration, NZU64}; +use commonware_storage::{archive::{Archive as ArchiveStore, Identifier as ArchiveIdentifier}, metadata::{self, Metadata}, queue}; +use commonware_utils::{sequence::U64, union, NZDuration}; use futures::lock::Mutex as AsyncMutex; use governor::clock::Clock as GClock; use nunchi_chain::engine::*; @@ -61,6 +54,7 @@ use nunchi_mempool::{Mempool, PoolConfig}; use rand::{CryptoRng, Rng}; use std::{ marker::PhantomData, + num::NonZeroU64, sync::Arc, time::{Duration, Instant}, }; @@ -84,13 +78,12 @@ pub struct Config, P: Manager, pub share: Option, pub peer_config: PeerConfig, + pub epoch_length: NonZeroU64, pub leader_timeout: Duration, pub certification_timeout: Duration, pub strategy: S, @@ -100,6 +93,7 @@ pub struct Config, P: Manager, pub indexer: Option, + pub indexer_spool_limits: indexer::SpoolLimits, } type DkgActor = nunchi_chain::DkgActor; @@ -110,8 +104,6 @@ type LimitedStatefulAppMailbox = VerifyLimiter>; type InlineApp = Inline, Block, FixedEpocher>; type Marshaled = BoxedAutomaton>; type SchemeProvider = Provider; -type FinalizationsArchive = immutable::Archive; -type BlocksArchive = immutable::Archive; type Marshal = MarshalActor< E, Standard, @@ -167,6 +159,7 @@ where stateful: StatefulApp, stateful_mailbox: StatefulAppMailbox, indexer_producer: Option, + indexer_producer_handle: Option>, indexer_consumer: Option>, } @@ -234,7 +227,7 @@ where max_supported_mode: MAX_SUPPORTED_MODE, namespace: NAMESPACE.to_vec(), storage_protector: dkg::StorageProtector::new(config.dkg_storage_key), - epoch_length: BLOCKS_PER_EPOCH, + epoch_length: config.epoch_length, }, ); @@ -251,88 +244,36 @@ where ); let start = Instant::now(); - let finalizations_by_height = immutable::Archive::init( - context.child("finalizations_by_height"), - immutable::Config { - metadata_partition: format!( - "{}-finalizations-by-height-metadata", - config.partition_prefix - ), - freezer_table_partition: format!( - "{}-finalizations-by-height-freezer-table", - config.partition_prefix - ), - freezer_table_initial_size: config.finalized_freezer_table_initial_size, - freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY, - freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE, - freezer_key_partition: format!( - "{}-finalizations-by-height-freezer-key", - config.partition_prefix - ), - freezer_key_page_cache: page_cache.clone(), - freezer_key_write_buffer: WRITE_BUFFER, - freezer_value_partition: format!( - "{}-finalizations-by-height-freezer-value", - config.partition_prefix - ), - freezer_value_write_buffer: WRITE_BUFFER, - freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE, - freezer_value_compression: FREEZER_VALUE_COMPRESSION, - ordinal_partition: format!( - "{}-finalizations-by-height-ordinal", - config.partition_prefix - ), - ordinal_write_buffer: WRITE_BUFFER, - items_per_section: IMMUTABLE_ITEMS_PER_SECTION, - codec_config: Scheme::certificate_codec_config_unbounded(), - replay_buffer: REPLAY_BUFFER, - }, - ) - .await - .expect("failed to initialize finalizations by height archive"); - info!(elapsed = ?start.elapsed(), "restored finalizations by height archive"); - - let start = Instant::now(); - let finalized_blocks = immutable::Archive::init( - context.child("finalized_blocks"), - immutable::Config { - metadata_partition: format!( - "{}-finalized_blocks-metadata", - config.partition_prefix - ), - freezer_table_partition: format!( - "{}-finalized_blocks-freezer-table", - config.partition_prefix - ), - freezer_table_initial_size: config.blocks_freezer_table_initial_size, - freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY, - freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE, - freezer_key_partition: format!( - "{}-finalized_blocks-freezer-key", - config.partition_prefix - ), - freezer_key_page_cache: page_cache.clone(), - freezer_key_write_buffer: WRITE_BUFFER, - freezer_value_partition: format!( - "{}-finalized_blocks-freezer-value", - config.partition_prefix - ), - freezer_value_write_buffer: WRITE_BUFFER, - freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE, - freezer_value_compression: FREEZER_VALUE_COMPRESSION, - ordinal_partition: format!("{}-finalized_blocks-ordinal", config.partition_prefix), - ordinal_write_buffer: WRITE_BUFFER, - items_per_section: IMMUTABLE_ITEMS_PER_SECTION, - codec_config: block_codec_config, - replay_buffer: REPLAY_BUFFER, - }, + let opened_history = history::open( + &context, + &config.partition_prefix, + page_cache.clone(), + block_codec_config, ) .await - .expect("failed to initialize finalized blocks archive"); - info!(elapsed = ?start.elapsed(), "restored finalized blocks archive"); + .unwrap_or_else(|error| panic!("failed to initialize bounded finalized history: {error}")); + let history::Opened { + finalizations: mut finalizations_by_height, + blocks: mut finalized_blocks, + partitions: history_partitions, + marker_status, + } = opened_history; + info!( + elapsed = ?start.elapsed(), + format_version = history::FORMAT_VERSION, + finalizations_key = %history_partitions.finalizations_key, + finalizations_value = %history_partitions.finalizations_value, + blocks_key = %history_partitions.blocks_key, + blocks_value = %history_partitions.blocks_value, + logical_retention = history::LOGICAL_RETENTION, + items_per_section = history::PRUNABLE_ITEMS_PER_SECTION.get(), + maximum_section_and_cadence_slack = history::MAX_RETAINED_HEIGHTS - history::LOGICAL_RETENTION, + ?marker_status, + "restored prunable finalized history" + ); let recovered_height = Certificates::last_index(&finalizations_by_height); - let mut recovered_floor = if let Some(height) = recovered_height { + let recovered_floor = if let Some(height) = recovered_height { Certificates::get( &finalizations_by_height, ArchiveIdentifier::Index(height.get()), @@ -425,15 +366,35 @@ where .expect("failed to initialize state database for startup probe"); state.sync_target() }; - if repair_marshal_progress_if_state_is_behind( + if let Some(processed_height) = validate_marshal_progress_against_state( &context, &config.partition_prefix, &finalized_blocks, ¤t_state_target, ) - .await + .await { + validate_history_through_processed( + &finalizations_by_height, + &finalized_blocks, + processed_height, + ) + .await; + if let Some(tip) = ArchiveStore::last_index(&finalized_blocks) { + let policy_floor = history::policy_floor(tip, history::LOGICAL_RETENTION); + let safe_floor = policy_floor.min(processed_height.get()); + finalizations_by_height + .prune(safe_floor) + .await + .expect("failed to startup-prune finalized certificates"); + finalized_blocks + .prune(safe_floor) + .await + .expect("failed to startup-prune finalized blocks"); + } + } else if ArchiveStore::last_index(&finalized_blocks).is_some() + || ArchiveStore::last_index(&finalizations_by_height).is_some() { - recovered_floor = None; + panic!("finalized history exists without persisted marshal processed height; rebuild or perform verified peer state sync"); } let applied_height = Arc::new(AsyncMutex::new(Height::zero())); let app = Application::with_consensus( @@ -497,7 +458,7 @@ where finalized_blocks, marshal::Config { provider: provider.clone(), - epocher: FixedEpocher::new(BLOCKS_PER_EPOCH), + epocher: FixedEpocher::new(config.epoch_length), start: marshal_start, partition_prefix: format!("{}_marshal", config.partition_prefix), mailbox_size: MAILBOX_SIZE, @@ -551,10 +512,10 @@ where APPLICATION_VERIFY_CONCURRENCY, ), marshal_mailbox.clone(), - FixedEpocher::new(BLOCKS_PER_EPOCH), + FixedEpocher::new(config.epoch_length), )); - let (indexer_producer, indexer_pusher, indexer_consumer) = + let (indexer_producer, indexer_producer_handle, indexer_pusher, indexer_consumer) = if let Some(client) = config.indexer.clone() { let indexer_context = context.child("indexer"); let indexer_metrics = indexer::IndexerMetrics::register(&indexer_context); @@ -562,10 +523,13 @@ where let queue = queue::shared::init( context.child("indexer_queue"), queue::Config { - partition: format!("{}-indexer-finalized-queue", config.partition_prefix), - items_per_section: NZU64!(128), + partition: format!( + "{}-indexer-finalized-payload-queue-v1", + config.partition_prefix + ), + items_per_section: indexer::SPOOL_ITEMS_PER_SECTION, compression: None, - codec_config: (), + codec_config: block_codec_config, page_cache: page_cache.clone(), write_buffer: WRITE_BUFFER, }, @@ -581,14 +545,20 @@ where mailbox_size: MAILBOX_SIZE, backfiller_max_active: commonware_utils::NZUsize!(16), backfiller_retry: Duration::from_millis(500), + spool_limits: config.indexer_spool_limits, metrics: indexer_metrics, }, ) .await; - let (producer, pusher, consumer) = indexer.split(); - (Some(producer), Some(pusher), Some(consumer)) + let (producer, producer_handle, pusher, consumer) = indexer.split(); + ( + Some(producer), + Some(producer_handle), + Some(pusher), + Some(consumer), + ) } else { - (None, None, None) + (None, None, None, None) }; let (orchestrator, orchestrator_mailbox) = orchestrator::Actor::new( @@ -605,7 +575,7 @@ where muxer_size: MAILBOX_SIZE.get(), mailbox_size: MAILBOX_SIZE, partition_prefix: format!("{}_consensus", config.partition_prefix), - epoch_length: BLOCKS_PER_EPOCH, + epoch_length: config.epoch_length, genesis_digest, recovered_floor, _phantom: PhantomData, @@ -629,6 +599,7 @@ where stateful, stateful_mailbox, indexer_producer, + indexer_producer_handle, indexer_consumer, }; (engine, node_handle) @@ -747,10 +718,13 @@ where .mempool .start_p2p(self.context.child("mempool"), mempool); let indexer_consumer_handle = self.indexer_consumer.map(indexer::Consumer::start); + let indexer_producer_handle = self.indexer_producer_handle; let clob_handle = self.clob.start_p2p(self.context.child("clob"), clob); let mut shutdown = self.context.stopped(); - if let Some(indexer_consumer_handle) = indexer_consumer_handle { + if let (Some(indexer_producer_handle), Some(indexer_consumer_handle)) = + (indexer_producer_handle, indexer_consumer_handle) + { commonware_macros::select! { stopped = &mut shutdown => match stopped { Ok(0) => { @@ -769,6 +743,7 @@ where result = orchestrator_handle => unexpected_exit("orchestrator", result), result = mempool_handle => unexpected_exit("mempool", result), result = clob_handle => unexpected_exit("clob", result), + result = indexer_producer_handle => unexpected_exit("indexer_producer", result), result = indexer_consumer_handle => unexpected_exit("indexer_consumer", result), } } else { @@ -806,14 +781,13 @@ fn unexpected_exit( } const MARSHAL_PROCESSED_KEY: U64 = U64::new(0xFF); -const STATE_SYNC_METADATA_SUFFIX: &str = "state_sync_metadata"; -async fn repair_marshal_progress_if_state_is_behind( +async fn validate_marshal_progress_against_state( context: &E, partition_prefix: &str, finalized_blocks: &BlocksArchive, current_target: &commonware_storage::qmdb::sync::Target, -) -> bool +) -> Option where E: BufferPooler + Clock @@ -837,63 +811,87 @@ where ) .await .expect("failed to initialize marshal progress metadata probe"); - let Some(processed_height) = metadata.get(&MARSHAL_PROCESSED_KEY).copied() else { - return false; - }; + let processed_height = metadata.get(&MARSHAL_PROCESSED_KEY).copied()?; drop(metadata); - let Some(processed_block) = finalized_blocks - .get(ArchiveIdentifier::Index(processed_height.get())) - .await - .expect("failed to read processed block for state repair") - else { - warn!( - %processed_height, - "marshal progress references a missing finalized block; clearing startup metadata to replay" - ); - remove_partition_if_exists(context, &marshal_metadata_partition).await; - clear_disabled_state_sync_metadata(context, partition_prefix).await; - return true; + let Some(tip) = ArchiveStore::last_index(finalized_blocks) else { + panic!("marshal progress references height {processed_height}, but finalized block history is empty; rebuild or perform verified peer state sync"); }; - let processed_target = >::sync_targets(&processed_block); - let current_size = current_target.range.end().as_u64(); - let processed_size = processed_target.range.end().as_u64(); - if current_size >= processed_size { - return false; + let search_end = tip.min( + processed_height + .get() + .saturating_add(MAX_PENDING_ACKS.get() as u64), + ); + for height in processed_height.get()..=search_end { + let Some(block) = ArchiveStore::get( + finalized_blocks, + ArchiveIdentifier::Index(height), + ) + .await + .expect("failed to read finalized block while validating QMDB startup target") + else { + continue; + }; + let target = >::sync_targets(&block); + if &target == current_target { + return Some(processed_height); + } } - warn!( - %processed_height, - current_size, - processed_size, - "state database is behind marshal progress; clearing startup metadata to replay finalized blocks" + panic!( + "QMDB committed target does not exactly match canonical finalized history from processed height {} through {}; rebuild or perform verified peer state sync", + processed_height, + search_end, ); - remove_partition_if_exists(context, &marshal_metadata_partition).await; - remove_partition_if_exists( - context, - &format!("{partition_prefix}{STATE_SYNC_METADATA_SUFFIX}"), - ) - .await; - true } -async fn clear_disabled_state_sync_metadata(context: &E, partition_prefix: &str) +async fn validate_history_through_processed( + finalizations: &FinalizationsArchive, + blocks: &BlocksArchive, + processed_height: Height, +) where - E: Storage, + E: BufferPooler + Metrics + Storage, { - remove_partition_if_exists( - context, - &format!("{partition_prefix}{STATE_SYNC_METADATA_SUFFIX}"), - ) - .await; -} + let processed = processed_height.get(); + let first = match (finalizations.first_index(), blocks.first_index()) { + (Some(finalization), Some(block)) => finalization.max(block), + _ => panic!( + "marshal processed height {processed_height} has incomplete finalized history; rebuild or perform verified peer state sync" + ), + }; + if first > processed { + panic!( + "marshal processed height {processed_height} is below retained finalized history floor {first}; rebuild or perform verified peer state sync" + ); + } -async fn remove_partition_if_exists(context: &E, partition: &str) -where - E: Storage, -{ - match context.remove(partition, None).await { - Ok(()) | Err(commonware_runtime::Error::PartitionMissing(_)) => {} - Err(error) => panic!("failed to remove partition {partition}: {error}"), + for height in first..=processed { + let certificate = finalizations + .get(ArchiveIdentifier::Index(height)) + .await + .expect("failed to validate finalized certificate archive"); + let block = blocks + .get(ArchiveIdentifier::Index(height)) + .await + .expect("failed to validate finalized block archive"); + if height == processed { + assert!( + block.is_some(), + "marshal processed height {processed_height} references a missing finalized block; rebuild or perform verified peer state sync" + ); + } + if let (Some(certificate), Some(block)) = (certificate, block) { + assert_eq!( + certificate.proposal.payload, + block.digest(), + "finalized block and certificate conflict at height {height}" + ); + assert_eq!( + certificate.proposal.round.epoch(), + block.context.round.epoch(), + "finalized block and certificate epochs conflict at height {height}" + ); + } } } diff --git a/examples/coins/chain/src/history.rs b/examples/coins/chain/src/history.rs new file mode 100644 index 0000000..6d39607 --- /dev/null +++ b/examples/coins/chain/src/history.rs @@ -0,0 +1,466 @@ +//! Bounded finalized-history storage for coins-chain validators. + +use crate::{Block, Finalization}; +use commonware_codec::Read; +use commonware_cryptography::sha256::Digest; +use commonware_runtime::{buffer::paged::CacheRef, BufferPooler, Clock, Metrics, Spawner, Storage}; +use commonware_storage::{ + archive::{prunable, Archive as _}, + metadata::{self, Metadata}, + translator::EightCap, +}; +use commonware_utils::{sequence::U64, NZU64}; +use nunchi_chain::engine::{ + MAX_PENDING_ACKS, PRUNE_MAINTENANCE_INTERVAL, PRUNE_RETAINED_MARSHAL_BLOCKS, REPLAY_BUFFER, + WRITE_BUFFER, +}; +use std::num::NonZeroU64; + +pub(crate) const FORMAT_VERSION: u64 = 1; +pub(crate) const PRUNABLE_ITEMS_PER_SECTION: NonZeroU64 = NZU64!(4_096); +pub(crate) const LOGICAL_RETENTION: u64 = + PRUNE_RETAINED_MARSHAL_BLOCKS as u64 + MAX_PENDING_ACKS.get() as u64 + 1; +pub(crate) const MAX_RETAINED_HEIGHTS: u64 = LOGICAL_RETENTION + + PRUNABLE_ITEMS_PER_SECTION.get() + - 1 + + PRUNE_MAINTENANCE_INTERVAL.get() as u64 + - 1; + +const MARKER_KEY: U64 = U64::new(0); + +pub(crate) type FinalizationsArchive = + prunable::Archive; +pub(crate) type BlocksArchive = prunable::Archive; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MarkerStatus { + Created, + RecoveredPartialInitialization, + Present, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum HistoryError { + #[error("legacy finalized-history partition exists: {partition}; stop the devnet and clear the complete validator storage directory before relaunch")] + LegacyPartition { partition: String }, + #[error("history format marker exists but required partition is missing: {partition}; clear the complete devnet validator storage directory before relaunch")] + MarkedPartitionMissing { partition: String }, + #[error("history format marker is missing but {partition} contains finalized data at height {height}; refusing to modify unmarked storage")] + NonEmptyUnmarked { partition: String, height: u64 }, + #[error("invalid history format marker in {partition}: expected version {expected}, found {found:?}")] + InvalidMarker { + partition: String, + expected: u64, + found: Option, + }, + #[error("failed to inspect history partition {partition}: {source}")] + Inspect { + partition: String, + #[source] + source: commonware_runtime::Error, + }, + #[error("failed to open history archive partition {partition}: {source}")] + Archive { + partition: String, + #[source] + source: commonware_storage::archive::Error, + }, + #[error("failed to access history marker partition {partition}: {source}")] + Marker { + partition: String, + #[source] + source: commonware_storage::metadata::Error, + }, + #[error("invalid history retention configuration: {0}")] + Retention(&'static str), +} + +#[derive(Clone, Debug)] +pub(crate) struct Partitions { + pub(crate) finalizations_key: String, + pub(crate) finalizations_value: String, + pub(crate) blocks_key: String, + pub(crate) blocks_value: String, + pub(crate) metadata: String, +} + +impl Partitions { + pub(crate) fn new(prefix: &str) -> Self { + Self { + finalizations_key: format!( + "{prefix}-finalizations-by-height-prunable-v1-key" + ), + finalizations_value: format!( + "{prefix}-finalizations-by-height-prunable-v1-value" + ), + blocks_key: format!("{prefix}-finalized-blocks-prunable-v1-key"), + blocks_value: format!("{prefix}-finalized-blocks-prunable-v1-value"), + metadata: format!("{prefix}-history-prunable-v1-metadata"), + } + } + + fn archives(&self) -> [&str; 4] { + [ + &self.finalizations_key, + &self.finalizations_value, + &self.blocks_key, + &self.blocks_value, + ] + } +} + +pub(crate) struct Opened +where + E: BufferPooler + Clock + Storage + Metrics, +{ + pub(crate) finalizations: FinalizationsArchive, + pub(crate) blocks: BlocksArchive, + pub(crate) partitions: Partitions, + pub(crate) marker_status: MarkerStatus, +} + +pub(crate) const fn policy_floor(tip: u64, retained: u64) -> u64 { + tip.saturating_add(1).saturating_sub(retained) +} + +pub(crate) fn validate_production_retention() -> Result<(), HistoryError> { + if MAX_PENDING_ACKS.get() != 16 { + return Err(HistoryError::Retention( + "MAX_PENDING_ACKS changed; update the mandatory rewind assertion", + )); + } + if LOGICAL_RETENTION != 217 { + return Err(HistoryError::Retention( + "logical finalized-history retention must be 217", + )); + } + if MAX_RETAINED_HEIGHTS != 4_343 { + return Err(HistoryError::Retention( + "section-granular finalized-history bound must be 4,343", + )); + } + Ok(()) +} + +pub(crate) async fn open( + context: &E, + prefix: &str, + page_cache: CacheRef, + block_codec_config: ::Cfg, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + validate_production_retention()?; + reject_legacy(context, prefix).await?; + + let partitions = Partitions::new(prefix); + let marker_existed = partition_exists(context, &partitions.metadata).await?; + let mut archive_existed = [false; 4]; + for (exists, partition) in archive_existed + .iter_mut() + .zip(partitions.archives()) + { + *exists = partition_exists(context, partition).await?; + } + let mut finalizations = + init_finalizations(context, &partitions, page_cache.clone()).await?; + let mut blocks = init_blocks( + context, + &partitions, + page_cache.clone(), + block_codec_config, + ) + .await?; + + let marker_status = if marker_existed { + validate_marker(context, &partitions.metadata).await?; + let archives_empty = finalizations.first_index().is_none() && blocks.first_index().is_none(); + let all_lazy = archive_existed.iter().all(|exists| !exists); + if !(archive_existed.iter().all(|exists| *exists) || archives_empty && all_lazy) { + let partition = partitions + .archives() + .into_iter() + .zip(archive_existed) + .find_map(|(partition, exists)| (!exists).then_some(partition)) + .expect("at least one marked archive partition is missing"); + return Err(HistoryError::MarkedPartitionMissing { + partition: partition.to_string(), + }); + } + MarkerStatus::Present + } else { + if let Some(height) = finalizations.first_index() { + return Err(HistoryError::NonEmptyUnmarked { + partition: partitions.finalizations_key.clone(), + height, + }); + } + if let Some(height) = blocks.first_index() { + return Err(HistoryError::NonEmptyUnmarked { + partition: partitions.blocks_key.clone(), + height, + }); + } + + // Ensure all newly-created archive blobs are durable and structurally reopenable before + // committing the format marker. + finalizations.sync().await.map_err(|source| HistoryError::Archive { + partition: partitions.finalizations_key.clone(), + source, + })?; + blocks.sync().await.map_err(|source| HistoryError::Archive { + partition: partitions.blocks_key.clone(), + source, + })?; + drop(finalizations); + drop(blocks); + finalizations = init_finalizations(context, &partitions, page_cache.clone()).await?; + blocks = init_blocks(context, &partitions, page_cache, block_codec_config).await?; + write_marker(context, &partitions.metadata).await?; + + if archive_existed.into_iter().any(|exists| exists) { + MarkerStatus::RecoveredPartialInitialization + } else { + MarkerStatus::Created + } + }; + + Ok(Opened { + finalizations, + blocks, + partitions, + marker_status, + }) +} + +async fn init_finalizations( + context: &E, + partitions: &Partitions, + page_cache: CacheRef, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + prunable::Archive::init( + context.child("finalizations_by_height"), + prunable::Config { + translator: EightCap, + key_partition: partitions.finalizations_key.clone(), + key_page_cache: page_cache, + value_partition: partitions.finalizations_value.clone(), + compression: Some(3), + codec_config: (), + items_per_section: PRUNABLE_ITEMS_PER_SECTION, + key_write_buffer: WRITE_BUFFER, + value_write_buffer: WRITE_BUFFER, + replay_buffer: REPLAY_BUFFER, + }, + ) + .await + .map_err(|source| HistoryError::Archive { + partition: partitions.finalizations_key.clone(), + source, + }) +} + +async fn init_blocks( + context: &E, + partitions: &Partitions, + page_cache: CacheRef, + codec_config: ::Cfg, +) -> Result, HistoryError> +where + E: BufferPooler + Clock + Spawner + Storage + Metrics, +{ + prunable::Archive::init( + context.child("finalized_blocks"), + prunable::Config { + translator: EightCap, + key_partition: partitions.blocks_key.clone(), + key_page_cache: page_cache, + value_partition: partitions.blocks_value.clone(), + compression: Some(3), + codec_config, + items_per_section: PRUNABLE_ITEMS_PER_SECTION, + key_write_buffer: WRITE_BUFFER, + value_write_buffer: WRITE_BUFFER, + replay_buffer: REPLAY_BUFFER, + }, + ) + .await + .map_err(|source| HistoryError::Archive { + partition: partitions.blocks_key.clone(), + source, + }) +} + +async fn partition_exists(context: &E, partition: &str) -> Result { + match context.scan(partition).await { + Ok(_) => Ok(true), + Err(commonware_runtime::Error::PartitionMissing(_)) => Ok(false), + Err(source) => Err(HistoryError::Inspect { + partition: partition.to_string(), + source, + }), + } +} + +async fn validate_marker(context: &E, partition: &str) -> Result<(), HistoryError> +where + E: BufferPooler + Clock + Storage + Metrics + Spawner, +{ + let marker = Metadata::::init( + context.child("history_marker"), + metadata::Config { + partition: partition.to_string(), + codec_config: (), + }, + ) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + })?; + let found = marker.get(&MARKER_KEY).copied(); + if found != Some(FORMAT_VERSION) { + return Err(HistoryError::InvalidMarker { + partition: partition.to_string(), + expected: FORMAT_VERSION, + found, + }); + } + Ok(()) +} + +async fn write_marker(context: &E, partition: &str) -> Result<(), HistoryError> +where + E: BufferPooler + Clock + Storage + Metrics + Spawner, +{ + let mut marker = Metadata::::init( + context.child("history_marker"), + metadata::Config { + partition: partition.to_string(), + codec_config: (), + }, + ) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + })?; + marker + .put_sync(MARKER_KEY, FORMAT_VERSION) + .await + .map_err(|source| HistoryError::Marker { + partition: partition.to_string(), + source, + }) +} + +async fn reject_legacy(context: &E, prefix: &str) -> Result<(), HistoryError> { + const FINALIZATION_SUFFIXES: [&str; 5] = [ + "finalizations-by-height-metadata", + "finalizations-by-height-freezer-table", + "finalizations-by-height-freezer-key", + "finalizations-by-height-freezer-value", + "finalizations-by-height-ordinal", + ]; + const BLOCK_SUFFIXES: [&str; 5] = [ + "finalized_blocks-metadata", + "finalized_blocks-freezer-table", + "finalized_blocks-freezer-key", + "finalized_blocks-freezer-value", + "finalized_blocks-ordinal", + ]; + + for suffix in FINALIZATION_SUFFIXES.into_iter().chain(BLOCK_SUFFIXES) { + let partition = format!("{prefix}-{suffix}"); + if partition_exists(context, &partition).await? { + return Err(HistoryError::LegacyPartition { partition }); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use commonware_runtime::{buffer::paged::CacheRef, deterministic, Runner as _}; + use commonware_utils::{NZU16, NZU32, NZUsize}; + + #[test] + fn production_retention_arithmetic() { + validate_production_retention().unwrap(); + assert_eq!(policy_floor(216, LOGICAL_RETENTION), 0); + assert_eq!(policy_floor(217, LOGICAL_RETENTION), 1); + assert_eq!(LOGICAL_RETENTION, 217); + assert_eq!(MAX_RETAINED_HEIGHTS, 4_343); + } + + #[test] + fn fresh_format_marker_is_durable_and_legacy_is_not_created() { + deterministic::Runner::default().start(|context| async move { + let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(32)); + let first = open( + &context, + "validator", + page_cache.clone(), + (NZU32!(1), ()), + ) + .await + .expect("initialize fresh history"); + assert_eq!(first.marker_status, MarkerStatus::Created); + assert_eq!(first.finalizations.first_index(), None); + assert_eq!(first.blocks.first_index(), None); + drop(first); + + let reopened = open( + &context, + "validator", + page_cache, + (NZU32!(1), ()), + ) + .await + .expect("reopen marked history"); + assert_eq!(reopened.marker_status, MarkerStatus::Present); + + for suffix in [ + "finalizations-by-height-metadata", + "finalizations-by-height-freezer-table", + "finalizations-by-height-freezer-key", + "finalizations-by-height-freezer-value", + "finalizations-by-height-ordinal", + "finalized_blocks-metadata", + "finalized_blocks-freezer-table", + "finalized_blocks-freezer-key", + "finalized_blocks-freezer-value", + "finalized_blocks-ordinal", + ] { + assert!(matches!( + context.scan(&format!("validator-{suffix}")).await, + Err(commonware_runtime::Error::PartitionMissing(_)) + )); + } + }); + } + + #[test] + fn refuses_known_legacy_partition() { + deterministic::Runner::default().start(|context| async move { + context + .open("validator-finalized_blocks-ordinal", b"legacy") + .await + .expect("create legacy partition"); + let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(32)); + let result = open( + &context, + "validator", + page_cache, + (NZU32!(1), ()), + ) + .await; + assert!(matches!(result, Err(HistoryError::LegacyPartition { .. }))); + }); + } +} diff --git a/examples/coins/chain/src/indexer/backfiller/consumer.rs b/examples/coins/chain/src/indexer/backfiller/consumer.rs index e714eb2..0bd66e8 100644 --- a/examples/coins/chain/src/indexer/backfiller/consumer.rs +++ b/examples/coins/chain/src/indexer/backfiller/consumer.rs @@ -1,17 +1,14 @@ -use super::{Decision, Entry, SharedState}; +use super::{ + producer::{Admission, AdmissionReceiver}, + Decision, Entry, SharedState, +}; use crate::indexer::{ metrics::{ - estimated_finalized_bytes, BackfillDecision, BackfillPhase, BackfillResetReason, - BackfillUploadGuard, BackfillWaitReason, BlockMetricSource, QueueReadSource, QueueStatus, - SharedCacheSource, + estimated_finalized_bytes, BackfillDecision, BackfillPhase, BackfillWaitReason, + BlockMetricSource, ProducerStatus, QueueReadSource, QueueStatus, }, - Client, IndexerMetrics, -}; -use crate::{Block, Finalized, Scheme}; -use commonware_consensus::marshal::{ - core::Mailbox as MarshalMailbox, standard::Standard, Identifier, + Client, IndexerMetrics, SpoolLimits, }; -use commonware_consensus::types::Height; use commonware_cryptography::sha256::Digest; use commonware_macros::select_loop; use commonware_runtime::{ @@ -22,7 +19,7 @@ use commonware_storage::queue; use commonware_utils::futures::{OptionFuture, Pool}; use std::{ num::NonZeroUsize, - time::{Duration, Instant}, + time::{Duration, Instant, UNIX_EPOCH}, }; use tracing::{debug, warn}; @@ -45,47 +42,39 @@ enum Completion { Skipped { position: u64, height: u64, - }, - Stale { - reason: BackfillResetReason, - first_height: u64, - first_digest: Digest, - attempts: u64, - elapsed: Duration, + digest: Digest, }, } pub(crate) struct Config { pub(crate) max_active: NonZeroUsize, pub(crate) retry: Duration, - pub(crate) missing_finalization_grace: Duration, - pub(crate) mismatched_finalization_grace: Duration, + pub(crate) spool_limits: SpoolLimits, } pub struct Consumer { context: ContextCell, client: C, - marshal: MarshalMailbox>, metrics: IndexerMetrics, upload_results: status::Counter, uploads: SharedState, writer: queue::Writer, reader: queue::Reader, + admission: AdmissionReceiver, active: Pool, max_active: NonZeroUsize, retry: Duration, - missing_finalization_grace: Duration, - mismatched_finalization_grace: Duration, + spool_limits: SpoolLimits, } impl Consumer { pub fn new( context: E, client: C, - marshal: MarshalMailbox>, metrics: IndexerMetrics, uploads: SharedState, backfiller: (queue::Writer, queue::Reader), + admission: AdmissionReceiver, config: Config, ) -> Self { let upload_results = context.register( @@ -96,25 +85,23 @@ impl Consumer< let Config { max_active, retry, - missing_finalization_grace, - mismatched_finalization_grace, + spool_limits, } = config; metrics.backfill_configured(max_active.get()); let (writer, reader) = backfiller; Self { context: ContextCell::new(context), client, - marshal, metrics, upload_results, uploads, writer, reader, + admission, active: Pool::default(), max_active, retry, - missing_finalization_grace, - mismatched_finalization_grace, + spool_limits, } } @@ -127,25 +114,24 @@ impl Consumer< self.context, on_start => { self.fill_slots().await; - if self.active.is_empty() { - let item = Self::recv_queue(&mut self.reader, self.metrics.clone()).await; - let Some((position, entry)) = item else { - warn!("consumer queue closed"); - break; - }; - self.start_upload(position, entry).await; - continue; - } let item = OptionFuture::from( (self.active.len() < self.max_active.get()).then(|| { Self::recv_queue(&mut self.reader, self.metrics.clone()) }), ); + let admission = self.admission.recv(); }, on_stopped => {}, completion = self.active.next_completed() => { self.complete(completion).await; }, + request = admission => { + let Some(request) = request else { + warn!("indexer spool admission coordinator closed"); + break; + }; + self.admit(request).await; + }, item = item => { match item { Some((position, entry)) => { @@ -160,6 +146,87 @@ impl Consumer< } } + async fn admit(&mut self, admission: Admission) { + let Admission { entry, response } = admission; + loop { + let now_millis: u64 = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + let next = { + let uploads = self.uploads.lock(); + let (entries, bytes) = uploads.spool_usage(); + uploads.oldest_queued().and_then( + |(digest, position, height, old_bytes, enqueued_at)| { + let age = Duration::from_millis( + now_millis.saturating_sub(enqueued_at), + ); + let status = if age >= self.spool_limits.max_age { + Some(ProducerStatus::ExpiredAge) + } else if entries >= self.spool_limits.max_entries { + Some(ProducerStatus::ExpiredEntries) + } else if bytes.saturating_add(entry.encoded_len) + > self.spool_limits.max_bytes + { + Some(ProducerStatus::ExpiredBytes) + } else { + None + }; + status.map(|status| { + (digest, position, height, old_bytes, age, status) + }) + }, + ) + }; + let Some((digest, position, height, encoded_len, age, status)) = next else { + break; + }; + + // Advancing the floor invalidates positions held by upload tasks. Cancel all of them + // and replay the still-live suffix so a stale completion cannot be applied twice. + self.active.cancel_all(); + self.reader + .ack_up_to(position.saturating_add(1)) + .await + .unwrap_or_else(|error| panic!("failed to expire indexer spool floor: {error:?}")); + let sync_started = Instant::now(); + self.writer.sync().await.unwrap_or_else(|error| { + panic!("failed to durably sync expired indexer spool floor: {error:?}") + }); + self.metrics + .queue_synced(QueueStatus::Success, sync_started.elapsed()); + self.reader.reset().await; + let expired = self.uploads.lock().expire_through(position); + for (_, _, _) in &expired { + self.metrics.producer_recorded(status, Duration::ZERO); + } + warn!( + height, + ?digest, + encoded_len, + ?age, + ?status, + expired_entries = expired.len(), + "terminally expired oldest finalized indexer payload before admission" + ); + } + + self.metrics.queue_entry(entry.height()); + let position = self.writer.enqueue(entry.clone()).await.unwrap_or_else(|error| { + self.metrics.queue_enqueued(QueueStatus::Failure); + panic!("failed to enqueue finalized indexer payload: {error:?}") + }); + self.uploads.lock().mark_queued(position, &entry); + self.metrics.queue_enqueued(QueueStatus::Success); + response + .send(()) + .expect("indexer spool producer stopped before admission completed"); + } + async fn fill_slots(&mut self) { while self.active.len() < self.max_active.get() { let item = Self::try_recv_queue(&mut self.reader, self.metrics.clone()).await; @@ -177,7 +244,7 @@ impl Consumer< match reader.recv().await { Ok(Some((position, entry))) => { metrics.queue_read(QueueReadSource::Recv, QueueStatus::Success); - metrics.queue_entry(entry.height); + metrics.queue_entry(entry.height()); Some((position, entry)) } Ok(None) => { @@ -198,7 +265,7 @@ impl Consumer< match reader.try_recv().await { Ok(Some((position, entry))) => { metrics.queue_read(QueueReadSource::TryRecv, QueueStatus::Success); - metrics.queue_entry(entry.height); + metrics.queue_entry(entry.height()); Some((position, entry)) } Ok(None) => { @@ -213,7 +280,26 @@ impl Consumer< } async fn start_upload(&mut self, position: u64, entry: Entry) { - let Entry { height, digest } = entry; + let height = entry.height(); + let digest = entry.digest(); + let enqueued_at_millis = entry.enqueued_at_millis; + let finalized = entry.finalized; + if self.payload_age(enqueued_at_millis) >= self.spool_limits.max_age { + self.metrics + .producer_recorded(ProducerStatus::ExpiredAge, Duration::ZERO); + warn!( + height, + ?digest, + "terminally expiring finalized indexer payload at configured age bound" + ); + self.complete(Completion::Skipped { + position, + height, + digest, + }) + .await; + return; + } let decision = self.uploads.lock().should_upload(&digest); self.metrics .backfill_decision((&decision).into(), BackfillPhase::Start); @@ -222,7 +308,11 @@ impl Consumer< let mut upload = self.metrics.start_backfill_upload(); upload.skipped(); } - self.complete(Completion::Skipped { position, height }) + self.complete(Completion::Skipped { + position, + height, + digest, + }) .await; debug!(?digest, "consumer skipping already-uploaded block"); return; @@ -235,82 +325,35 @@ impl Consumer< .with_attribute("digest", digest) .with_attribute("height", height); let client = self.client.clone(); - let marshal = self.marshal.clone(); let metrics = self.metrics.clone(); let upload_results = self.upload_results.clone(); let uploads = self.uploads.clone(); let retry = self.retry; - let missing_finalization_grace = self.missing_finalization_grace; - let mismatched_finalization_grace = self.mismatched_finalization_grace; + let max_age = self.spool_limits.max_age; async move { let mut upload = metrics.start_backfill_upload(); - let Some(block) = - Self::wait_for_uploadable_block( - &context, &marshal, &metrics, &uploads, &mut upload, digest, retry, - ) - .await - else { - upload.skipped(); - debug!(?digest, "skipping previously uploaded block"); - return Completion::Skipped { position, height }; - }; + metrics.observe_block(BlockMetricSource::ConsumerCached, &finalized.block); + upload.hold_block(&finalized.block); - let mut proof_failure = ProofFailure::default(); - let finalized = loop { - let Some(proof) = marshal.get_finalization(Height::new(height)).await else { - if let Some(stale) = proof_failure.observe( - &context, - BackfillResetReason::MissingFinalization, - missing_finalization_grace, - height, - digest, - ) { - upload.abandoned(); - return stale; - } - warn!( - height, - ?digest, - "consumer could not find finalization, retrying" - ); - Self::wait( - &context, - &metrics, - BackfillWaitReason::MissingFinalization, - retry, - ) - .await; - continue; - }; - if proof.proposal.payload != digest { - if let Some(stale) = proof_failure.observe( - &context, - BackfillResetReason::MismatchedFinalization, - mismatched_finalization_grace, + loop { + let now_millis: u64 = context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + // Backward wall-clock movement produces age zero until the clock catches up. + let age = Duration::from_millis(now_millis.saturating_sub(enqueued_at_millis)); + if age >= max_age { + metrics.producer_recorded(ProducerStatus::ExpiredAge, Duration::ZERO); + warn!(height, ?digest, ?age, "terminally expiring active finalized indexer upload at configured age bound"); + return Completion::Skipped { + position, height, digest, - ) { - upload.abandoned(); - return stale; - } - warn!( - height, - ?digest, - "consumer found mismatched finalization, retrying" - ); - Self::wait( - &context, - &metrics, - BackfillWaitReason::MismatchedFinalization, - retry, - ) - .await; - continue; + }; } - break Finalized::new(proof, block.clone()); - }; - - loop { let decision = { let uploads = uploads.lock(); uploads.should_upload(&digest) @@ -320,7 +363,11 @@ impl Consumer< Decision::Skip => { upload.skipped(); debug!(?digest, "skipping previously uploaded block"); - return Completion::Skipped { position, height }; + return Completion::Skipped { + position, + height, + digest, + }; } Decision::Wait => { Self::wait( @@ -363,86 +410,17 @@ impl Consumer< }); } - async fn wait_for_uploadable_block( - context: &E, - marshal: &MarshalMailbox>, - metrics: &IndexerMetrics, - uploads: &SharedState, - upload: &mut BackfillUploadGuard, - digest: Digest, - retry: Duration, - ) -> Option { - enum NextBlock { - AlreadyUploaded, - WaitForCertificate, - Ready(Box), - FetchFromMarshal, - } - - loop { - let next = { - let uploads = uploads.lock(); - match uploads.should_upload(&digest) { - Decision::Skip => { - metrics.backfill_decision( - BackfillDecision::Skip, - BackfillPhase::BeforeBlock, - ); - NextBlock::AlreadyUploaded - } - Decision::Wait => { - metrics.backfill_decision( - BackfillDecision::Wait, - BackfillPhase::BeforeBlock, - ); - NextBlock::WaitForCertificate - } - Decision::Proceed => { - metrics.backfill_decision( - BackfillDecision::Proceed, - BackfillPhase::BeforeBlock, - ); - uploads - .cached_block(&digest) - .map(|block| NextBlock::Ready(Box::new(block))) - .unwrap_or(NextBlock::FetchFromMarshal) - } - } - }; - - match next { - NextBlock::AlreadyUploaded => return None, - NextBlock::WaitForCertificate => { - Self::wait( - context, - metrics, - BackfillWaitReason::CertificateUpload, - retry, - ) - .await; - } - NextBlock::Ready(block) => { - metrics.observe_block(BlockMetricSource::ConsumerCached, &block); - upload.hold_block(&block); - return Some(*block); - } - NextBlock::FetchFromMarshal => { - if let Some(block) = marshal.get_block(Identifier::Digest(digest)).await { - metrics.observe_block(BlockMetricSource::ConsumerMarshal, &block); - uploads - .lock() - .cache_block(block.clone(), SharedCacheSource::ConsumerMarshal); - upload.hold_block(&block); - return Some(block); - } - warn!( - ?digest, - "consumer could not find block in marshal, retrying" - ); - Self::wait(context, metrics, BackfillWaitReason::MissingBlock, retry).await; - } - } - } + fn payload_age(&self, enqueued_at_millis: u64) -> Duration { + let now_millis: u64 = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + // Backward wall-clock movement produces age zero until the clock catches up. + Duration::from_millis(now_millis.saturating_sub(enqueued_at_millis)) } async fn wait( @@ -462,28 +440,22 @@ impl Consumer< } async fn complete(&mut self, completion: Completion) { - let (position, height) = match completion { + let (position, height, digest) = match completion { Completion::Uploaded { position, height, digest, } => { self.uploads.lock().mark_uploaded(digest, height); - (position, height) - } - Completion::Skipped { position, height } => (position, height), - Completion::Stale { - reason, - first_height, - first_digest, - attempts, - elapsed, - } => { - self.reset_stale_queue(reason, first_height, first_digest, attempts, elapsed) - .await; - return; + (position, height, digest) } + Completion::Skipped { + position, + height, + digest, + } => (position, height, digest), }; + self.uploads.lock().finish_queued(&digest); let floor = self.reader.ack_floor().await; match self.reader.ack(position).await { @@ -511,109 +483,4 @@ impl Consumer< } } - async fn reset_stale_queue( - &mut self, - reason: BackfillResetReason, - first_height: u64, - first_digest: Digest, - attempts: u64, - elapsed: Duration, - ) { - let queue_size = self.writer.size().await; - let ack_floor = self.reader.ack_floor().await; - let abandoned_entries = queue_size.saturating_sub(ack_floor); - let tip = Self::latest_proof_bearing_tip(self.marshal.clone()).await; - let (tip_height, tip_digest) = tip.unwrap_or((0, first_digest)); - let abandoned_height_span = tip_height.saturating_sub(first_height); - - warn!( - ?reason, - first_height, - ?first_digest, - tip_height, - ?tip_digest, - attempts, - ?elapsed, - abandoned_entries, - abandoned_height_span, - "resetting stale durable backfill queue" - ); - - self.active.cancel_all(); - match self.reader.ack_up_to(queue_size).await { - Ok(()) => self.metrics.queue_acked(QueueStatus::Success), - Err(err) => { - self.metrics.queue_acked(QueueStatus::Failure); - panic!("failed to ack stale finalized queue: {err:?}"); - } - } - let sync_started = Instant::now(); - match self.writer.sync().await { - Ok(()) => self - .metrics - .queue_synced(QueueStatus::Success, sync_started.elapsed()), - Err(err) => { - self.metrics - .queue_synced(QueueStatus::Failure, sync_started.elapsed()); - panic!("failed to sync stale finalized queue reset: {err:?}"); - } - } - self.reader.reset().await; - self.metrics.queue_ack_floor(tip_height); - self.metrics.backfill_queue_reset( - reason, - abandoned_entries, - abandoned_height_span, - ); - self.uploads.lock().advance_queue_floor(tip_height); - self.uploads.lock().restart_above(tip_height); - } - - async fn latest_proof_bearing_tip( - marshal: MarshalMailbox>, - ) -> Option<(u64, Digest)> { - let (height, _) = marshal.get_info(Identifier::Latest).await?; - for height in (0..=height.get()).rev() { - if let Some(proof) = marshal.get_finalization(Height::new(height)).await { - return Some((height, proof.proposal.payload)); - } - } - None - } -} - -#[derive(Default)] -struct ProofFailure { - reason: Option, - first_seen: Option, - attempts: u64, -} - -impl ProofFailure { - fn observe( - &mut self, - context: &E, - reason: BackfillResetReason, - grace: Duration, - height: u64, - digest: Digest, - ) -> Option { - if self.reason != Some(reason) { - self.reason = Some(reason); - self.first_seen = Some(context.current()); - self.attempts = 0; - } - self.attempts = self.attempts.saturating_add(1); - let elapsed = context - .current() - .duration_since(self.first_seen.expect("proof failure start")) - .unwrap_or_default(); - (elapsed >= grace).then_some(Completion::Stale { - reason, - first_height: height, - first_digest: digest, - attempts: self.attempts, - elapsed, - }) - } } diff --git a/examples/coins/chain/src/indexer/backfiller/producer.rs b/examples/coins/chain/src/indexer/backfiller/producer.rs index 88b01e4..fccdb5d 100644 --- a/examples/coins/chain/src/indexer/backfiller/producer.rs +++ b/examples/coins/chain/src/indexer/backfiller/producer.rs @@ -3,23 +3,32 @@ use crate::{ indexer::{ metrics::{ estimated_block_bytes, BlockMetricSource, ProducerActivity, ProducerStatus, - QueueStatus, }, - IndexerMetrics, + IndexerMetrics, SpoolLimits, }, - Block, + Block, Finalized, Scheme, }; use commonware_actor::{ mailbox::{self, Overflow, Policy}, Feedback, }; -use commonware_consensus::{marshal::Update, Reporter}; +use commonware_consensus::{ + marshal::{core::Mailbox as MarshalMailbox, standard::Standard, Update}, + types::Height, + Reporter, +}; use commonware_runtime::{ spawn_cell, BufferPooler, Clock, ContextCell, Handle, Metrics, Spawner, Storage, }; -use commonware_storage::queue; use commonware_utils::{acknowledgement::Exact, Acknowledgement}; -use std::{collections::VecDeque, num::NonZeroUsize, sync::Arc, time::Instant}; +use commonware_utils::channel::oneshot; +use std::{ + collections::VecDeque, + num::NonZeroUsize, + sync::Arc, + time::{Duration, Instant, UNIX_EPOCH}, +}; +use tracing::warn; #[derive(Clone)] pub struct Producer { @@ -27,6 +36,22 @@ pub struct Producer { metrics: IndexerMetrics, } +pub(crate) struct Admission { + pub(crate) entry: Entry, + pub(crate) response: oneshot::Sender<()>, +} + +impl Policy for Admission { + type Overflow = VecDeque; + + fn handle(overflow: &mut Self::Overflow, message: Self) { + overflow.push_back(message); + } +} + +pub(crate) type AdmissionSender = mailbox::Sender; +pub(crate) type AdmissionReceiver = mailbox::Receiver; + struct Message { block: Arc, block_estimated_bytes: u64, @@ -87,8 +112,21 @@ struct Actor { context: ContextCell, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, + marshal: MarshalMailbox>, + admission: AdmissionSender, receiver: mailbox::Receiver, + retry: Duration, + missing_finalization_grace: Duration, + mismatched_finalization_grace: Duration, + spool_limits: SpoolLimits, +} + +pub(crate) struct Config { + pub(crate) mailbox_size: NonZeroUsize, + pub(crate) retry: Duration, + pub(crate) missing_finalization_grace: Duration, + pub(crate) mismatched_finalization_grace: Duration, + pub(crate) spool_limits: SpoolLimits, } impl Reporter for Producer { @@ -123,16 +161,29 @@ impl Actor { context: E, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, - mailbox_size: NonZeroUsize, + marshal: MarshalMailbox>, + admission: AdmissionSender, + config: Config, ) -> (Self, Producer) { + let Config { + mailbox_size, + retry, + missing_finalization_grace, + mismatched_finalization_grace, + spool_limits, + } = config; let (sender, receiver) = mailbox::new(context.child("mailbox"), mailbox_size); let actor = Self { context: ContextCell::new(context), uploads, metrics: metrics.clone(), - writer, + marshal, + admission, receiver, + retry, + missing_finalization_grace, + mismatched_finalization_grace, + spool_limits, }; ( actor, @@ -158,25 +209,91 @@ impl Actor { let started = Instant::now(); self.metrics .observe_block(BlockMetricSource::ProducerRecord, block); - let Some(entry) = self.uploads.lock().record(block) else { + let Some(candidate) = self.uploads.lock().record(block) else { self.metrics.producer_recorded( ProducerStatus::AlreadyUploaded, started.elapsed(), ); return; }; - self.metrics.queue_entry(entry.height); - match self.writer.enqueue(entry).await { - Ok(_) => { - self.metrics.queue_enqueued(QueueStatus::Success); - self.metrics - .producer_recorded(ProducerStatus::Recorded, started.elapsed()); - } - Err(err) => { - self.metrics.queue_enqueued(QueueStatus::Failure); - panic!("failed to enqueue finalized digest: {err:?}"); + let first_seen = self.context.current(); + self.metrics.certificate_request_started(); + let proof = loop { + let Some(proof) = self + .marshal + .get_finalization(Height::new(candidate.height)) + .await + else { + let elapsed = self + .context + .current() + .duration_since(first_seen) + .unwrap_or_default(); + assert!( + elapsed < self.missing_finalization_grace, + "marshal has no finalization certificate for durable indexer payload at height {} after {:?}", + candidate.height, + elapsed, + ); + warn!(height = candidate.height, ?elapsed, "waiting for finalized certificate before spooling indexer payload"); + self.context.sleep(self.retry).await; + continue; + }; + if proof.proposal.payload != candidate.digest + || proof.proposal.round.epoch() != block.context.round.epoch() + { + let elapsed = self + .context + .current() + .duration_since(first_seen) + .unwrap_or_default(); + assert!( + elapsed < self.mismatched_finalization_grace, + "marshal finalization certificate conflicts with block at height {} after {:?}", + candidate.height, + elapsed, + ); + warn!(height = candidate.height, ?elapsed, "waiting for matching finalized certificate before spooling indexer payload"); + self.context.sleep(self.retry).await; + continue; } + break proof; + }; + self.metrics.certificate_request_finished(); + let enqueued_at_millis = self + .context + .current() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX); + let entry = Entry::new(enqueued_at_millis, Finalized::new(proof, block.clone())); + let expiry = if entry.encoded_len > self.spool_limits.max_payload_bytes + || entry.encoded_len > self.spool_limits.max_bytes + { + Some(ProducerStatus::ExpiredOversized) + } else { + None + }; + if let Some(status) = expiry { + self.metrics.producer_recorded(status, started.elapsed()); + warn!( + height = entry.height(), + digest = ?entry.digest(), + encoded_len = entry.encoded_len, + ?status, + "terminally expiring finalized indexer payload to preserve spool availability bound" + ); + return; } + let (response, completed) = oneshot::channel(); + let _ = self.admission.enqueue(Admission { entry, response }); + completed + .await + .expect("indexer spool admission coordinator stopped"); + self.metrics + .producer_recorded(ProducerStatus::Recorded, started.elapsed()); } } @@ -184,15 +301,23 @@ pub fn init( context: E, uploads: SharedState, metrics: IndexerMetrics, - writer: queue::Writer, - mailbox_size: NonZeroUsize, -) -> Producer + marshal: MarshalMailbox>, + admission: AdmissionSender, + config: Config, +) -> (Producer, Handle<()>) where E: BufferPooler + Clock + Storage + Metrics + Spawner, { - let (actor, producer) = Actor::new(context, uploads, metrics, writer, mailbox_size); - actor.start(); - producer + let (actor, producer) = Actor::new( + context, + uploads, + metrics, + marshal, + admission, + config, + ); + let handle = actor.start(); + (producer, handle) } #[cfg(test)] @@ -200,16 +325,12 @@ mod tests { use super::*; use crate::{StateCommitment, Transaction, EPOCH}; use commonware_consensus::types::{Height, Round, View}; - use commonware_cryptography::{ed25519, Digestible, Hasher, Sha256, Signer}; - use commonware_runtime::{ - buffer::paged::CacheRef, deterministic, Runner as _, Supervisor as _, - }; + use commonware_cryptography::{ed25519, Hasher, Sha256, Signer}; + use commonware_runtime::{deterministic, Runner as _, Supervisor as _}; use commonware_storage::mmr::Location; use commonware_utils::{ - acknowledgement::Exact, range::NonEmptyRange, sync::Mutex, NZUsize, NZU16, NZU64, + acknowledgement::Exact, range::NonEmptyRange, NZUsize, }; - use futures::FutureExt; - use std::{sync::Arc, time::Duration}; fn state(height: u64) -> StateCommitment { StateCommitment { @@ -239,60 +360,6 @@ mod tests { ) } - #[test] - fn queues_finalized_block_before_acknowledging() { - deterministic::Runner::default().start(|context| async move { - let page_cache = CacheRef::from_pooler(&context, NZU16!(4_096), NZUsize!(128)); - let (writer, mut reader) = queue::shared::init( - context.child("queue"), - queue::Config { - partition: "indexer-producer-test".to_string(), - items_per_section: NZU64!(16), - compression: None, - codec_config: (), - page_cache, - write_buffer: NZUsize!(1024), - }, - ) - .await - .expect("init queue"); - let uploads = Arc::new(Mutex::new(crate::indexer::backfiller::State::new())); - let metrics = IndexerMetrics::register(&context.child("indexer")); - let mut producer = init(context.child("producer"), uploads, metrics, writer, NZUsize!(4)); - let block = block(2, 2, b"block"); - let digest = block.digest(); - let (ack, waiter) = Exact::handle(); - - assert!(producer - .report(commonware_consensus::marshal::Update::Block( - block.into(), - ack, - )) - .accepted()); - commonware_macros::select! { - result = waiter.fuse() => result.expect("acknowledged"), - _ = context.sleep(Duration::from_secs(1)) => panic!("ack timed out"), - } - - let (_, entry) = reader - .recv() - .await - .expect("read queue") - .expect("queued entry"); - assert_eq!(entry.height, 2); - assert_eq!(entry.digest, digest); - - let encoded = context.encode(); - assert!(encoded.contains( - "indexer_producer_report_total{activity=\"block\",status=\"enqueued\"} 1", - )); - assert!(encoded.contains( - "indexer_producer_report_total{activity=\"block\",status=\"recorded\"} 1", - )); - assert!(encoded.contains("indexer_producer_record_duration_seconds_bucket")); - }); - } - #[test] fn mailbox_overflow_tracks_retained_block_pressure() { deterministic::Runner::default().start(|context| async move { diff --git a/examples/coins/chain/src/indexer/backfiller/state.rs b/examples/coins/chain/src/indexer/backfiller/state.rs index 217b70f..e52c24c 100644 --- a/examples/coins/chain/src/indexer/backfiller/state.rs +++ b/examples/coins/chain/src/indexer/backfiller/state.rs @@ -5,10 +5,10 @@ use crate::{ }, IndexerMetrics, }, - Block, + Block, Finalized, }; use bytes::{Buf, BufMut}; -use commonware_codec::{self, FixedSize, Read, Write}; +use commonware_codec::{self, EncodeSize, FixedSize, Read, Write}; use commonware_cryptography::{sha256::Digest, Digestible}; use commonware_utils::{sync::Mutex, PrioritySet}; use std::{collections::BTreeMap, sync::Arc}; @@ -19,30 +19,84 @@ pub enum Decision { Proceed, } -#[derive(Clone, Copy, PartialEq, Eq)] -pub struct Entry { +pub struct Candidate { pub height: u64, pub digest: Digest, } -impl FixedSize for Entry { - const SIZE: usize = u64::SIZE + Digest::SIZE; +const ENTRY_VERSION: u8 = 1; + +#[derive(Clone)] +pub struct Entry { + pub version: u8, + pub enqueued_at_millis: u64, + pub encoded_len: u64, + pub finalized: Finalized, +} + +impl Entry { + pub fn new(enqueued_at_millis: u64, finalized: Finalized) -> Self { + let mut entry = Self { + version: ENTRY_VERSION, + enqueued_at_millis, + encoded_len: 0, + finalized, + }; + entry.encoded_len = entry.encode_size() as u64; + entry + } + + pub fn height(&self) -> u64 { + self.finalized.block.height.get() + } + + pub fn digest(&self) -> Digest { + self.finalized.block.digest() + } } impl Write for Entry { fn write(&self, buf: &mut impl BufMut) { - self.height.write(buf); - self.digest.write(buf); + self.version.write(buf); + self.enqueued_at_millis.write(buf); + self.encoded_len.write(buf); + self.finalized.write(buf); } } impl Read for Entry { - type Cfg = (); + type Cfg = ::Cfg; + + fn read_cfg(buf: &mut impl Buf, cfg: &Self::Cfg) -> Result { + let version = u8::read_cfg(buf, &())?; + if version != ENTRY_VERSION { + return Err(commonware_codec::Error::Invalid( + "indexer spool entry", + "unsupported version", + )); + } + let enqueued_at_millis = u64::read_cfg(buf, &())?; + let encoded_len = u64::read_cfg(buf, &())?; + let finalized = Finalized::read_cfg(buf, cfg)?; + let entry = Self { + version, + enqueued_at_millis, + encoded_len, + finalized, + }; + if entry.encode_size() as u64 != encoded_len { + return Err(commonware_codec::Error::Invalid( + "indexer spool entry", + "encoded length mismatch", + )); + } + Ok(entry) + } +} - fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { - let height = u64::read_cfg(buf, &())?; - let digest = Digest::read_cfg(buf, &())?; - Ok(Self { height, digest }) +impl EncodeSize for Entry { + fn encode_size(&self) -> usize { + u8::SIZE + u64::SIZE + u64::SIZE + self.finalized.encode_size() } } @@ -55,6 +109,8 @@ pub struct State { cached_block_estimated_bytes: u64, certificate_uploads: BTreeMap, certificate_upload_refs: usize, + queued: BTreeMap, + queued_bytes: u64, metrics: Option, } @@ -74,6 +130,8 @@ impl State { cached_block_estimated_bytes: 0, certificate_uploads: BTreeMap::new(), certificate_upload_refs: 0, + queued: BTreeMap::new(), + queued_bytes: 0, metrics: None, } } @@ -89,8 +147,8 @@ impl State { self.uploaded.contains(digest) } - pub fn record(&mut self, block: &Block) -> Option { - let entry = Entry { + pub fn record(&mut self, block: &Block) -> Option { + let entry = Candidate { height: block.height.get(), digest: block.digest(), }; @@ -99,7 +157,7 @@ impl State { self.sync_metrics(); return None; } - if self.is_uploaded(&entry.digest) { + if self.is_uploaded(&entry.digest) || self.queued(&entry.digest) { self.sync_metrics(); return None; } @@ -108,12 +166,74 @@ impl State { Some(entry) } + pub fn queued(&self, digest: &Digest) -> bool { + self.queued.contains_key(digest) + } + + pub fn mark_queued(&mut self, position: u64, entry: &Entry) { + if let Some((_, _, old_bytes, _)) = + self.queued + .insert( + entry.digest(), + ( + position, + entry.height(), + entry.encoded_len, + entry.enqueued_at_millis, + ), + ) + { + self.queued_bytes = self.queued_bytes.saturating_sub(old_bytes); + } + self.queued_bytes = self.queued_bytes.saturating_add(entry.encoded_len); + self.sync_metrics(); + } + + pub fn recover_queued(&mut self, position: u64, entry: &Entry) { + self.observe_finalization(entry.height()); + self.mark_queued(position, entry); + } + + pub fn finish_queued(&mut self, digest: &Digest) { + if let Some((_, _, bytes, _)) = self.queued.remove(digest) { + self.queued_bytes = self.queued_bytes.saturating_sub(bytes); + } + self.sync_metrics(); + } + + pub fn spool_usage(&self) -> (u64, u64) { + (self.queued.len() as u64, self.queued_bytes) + } + + pub fn oldest_queued(&self) -> Option<(Digest, u64, u64, u64, u64)> { + self.queued + .iter() + .min_by_key(|(_, (position, _, _, _))| *position) + .map(|(digest, (position, height, bytes, enqueued_at))| { + (*digest, *position, *height, *bytes, *enqueued_at) + }) + } + + pub fn expire_through(&mut self, position: u64) -> Vec<(Digest, u64, u64)> { + let expired: Vec<_> = self + .queued + .iter() + .filter(|(_, (queued_position, _, _, _))| *queued_position <= position) + .map(|(digest, (_, height, bytes, _))| (*digest, *height, *bytes)) + .collect(); + for (digest, _, _) in &expired { + self.finish_queued(digest); + } + expired + } + pub fn advance_queue_floor(&mut self, height: u64) { self.acked_through = self.acked_through.max(height); self.prune(); self.sync_metrics(); } + #[cfg(test)] pub fn restart_above(&mut self, height: u64) { self.restart_watermark = self.restart_watermark.max(height); let pruned = self @@ -145,6 +265,7 @@ impl State { self.sync_metrics(); } + #[cfg(test)] pub fn cached_block(&self, digest: &Digest) -> Option { self.cached_blocks .get(digest) @@ -279,6 +400,20 @@ impl State { uploaded_digests: self.uploaded.len(), latest_finalized_height: self.latest_finalized, acked_through_height: self.acked_through, + spool_entries: self.queued.len(), + spool_logical_bytes: self.queued_bytes, + spool_oldest_height: self + .queued + .values() + .map(|(_, height, _, _)| *height) + .min() + .unwrap_or(0), + spool_oldest_enqueued_at_millis: self + .queued + .values() + .map(|(_, _, _, enqueued_at)| *enqueued_at) + .min() + .unwrap_or(0), }); } } @@ -290,7 +425,6 @@ pub type SharedState = Arc>; mod tests { use super::*; use crate::{StateCommitment, Transaction, EPOCH}; - use commonware_codec::{DecodeExt, Encode}; use commonware_consensus::types::{Height, Round, View}; use commonware_cryptography::{ed25519, Hasher, Sha256, Signer}; use commonware_runtime::{deterministic, Metrics as _, Runner as _, Supervisor as _}; @@ -325,20 +459,6 @@ mod tests { ) } - #[test] - fn entry_codec_roundtrips() { - let entry = Entry { - height: 7, - digest: Sha256::hash(b"block"), - }; - - let encoded = entry.encode(); - let decoded = Entry::decode(encoded).expect("decode entry"); - - assert_eq!(decoded.height, entry.height); - assert_eq!(decoded.digest, entry.digest); - } - #[test] fn record_caches_block_until_uploaded() { let mut state = State::new(); diff --git a/examples/coins/chain/src/indexer/metrics.rs b/examples/coins/chain/src/indexer/metrics.rs index 18d2f08..ca211ea 100644 --- a/examples/coins/chain/src/indexer/metrics.rs +++ b/examples/coins/chain/src/indexer/metrics.rs @@ -163,7 +163,6 @@ pub(crate) enum BlockMetricSource { ProducerRecord, LiveCertificate, ConsumerCached, - ConsumerMarshal, } impl BlockMetricSource { @@ -172,7 +171,6 @@ impl BlockMetricSource { Self::ProducerRecord => "producer_record", Self::LiveCertificate => "live_certificate", Self::ConsumerCached => "consumer_cached", - Self::ConsumerMarshal => "consumer_marshal", } } } @@ -192,7 +190,6 @@ impl EncodeLabelValueTrait for BlockMetricSource { pub(crate) enum SharedCacheSource { ProducerRecord, LiveCertificate, - ConsumerMarshal, } impl SharedCacheSource { @@ -200,7 +197,6 @@ impl SharedCacheSource { match self { Self::ProducerRecord => "producer_record", Self::LiveCertificate => "live_certificate", - Self::ConsumerMarshal => "consumer_marshal", } } } @@ -248,7 +244,6 @@ impl EncodeLabelValueTrait for SharedRetentionReason { pub(crate) enum BackfillStatus { Uploaded, Skipped, - Abandoned, } impl BackfillStatus { @@ -256,7 +251,6 @@ impl BackfillStatus { match self { Self::Uploaded => "uploaded", Self::Skipped => "skipped", - Self::Abandoned => "abandoned", } } } @@ -303,7 +297,6 @@ impl EncodeLabelValueTrait for BackfillDecision { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum BackfillPhase { Start, - BeforeBlock, BeforeAttempt, } @@ -311,7 +304,6 @@ impl BackfillPhase { const fn as_str(self) -> &'static str { match self { Self::Start => "start", - Self::BeforeBlock => "before_block", Self::BeforeAttempt => "before_attempt", } } @@ -331,9 +323,6 @@ impl EncodeLabelValueTrait for BackfillPhase { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum BackfillWaitReason { CertificateUpload, - MissingBlock, - MissingFinalization, - MismatchedFinalization, HttpError, } @@ -341,40 +330,11 @@ impl BackfillWaitReason { const fn as_str(self) -> &'static str { match self { Self::CertificateUpload => "certificate_upload", - Self::MissingBlock => "missing_block", - Self::MissingFinalization => "missing_finalization", - Self::MismatchedFinalization => "mismatched_finalization", Self::HttpError => "http_error", } } } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub(crate) enum BackfillResetReason { - MissingFinalization, - MismatchedFinalization, -} - -impl BackfillResetReason { - const fn as_str(self) -> &'static str { - match self { - Self::MissingFinalization => "missing_finalization", - Self::MismatchedFinalization => "mismatched_finalization", - } - } -} - -impl EncodeLabelValueTrait for BackfillResetReason { - fn encode( - &self, - encoder: &mut commonware_runtime::telemetry::metrics::LabelValueEncoder, - ) -> Result<(), fmt::Error> { - use fmt::Write as _; - - encoder.write_str(self.as_str()) - } -} - impl EncodeLabelValueTrait for BackfillWaitReason { fn encode( &self, @@ -475,6 +435,10 @@ pub(crate) enum ProducerStatus { Ignored, Recorded, AlreadyUploaded, + ExpiredOversized, + ExpiredEntries, + ExpiredBytes, + ExpiredAge, } impl ProducerStatus { @@ -485,6 +449,10 @@ impl ProducerStatus { Self::Ignored => "ignored", Self::Recorded => "recorded", Self::AlreadyUploaded => "already_uploaded", + Self::ExpiredOversized => "expired_oversized", + Self::ExpiredEntries => "expired_entries", + Self::ExpiredBytes => "expired_bytes", + Self::ExpiredAge => "expired_age", } } } @@ -538,6 +506,10 @@ pub(crate) struct SharedStateSnapshot { pub(crate) uploaded_digests: usize, pub(crate) latest_finalized_height: u64, pub(crate) acked_through_height: u64, + pub(crate) spool_entries: usize, + pub(crate) spool_logical_bytes: u64, + pub(crate) spool_oldest_height: u64, + pub(crate) spool_oldest_enqueued_at_millis: u64, } #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] @@ -599,11 +571,6 @@ struct BackfillWaitReasonLabel { reason: BackfillWaitReason, } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] -struct BackfillResetReasonLabel { - reason: BackfillResetReason, -} - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EncodeLabelSet)] struct QueueStatusLabel { status: QueueStatus, @@ -666,6 +633,11 @@ pub(crate) struct Inner { shared_uploaded_digests: Gauge, shared_latest_finalized_height: Gauge, shared_acked_through_height: Gauge, + spool_entries: Gauge, + spool_logical_bytes: Gauge, + spool_oldest_height: Gauge, + spool_oldest_enqueued_at_millis: Gauge, + producer_certificate_requests: Gauge, shared_pruned: CounterFamily, shared_cache_inserted: CounterFamily, shared_cache_removed: CounterFamily, @@ -677,9 +649,6 @@ pub(crate) struct Inner { backfill_decision: CounterFamily, backfill_wait_duration: HistogramFamily, backfill_retry: CounterFamily, - backfill_queue_reset: CounterFamily, - backfill_queue_reset_abandoned_entries: HistogramFamily, - backfill_queue_reset_abandoned_height_span: HistogramFamily, backfill_active_block_estimated_bytes: Gauge, backfill_active_body_estimated_bytes: Gauge, queue_enqueue: CounterFamily, @@ -817,6 +786,26 @@ impl IndexerMetrics { "shared_acked_through_height", "Queue acknowledgement floor tracked by shared indexer state", ), + spool_entries: context.gauge( + "spool_entries", + "Current number of self-contained finalized payloads in the durable spool", + ), + spool_logical_bytes: context.gauge( + "spool_logical_bytes", + "Current encoded bytes in self-contained finalized spool payloads", + ), + spool_oldest_height: context.gauge( + "spool_oldest_height", + "Oldest finalized height retained in the durable spool", + ), + spool_oldest_enqueued_at_millis: context.gauge( + "spool_oldest_enqueued_at_millis", + "Wall-clock Unix millisecond timestamp of the oldest durable spool payload", + ), + producer_certificate_requests: context.gauge( + "spool_certificate_requests", + "Current finalized blocks awaiting a marshal certificate before spool admission", + ), shared_pruned: context.family( "shared_prune", "Total shared indexer state retention removals by reason", @@ -867,24 +856,6 @@ impl IndexerMetrics { "backfill_retry", "Total durable backfill retries by reason", ), - backfill_queue_reset: context.family( - "backfill_queue_reset", - "Total durable backfill queue resets by reason", - ), - backfill_queue_reset_abandoned_entries: context.register( - "backfill_queue_reset_abandoned_entries", - "Durable backfill entries abandoned by queue reset", - raw::Family:: raw::Histogram>::new_with_constructor( - local_histogram, - ), - ), - backfill_queue_reset_abandoned_height_span: context.register( - "backfill_queue_reset_abandoned_height_span", - "Durable backfill height span abandoned by queue reset", - raw::Family:: raw::Histogram>::new_with_constructor( - local_histogram, - ), - ), backfill_active_block_estimated_bytes: context.gauge( "backfill_active_block_estimated_bytes", "Current estimated encoded block bytes held by durable backfill upload tasks", @@ -1074,6 +1045,16 @@ impl IndexerMetrics { let _ = self .shared_acked_through_height .try_set(snapshot.acked_through_height); + let _ = self.spool_entries.try_set(snapshot.spool_entries); + let _ = self + .spool_logical_bytes + .try_set(snapshot.spool_logical_bytes); + let _ = self + .spool_oldest_height + .try_set(snapshot.spool_oldest_height); + let _ = self + .spool_oldest_enqueued_at_millis + .try_set(snapshot.spool_oldest_enqueued_at_millis); self.queue_ack_floor(snapshot.acked_through_height); let lag = snapshot .latest_finalized_height @@ -1081,6 +1062,14 @@ impl IndexerMetrics { let _ = self.queue_lag_height.try_set(lag); } + pub(crate) fn certificate_request_started(&self) { + self.producer_certificate_requests.inc(); + } + + pub(crate) fn certificate_request_finished(&self) { + self.producer_certificate_requests.dec(); + } + pub(crate) fn shared_cache_inserted(&self, source: SharedCacheSource) { self.shared_cache_inserted .get_or_create(&SharedCacheSourceLabel { source }) @@ -1132,22 +1121,6 @@ impl IndexerMetrics { .observe(duration.as_secs_f64()); } - pub(crate) fn backfill_queue_reset( - &self, - reason: BackfillResetReason, - abandoned_entries: u64, - abandoned_height_span: u64, - ) { - let label = BackfillResetReasonLabel { reason }; - self.backfill_queue_reset.get_or_create(&label).inc(); - self.backfill_queue_reset_abandoned_entries - .get_or_create(&label) - .observe(abandoned_entries as f64); - self.backfill_queue_reset_abandoned_height_span - .get_or_create(&label) - .observe(abandoned_height_span as f64); - } - pub(crate) fn start_backfill_body(&self, bytes: u64) -> BackfillBodyGuard { let bytes = gauge_u64(bytes); self.backfill_active_body_estimated_bytes.inc_by(bytes); @@ -1403,9 +1376,6 @@ impl BackfillUploadGuard { self.status = Some(BackfillStatus::Skipped); } - pub(crate) const fn abandoned(&mut self) { - self.status = Some(BackfillStatus::Abandoned); - } } impl Drop for BackfillUploadGuard { diff --git a/examples/coins/chain/src/indexer/mod.rs b/examples/coins/chain/src/indexer/mod.rs index 2604221..fac82a2 100644 --- a/examples/coins/chain/src/indexer/mod.rs +++ b/examples/coins/chain/src/indexer/mod.rs @@ -6,9 +6,9 @@ use commonware_consensus::types::Epoch; use commonware_cryptography::bls12381::{ dkg::feldman_desmedt::Output as DkgOutput, primitives::variant::MinSig, }; -use commonware_runtime::{BufferPooler, Clock, Metrics, Spawner, Storage}; +use commonware_runtime::{BufferPooler, Clock, Handle, Metrics, Spawner, Storage}; use commonware_storage::queue; -use commonware_utils::sync::Mutex; +use commonware_utils::{sync::Mutex, NZU64}; use std::{future::Future, num::NonZeroUsize, sync::Arc, time::Duration}; use thiserror::Error; @@ -21,7 +21,7 @@ use backfiller::{SharedState, State}; pub(crate) use metrics::{DkgUploadStatus, IndexerMetrics}; #[cfg(test)] pub(crate) use metrics::{ - BackfillDecision, BackfillPhase, BackfillResetReason, BackfillWaitReason, BlockMetricSource, + BackfillDecision, BackfillPhase, BackfillWaitReason, BlockMetricSource, HttpArtifact, LiveUploadArtifact, ProducerActivity, ProducerStatus, QueueReadSource, QueueStatus, SharedCacheSource, SharedRetentionReason, SharedStateSnapshot, }; @@ -32,6 +32,64 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const MISSING_FINALIZATION_GRACE: Duration = Duration::from_secs(120); const MISMATCHED_FINALIZATION_GRACE: Duration = Duration::from_secs(15); +pub(crate) const SPOOL_ITEMS_PER_SECTION: std::num::NonZeroU64 = NZU64!(128); + +/// Availability window for durable finalized uploads while the external indexer is unavailable. +#[derive(Clone, Copy, Debug)] +pub struct SpoolLimits { + pub max_entries: u64, + pub max_bytes: u64, + pub max_payload_bytes: u64, + pub max_age: Duration, +} + +impl Default for SpoolLimits { + fn default() -> Self { + Self { + max_entries: crate::BLOCKS_PER_EPOCH.get(), + max_bytes: 2 * 1024 * 1024 * 1024, + max_payload_bytes: 16 * 1024 * 1024, + max_age: Duration::from_secs(24 * 60 * 60), + } + } +} + +impl SpoolLimits { + fn validate(self) { + assert!(self.max_entries > 0, "indexer spool max_entries must be non-zero"); + assert!(self.max_bytes > 0, "indexer spool max_bytes must be non-zero"); + assert!( + self.max_payload_bytes > 0 && self.max_payload_bytes <= self.max_bytes, + "indexer spool max_payload_bytes must be in 1..=max_bytes" + ); + assert!(!self.max_age.is_zero(), "indexer spool max_age must be non-zero"); + self.max_encoded_payload_bytes() + .expect("indexer spool encoded payload bound overflows u64"); + } + + pub fn max_encoded_payload_bytes(self) -> Option { + (SPOOL_ITEMS_PER_SECTION.get() - 1) + .checked_mul(self.max_payload_bytes)? + .checked_add(self.max_bytes) + } +} + +#[cfg(test)] +mod spool_tests { + use super::*; + + #[test] + fn default_spool_bound_includes_section_slack() { + let limits = SpoolLimits::default(); + assert_eq!(limits.max_entries, crate::BLOCKS_PER_EPOCH.get()); + assert_eq!(limits.max_bytes, 2 * 1024 * 1024 * 1024); + assert_eq!(limits.max_age, Duration::from_secs(24 * 60 * 60)); + assert_eq!( + limits.max_encoded_payload_bytes(), + Some(limits.max_bytes + 127 * limits.max_payload_bytes) + ); + } +} /// Errors returned by the HTTP indexer client. #[derive(Debug, Error)] @@ -184,11 +242,13 @@ pub(crate) struct Config { pub(crate) mailbox_size: NonZeroUsize, pub(crate) backfiller_max_active: NonZeroUsize, pub(crate) backfiller_retry: Duration, + pub(crate) spool_limits: SpoolLimits, pub(crate) metrics: IndexerMetrics, } pub(crate) struct Indexer { producer: Producer, + producer_handle: Handle<()>, pusher: Pusher, consumer: Consumer, } @@ -205,8 +265,10 @@ impl Indexer Indexer (Producer, Pusher, Consumer) { + pub(crate) fn split(self) -> (Producer, Handle<()>, Pusher, Consumer) { let Self { producer, + producer_handle, pusher, consumer, } = self; - (producer, pusher, consumer) + (producer, producer_handle, pusher, consumer) } } diff --git a/examples/coins/chain/src/lib.rs b/examples/coins/chain/src/lib.rs index b034460..359f7f8 100644 --- a/examples/coins/chain/src/lib.rs +++ b/examples/coins/chain/src/lib.rs @@ -19,6 +19,7 @@ pub mod application; pub mod engine; pub mod execution; pub mod genesis; +pub(crate) mod history; pub mod indexer; pub mod rpc; pub mod runtime; diff --git a/examples/coins/chain/src/testnet.rs b/examples/coins/chain/src/testnet.rs index a1fc1ea..feaf62a 100644 --- a/examples/coins/chain/src/testnet.rs +++ b/examples/coins/chain/src/testnet.rs @@ -10,7 +10,7 @@ use crate::{ channels, engine::{Config as EngineConfig, Engine}, genesis::{ChainGenesis, GenesisError}, - indexer, rpc, PublicKey, NAMESPACE, + indexer, rpc, PublicKey, BLOCKS_PER_EPOCH, NAMESPACE, }; use commonware_codec::{Decode, DecodeExt, Encode, EncodeSize}; use commonware_consensus::marshal; @@ -29,7 +29,7 @@ use commonware_p2p::{ use commonware_runtime::{ tokio, Clock as _, Handle, Runner as _, Spawner as _, Strategizer as _, Supervisor as _, }; -use commonware_utils::{ordered::Set, N3f1, NZUsize, NZU32}; +use commonware_utils::{ordered::Set, Hostname, N3f1, NZUsize, NZU32}; use governor::Quota; use nunchi_dkg::{ ContinueOnUpdate, PeerConfig, Storage as DkgStorage, StorageKey, StorageProtector, @@ -48,7 +48,6 @@ use std::{ }; use tracing::{info, warn, Level}; -const FREEZER_TABLE_INITIAL_SIZE: u32 = 2u32.pow(14); const DEFAULT_MAX_BLOCK_TRANSACTIONS: usize = 4_096; const DEFAULT_MAX_MESSAGE_SIZE: u32 = 1024 * 1024; const DEFAULT_CHANNEL_BACKLOG: usize = 1024; @@ -124,7 +123,12 @@ pub struct NodeConfig { pub share: String, pub peer_config: PeerConfig, pub listen_address: SocketAddr, - pub dialable_address: SocketAddr, + /// Address advertised to peers. IP literals use socket syntax (with brackets around IPv6), + /// while DNS names use `hostname:port` without a URL scheme or path. DNS is resolved again + /// whenever a disconnected peer retries this node, so operators should use a stable hostname + /// and an appropriate TTL. + #[serde(with = "ingress_serde")] + pub dialable_address: Ingress, pub rpc_address: SocketAddr, pub metrics_address: SocketAddr, pub bootstrappers: Vec, @@ -132,6 +136,18 @@ pub struct NodeConfig { pub genesis_path: Option, #[serde(default)] pub indexer_url: Option, + /// Maximum self-contained finalized payloads retained while the indexer is unavailable. + #[serde(default = "default_indexer_spool_max_entries")] + pub indexer_spool_max_entries: u64, + /// Maximum logical encoded payload bytes retained by the indexer spool. + #[serde(default = "default_indexer_spool_max_bytes")] + pub indexer_spool_max_bytes: u64, + /// Maximum encoded size accepted for one finalized payload. + #[serde(default = "default_indexer_spool_max_payload_bytes")] + pub indexer_spool_max_payload_bytes: u64, + /// Maximum payload age before visible terminal expiry. + #[serde(default = "default_indexer_spool_max_age_seconds")] + pub indexer_spool_max_age_seconds: u64, pub consensus: ConsensusConfig, pub networking: NetworkConfig, /// Enable one-time peer QMDB state sync for a fresh joining node. @@ -152,10 +168,83 @@ impl NodeConfig { } } +fn default_indexer_spool_max_entries() -> u64 { + indexer::SpoolLimits::default().max_entries +} + +fn default_indexer_spool_max_bytes() -> u64 { + indexer::SpoolLimits::default().max_bytes +} + +fn default_indexer_spool_max_payload_bytes() -> u64 { + indexer::SpoolLimits::default().max_payload_bytes +} + +fn default_indexer_spool_max_age_seconds() -> u64 { + indexer::SpoolLimits::default().max_age.as_secs() +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct BootstrapperConfig { pub public_key: String, - pub address: SocketAddr, + /// Address used to dial the bootstrapper. IP literals use socket syntax (with brackets around + /// IPv6), while DNS names use `hostname:port` without a URL scheme or path. DNS is resolved on + /// each reconnect attempt rather than continuously while a connection is healthy. + #[serde(with = "ingress_serde")] + pub address: Ingress, +} + +mod ingress_serde { + use super::*; + use serde::{de::Error as _, Deserializer, Serializer}; + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + if let Ok(address) = SocketAddr::from_str(&value) { + return Ok(Ingress::Socket(address)); + } + if value.contains("://") { + return Err(D::Error::custom( + "peer address must not contain a URL scheme or path", + )); + } + + let (host, port) = value + .rsplit_once(':') + .ok_or_else(|| D::Error::custom("peer address is missing a port"))?; + if host.is_empty() { + return Err(D::Error::custom("peer address is missing a host")); + } + if port.is_empty() { + return Err(D::Error::custom("peer address is missing a port")); + } + if host.contains(':') { + return Err(D::Error::custom( + "IPv6 peer addresses must use bracketed socket syntax", + )); + } + let port = port + .parse::() + .map_err(|error| D::Error::custom(format!("invalid peer address port: {error}")))?; + let host = Hostname::new(host) + .map_err(|error| D::Error::custom(format!("invalid peer address hostname: {error}")))?; + Ok(Ingress::Dns { host, port }) + } + + pub fn serialize(ingress: &Ingress, serializer: S) -> Result + where + S: Serializer, + { + match ingress { + Ingress::Socket(address) => serializer.serialize_str(&address.to_string()), + Ingress::Dns { host, port } => { + serializer.serialize_str(&format!("{host}:{port}")) + } + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -284,14 +373,14 @@ pub fn generate_local_testnet(config: LocalTestnetConfig) -> Result Result, Error>>()?; @@ -328,6 +417,10 @@ pub fn generate_local_testnet(config: LocalTestnetConfig) -> Result(&bootstrapper.public_key, "bootstrapper.public_key")?, - Ingress::from(bootstrapper.address), + bootstrapper.address.clone(), )) }) .collect::, Error>>()?; @@ -486,6 +579,7 @@ async fn start_node( node = %config.name, public_key = %public_key, listen = %config.listen_address, + dialable = ?config.dialable_address, rpc = %config.rpc_address, metrics = %config.metrics_address, "starting coins-chain validator" @@ -495,7 +589,7 @@ async fn start_node( private_key.clone(), NAMESPACE, config.listen_address, - config.dialable_address, + config.dialable_address.clone(), bootstrappers, config.networking.max_message_size, ); @@ -541,13 +635,12 @@ async fn start_node( blocker: oracle.clone(), manager: oracle.clone(), partition_prefix: config.name.clone(), - blocks_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, - finalized_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, signer: private_key, dkg_storage_key, output, share: Some(share), peer_config: config.peer_config.clone(), + epoch_length: BLOCKS_PER_EPOCH, leader_timeout: Duration::from_millis(config.consensus.leader_timeout_ms), certification_timeout: Duration::from_millis(config.consensus.certification_timeout_ms), strategy: context @@ -557,6 +650,12 @@ async fn start_node( pool_config: PoolConfig::default(), genesis: read_genesis(config.genesis_path.as_ref())?, indexer: indexer_client.map(|(client, _metrics)| client), + indexer_spool_limits: indexer::SpoolLimits { + max_entries: config.indexer_spool_max_entries, + max_bytes: config.indexer_spool_max_bytes, + max_payload_bytes: config.indexer_spool_max_payload_bytes, + max_age: Duration::from_secs(config.indexer_spool_max_age_seconds), + }, }; let dkg_callback: Box> = ContinueOnUpdate::boxed(); diff --git a/examples/coins/chain/src/tests/indexer.rs b/examples/coins/chain/src/tests/indexer.rs index b26e49e..d26399b 100644 --- a/examples/coins/chain/src/tests/indexer.rs +++ b/examples/coins/chain/src/tests/indexer.rs @@ -1,6 +1,6 @@ use crate::{ indexer::{ - BackfillDecision, BackfillPhase, BackfillResetReason, BackfillWaitReason, + BackfillDecision, BackfillPhase, BackfillWaitReason, BlockMetricSource, DkgUploadStatus, HttpArtifact, IndexerMetrics, LiveUploadArtifact, ProducerActivity, ProducerStatus, QueueReadSource, QueueStatus, SharedCacheSource, SharedRetentionReason, @@ -346,10 +346,13 @@ fn shared_state_metrics_use_expected_labels() { uploaded_digests: 4, latest_finalized_height: 8, acked_through_height: 6, + spool_entries: 5, + spool_logical_bytes: 512, + spool_oldest_height: 2, + spool_oldest_enqueued_at_millis: 1_000, }); metrics.shared_cache_inserted(SharedCacheSource::ProducerRecord); metrics.shared_cache_inserted(SharedCacheSource::LiveCertificate); - metrics.shared_cache_inserted(SharedCacheSource::ConsumerMarshal); metrics.shared_cache_removed(SharedRetentionReason::Uploaded); metrics.shared_cache_removed(SharedRetentionReason::CertificateFinished); metrics.shared_cache_removed(SharedRetentionReason::Pruned); @@ -373,9 +376,6 @@ fn shared_state_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_shared_cache_insert_total{source=\"live_certificate\"} 1", )); - assert!(encoded.contains( - "indexer_shared_cache_insert_total{source=\"consumer_marshal\"} 1", - )); assert!(encoded.contains( "indexer_shared_cache_remove_total{reason=\"uploaded\"} 1", )); @@ -402,7 +402,6 @@ fn block_metrics_use_expected_sources() { metrics.observe_block(BlockMetricSource::ProducerRecord, &block); metrics.observe_block(BlockMetricSource::LiveCertificate, &block); metrics.observe_block(BlockMetricSource::ConsumerCached, &block); - metrics.observe_block(BlockMetricSource::ConsumerMarshal, &block); let encoded = context.encode(); assert!(encoded.contains("indexer_block_estimated_bytes_bucket")); @@ -410,7 +409,6 @@ fn block_metrics_use_expected_sources() { assert!(encoded.contains("source=\"producer_record\"")); assert!(encoded.contains("source=\"live_certificate\"")); assert!(encoded.contains("source=\"consumer_cached\"")); - assert!(encoded.contains("source=\"consumer_marshal\"")); }); } @@ -433,20 +431,11 @@ fn backfill_metrics_use_expected_labels() { upload.skipped(); } metrics.backfill_decision(BackfillDecision::Skip, BackfillPhase::Start); - metrics.backfill_decision(BackfillDecision::Wait, BackfillPhase::BeforeBlock); metrics.backfill_decision(BackfillDecision::Proceed, BackfillPhase::BeforeAttempt); metrics.backfill_retry(BackfillWaitReason::CertificateUpload); - metrics.backfill_retry(BackfillWaitReason::MissingBlock); - metrics.backfill_retry(BackfillWaitReason::MissingFinalization); - metrics.backfill_retry(BackfillWaitReason::MismatchedFinalization); metrics.backfill_retry(BackfillWaitReason::HttpError); metrics.backfill_waited(BackfillWaitReason::CertificateUpload, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MissingBlock, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MissingFinalization, Duration::from_millis(1)); - metrics.backfill_waited(BackfillWaitReason::MismatchedFinalization, Duration::from_millis(1)); metrics.backfill_waited(BackfillWaitReason::HttpError, Duration::from_millis(1)); - metrics.backfill_queue_reset(BackfillResetReason::MissingFinalization, 32, 31); - metrics.backfill_queue_reset(BackfillResetReason::MismatchedFinalization, 2, 1); metrics.queue_enqueued(QueueStatus::Success); metrics.queue_enqueued(QueueStatus::Failure); metrics.queue_acked(QueueStatus::Success); @@ -486,9 +475,6 @@ fn backfill_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_backfill_decision_total{decision=\"skip\",phase=\"start\"} 1", )); - assert!(encoded.contains( - "indexer_backfill_decision_total{decision=\"wait\",phase=\"before_block\"} 1", - )); assert!(encoded.contains( "indexer_backfill_decision_total{decision=\"proceed\",phase=\"before_attempt\"} 1", )); @@ -496,20 +482,7 @@ fn backfill_metrics_use_expected_labels() { assert!(encoded.contains( "indexer_backfill_retry_total{reason=\"certificate_upload\"} 1", )); - assert!(encoded.contains("indexer_backfill_retry_total{reason=\"missing_block\"} 1")); - assert!(encoded - .contains("indexer_backfill_retry_total{reason=\"missing_finalization\"} 1")); - assert!(encoded - .contains("indexer_backfill_retry_total{reason=\"mismatched_finalization\"} 1")); assert!(encoded.contains("indexer_backfill_retry_total{reason=\"http_error\"} 1")); - assert!(encoded.contains( - "indexer_backfill_queue_reset_total{reason=\"missing_finalization\"} 1", - )); - assert!(encoded.contains( - "indexer_backfill_queue_reset_total{reason=\"mismatched_finalization\"} 1", - )); - assert!(encoded.contains("indexer_backfill_queue_reset_abandoned_entries_bucket")); - assert!(encoded.contains("indexer_backfill_queue_reset_abandoned_height_span_bucket")); assert!(encoded.contains("indexer_backfill_active_block_estimated_bytes 0")); assert!(encoded.contains("indexer_backfill_active_body_estimated_bytes 0")); assert!(encoded.contains("indexer_queue_enqueue_total{status=\"success\"} 1")); diff --git a/examples/coins/chain/src/tests/testnet.rs b/examples/coins/chain/src/tests/testnet.rs index ac240e4..cd701fb 100644 --- a/examples/coins/chain/src/tests/testnet.rs +++ b/examples/coins/chain/src/tests/testnet.rs @@ -5,16 +5,24 @@ //! as `narae` consume. [`run_node`] boots a single validator from one of those configs on the //! tokio runtime with authenticated peer discovery, and serves the aggregated JSON-RPC module. -use commonware_cryptography::{bls12381::primitives::group, ed25519}; +use commonware_cryptography::{bls12381::primitives::group, ed25519, Signer as _}; +use commonware_macros::select; +use commonware_p2p::{ + authenticated::discovery::{self, Network}, + Ingress, Manager, Receiver as _, Recipients, Sender as _, +}; +use commonware_runtime::{deterministic, Clock as _, Quota, Runner as _, Supervisor as _}; +use commonware_utils::{ordered::Set, Hostname, NZU32}; use std::collections::HashSet; use std::{ fs, - net::{IpAddr, Ipv4Addr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, num::NonZeroU32, path::PathBuf, + time::Duration, }; -use crate::testnet::*; +use crate::{testnet::*, NAMESPACE}; #[test] fn generated_testnet_has_unique_ports_dirs_and_complete_peer_sets() { @@ -124,17 +132,20 @@ fn generated_testnet_can_advertise_remote_hosts() { let first = NodeConfig::read(&manifest.nodes[0].config_path).expect("read first config"); let second = NodeConfig::read(&manifest.nodes[1].config_path).expect("read second config"); + let first_raw = fs::read_to_string(&manifest.nodes[0].config_path).expect("read first config"); assert_eq!(first.listen_address.ip(), IpAddr::V4(Ipv4Addr::UNSPECIFIED)); - assert_eq!(first.dialable_address.ip(), public_ips[0]); - assert_eq!(first.bootstrappers[0].address.ip(), public_ips[1]); + assert_eq!(first.dialable_address.ip(), Some(public_ips[0])); + assert_eq!(first.bootstrappers[0].address.ip(), Some(public_ips[1])); + assert!(first_raw.contains("dialable_address = \"192.0.2.10:30000\"")); + assert!(first_raw.contains("address = \"192.0.2.11:30001\"")); assert_eq!(first.storage_dir, storage_dir); assert_eq!( first.indexer_url.as_deref(), Some("https://indexer.example.com/coins-chain") ); - assert_eq!(second.dialable_address.ip(), public_ips[1]); - assert_eq!(second.bootstrappers[0].address.ip(), public_ips[0]); + assert_eq!(second.dialable_address.ip(), Some(public_ips[1])); + assert_eq!(second.bootstrappers[0].address.ip(), Some(public_ips[0])); assert_eq!( second.indexer_url.as_deref(), Some("https://indexer.example.com/coins-chain") @@ -142,3 +153,454 @@ fn generated_testnet_can_advertise_remote_hosts() { let _ = fs::remove_dir_all(dir); } + +#[test] +fn peer_addresses_round_trip_dns_and_ipv6_as_strings() { + let dir = std::env::temp_dir().join(format!( + "coins-chain-address-round-trip-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 2, + base_port: 32_000, + base_rpc_port: 33_000, + base_metrics_port: 34_000, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed: 9, + }) + .expect("generate testnet"); + let path = &manifest.nodes[0].config_path; + let mut config = NodeConfig::read(path).expect("read generated config"); + config.dialable_address = Ingress::Dns { + host: Hostname::new("validator-0.example.com").unwrap(), + port: 30_000, + }; + config.bootstrappers[0].address = Ingress::Dns { + host: Hostname::new("validator-1.example.com").unwrap(), + port: 30_001, + }; + config.write(path).expect("write DNS config"); + + let raw = fs::read_to_string(path).expect("read DNS config"); + assert!(raw.contains("dialable_address = \"validator-0.example.com:30000\"")); + assert!(raw.contains("address = \"validator-1.example.com:30001\"")); + let decoded = NodeConfig::read(path).expect("read DNS config"); + assert_eq!(decoded.dialable_address, config.dialable_address); + assert_eq!( + decoded.bootstrappers[0].address, + config.bootstrappers[0].address + ); + + config.dialable_address = Ingress::Socket(SocketAddr::new( + IpAddr::V6("2001:db8::10".parse::().unwrap()), + 30_000, + )); + config.write(path).expect("write IPv6 config"); + let raw = fs::read_to_string(path).expect("read IPv6 config"); + assert!(raw.contains("dialable_address = \"[2001:db8::10]:30000\"")); + assert_eq!( + NodeConfig::read(path) + .expect("read IPv6 config") + .dialable_address, + config.dialable_address + ); + + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn invalid_peer_addresses_are_rejected_through_node_config_read() { + let dir = std::env::temp_dir().join(format!( + "coins-chain-invalid-addresses-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 1, + base_port: 35_000, + base_rpc_port: 36_000, + base_metrics_port: 37_000, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed: 10, + }) + .expect("generate testnet"); + let valid_path = &manifest.nodes[0].config_path; + let valid = fs::read_to_string(valid_path).expect("read generated config"); + let valid_address = "dialable_address = \"127.0.0.1:35000\""; + assert!(valid.contains(valid_address)); + + let cases = [ + ("validator.example.com", "missing a port"), + ("validator.example.com:not-a-port", "invalid peer address port"), + ("validator.example.com:65536", "invalid peer address port"), + ("-validator.example.com:30000", "invalid peer address hostname"), + ( + "https://validator.example.com:30000", + "must not contain a URL scheme or path", + ), + ("2001:db8::10:30000", "must use bracketed socket syntax"), + ]; + for (index, (address, expected)) in cases.into_iter().enumerate() { + let path = dir.join(format!("invalid-{index}.toml")); + let invalid = valid.replace( + valid_address, + &format!("dialable_address = \"{address}\""), + ); + fs::write(&path, invalid).expect("write invalid config"); + let error = NodeConfig::read(&path).expect_err("invalid address should fail"); + assert!( + error.to_string().contains(expected), + "unexpected error for {address}: {error}" + ); + } + + let _ = fs::remove_dir_all(dir); +} + +fn parse_dns_address_through_node_config(address: &str, seed: u64) -> Ingress { + let dir = std::env::temp_dir().join(format!( + "coins-chain-parse-dns-{seed}-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let manifest = generate_local_testnet(LocalTestnetConfig { + validators: 1, + base_port: 38_000, + base_rpc_port: 38_100, + base_metrics_port: 38_200, + base_data_dir: dir.clone(), + bind_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + public_ips: None, + storage_dir: None, + genesis_path: None, + indexer_url: None, + seed, + }) + .expect("generate parser fixture"); + let path = &manifest.nodes[0].config_path; + let valid = fs::read_to_string(path).expect("read parser fixture"); + let dns = valid.replace( + "dialable_address = \"127.0.0.1:38000\"", + &format!("dialable_address = \"{address}\""), + ); + fs::write(path, dns).expect("write parser fixture"); + let ingress = NodeConfig::read(path) + .expect("parse DNS address") + .dialable_address; + let _ = fs::remove_dir_all(dir); + ingress +} + +fn reconnect_config( + key: ed25519::PrivateKey, + listen: SocketAddr, + dialable: Ingress, + bootstrappers: Vec<(ed25519::PublicKey, Ingress)>, +) -> discovery::Config { + let mut config = discovery::Config::local( + key, + NAMESPACE, + listen, + dialable, + bootstrappers, + 1024 * 1024, + ); + config.peer_connection_cooldown = Duration::from_millis(100); + config.dial_frequency = Duration::from_millis(50); + config.gossip_bit_vec_frequency = Duration::from_millis(100); + config +} + +fn run_dns_bootstrapper_redeployment(seed: u64, dns: Ingress) -> String { + let runner = deterministic::Runner::new( + deterministic::Config::new() + .with_seed(seed) + .with_timeout(Some(Duration::from_secs(20))), + ); + runner.start(|context| async move { + let key_a = ed25519::PrivateKey::from_seed(100); + let key_b = ed25519::PrivateKey::from_seed(101); + let public_a = key_a.public_key(); + let public_b = key_b.public_key(); + let peers = Set::try_from(vec![public_a.clone(), public_b.clone()]).unwrap(); + let socket_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 10)), 39_000); + let socket_b1 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 11)), 39_000); + let socket_b2 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 12)), 39_000); + let unreachable_a = Ingress::Socket(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 0, 250)), + 49_000, + )); + context.resolver_register("validator-b-bootstrap.test", Some(vec![socket_b1.ip()])); + + let config_b = reconnect_config(key_b.clone(), socket_b1, dns.clone(), vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_initial").child("network"), config_b); + oracle_b.track(0, peers.clone()); + let (mut sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let handle_b = network_b.start(); + + let config_a = reconnect_config( + key_a, + socket_a, + unreachable_a, + vec![(public_b.clone(), dns.clone())], + ); + assert!(config_a.allow_dns); + let (mut network_a, mut oracle_a) = + Network::new(context.child("a").child("network"), config_a); + oracle_a.track(0, peers.clone()); + let (mut sender_a, mut receiver_a) = + network_a.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_a = network_a.start(); + + let initial_exchange = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-before".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + loop { + sender_b.send( + Recipients::One(public_a.clone()), + b"b-before".to_vec(), + true, + ); + select! { + result = receiver_a.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_b); + assert_eq!(message.as_ref(), b"b-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = initial_exchange => {}, + _ = context.sleep(Duration::from_secs(5)) => panic!("initial DNS connection timed out"), + } + + handle_b.abort(); + drop(sender_b); + drop(receiver_b); + context.resolver_register("validator-b-bootstrap.test", Some(vec![socket_b2.ip()])); + + // The restarted peer has no bootstrappers and A advertises an unreachable address, so B + // cannot initiate the replacement connection. + let config_b = reconnect_config(key_b, socket_b2, dns, vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_restarted").child("network"), config_b); + oracle_b.track(0, peers); + let (_sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_b = network_b.start(); + + let reconnected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-after".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-after"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = reconnected => {}, + _ = context.sleep(Duration::from_secs(5)) => panic!("DNS bootstrapper did not reconnect"), + } + context.auditor().state() + }) +} + +#[test] +fn dns_bootstrapper_is_resolved_again_after_redeployment() { + let dns = parse_dns_address_through_node_config("validator-b-bootstrap.test:39000", 11); + let first = run_dns_bootstrapper_redeployment(42, dns.clone()); + let second = run_dns_bootstrapper_redeployment(42, dns); + assert_eq!(first, second); +} + +fn run_discovered_dns_redeployment(seed: u64, dns: Ingress) -> String { + let runner = deterministic::Runner::new( + deterministic::Config::new() + .with_seed(seed) + .with_timeout(Some(Duration::from_secs(20))), + ); + runner.start(|context| async move { + let key_a = ed25519::PrivateKey::from_seed(200); + let key_b = ed25519::PrivateKey::from_seed(201); + let key_c = ed25519::PrivateKey::from_seed(202); + let public_a = key_a.public_key(); + let public_b = key_b.public_key(); + let public_c = key_c.public_key(); + let peers = Set::try_from(vec![ + public_a.clone(), + public_b.clone(), + public_c.clone(), + ]) + .unwrap(); + let socket_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 10)), 40_000); + let socket_b1 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 11)), 40_000); + let socket_b2 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 12)), 40_000); + let socket_c = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 13)), 40_000); + let unreachable_a = Ingress::Socket(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 1, 250)), + 50_000, + )); + context.resolver_register("validator-b-discovered.test", Some(vec![socket_b1.ip()])); + + let config_c = reconnect_config(key_c, socket_c, socket_c.into(), vec![]); + let (mut network_c, mut oracle_c) = + Network::new(context.child("c").child("network"), config_c); + oracle_c.track(0, peers.clone()); + let (_sender_c, _receiver_c) = + network_c.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_c = network_c.start(); + + let config_b = reconnect_config( + key_b.clone(), + socket_b1, + dns.clone(), + vec![(public_c.clone(), socket_c.into())], + ); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_initial").child("network"), config_b); + oracle_b.track(0, peers.clone()); + let (mut sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let handle_b = network_b.start(); + + // A only knows C initially. It must learn B's DNS ingress through discovery. + let config_a = reconnect_config( + key_a, + socket_a, + unreachable_a, + vec![(public_c, socket_c.into())], + ); + let (mut network_a, mut oracle_a) = + Network::new(context.child("a").child("network"), config_a); + oracle_a.track(0, peers.clone()); + let (mut sender_a, mut receiver_a) = + network_a.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_a = network_a.start(); + + let learned_and_connected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-before".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + loop { + sender_b.send( + Recipients::One(public_a.clone()), + b"b-before".to_vec(), + true, + ); + select! { + result = receiver_a.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_b); + assert_eq!(message.as_ref(), b"b-before"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = learned_and_connected => {}, + _ = context.sleep(Duration::from_secs(7)) => panic!("A did not discover B's DNS ingress"), + } + + handle_b.abort(); + drop(sender_b); + drop(receiver_b); + context.resolver_register("validator-b-discovered.test", Some(vec![socket_b2.ip()])); + + // B has no bootstrapper after restart and A's advertised address is unreachable. A must + // retain the discovered hostname, resolve it again, and dial B at its replacement IP. + let config_b = reconnect_config(key_b, socket_b2, dns, vec![]); + let (mut network_b, mut oracle_b) = + Network::new(context.child("b_restarted").child("network"), config_b); + oracle_b.track(0, peers); + let (_sender_b, mut receiver_b) = + network_b.register(0, Quota::per_second(NZU32!(100)), 128); + let _handle_b = network_b.start(); + + let reconnected = async { + loop { + sender_a.send( + Recipients::One(public_b.clone()), + b"a-after".to_vec(), + true, + ); + select! { + result = receiver_b.recv() => { + let (from, message) = result.unwrap(); + assert_eq!(from, public_a); + assert_eq!(message.as_ref(), b"a-after"); + break; + }, + _ = context.sleep(Duration::from_millis(50)) => {}, + } + } + }; + select! { + _ = reconnected => {}, + _ = context.sleep(Duration::from_secs(7)) => panic!("discovered DNS peer did not reconnect"), + } + context.auditor().state() + }) +} + +#[test] +fn discovered_dns_address_is_resolved_again_after_redeployment() { + let dns = parse_dns_address_through_node_config("validator-b-discovered.test:40000", 12); + let first = run_discovered_dns_redeployment(84, dns.clone()); + let second = run_discovered_dns_redeployment(84, dns); + assert_eq!(first, second); +} diff --git a/examples/coins/chain/tests/coins.rs b/examples/coins/chain/tests/coins.rs index 9051289..da9ec09 100644 --- a/examples/coins/chain/tests/coins.rs +++ b/examples/coins/chain/tests/coins.rs @@ -8,7 +8,8 @@ use commonware_cryptography::Signer as _; use commonware_cryptography::{Hasher, Sha256}; use commonware_macros::{select, test_traced}; use commonware_p2p::simulated::Link; -use commonware_runtime::{deterministic, Clock, Runner as _}; +use commonware_runtime::{deterministic, Clock, Runner as _, Spawner as _, Supervisor as _}; +use commonware_utils::NZU64; use nunchi_authority::{ proposal_id, AuthorityOperation, MultisigPolicy, RegistryChange, Transaction as AuthorityTransaction, @@ -104,6 +105,56 @@ fn reaches_height_100() { }); } +#[test_traced] +fn crosses_epoch_boundary_and_reclaims_retired_partition() { + with_large_stack(|| { + let executor = deterministic::Runner::timed(Duration::from_secs(60)); + executor.start(|mut context| async move { + let cfg = ValidatorConfig { + epoch_length: NZU64!(20), + ..ValidatorConfig::default() + }; + let mut network = TestNetworkBuilder::new(VALIDATORS) + .with_initial_link(reliable_link()) + .with_validator_config(cfg) + .build(&mut context) + .await; + network.start_all().await; + + let validator_task_prefix = "validator"; + assert!(network.running_tasks(validator_task_prefix) > 0); + + network.run_until_height(25).await; + + network.assert_validator_metric("engine_orchestrator_latest_epoch", &[], 1); + network.assert_validator_metric( + "engine_orchestrator_consensus_partitions_active", + &[], + 1, + ); + network.assert_validator_metric( + "engine_orchestrator_consensus_partition_cleanup_watermark", + &[], + 1, + ); + network.assert_validator_metric( + "engine_orchestrator_consensus_partition_cleanup_total", + &[("status", "removed")], + 1, + ); + network.assert_consensus_partition_missing(0).await; + assert!(network.running_tasks(validator_task_prefix) > 0); + + let shutdown = network.context().child("shutdown"); + shutdown + .stop(0, Some(Duration::from_secs(10))) + .await + .expect("validator tasks should stop cleanly"); + assert_eq!(network.running_tasks(validator_task_prefix), 0); + }); + }); +} + #[test_traced] fn state_syncs_late_validator() { with_large_stack(|| { @@ -154,6 +205,7 @@ fn recovers_unclean_shutdown() { let cfg = ValidatorConfig { leader_timeout: Duration::from_millis(250), certification_timeout: Duration::from_millis(500), + ..ValidatorConfig::default() }; let wait = diff --git a/examples/coins/chain/tests/common/network.rs b/examples/coins/chain/tests/common/network.rs index 04495cb..21d473f 100644 --- a/examples/coins/chain/tests/common/network.rs +++ b/examples/coins/chain/tests/common/network.rs @@ -15,7 +15,7 @@ use commonware_p2p::{ use commonware_parallel::Sequential; use commonware_runtime::{ deterministic::{self, Runner}, - Clock, Metrics, Runner as _, Supervisor, + Clock, Error as RuntimeError, Metrics, Runner as _, Storage, Supervisor, }; use commonware_utils::{ ordered::{Map, Set}, @@ -27,7 +27,7 @@ use nunchi_coins::{Address, Ledger}; use nunchi_coins_chain::{ engine::{Config, Engine}, execution::NodeHandle, - PublicKey, Transaction, + PublicKey, Transaction, BLOCKS_PER_EPOCH, }; use nunchi_common::QmdbReader; use nunchi_dkg::{ContinueOnUpdate, PeerConfig}; @@ -35,10 +35,10 @@ use nunchi_mempool::{MempoolHandle, PoolConfig}; use nunchi_oracle::OracleLedger; use std::{ collections::{HashMap, HashSet}, + num::NonZeroU64, time::Duration, }; -const FREEZER_TABLE_INITIAL_SIZE: u32 = 2u32.pow(14); // 1MB const TEST_QUOTA: Quota = Quota::per_second(NZU32!(u32::MAX)); const MAX_BLOCK_TRANSACTIONS: usize = 256; @@ -109,6 +109,7 @@ pub(crate) fn lossy_link() -> Link { #[derive(Clone)] pub(crate) struct ValidatorConfig { + pub(crate) epoch_length: NonZeroU64, pub(crate) leader_timeout: Duration, pub(crate) certification_timeout: Duration, } @@ -116,6 +117,7 @@ pub(crate) struct ValidatorConfig { impl Default for ValidatorConfig { fn default() -> Self { Self { + epoch_length: BLOCKS_PER_EPOCH, leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(2), } @@ -356,6 +358,83 @@ impl TestNetwork<'_> { } } + pub(crate) fn assert_validator_metric( + &self, + suffix: &str, + required_labels: &[(&str, &str)], + expected_value: u64, + ) { + let expected = self.started_validator_ids(); + let mut observed = HashSet::new(); + let metrics = self.context.encode(); + for line in metrics.lines() { + let Some((metric, labels, value)) = validator_metric_sample(line) else { + continue; + }; + if !metric.ends_with(suffix) + || !required_labels + .iter() + .all(|(name, expected)| metric_label(labels, name) == Some(*expected)) + { + continue; + } + let Some(id) = metric_label(labels, "id") else { + continue; + }; + if !expected.contains(id) { + continue; + } + assert_eq!( + value.parse::().unwrap(), + expected_value, + "unexpected {suffix} value for {id}: {line}", + ); + assert!( + observed.insert(id.to_string()), + "duplicate {suffix} for {id}" + ); + } + assert_eq!( + observed, expected, + "missing {suffix} samples in metrics: {metrics}", + ); + } + + pub(crate) async fn assert_consensus_partition_missing(&self, epoch: u64) { + for signer in &self.private_keys { + let public_key = signer.public_key(); + if self.registrations.contains_key(&public_key) { + continue; + } + let partition = format!("validator_{public_key}_consensus_consensus_{epoch}"); + let result = self.context.scan(&partition).await; + assert!( + matches!( + result, + Err(RuntimeError::PartitionMissing(ref missing)) if missing == &partition + ), + "retired consensus partition {partition} still exists: {result:?}", + ); + } + } + + pub(crate) fn running_tasks(&self, prefix: &str) -> usize { + self.context + .encode() + .lines() + .filter_map(|line| { + if !line.starts_with("runtime_tasks_running{") { + return None; + } + let name = line.split("name=\"").nth(1)?.split('"').next()?; + if !name.starts_with(prefix) { + return None; + } + line.rsplit(' ').next()?.parse::().ok() + }) + .sum() + } + fn started_validator_ids(&self) -> HashSet { self.private_keys .iter() @@ -512,13 +591,12 @@ async fn start_validator( blocker: oracle.control(public_key.clone()), manager: oracle.manager(), partition_prefix: uid.clone(), - blocks_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, - finalized_freezer_table_initial_size: FREEZER_TABLE_INITIAL_SIZE, signer: signer.clone(), dkg_storage_key: [9u8; 32], output, share: Some(share), peer_config, + epoch_length: cfg.epoch_length, leader_timeout: cfg.leader_timeout, certification_timeout: cfg.certification_timeout, strategy: Sequential, @@ -527,6 +605,7 @@ async fn start_validator( pool_config: PoolConfig::default(), genesis: None, indexer: None, + indexer_spool_limits: Default::default(), }; let validator_context = context.child("validator").with_attribute("id", &uid);