diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock new file mode 100644 index 00000000000..d2c9a3c6432 --- /dev/null +++ b/.claude/scheduled_tasks.lock @@ -0,0 +1 @@ +{"sessionId":"5d11d1e0-123c-4116-9054-2646710b8c68","pid":2222350,"procStart":"11975237","acquiredAt":1780623034771} \ No newline at end of file diff --git a/.codespellrc b/.codespellrc index 74b16375709..6358b6ae3cc 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,4 +1,4 @@ [codespell] -ignore-words-list = crate,Sur,inout,Groth,groth,re-use,abl,zcash,zcashd,zebrad,zebra,utxo,utxos,nullifier,nullifiers,sapling,orchard,sprout,backpressure,proptest,thiserror,rocksdb,libsecp,fullnode,peerset,threadsafe,unrepresentable,getblocktemplate,syncer,Actix,Akka,mermaid,println,eprintln,usize,nocapture,Parallelizable,invis,UTXO,Zcash,Zaino,Zallet,librustzcash,Mainnet,Testnet,mainnet,testnet,idents,reentrancy,serializable,deserializable,deserialization,zkSNARK,zkSNARKs,lightwalletd,statics,ser,endianity,aranges +ignore-words-list = crate,Sur,inout,Groth,groth,re-use,abl,zcash,zcashd,zebrad,zebra,utxo,utxos,nullifier,nullifiers,sapling,orchard,sprout,backpressure,proptest,thiserror,rocksdb,libsecp,fullnode,peerset,threadsafe,unrepresentable,getblocktemplate,syncer,Actix,Akka,mermaid,println,eprintln,usize,nocapture,Parallelizable,invis,UTXO,Zcash,Zaino,Zallet,librustzcash,Mainnet,Testnet,mainnet,testnet,idents,reentrancy,serializable,deserializable,deserialization,zkSNARK,zkSNARKs,lightwalletd,statics,ser,endianity,aranges,daa exclude-file = book/mermaid.min.js skip = ./zebra-rpc/qa/rpc-tests,./supply-chain,./target diff --git a/CHANGELOG.md b/CHANGELOG.md index 881932c6cd2..71c026fdf70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Added bounded Zakura P2P v2 upgrade prelude and control-handshake wire types, including transcript binding, native-vs-upgraded control validation, and duplicate-peer handling scaffolding. +- Added bounded Zakura header-sync stream-5 wire messages, stateless header + validation, and the default `network.zakura.header_sync` config surface. - Include the `zebra-rollback-state` and `zebra-prune-state` utilities alongside `zebrad` in release Docker images and Docker CI builds. - Use the `5.0.0-rc.3` release identity for this fork's v5 rollback build. diff --git a/zebra-network/CHANGELOG.md b/zebra-network/CHANGELOG.md index f406d9dbccd..feccca17cc1 100644 --- a/zebra-network/CHANGELOG.md +++ b/zebra-network/CHANGELOG.md @@ -38,6 +38,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 legacy stream is dropped, so mutually capable peers move their gossip and inventory traffic onto Zakura with no configured bootstrap peers. Any neutral problem (no live endpoint, malformed/rejected prelude) falls back to legacy. +- Added Zakura header-sync stream-5 wire constants, bounded message codecs, + stateless header validation, and the default `network.zakura.header_sync` + config surface. ### Changed diff --git a/zebra-network/src/config/tests/vectors.rs b/zebra-network/src/config/tests/vectors.rs index 73ca73c619a..144efefdae6 100644 --- a/zebra-network/src/config/tests/vectors.rs +++ b/zebra-network/src/config/tests/vectors.rs @@ -1,5 +1,7 @@ //! Fixed test vectors for zebra-network configuration. +use std::time::Duration; + use static_assertions::const_assert; use zebra_chain::{ block::Height, @@ -11,6 +13,7 @@ use zebra_chain::{ use crate::{ constants::{INBOUND_PEER_LIMIT_MULTIPLIER, OUTBOUND_PEER_LIMIT_MULTIPLIER}, + zakura::{DEFAULT_HS_MAX_INFLIGHT, DEFAULT_HS_RANGE}, Config, }; @@ -161,6 +164,20 @@ fn p2p_v2_old_config_without_zakura_fields_uses_safe_defaults() { assert!(config.zakura.bootstrap_peers.is_empty()); assert!(config.zakura.max_connections > 0); assert!(config.zakura.max_pending_handshakes > 0); + assert_eq!( + config.zakura.header_sync.max_headers_per_response, + DEFAULT_HS_RANGE + ); + assert_eq!( + config.zakura.header_sync.max_inflight_requests, + DEFAULT_HS_MAX_INFLIGHT + ); + assert_eq!( + config.zakura.header_sync.status_refresh_interval, + Duration::from_secs(30) + ); + assert_eq!(config.zakura.header_sync.anchor_height, None); + assert_eq!(config.zakura.header_sync.anchor_hash, None); } #[test] @@ -185,6 +202,18 @@ fn p2p_v2_unknown_future_config_fields_are_rejected() { nested.to_string().contains("unknown field"), "unexpected error for unknown nested field: {nested}", ); + + let header_sync = toml::from_str::( + r#" + [zakura.header_sync] + future_field = true + "#, + ) + .expect_err("deny_unknown_fields rejects unknown header-sync fields"); + assert!( + header_sync.to_string().contains("unknown field"), + "unexpected error for unknown header-sync field: {header_sync}", + ); } #[test] @@ -203,6 +232,11 @@ fn p2p_v2_config_roundtrip_keeps_dconfig_zakura_fields() { stream_open_rate_per_second = 11 message_rate_per_second = 13 trace_dir = "target/zakura-test-traces" + + [zakura.header_sync] + max_headers_per_response = 333 + max_inflight_requests = 9 + status_refresh_interval = "45s" "#, ) .unwrap(); @@ -214,6 +248,10 @@ fn p2p_v2_config_roundtrip_keeps_dconfig_zakura_fields() { assert!(serialized.contains("bootstrap_peers")); assert!(serialized.contains("max_connections = 7")); assert!(serialized.contains("trace_dir = \"target/zakura-test-traces\"")); + assert!(serialized.contains("[zakura.header_sync]")); + assert!(serialized.contains("max_headers_per_response = 333")); + assert!(serialized.contains("max_inflight_requests = 9")); + assert!(serialized.contains("status_refresh_interval = \"45s\"")); assert_eq!(toml::from_str::(&serialized).unwrap(), config); } diff --git a/zebra-network/src/lib.rs b/zebra-network/src/lib.rs index 804b5b9cb8a..27faa6bdac5 100644 --- a/zebra-network/src/lib.rs +++ b/zebra-network/src/lib.rs @@ -188,7 +188,7 @@ pub use crate::{ isolated::{connect_isolated, connect_isolated_tcp_direct}, meta_addr::{PeerAddrState, PeerSocketAddr}, peer::{Client, ConnectedAddr, ConnectionInfo, HandshakeError, PeerError, SharedPeerError}, - peer_set::init, + peer_set::{init, init_with_zakura_header_sync}, policies::RetryLimit, protocol::{ external::{Version, VersionMessage, MAX_TX_INV_IN_SENT_MESSAGE}, diff --git a/zebra-network/src/peer_set.rs b/zebra-network/src/peer_set.rs index 187b1d32056..a1f5d8c8733 100644 --- a/zebra-network/src/peer_set.rs +++ b/zebra-network/src/peer_set.rs @@ -13,4 +13,4 @@ pub(crate) use limit::{ActiveConnectionCounter, ConnectionTracker}; use inventory_registry::InventoryRegistry; pub(crate) use set::PeerSet; -pub use initialize::init; +pub use initialize::{init, init_with_zakura_header_sync}; diff --git a/zebra-network/src/peer_set/initialize.rs b/zebra-network/src/peer_set/initialize.rs index ad5fd8ec50e..94e8c443caf 100644 --- a/zebra-network/src/peer_set/initialize.rs +++ b/zebra-network/src/peer_set/initialize.rs @@ -111,19 +111,27 @@ where S::Future: Send + 'static, C: ChainTip + Clone + Send + Sync + 'static, { - let (peer_set, address_book, misbehavior_tx, _zakura_endpoint) = - init_with_zakura_endpoint(config, inbound_service, latest_chain_tip, user_agent).await; + let (peer_set, address_book, misbehavior_tx, _zakura_endpoint) = init_with_zakura_header_sync( + config, + inbound_service, + latest_chain_tip, + user_agent, + advertised_services, + None, + ) + .await; (peer_set, address_book, misbehavior_tx) } -/// Initialize a peer set and expose the live Zakura endpoint (used by the -/// legacy->Zakura upgrade handshake connector). -async fn init_with_zakura_endpoint( +/// Initialize a peer set and optionally expose a real-driver Zakura header-sync endpoint. +pub async fn init_with_zakura_header_sync( config: Config, inbound_service: S, latest_chain_tip: C, user_agent: String, + advertised_services: PeerServices, + header_sync_driver_startup: Option, ) -> ( Buffer, Request>, Arc>, @@ -146,16 +154,19 @@ where // handshake builder consumes the original below. The factory only runs when // `v2_p2p` is enabled; otherwise the endpoint is `None` and the clone drops. let inbound_for_zakura_sink = inbound_service.clone(); - let zakura_endpoint = - crate::zakura::spawn_zakura_endpoint(&config, move |supervisor, trace| { + let zakura_endpoint = crate::zakura::spawn_zakura_endpoint_with_header_sync_driver( + &config, + move |supervisor, trace| { Arc::new(crate::zakura::LegacyGossipSink::spawn_with_trace( inbound_for_zakura_sink, supervisor, trace, )) as Arc - }) - .await - .expect("Zakura endpoint should start when P2P v2 is enabled"); + }, + header_sync_driver_startup, + ) + .await + .expect("Zakura endpoint should start when P2P v2 is enabled"); let ( address_book, diff --git a/zebra-network/src/zakura.rs b/zebra-network/src/zakura.rs index 82cdf1dfbb9..36b2a28b671 100644 --- a/zebra-network/src/zakura.rs +++ b/zebra-network/src/zakura.rs @@ -17,6 +17,7 @@ use crate::{ mod discovery; mod handler; mod handshake; +mod header_sync; mod legacy_gossip; #[cfg(any(test, feature = "zakura-testkit"))] pub mod testkit; @@ -26,11 +27,12 @@ pub mod transport; pub use discovery::*; pub use handler::*; pub use handshake::*; +pub use header_sync::*; pub use legacy_gossip::*; pub use trace::{ peer_label as zakura_trace_peer_label, reject_reason_label as zakura_trace_reject_reason_label, - ZakuraTrace, ZakuraTraceEvent, CONN_TABLE, HANDSHAKE_TABLE, LEGACY_REQUEST_TABLE, - RATELIMIT_TABLE, STREAM_TABLE, + ZakuraTrace, ZakuraTraceEvent, CONN_TABLE, HANDSHAKE_TABLE, HEADER_SYNC_TABLE, + LEGACY_REQUEST_TABLE, RATELIMIT_TABLE, STREAM_TABLE, }; pub use transport::*; @@ -49,6 +51,9 @@ pub const IROH_VERSION: &str = "0.92.0"; /// Capability bit for the legacy gossip compatibility service. pub const ZAKURA_CAP_LEGACY_GOSSIP: u64 = 1 << 0; +/// Capability bit for the native header-sync service. +pub const ZAKURA_CAP_HEADER_SYNC: u64 = 1 << 1; + /// Capability bit for the native discovery service. pub const ZAKURA_CAP_DISCOVERY: u64 = 1 << 2; diff --git a/zebra-network/src/zakura/handler.rs b/zebra-network/src/zakura/handler.rs index 377445268e0..a275d141eea 100644 --- a/zebra-network/src/zakura/handler.rs +++ b/zebra-network/src/zakura/handler.rs @@ -25,12 +25,12 @@ use rand::{rngs::OsRng, RngCore}; use thiserror::Error; use tokio::{ sync::{mpsc, oneshot, watch, Mutex, OwnedSemaphorePermit, Semaphore}, - task::JoinSet, + task::{JoinHandle, JoinSet}, time::{timeout, Instant}, }; use tokio_util::sync::CancellationToken; use zebra_chain::{ - block::{Block, CountedHeader}, + block::{self, Block, CountedHeader}, parameters::Network, serialization::{CompactSizeMessage, ZcashDeserialize, MAX_HEADERS_PER_MESSAGE}, transaction::Transaction, @@ -47,14 +47,19 @@ use super::{ use crate::{ protocol::external::InventoryHash, zakura::{ - direct_endpoint_builder, Clock, Frame, FramedRecv, FramedSend, Peer, RealClock, Service, - ServiceRegistry, SinkReject, Stream, StreamMode, StreamPrelude, ZakuraAcceptedLimits, - ZakuraControlAck, ZakuraControlHello, ZakuraControlRole, ZakuraControlValidation, - ZakuraHandshakeConfig, ZakuraHandshakePath, ZakuraInitialLimits, ZakuraLimits, - ZakuraPeerId, ZakuraPeerSupervisor, ZakuraProtocolError, ZakuraRejectReason, - ZakuraUpgradeOutcome, CONTROL_ACK_MAGIC, CONTROL_HELLO_MAGIC, CONTROL_VERSION, - FRAME_HEADER_BYTES, LOCAL_MAX_CONTROL_FRAME_BYTES, P2P_V2_ALPN, STREAM_PRELUDE_MAGIC, - TRANSCRIPT_HASH_BYTES, ZAKURA_PROTOCOL_VERSION_1, + direct_endpoint_builder, drive_header_sync_actions, header_sync_outbound_for_supervisor, + send_header_sync_message as send_header_sync_service_message, spawn_header_sync_reactor, + Clock, Frame, FramedRecv, FramedSend, HeaderSyncAction, HeaderSyncFrontiers, + HeaderSyncMessage, HeaderSyncPassthroughService, HeaderSyncService, HeaderSyncStartup, + Peer, RealClock, Service, ServiceRegistry, SinkReject, Stream, StreamMode, StreamPrelude, + ZakuraAcceptedLimits, ZakuraControlAck, ZakuraControlHello, ZakuraControlRole, + ZakuraControlValidation, ZakuraHandshakeConfig, ZakuraHandshakePath, + ZakuraHeaderSyncConfig, ZakuraInitialLimits, ZakuraLimits, ZakuraPeerId, + ZakuraPeerSupervisor, ZakuraProtocolError, ZakuraRejectReason, ZakuraUpgradeOutcome, + CONTROL_ACK_MAGIC, CONTROL_HELLO_MAGIC, CONTROL_VERSION, FRAME_HEADER_BYTES, + LOCAL_MAX_CONTROL_FRAME_BYTES, MAX_HS_MESSAGE_BYTES, P2P_V2_ALPN, STREAM_PRELUDE_MAGIC, + TRANSCRIPT_HASH_BYTES, ZAKURA_HEADER_SYNC_STREAM_VERSION, ZAKURA_PROTOCOL_VERSION_1, + ZAKURA_STREAM_HEADER_SYNC, }, }; use crate::{BoxError, Config, MAX_TX_INV_IN_SENT_MESSAGE}; @@ -104,6 +109,7 @@ const OUTBOUND_REQUEST_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30); const LEGACY_GOSSIP_STREAM_KIND: u16 = 2; const LEGACY_REQUEST_STREAM_KIND: u16 = 3; const DISCOVERY_STREAM_KIND: u16 = 4; +const HEADER_SYNC_STREAM_KIND: u16 = ZAKURA_STREAM_HEADER_SYNC; const LEGACY_REQUEST_BLOCKS_BY_HASH: u16 = 3; const LEGACY_REQUEST_TRANSACTIONS_BY_ID: u16 = 4; const LEGACY_RESPONSE_BLOCK: u16 = 5; @@ -130,6 +136,8 @@ const _: () = assert!(LEGACY_GOSSIP_STREAM_KIND == super::legacy_gossip::ZAKURA_ const _: () = assert!(LEGACY_REQUEST_STREAM_KIND == super::legacy_gossip::ZAKURA_STREAM_LEGACY_REQUESTS); const _: () = assert!(DISCOVERY_STREAM_KIND == super::discovery::ZAKURA_STREAM_DISCOVERY); +const _: () = assert!(HEADER_SYNC_STREAM_KIND == super::header_sync::ZAKURA_STREAM_HEADER_SYNC); +const _: () = assert!(ZAKURA_STREAM_VERSION_1 == ZAKURA_HEADER_SYNC_STREAM_VERSION); const _: () = assert!(LEGACY_REQUEST_BLOCKS_BY_HASH == super::legacy_gossip::MSG_REQUEST_BLOCKS_BY_HASH); const _: () = assert!( @@ -205,6 +213,8 @@ pub struct ZakuraConfig { /// When unset, Zakura trace emission is disabled. When set, the native /// Zakura endpoint writes the production trace schema into this directory. pub trace_dir: Option, + /// Native stream-5 header-sync wire settings. + pub header_sync: ZakuraHeaderSyncConfig, } impl Default for ZakuraConfig { @@ -217,6 +227,7 @@ impl Default for ZakuraConfig { stream_open_rate_per_second: DEFAULT_ZAKURA_STREAM_OPEN_RATE_PER_SECOND, message_rate_per_second: DEFAULT_ZAKURA_MESSAGE_RATE_PER_SECOND, trace_dir: None, + header_sync: ZakuraHeaderSyncConfig::default(), } } } @@ -362,9 +373,27 @@ pub struct ZakuraEndpoint { router: Router, supervisor: ZakuraSupervisorHandle, handler: ZakuraProtocolHandler, + header_sync: Option, + header_sync_tasks: Option>, + header_sync_actions: Option>>>>, upgrade_dials: Arc>>, } +#[derive(Debug)] +struct HeaderSyncBackgroundTasks { + shutdown: CancellationToken, + tasks: Mutex>>, +} + +/// Durable state facts required before attaching the production header-sync driver. +#[derive(Clone, Debug)] +pub struct ZakuraHeaderSyncDriverStartup { + /// Durable state frontiers loaded at node startup. + pub frontiers: HeaderSyncFrontiers, + /// Durable best header tip loaded from state. + pub best_header_tip: Option<(block::Height, block::Hash)>, +} + impl ZakuraEndpoint { /// Returns the connector injected into the legacy handshake path. pub fn connector(&self) -> super::ZakuraHandshakeConnector { @@ -398,6 +427,36 @@ impl ZakuraEndpoint { self.handler.trace.clone() } + /// Returns the header-sync handle when native header sync is active. + pub fn header_sync(&self) -> Option { + self.header_sync.clone() + } + + /// Take the header-sync action receiver when this endpoint was started in external-driver mode. + pub async fn take_header_sync_actions(&self) -> Option> { + let actions = self.header_sync_actions.as_ref()?; + actions.lock().await.take() + } + + /// Returns the endpoint-owned header-sync shutdown token. + pub fn header_sync_shutdown(&self) -> Option { + self.header_sync_tasks + .as_ref() + .map(|tasks| tasks.shutdown.clone()) + } + + /// Track a header-sync integration task under the endpoint shutdown owner. + pub async fn push_header_sync_task(&self, task: JoinHandle<()>) { + if let Some(tasks) = self.header_sync_tasks.as_ref() { + tasks.tasks.lock().await.push(task); + } + } + + /// Send a stream-5 header-sync message to a registered Zakura peer. + pub async fn send_header_sync_message(&self, peer: &ZakuraPeerId, msg: HeaderSyncMessage) { + send_header_sync_service_message(&self.supervisor, peer, msg).await; + } + /// Returns the endpoint's current direct node address. pub async fn node_addr(&self) -> NodeAddr { self.router.endpoint().node_addr().initialized().await @@ -480,6 +539,16 @@ impl ZakuraEndpoint { /// Shut down the Router's ordered accept/handler lifecycle. pub async fn shutdown(&self) { + if let Some(tasks) = &self.header_sync_tasks { + tasks.shutdown.cancel(); + let mut tasks = tasks.tasks.lock().await; + for mut task in tasks.drain(..) { + if timeout(Duration::from_secs(1), &mut task).await.is_err() { + task.abort(); + let _ = task.await; + } + } + } self.supervisor.shutdown(); let _ = self.router.shutdown().await; } @@ -494,6 +563,33 @@ impl ZakuraEndpoint { router, supervisor, handler, + header_sync: None, + header_sync_tasks: None, + header_sync_actions: None, + upgrade_dials: Arc::new(StdMutex::new(HashSet::new())), + } + } + + #[cfg(any(test, feature = "zakura-testkit"))] + pub(crate) fn from_parts_with_header_sync( + router: Router, + supervisor: ZakuraSupervisorHandle, + handler: ZakuraProtocolHandler, + header_sync: super::HeaderSyncHandle, + shutdown: CancellationToken, + tasks: Vec>, + actions: Option>, + ) -> Self { + Self { + router, + supervisor, + handler, + header_sync: Some(header_sync), + header_sync_tasks: Some(Arc::new(HeaderSyncBackgroundTasks { + shutdown, + tasks: Mutex::new(tasks), + })), + header_sync_actions: actions.map(|actions| Arc::new(Mutex::new(Some(actions)))), upgrade_dials: Arc::new(StdMutex::new(HashSet::new())), } } @@ -934,11 +1030,19 @@ pub(crate) struct NativeHandshakeNegotiated { } pub(crate) fn service_registry( - _supervisor: &ZakuraSupervisorHandle, + supervisor: &ZakuraSupervisorHandle, + header_sync: Option, legacy_service: Arc, discovery_service: Arc, ) -> Result, BoxError> { - let services = vec![legacy_service, discovery_service]; + let mut services = vec![legacy_service.clone(), discovery_service]; + if let Some(header_sync) = header_sync { + let outbound = header_sync_outbound_for_supervisor(supervisor); + services.push(Arc::new(HeaderSyncService::new(header_sync, outbound)) as Arc); + } else { + services + .push(Arc::new(HeaderSyncPassthroughService::new(legacy_service)) as Arc); + } Ok(Arc::new( ServiceRegistry::new(services).map_err(|error| -> BoxError { Box::new(error) })?, @@ -1811,6 +1915,15 @@ impl ProtocolHandler for ZakuraProtocolHandler { pub async fn spawn_zakura_endpoint( config: &Config, sink_factory: impl FnOnce(ZakuraSupervisorHandle, ZakuraTrace) -> Arc, +) -> Result, BoxError> { + spawn_zakura_endpoint_with_header_sync_driver(config, sink_factory, None).await +} + +/// Start a Zakura endpoint with an externally driven header-sync reactor. +pub async fn spawn_zakura_endpoint_with_header_sync_driver( + config: &Config, + sink_factory: impl FnOnce(ZakuraSupervisorHandle, ZakuraTrace) -> Arc, + header_sync_driver_startup: Option, ) -> Result, BoxError> { if !config.v2_p2p { return Ok(None); @@ -1850,8 +1963,58 @@ pub async fn spawn_zakura_endpoint( )?; let discovery_service = Arc::new(super::DiscoveryService::new(discovery.clone())) as Arc; + let anchor = config.zakura.header_sync.anchor(&config.network)?; + let frontiers = header_sync_driver_startup.as_ref().map_or( + HeaderSyncFrontiers { + finalized_height: anchor.0, + verified_block_tip: anchor.0, + }, + |startup| startup.frontiers, + ); + let best_header_tip = header_sync_driver_startup + .as_ref() + .map_or(Some(anchor), |startup| startup.best_header_tip); + let mut startup = HeaderSyncStartup::new( + config.network.clone(), + anchor, + frontiers, + best_header_tip, + config.zakura.header_sync.clone(), + limits.max_frame_bytes, + ); + startup.status_refresh_interval = config.zakura.header_sync.status_refresh_interval; + startup.trace = trace.clone(); + let header_sync_shutdown = CancellationToken::new(); + startup.shutdown = header_sync_shutdown.clone(); + if header_sync_driver_startup.is_some() { + startup.range_state_actions_enabled = true; + startup.inbound_new_block_acceptance_enabled = true; + } + let (header_sync, header_sync_actions, header_sync_task) = spawn_header_sync_reactor(startup)?; let legacy_service = sink_factory(supervisor.clone(), trace.clone()); - let registry = service_registry(&supervisor, legacy_service, discovery_service)?; + let registry = service_registry( + &supervisor, + Some(header_sync.clone()), + legacy_service, + discovery_service, + )?; + let mut tasks = vec![header_sync_task]; + let header_sync_actions = if header_sync_driver_startup.is_some() { + Some(Arc::new(Mutex::new(Some(header_sync_actions)))) + } else { + let action_driver_task = tokio::spawn(drive_header_sync_actions( + header_sync_actions, + header_sync.clone(), + supervisor.clone(), + header_sync_shutdown.clone(), + )); + tasks.push(action_driver_task); + None + }; + let header_sync_tasks = Arc::new(HeaderSyncBackgroundTasks { + shutdown: header_sync_shutdown, + tasks: Mutex::new(tasks), + }); let handler = ZakuraProtocolHandler::new_with_registry_and_trace( supervisor.clone(), config.network.clone(), @@ -1867,6 +2030,9 @@ pub async fn spawn_zakura_endpoint( router, supervisor, handler, + header_sync: Some(header_sync), + header_sync_tasks: Some(header_sync_tasks), + header_sync_actions, upgrade_dials: Arc::new(StdMutex::new(HashSet::new())), }; @@ -3044,14 +3210,22 @@ fn stream_kind_label(stream_kind: u16) -> &'static str { LEGACY_GOSSIP_STREAM_KIND => "gossip", LEGACY_REQUEST_STREAM_KIND => "legacy_request", DISCOVERY_STREAM_KIND => "discovery", + HEADER_SYNC_STREAM_KIND => "header_sync", _ => "unknown", } } -fn app_frame_cap_for_stream_kind(limits: &ZakuraConnectionLimits, _stream_kind: u16) -> u32 { - limits - .max_frame_bytes - .clamp(1, LOCAL_MAX_CONTROL_FRAME_BYTES) +fn app_frame_cap_for_stream_kind(limits: &ZakuraConnectionLimits, stream_kind: u16) -> u32 { + match stream_kind { + HEADER_SYNC_STREAM_KIND => { + let header_sync_cap = + u32::try_from(MAX_HS_MESSAGE_BYTES.saturating_add(FRAME_HEADER_BYTES)) + .expect("header-sync frame cap fits in u32"); + limits.max_frame_bytes.min(header_sync_cap) + } + _ => limits.max_frame_bytes.min(LOCAL_MAX_CONTROL_FRAME_BYTES), + } + .max(1) } fn per_stream_inbound_queue_depth( @@ -3237,7 +3411,9 @@ mod tests { protocol::internal::{InventoryResponse, Response}, zakura::{ legacy_gossip::{LegacyRequestFrame, LegacyRequestKind, LegacyResponseCodec}, - ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_LEGACY_GOSSIP, + HeaderSyncEvent, HeaderSyncMisbehavior, HeaderSyncOutbound, HeaderSyncStatus, + LOCAL_MAX_MESSAGE_BYTES, MAX_HS_MESSAGE_BYTES, MSG_HS_STATUS, ZAKURA_CAP_DISCOVERY, + ZAKURA_CAP_HEADER_SYNC, ZAKURA_CAP_LEGACY_GOSSIP, }, }; use zebra_chain::{ @@ -3247,6 +3423,64 @@ mod tests { }; use zebra_test::vectors::BLOCK_TESTNET_141042_BYTES; + #[derive(Debug, Default)] + struct RecordingService { + deliveries: std::sync::Mutex>, + } + + impl RecordingService { + fn deliveries(&self) -> Vec<(ZakuraPeerId, u16, u16)> { + self.deliveries + .lock() + .expect("recording sink mutex is never poisoned") + .clone() + } + } + + impl Service for RecordingService { + fn name(&self) -> &'static str { + "recording" + } + + fn streams(&self) -> &[Stream] { + crate::zakura::legacy_gossip::legacy_gossip_streams() + } + + fn add_peer(&self, _peer: Peer) {} + + fn remove_peer(&self, _peer: &ZakuraPeerId) {} + + fn deliver_frame( + &self, + peer_id: ZakuraPeerId, + stream_kind: u16, + frame: Frame, + ) -> Result<(), SinkReject> { + self.deliveries + .lock() + .map_err(|error| SinkReject::local(format!("recording sink poisoned: {error}")))? + .push((peer_id, stream_kind, frame.message_type)); + Ok(()) + } + } + + #[derive(Debug)] + struct NoopService; + + impl Service for NoopService { + fn name(&self) -> &'static str { + "noop" + } + + fn streams(&self) -> &[Stream] { + &[] + } + + fn add_peer(&self, _peer: Peer) {} + + fn remove_peer(&self, _peer: &ZakuraPeerId) {} + } + #[derive(Debug)] struct DeclaredStreamService { streams: Vec, @@ -3270,6 +3504,42 @@ mod tests { ZakuraPeerId::new(vec![byte; 32]).expect("32-byte node id is valid") } + fn header_sync_startup(shutdown: CancellationToken) -> HeaderSyncStartup { + let network = Network::Mainnet; + let anchor = (block::Height(0), network.genesis_hash()); + let mut startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: anchor.0, + verified_block_tip: anchor.0, + }, + Some(anchor), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.shutdown = shutdown; + startup + } + + async fn next_header_sync_action( + actions: &mut mpsc::Receiver, + ) -> HeaderSyncAction { + tokio::time::timeout(Duration::from_secs(2), actions.recv()) + .await + .expect("header-sync action arrives before timeout") + .expect("header-sync action channel stays open") + } + + fn status_at_genesis(network: &Network) -> HeaderSyncStatus { + HeaderSyncStatus { + tip_height: block::Height(0), + tip_hash: network.genesis_hash(), + anchor_height: block::Height(0), + ..HeaderSyncStatus::default() + } + } + async fn register_test_peer( supervisor: &ZakuraSupervisorHandle, peer: ZakuraPeerId, @@ -3284,7 +3554,7 @@ mod tests { [peer.as_bytes()[0]; TRANSCRIPT_HASH_BYTES], outbound_handle, disconnect_token, - ZAKURA_CAP_LEGACY_GOSSIP, + ZAKURA_CAP_LEGACY_GOSSIP | ZAKURA_CAP_HEADER_SYNC, ) .await; @@ -3334,6 +3604,320 @@ mod tests { assert!(should_run_freshness_reaper(0, 0)); } + #[tokio::test] + async fn v2_p2p_false_leaves_header_sync_disabled() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let config = Config { + v2_p2p: false, + ..Config::default() + }; + + let endpoint = spawn_zakura_endpoint(&config, |_supervisor, _trace| { + Arc::new(NoopService) as Arc + }) + .await?; + + assert!(endpoint.is_none()); + Ok(()) + } + + #[tokio::test] + async fn v2_p2p_true_starts_header_sync_handle() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let config = Config::default(); + + let endpoint = spawn_zakura_endpoint(&config, |_supervisor, _trace| { + Arc::new(NoopService) as Arc + }) + .await? + .expect("v2_p2p is enabled by default"); + + assert!(endpoint.header_sync().is_some()); + endpoint.shutdown().await; + Ok(()) + } + + #[tokio::test] + async fn endpoint_shutdown_stops_header_sync_task() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let endpoint = spawn_zakura_endpoint(&Config::default(), |_supervisor, _trace| { + Arc::new(NoopService) as Arc + }) + .await? + .expect("v2_p2p is enabled by default"); + let header_sync = endpoint + .header_sync() + .expect("header sync handle exists for a v2 endpoint"); + + endpoint.shutdown().await; + + let send_result = tokio::time::timeout( + Duration::from_secs(1), + header_sync.send(HeaderSyncEvent::PeerConnected( + ZakuraPeerId::new(vec![5u8; 32]).expect("32-byte node id is valid"), + )), + ) + .await + .expect("send returns promptly after header-sync shutdown"); + assert!(send_result.is_err()); + + Ok(()) + } + + #[tokio::test] + async fn registry_routes_legacy_and_header_sync_and_drops_unknown() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let shutdown = CancellationToken::new(); + let startup = header_sync_startup(shutdown.clone()); + let (header_sync, mut actions, task) = spawn_header_sync_reactor(startup)?; + let recorder = Arc::new(RecordingService::default()); + let supervisor = ZakuraSupervisorHandle::new(1); + let registry = service_registry(&supervisor, Some(header_sync.clone()), recorder.clone())?; + let peer = test_peer(6); + + header_sync + .send(HeaderSyncEvent::PeerConnected(peer.clone())) + .await?; + assert!(matches!( + next_header_sync_action(&mut actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); + + let gossip_frame = Frame { + message_type: 11, + flags: 0, + payload: Vec::new(), + }; + let request_frame = Frame { + message_type: 12, + flags: 0, + payload: Vec::new(), + }; + let get_headers_frame = HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 1, + } + .encode_frame()?; + + let (gossip_result, request_result, header_sync_result) = tokio::join!( + async { registry.deliver(peer.clone(), LEGACY_GOSSIP_STREAM_KIND, gossip_frame) }, + async { registry.deliver(peer.clone(), LEGACY_REQUEST_STREAM_KIND, request_frame) }, + async { registry.deliver(peer.clone(), HEADER_SYNC_STREAM_KIND, get_headers_frame) }, + ); + + gossip_result?; + request_result?; + header_sync_result?; + registry.deliver( + peer.clone(), + 99, + Frame { + message_type: 13, + flags: 0, + payload: Vec::new(), + }, + )?; + + let action = next_header_sync_action(&mut actions).await; + assert!( + matches!( + action, + HeaderSyncAction::Misbehavior { + reason: HeaderSyncMisbehavior::GetHeadersSpam, + .. + } + ), + "kind-5 GetHeaders must reach the header-sync reactor, got {action:?}" + ); + + let deliveries = recorder.deliveries(); + assert_eq!(deliveries.len(), 2); + assert!(deliveries.iter().any(|(_, kind, message_type)| *kind + == LEGACY_GOSSIP_STREAM_KIND + && *message_type == 11)); + assert!(deliveries.iter().any(|(_, kind, message_type)| *kind + == LEGACY_REQUEST_STREAM_KIND + && *message_type == 12)); + assert!(!deliveries + .iter() + .any(|(_, kind, _)| *kind == HEADER_SYNC_STREAM_KIND)); + + let rejected = registry + .request( + peer, + HEADER_SYNC_STREAM_KIND, + 99, + LOCAL_MAX_CONTROL_FRAME_BYTES, + Frame { + message_type: 1, + flags: 0, + payload: Vec::new(), + }, + ) + .await; + assert!(matches!(rejected, Err(SinkReject::Protocol(_)))); + + shutdown.cancel(); + task.await?; + Ok(()) + } + + #[tokio::test] + async fn malformed_header_sync_frame_is_protocol_reject_when_reactor_queue_closed( + ) -> Result<(), BoxError> { + let shutdown = CancellationToken::new(); + let startup = header_sync_startup(shutdown.clone()); + let (header_sync, _actions, task) = spawn_header_sync_reactor(startup)?; + let service = HeaderSyncService::new(header_sync, HeaderSyncOutbound::default()); + let peer = test_peer(11); + + shutdown.cancel(); + task.await?; + + let valid_status_frame = + HeaderSyncMessage::Status(status_at_genesis(&Network::Mainnet)).encode_frame()?; + let valid_result = + service.deliver_frame(peer.clone(), HEADER_SYNC_STREAM_KIND, valid_status_frame); + assert!( + matches!(valid_result, Err(SinkReject::Local(_))), + "valid stream-5 frames depend on local reactor queue availability" + ); + + let malformed_frame = Frame { + message_type: u16::from(MSG_HS_STATUS), + flags: 0, + payload: Vec::new(), + }; + let malformed_result = + service.deliver_frame(peer, HEADER_SYNC_STREAM_KIND, malformed_frame); + assert!( + matches!(malformed_result, Err(SinkReject::Protocol(_))), + "malformed stream-5 frames must disconnect independently of reactor queue availability" + ); + + Ok(()) + } + + #[tokio::test] + async fn header_sync_peer_connected_has_ready_outbound_source() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let shutdown = CancellationToken::new(); + let startup = header_sync_startup(shutdown.clone()); + let (header_sync, mut actions, reactor_task) = spawn_header_sync_reactor(startup)?; + let outbound = HeaderSyncOutbound::default(); + let service = HeaderSyncService::new(header_sync, outbound.clone()); + let peer = test_peer(12); + let cancel_token = CancellationToken::new(); + let (_inbound_tx, inbound_rx) = crate::zakura::framed_channel(1); + let (outbound_tx, mut outbound_rx) = crate::zakura::framed_channel(1); + + let mut streams = HashMap::new(); + streams.insert(HEADER_SYNC_STREAM_KIND, (inbound_rx, outbound_tx)); + service.add_peer(Peer::new( + peer.clone(), + None, + ZAKURA_CAP_HEADER_SYNC, + streams, + cancel_token.clone(), + )); + + let queued = Frame { + message_type: u16::from(MSG_HS_STATUS), + flags: 0, + payload: Vec::new(), + }; + outbound.send(&peer, queued.clone()).await?; + let received = tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv()) + .await + .expect("header-sync outbound source is immediately ready") + .expect("header-sync outbound receiver stays open"); + assert_eq!(received.message_type, queued.message_type); + + assert!(matches!( + next_header_sync_action(&mut actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); + + cancel_token.cancel(); + service.remove_peer(&peer); + shutdown.cancel(); + reactor_task.await?; + Ok(()) + } + + #[tokio::test] + async fn registry_add_remove_updates_header_sync_peers() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let shutdown = CancellationToken::new(); + let startup = header_sync_startup(shutdown.clone()); + let (header_sync, mut actions, reactor_task) = spawn_header_sync_reactor(startup)?; + let peer = test_peer(7); + let supervisor = ZakuraSupervisorHandle::new(1); + let registry = service_registry( + &supervisor, + Some(header_sync.clone()), + Arc::new(RecordingService::default()), + )?; + let (_inbound_tx, inbound_rx) = crate::zakura::framed_channel(1); + let (outbound_tx, _outbound_rx) = crate::zakura::framed_channel(1); + let mut streams = HashMap::new(); + streams.insert(HEADER_SYNC_STREAM_KIND, (inbound_rx, outbound_tx)); + registry.add_peer(Peer::new( + peer.clone(), + None, + ZAKURA_CAP_HEADER_SYNC, + streams, + CancellationToken::new(), + )); + assert!(matches!( + next_header_sync_action(&mut actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); + + header_sync + .send(HeaderSyncEvent::WireMessage { + peer: peer.clone(), + msg: HeaderSyncMessage::Status(status_at_genesis(&Network::Mainnet)), + }) + .await?; + registry.remove_peer(&peer, ZAKURA_CAP_HEADER_SYNC); + tokio::time::sleep(Duration::from_millis(50)).await; + header_sync + .send(HeaderSyncEvent::WireMessage { + peer: peer.clone(), + msg: HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 1, + }, + }) + .await?; + + let action = next_header_sync_action(&mut actions).await; + assert!( + matches!( + action, + HeaderSyncAction::Misbehavior { + reason: HeaderSyncMisbehavior::GetHeadersSpam, + .. + } + ), + "deregistered peer must be removed from header-sync state, got {action:?}" + ); + + shutdown.cancel(); + reactor_task.await?; + Ok(()) + } + #[tokio::test] async fn supervisor_disconnect_peer_cancels_registered_token() -> Result<(), BoxError> { let supervisor = ZakuraSupervisorHandle::new(4); @@ -3350,6 +3934,54 @@ mod tests { Ok(()) } + #[tokio::test] + async fn header_sync_misbehavior_action_disconnects_peer() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let reactor_shutdown = CancellationToken::new(); + let startup = header_sync_startup(reactor_shutdown.clone()); + let (header_sync, _reactor_actions, reactor_task) = spawn_header_sync_reactor(startup)?; + let supervisor = ZakuraSupervisorHandle::new(4); + let peer = test_peer(10); + let disconnect_token = CancellationToken::new(); + register_test_peer(&supervisor, peer.clone(), disconnect_token.clone()).await; + + let (actions_tx, actions_rx) = mpsc::channel(4); + let driver_shutdown = CancellationToken::new(); + let driver_task = tokio::spawn(drive_header_sync_actions( + actions_rx, + header_sync, + supervisor, + driver_shutdown.clone(), + )); + actions_tx + .send(HeaderSyncAction::Misbehavior { + peer, + reason: HeaderSyncMisbehavior::MalformedMessage, + }) + .await?; + + tokio::time::timeout(Duration::from_secs(1), disconnect_token.cancelled()) + .await + .expect("misbehavior action cancels the registered connection"); + + driver_shutdown.cancel(); + driver_task.await?; + reactor_shutdown.cancel(); + reactor_task.await?; + Ok(()) + } + + #[test] + fn local_application_frame_cap_admits_default_header_sync_response() { + let limits = ZakuraLocalLimits::from_config(&Config::default()); + let default_header_sync_frame_bytes = + u32::try_from(MAX_HS_MESSAGE_BYTES.saturating_add(FRAME_HEADER_BYTES)) + .expect("header-sync frame cap fits in u32"); + + assert!(limits.max_frame_bytes >= default_header_sync_frame_bytes); + assert!(limits.initial_limits().max_frame_bytes >= default_header_sync_frame_bytes); + } + #[test] fn stream_specific_application_frame_caps_keep_gossip_and_discovery_tight() { let limits = ZakuraLocalLimits::from_config(&Config::default()); @@ -3360,6 +3992,9 @@ mod tests { max_inbound_queue_depth: u16::MAX, idle_timeout_millis: u32::MAX, }); + let header_sync_frame_bytes = + u32::try_from(MAX_HS_MESSAGE_BYTES.saturating_add(FRAME_HEADER_BYTES)) + .expect("header-sync frame cap fits in u32"); assert_eq!( app_frame_cap_for_stream_kind(&negotiated, LEGACY_GOSSIP_STREAM_KIND), @@ -3373,13 +4008,23 @@ mod tests { app_frame_cap_for_stream_kind(&negotiated, DISCOVERY_STREAM_KIND), LOCAL_MAX_CONTROL_FRAME_BYTES ); + assert_eq!( + app_frame_cap_for_stream_kind(&negotiated, HEADER_SYNC_STREAM_KIND), + header_sync_frame_bytes + ); let over_tight_cap = usize::try_from(LOCAL_MAX_CONTROL_FRAME_BYTES).unwrap() + 1; + let header_sync_cap = usize::try_from(header_sync_frame_bytes).unwrap(); let gossip_frame = Frame { message_type: 1, flags: 0, payload: vec![0; over_tight_cap.saturating_sub(FRAME_HEADER_BYTES)], }; + let header_sync_frame = Frame { + message_type: 1, + flags: 0, + payload: vec![0; header_sync_cap.saturating_sub(FRAME_HEADER_BYTES)], + }; assert!( gossip_frame @@ -3408,6 +4053,15 @@ mod tests { .is_err(), "legacy request frames over the tight stream cap must be rejected" ); + assert!( + header_sync_frame + .encode(app_frame_cap_for_stream_kind( + &negotiated, + HEADER_SYNC_STREAM_KIND + )) + .is_ok(), + "header-sync frames up to MAX_HS_MESSAGE_BYTES must still be accepted" + ); } #[test] @@ -3450,6 +4104,13 @@ mod tests { capability: ZAKURA_CAP_DISCOVERY, mode: StreamMode::Ordered, }, + Stream { + kind: HEADER_SYNC_STREAM_KIND, + version: ZAKURA_STREAM_VERSION_1, + frame_cap: 1024, + capability: ZAKURA_CAP_HEADER_SYNC, + mode: StreamMode::Ordered, + }, ], }) as Arc]) .expect("test registry declares unique stream kinds"); @@ -3458,6 +4119,7 @@ mod tests { LEGACY_GOSSIP_STREAM_KIND, LEGACY_REQUEST_STREAM_KIND, DISCOVERY_STREAM_KIND, + HEADER_SYNC_STREAM_KIND, ] { assert!( is_supported_stream(®istry, kind, ZAKURA_STREAM_VERSION_1), @@ -3476,8 +4138,9 @@ mod tests { assert_eq!(stream_kind_label(2), "gossip"); assert_eq!(stream_kind_label(3), "legacy_request"); assert_eq!(stream_kind_label(4), "discovery"); + assert_eq!(stream_kind_label(5), "header_sync"); - for kind in [0u16, 1, 5, 6, 7, 255, u16::MAX] { + for kind in [0u16, 1, 6, 7, 255, u16::MAX] { assert!( !is_supported_stream(®istry, kind, ZAKURA_STREAM_VERSION_1), "unknown kind {kind} must be rejected even at version 1" diff --git a/zebra-network/src/zakura/header_sync/config.rs b/zebra-network/src/zakura/header_sync/config.rs new file mode 100644 index 00000000000..6f991aa3c3d --- /dev/null +++ b/zebra-network/src/zakura/header_sync/config.rs @@ -0,0 +1,177 @@ +use super::{error::*, validation::*, wire::*, *}; + +/// Header-sync peer status advertisement. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct HeaderSyncStatus { + /// Sender's best known tip height. + pub tip_height: block::Height, + /// Sender's best known tip hash. + pub tip_hash: block::Hash, + /// Sender's lowest contiguous header height. + pub anchor_height: block::Height, + /// Maximum headers the sender will serve per response. + pub max_headers_per_response: u32, + /// Maximum concurrent `GetHeaders` requests the sender will service. + pub max_inflight_requests: u16, +} + +impl HeaderSyncStatus { + pub(super) fn encode_to(&self, writer: &mut W) -> Result<(), HeaderSyncWireError> { + write_height(writer, self.tip_height)?; + self.tip_hash.zcash_serialize(&mut *writer)?; + write_height(writer, self.anchor_height)?; + writer.write_u32::(clamp_advertised_range(self.max_headers_per_response))?; + writer.write_u16::(self.max_inflight_requests)?; + Ok(()) + } + + pub(super) fn decode_from(reader: &mut R) -> Result { + Ok(Self { + tip_height: read_height(reader)?, + tip_hash: block::Hash::zcash_deserialize(&mut *reader)?, + anchor_height: read_height(reader)?, + max_headers_per_response: clamp_advertised_range(reader.read_u32::()?), + max_inflight_requests: reader.read_u16::()?, + }) + } +} + +impl Default for HeaderSyncStatus { + fn default() -> Self { + Self { + tip_height: block::Height::MIN, + tip_hash: block::Hash([0; 32]), + anchor_height: block::Height::MIN, + max_headers_per_response: DEFAULT_HS_RANGE, + max_inflight_requests: DEFAULT_HS_MAX_INFLIGHT, + } + } +} + +/// Header-sync configuration nested under the Zakura P2P-v2 config. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct ZakuraHeaderSyncConfig { + /// Maximum headers this node advertises per `GetHeaders` response. + pub max_headers_per_response: u32, + /// Maximum concurrent `GetHeaders` requests this node advertises per peer. + pub max_inflight_requests: u16, + /// How often this node sends unsolicited status refreshes after local frontier changes. + #[serde(with = "humantime_serde")] + pub status_refresh_interval: Duration, + /// Optional trusted header-sync anchor height. + /// + /// When unset, header sync starts from genesis. When set, [`anchor_hash`](Self::anchor_hash) + /// must also be set and must match genesis or a configured checkpoint. + pub anchor_height: Option, + /// Optional trusted header-sync anchor hash. + /// + /// When unset, header sync starts from genesis. When set, [`anchor_height`](Self::anchor_height) + /// must also be set and must match genesis or a configured checkpoint. + pub anchor_hash: Option, +} + +impl Default for ZakuraHeaderSyncConfig { + fn default() -> Self { + Self { + max_headers_per_response: DEFAULT_HS_RANGE, + max_inflight_requests: DEFAULT_HS_MAX_INFLIGHT, + status_refresh_interval: DEFAULT_HS_STATUS_REFRESH_INTERVAL, + anchor_height: None, + anchor_hash: None, + } + } +} + +impl ZakuraHeaderSyncConfig { + /// Return the clamped served-range advertisement for wire status messages. + pub fn advertised_max_headers_per_response(&self) -> u32 { + clamp_advertised_range(self.max_headers_per_response) + } + + /// Return the locally capped in-flight advertisement for status messages. + pub fn advertised_max_inflight_requests(&self) -> u16 { + self.max_inflight_requests + .clamp(1, LOCAL_MAX_HS_INFLIGHT_PER_PEER) + } + + /// Return the configured trusted anchor, or genesis when no override is configured. + pub fn anchor( + &self, + network: &Network, + ) -> Result<(block::Height, block::Hash), HeaderSyncStartError> { + match (self.anchor_height, self.anchor_hash) { + (Some(height), Some(hash)) => Ok((height, hash)), + (None, None) => Ok((block::Height(0), network.genesis_hash())), + _ => Err(HeaderSyncStartError::IncompleteAnchor), + } + } +} + +/// Returns the serialized byte length of a stream-5 header on `network`. +pub fn header_sync_header_bytes_for_network(network: &Network) -> usize { + if network + .parameters() + .is_some_and(|parameters| parameters.is_regtest()) + { + REGTEST_HEADER_BYTES + } else { + COMMON_HEADER_BYTES + } +} + +/// Maximum `Headers` count that fits both the stream-5 payload cap and the app frame cap. +pub fn header_sync_count_by_byte_budget(network: &Network, max_frame_bytes: u32) -> u32 { + let frame_payload_cap = usize::try_from(max_frame_bytes) + .unwrap_or(usize::MAX) + .saturating_sub(FRAME_HEADER_BYTES); + let payload_cap = MAX_HS_MESSAGE_BYTES.min(frame_payload_cap); + let header_bytes = header_sync_header_bytes_for_network(network); + let count = payload_cap + .saturating_sub(HEADER_SYNC_MESSAGE_TYPE_BYTES + HEADER_SYNC_COUNT_BYTES) + / header_bytes; + + u32::try_from(count) + .unwrap_or(u32::MAX) + .clamp(1, MAX_HS_RANGE) +} + +/// Clamp an outbound `GetHeaders.count` by peer, hard, payload, and frame caps. +pub fn clamp_header_sync_request_count( + desired_count: u32, + peer_max_headers_per_response: u32, + network: &Network, + max_frame_bytes: u32, +) -> u32 { + desired_count + .min(clamp_advertised_range(peer_max_headers_per_response)) + .min(MAX_HS_RANGE) + .min(header_sync_count_by_byte_budget(network, max_frame_bytes)) + .max(1) +} + +/// Maximum inbound `GetHeaders.count` this node will serve. +pub fn inbound_get_headers_count_limit( + config: &ZakuraHeaderSyncConfig, + network: &Network, + max_frame_bytes: u32, +) -> u32 { + clamp_header_sync_request_count( + u32::MAX, + config.advertised_max_headers_per_response(), + network, + max_frame_bytes, + ) +} + +/// Truncate a served header run so the encoded `Headers` response fits the byte budgets. +pub fn truncate_headers_to_byte_budget( + mut headers: Vec>, + network: &Network, + max_frame_bytes: u32, +) -> Vec> { + let max_count = usize::try_from(header_sync_count_by_byte_budget(network, max_frame_bytes)) + .expect("header-sync byte-budget count fits in usize"); + headers.truncate(max_count); + headers +} diff --git a/zebra-network/src/zakura/header_sync/error.rs b/zebra-network/src/zakura/header_sync/error.rs new file mode 100644 index 00000000000..4b4f5b4f609 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/error.rs @@ -0,0 +1,124 @@ +use super::*; + +/// Errors that prevent the header-sync reactor from starting. +#[derive(Debug, Error)] +pub enum HeaderSyncStartError { + /// The configured anchor is neither genesis nor a hash-matching checkpoint. + #[error("invalid Zakura header-sync anchor at height {anchor:?}")] + InvalidAnchor { + /// Rejected anchor. + anchor: (block::Height, block::Hash), + }, + + /// Only one anchor field was configured. + #[error("Zakura header-sync anchor_height and anchor_hash must be configured together")] + IncompleteAnchor, +} + +/// Structured wire and stateless-validation errors for stream 5. +#[derive(Debug, Error)] +pub enum HeaderSyncWireError { + /// A payload or peer-controlled count exceeded its cap. + #[error("Zakura header-sync payload length {actual} exceeds cap {max}")] + OversizedPayload { + /// Actual payload length. + actual: usize, + /// Maximum allowed payload length. + max: usize, + }, + + /// A decoded header count exceeded its contract. + #[error("Zakura header-sync header count {actual} exceeds cap {max}")] + HeaderCountLimit { + /// Actual header count. + actual: usize, + /// Maximum allowed header count. + max: usize, + }, + + /// An inbound `Headers` response did not match an in-flight request. + #[error("unsolicited Zakura header-sync Headers response")] + UnsolicitedHeaders, + + /// A `GetHeaders` request asked for zero headers. + #[error("Zakura header-sync GetHeaders count must be non-zero")] + ZeroHeaderRequestCount, + + /// A height exceeded Zebra's supported height range. + #[error("Zakura header-sync height {0} exceeds supported range")] + HeightOutOfRange(u32), + + /// A payload used an unknown stream-5 message discriminator. + #[error("unknown Zakura header-sync message type {0}")] + UnknownMessageType(u8), + + /// A frame used a message type that does not fit stream-5's u8 discriminator. + #[error("unknown Zakura header-sync frame message type {0}")] + UnknownFrameMessageType(u16), + + /// Frame flags are reserved in stream 5. + #[error("unsupported Zakura header-sync frame flags {0}")] + UnsupportedFlags(u16), + + /// Frame and payload message types disagreed. + #[error("Zakura header-sync frame type {frame} disagrees with payload type {payload}")] + MismatchedFrameMessageType { + /// Outer frame message type. + frame: u16, + /// Inner payload message type. + payload: u8, + }, + + /// A decoded payload had trailing bytes. + #[error("trailing bytes in Zakura header-sync payload")] + TrailingBytes, + + /// Adjacent headers did not hash-link. + #[error("non-contiguous Zakura header-sync header run")] + NonContiguousHeaders, + + /// The first header in a range did not link to its anchor. + #[error("first Zakura header-sync range header does not link to anchor")] + FirstHeaderDoesNotLink, + + /// Equihash solution size did not match the active network. + #[error("Zakura header-sync Equihash solution size does not match the active network")] + WrongEquihashSolutionSize, + + /// The compact difficulty threshold did not expand to a valid target. + #[error("invalid Zakura header-sync compact difficulty threshold")] + InvalidDifficultyThreshold, + + /// The header hash failed the context-free difficulty filter. + #[error("Zakura header-sync hash {hash:?} exceeds difficulty threshold {threshold:?}")] + DifficultyFilter { + /// Header hash. + hash: block::Hash, + /// Expanded threshold from the header. + threshold: ExpandedDifficulty, + }, + + /// A numeric conversion failed while handling bounded data. + #[error("numeric overflow while encoding Zakura header-sync {0}")] + NumericOverflow(&'static str), + + /// An I/O error while encoding or decoding. + #[error("Zakura header-sync wire I/O error: {0}")] + Io(#[from] io::Error), + + /// Zcash serialization failed. + #[error("Zakura header-sync Zcash serialization error: {0}")] + Serialization(#[from] SerializationError), + + /// Header time was too far in the future. + #[error(transparent)] + Time(#[from] BlockTimeError), + + /// Equihash verification failed. + #[error(transparent)] + Equihash(#[from] equihash::Error), + + /// The blocking validation task failed. + #[error("Zakura header-sync blocking validation task failed: {0}")] + BlockingTask(#[from] JoinError), +} diff --git a/zebra-network/src/zakura/header_sync/events.rs b/zebra-network/src/zakura/header_sync/events.rs new file mode 100644 index 00000000000..91876fdd2c3 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/events.rs @@ -0,0 +1,350 @@ +use super::{config::*, error::*, validation::*, wire::*, *}; + +/// Cached state frontiers used by the header-sync reactor. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct HeaderSyncFrontiers { + /// Shared finalized height `F`, supplied by state. + pub finalized_height: block::Height, + /// Highest verified block body height, supplied by state. + pub verified_block_tip: block::Height, +} + +/// Startup inputs for the dependency-neutral header-sync reactor. +#[derive(Clone, Debug)] +pub struct HeaderSyncStartup { + /// Active network. + pub network: Network, + /// Trusted anchor height and hash. + pub anchor: (block::Height, block::Hash), + /// Cached state frontiers at startup. + pub frontiers: HeaderSyncFrontiers, + /// Durable best header tip loaded from storage at startup. + pub best_header_tip: Option<(block::Height, block::Hash)>, + /// Local stream-5 advertisement. + pub config: ZakuraHeaderSyncConfig, + /// Negotiated or local application frame cap for header-sync responses. + pub max_frame_bytes: u32, + /// Per-request timeout. + pub request_timeout: Duration, + /// Minimum interval between unsolicited status refreshes to a peer. + pub status_refresh_interval: Duration, + /// Optional JSONL trace emitter for header-sync runtime events. + pub trace: ZakuraTrace, + /// Shared shutdown signal owned by the embedding endpoint or test harness. + pub shutdown: CancellationToken, + /// Enables outbound range scheduling and state-backed header actions. + pub range_state_actions_enabled: bool, + /// Enables relaying inbound `NewBlock` messages after local block acceptance is wired. + pub inbound_new_block_acceptance_enabled: bool, +} + +impl HeaderSyncStartup { + /// Build a startup config from the active network and durable/frontier facts. + pub fn new( + network: Network, + anchor: (block::Height, block::Hash), + frontiers: HeaderSyncFrontiers, + best_header_tip: Option<(block::Height, block::Hash)>, + config: ZakuraHeaderSyncConfig, + max_frame_bytes: u32, + ) -> Self { + Self { + network, + anchor, + frontiers, + best_header_tip, + config, + max_frame_bytes, + request_timeout: DEFAULT_HS_REQUEST_TIMEOUT, + status_refresh_interval: DEFAULT_HS_STATUS_REFRESH_INTERVAL, + trace: ZakuraTrace::noop(), + shutdown: CancellationToken::new(), + range_state_actions_enabled: false, + inbound_new_block_acceptance_enabled: false, + } + } +} + +/// Cheap cloneable handle used by other services to inform header sync. +#[derive(Clone, Debug)] +pub struct HeaderSyncHandle { + pub(super) events: mpsc::Sender, + pub(super) lifecycle: mpsc::UnboundedSender, + pub(super) tip: watch::Receiver<(block::Height, block::Hash)>, +} + +impl HeaderSyncHandle { + /// Send a fact/event to the header-sync reactor. + pub async fn send( + &self, + event: HeaderSyncEvent, + ) -> Result<(), mpsc::error::SendError> { + self.events.send(event).await + } + + /// Try to send a fact/event without awaiting. + pub fn try_send( + &self, + event: HeaderSyncEvent, + ) -> Result<(), mpsc::error::TrySendError> { + self.events.try_send(event) + } + + /// Send a peer lifecycle event without sharing the bounded wire-event queue. + pub fn send_lifecycle( + &self, + event: HeaderSyncEvent, + ) -> Result<(), mpsc::error::SendError> { + self.lifecycle + .send(event) + .map_err(|error| mpsc::error::SendError(error.0)) + } + + /// Subscribe to best-header frontier updates. + pub fn subscribe_tip(&self) -> watch::Receiver<(block::Height, block::Hash)> { + self.tip.clone() + } + + /// Return the currently cached best-header frontier. + pub fn best_header_tip(&self) -> (block::Height, block::Hash) { + *self.tip.borrow() + } +} + +/// Facts accepted by the header-sync reactor. +#[derive(Clone, Debug)] +pub enum HeaderSyncEvent { + /// A peer became available for stream-5 header sync. + PeerConnected(ZakuraPeerId), + /// A peer disconnected; all of its outstanding work is dropped. + PeerDisconnected(ZakuraPeerId), + /// State committed a full block. + FullBlockCommitted { + /// Committed block height. + height: block::Height, + /// Committed block hash. + hash: block::Hash, + /// Committed block header. Transient only; not retained by runtime state. + header: Arc, + }, + /// The node's block pipeline accepted an inbound `NewBlock` body. + NewBlockAccepted { + /// Source peer. + peer: ZakuraPeerId, + /// Accepted block height. + height: block::Height, + /// Accepted block hash. + hash: block::Hash, + /// Accepted full block. + block: Arc, + }, + /// The node's block pipeline reported an inbound `NewBlock` was already known. + NewBlockDuplicate { + /// Source peer. + peer: ZakuraPeerId, + /// Duplicate block height. + height: block::Height, + /// Duplicate block hash. + hash: block::Hash, + }, + /// The node's block pipeline rejected an inbound `NewBlock` body. + NewBlockRejected { + /// Source peer. + peer: ZakuraPeerId, + /// Rejected block hash. + hash: block::Hash, + }, + /// Inbound stream-5 message from `peer`. + WireMessage { + /// Serving peer. + peer: ZakuraPeerId, + /// Decoded stream-5 message. + msg: HeaderSyncMessage, + }, + /// Inbound stream-5 frame from `peer` whose decode depends on reactor state. + WireFrame { + /// Serving peer. + peer: ZakuraPeerId, + /// Raw stream-5 frame. + frame: Frame, + }, + /// Stream-5 frame decoding failed after handler admission. + WireDecodeFailed { + /// Peer that sent the malformed frame. + peer: ZakuraPeerId, + /// Decode/validation error. + error: Arc, + }, + /// State finalized or verified-body frontiers changed. + StateFrontiersChanged(HeaderSyncFrontiers), + /// State successfully committed a header range. + HeaderRangeCommitted { + /// First committed height. + start_height: block::Height, + /// New best header tip height. + tip_height: block::Height, + /// New best header tip hash. + tip_hash: block::Hash, + }, + /// State rejected a previously requested range. + HeaderRangeCommitFailed { + /// Peer that supplied the failed range. + peer: ZakuraPeerId, + /// First failed range height. + start_height: block::Height, + /// Failed range count. + count: u32, + /// Whether state rejected peer data or hit a local resource/channel failure. + kind: HeaderSyncCommitFailureKind, + }, + /// Node wiring finished or abandoned a `Headers` response to an inbound `GetHeaders`. + HeaderRangeResponseFinished { + /// Peer whose served-response slot can be released. + peer: ZakuraPeerId, + /// First requested height. + start_height: block::Height, + /// Requested header count. + requested_count: u32, + /// Number of headers read from state and sent in the response. + returned_count: u32, + }, +} + +/// Actions emitted by the header-sync reactor for the eventual node wiring. +#[derive(Clone, Debug)] +pub enum HeaderSyncAction { + /// Send a stream-5 message to a peer. + SendMessage { + /// Destination peer. + peer: ZakuraPeerId, + /// Message to send. + msg: HeaderSyncMessage, + }, + /// Ask state to commit a contiguous header range. + CommitHeaderRange { + /// Peer that supplied the range. + peer: ZakuraPeerId, + /// Parent anchor hash for the first header. + anchor: block::Hash, + /// First header height. + start_height: block::Height, + /// Headers to commit. This is an output payload, not reactor state. + headers: Vec>, + /// Whether the range is expected to be finalized by checkpoint policy. + finalized: bool, + }, + /// Ask state for the durable best header tip. + QueryBestHeaderTip, + /// Ask state for a bounded contiguous range of headers. + QueryHeadersByHeightRange { + /// Peer that requested the range. + peer: ZakuraPeerId, + /// First height. + start: block::Height, + /// Maximum count. + count: u32, + }, + /// Ask state for missing block-body gaps. + QueryMissingBlockBodies { + /// First height to consider. + from: block::Height, + /// Maximum number of heights. + limit: u32, + }, + /// Report peer misbehavior to the supervisor. + Misbehavior { + /// Misbehaving peer. + peer: ZakuraPeerId, + /// Reason for reporting. + reason: HeaderSyncMisbehavior, + }, + /// Notify body download wiring that header-known body gaps exist. + BodyGaps { + /// First missing height. + from: block::Height, + /// Last missing height. + to: block::Height, + }, + /// Forward an unseen valid full tip block to one eligible stream-5 peer. + ForwardNewBlock { + /// Source peer, if the block was received from the network. + source: Option, + /// Destination peer. + peer: ZakuraPeerId, + /// Block height from the coinbase transaction. + height: block::Height, + /// Block hash used for deduplication. + hash: block::Hash, + /// Full block to forward. + block: Arc, + }, + /// Inform later block-pipeline wiring that a validated tip block arrived. + NewBlockReceived { + /// Source peer. + peer: ZakuraPeerId, + /// Block height from the coinbase transaction. + height: block::Height, + /// Block hash used for deduplication. + hash: block::Hash, + /// Full block received from the peer. + block: Arc, + }, +} + +/// Header-sync peer-accounting violations. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum HeaderSyncMisbehavior { + /// Peer status is internally impossible. + InvalidStatus, + /// `Headers` arrived without an outstanding request. + UnsolicitedHeaders, + /// `Headers` was empty and made no progress. + EmptyHeaders, + /// `Headers` exceeded the outstanding request contract. + ResponseTooLong, + /// Peer supplied a range that failed state/contextual commit. + InvalidRange, + /// A stream-5 payload was malformed before semantic handling. + MalformedMessage, + /// A peer sent semantic `Status` messages faster than the v1 budget. + StatusSpam, + /// A peer sent semantic `NewBlock` messages faster than the v1 budget. + NewBlockSpam, + /// A peer exceeded this node's inbound `GetHeaders` serving budget. + GetHeadersSpam, + /// A peer requested more headers than this node advertised it can serve. + GetHeadersTooLong, + /// A stream-5 message came from a peer with no active header-sync state. + UnknownPeer, + /// A full-block tip flood failed stateless validation. + InvalidNewBlock, +} + +/// State commit failure classification returned to the reactor by node wiring. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum HeaderSyncCommitFailureKind { + /// The supplied headers failed contextual validation or checkpoint consistency. + InvalidPeerRange, + /// Local storage/channel/resource failure; do not score the peer. + Local, +} + +/// A single outbound `GetHeaders` range used to validate the next response. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct HeaderSyncRequestContract { + /// First requested height. + pub start_height: block::Height, + /// Requested header count. + pub count: u32, +} + +impl HeaderSyncRequestContract { + /// Create a bounded request contract. + pub fn new(start_height: block::Height, count: u32) -> Result { + validate_get_headers_count(count)?; + Ok(Self { + start_height, + count, + }) + } +} diff --git a/zebra-network/src/zakura/header_sync/mod.rs b/zebra-network/src/zakura/header_sync/mod.rs new file mode 100644 index 00000000000..026eeee7e35 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/mod.rs @@ -0,0 +1,70 @@ +//! Native Zakura header-sync stream messages and stateless guards. + +use std::{ + cmp::min, + collections::{HashMap, HashSet, VecDeque}, + io::{self, Cursor, Read, Write}, + sync::Arc, + time::Duration, +}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use chrono::{DateTime, Utc}; +use serde_json::{Number, Value}; +use thiserror::Error; +use tokio::{ + sync::{mpsc, watch}, + task::{JoinError, JoinHandle}, + time::{self, Instant}, +}; +use tokio_util::sync::CancellationToken; +use zebra_chain::{ + block::{self, BlockTimeError}, + parameters::Network, + serialization::{SerializationError, ZcashDeserialize, ZcashSerialize}, + work::{difficulty::CompactDifficulty, difficulty::ExpandedDifficulty, equihash}, +}; + +use super::{ + trace::{header_sync_trace as hs_trace, peer_label as trace_peer_label, HEADER_SYNC_TABLE}, + Frame, ZakuraPeerId, ZakuraTrace, FRAME_HEADER_BYTES, LOCAL_MAX_MESSAGE_BYTES, +}; + +mod config; +mod error; +mod events; +mod reactor; +mod scheduler; +mod service; +mod state; +#[cfg(test)] +mod tests; +mod validation; +mod wire; + +pub use config::{ + clamp_header_sync_request_count, header_sync_count_by_byte_budget, + header_sync_header_bytes_for_network, inbound_get_headers_count_limit, + truncate_headers_to_byte_budget, HeaderSyncStatus, ZakuraHeaderSyncConfig, +}; +pub use error::{HeaderSyncStartError, HeaderSyncWireError}; +pub use events::{ + HeaderSyncAction, HeaderSyncCommitFailureKind, HeaderSyncEvent, HeaderSyncFrontiers, + HeaderSyncHandle, HeaderSyncMisbehavior, HeaderSyncRequestContract, HeaderSyncStartup, +}; +pub use reactor::spawn_header_sync_reactor; +#[cfg(test)] +pub(crate) use service::HeaderSyncOutbound; +pub(crate) use service::{ + drive_header_sync_actions, header_sync_outbound_for_supervisor, send_header_sync_message, + HeaderSyncPassthroughService, HeaderSyncService, +}; +pub use validation::{ + validate_header_range_links, validate_headers_stateless, validate_new_block_stateless, + HeaderSyncDecodeContext, HeaderSyncValidationContext, +}; +pub use wire::{ + HeaderSyncMessage, DEFAULT_HS_MAX_INFLIGHT, DEFAULT_HS_RANGE, MAX_HS_MESSAGE_BYTES, + MAX_HS_RANGE, MSG_HS_GET_HEADERS, MSG_HS_HEADERS, MSG_HS_NEW_BLOCK, MSG_HS_STATUS, + ZAKURA_HEADER_SYNC_STREAM_VERSION, ZAKURA_STREAM_HEADER_SYNC, +}; diff --git a/zebra-network/src/zakura/header_sync/reactor.rs b/zebra-network/src/zakura/header_sync/reactor.rs new file mode 100644 index 00000000000..200abeb7488 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/reactor.rs @@ -0,0 +1,1163 @@ +use super::{config::*, error::*, events::*, scheduler::*, state::*, validation::*, wire::*, *}; + +/// Spawn a header-sync reactor and return its handle plus action stream. +pub fn spawn_header_sync_reactor( + startup: HeaderSyncStartup, +) -> Result< + ( + HeaderSyncHandle, + mpsc::Receiver, + JoinHandle<()>, + ), + HeaderSyncStartError, +> { + let state = HeaderSyncState::new(&startup)?; + let (events_tx, events_rx) = mpsc::channel(128); + let (lifecycle_tx, lifecycle_rx) = mpsc::unbounded_channel(); + let (actions_tx, actions_rx) = mpsc::channel(128); + let (tip_tx, tip_rx) = watch::channel((state.best_header_tip, state.best_header_hash)); + let handle = HeaderSyncHandle { + events: events_tx, + lifecycle: lifecycle_tx, + tip: tip_rx, + }; + let reactor = HeaderSyncReactor { + startup, + state, + events: events_rx, + lifecycle: lifecycle_rx, + actions: actions_tx, + tip: tip_tx, + }; + let task = tokio::spawn(reactor.run()); + + Ok((handle, actions_rx, task)) +} + +#[derive(Debug)] +pub(super) struct HeaderSyncReactor { + startup: HeaderSyncStartup, + state: HeaderSyncState, + events: mpsc::Receiver, + lifecycle: mpsc::UnboundedReceiver, + actions: mpsc::Sender, + tip: watch::Sender<(block::Height, block::Hash)>, +} + +impl HeaderSyncReactor { + async fn run(mut self) { + if self.startup.range_state_actions_enabled { + let _ = self + .actions + .send(HeaderSyncAction::QueryBestHeaderTip) + .await; + let _ = self + .actions + .send(HeaderSyncAction::QueryMissingBlockBodies { + from: next_height(self.state.verified_block_tip) + .unwrap_or(self.state.verified_block_tip), + limit: DEFAULT_HS_RANGE, + }) + .await; + } + + let mut ticks = time::interval(self.empty_headers_retry_delay()); + loop { + tokio::select! { + biased; + _ = self.startup.shutdown.cancelled() => { + break; + } + event = self.lifecycle.recv() => { + let Some(event) = event else { + break; + }; + self.handle_event(event).await; + } + event = self.events.recv() => { + let Some(event) = event else { + break; + }; + self.handle_event(event).await; + } + _ = ticks.tick() => { + self.handle_timeouts().await; + } + } + } + } + + async fn handle_event(&mut self, event: HeaderSyncEvent) { + match event { + HeaderSyncEvent::PeerConnected(peer) => self.handle_peer_connected(peer).await, + HeaderSyncEvent::PeerDisconnected(peer) => self.handle_peer_disconnected(peer), + HeaderSyncEvent::FullBlockCommitted { + height, + hash, + header: _, + } => self.handle_full_block_committed(height, hash).await, + HeaderSyncEvent::NewBlockAccepted { + peer, + height, + hash, + block, + } => { + self.handle_new_block_accepted(peer, height, hash, block) + .await + } + HeaderSyncEvent::NewBlockDuplicate { peer, height, hash } => { + self.handle_new_block_duplicate(peer, height, hash) + } + HeaderSyncEvent::NewBlockRejected { peer, hash } => { + self.handle_new_block_rejected(peer, hash).await + } + HeaderSyncEvent::WireMessage { peer, msg } => { + self.handle_wire_message(peer, msg).await; + } + HeaderSyncEvent::WireFrame { peer, frame } => { + self.handle_wire_frame(peer, frame).await; + } + HeaderSyncEvent::WireDecodeFailed { peer, error } => { + self.handle_wire_decode_failed(peer, error).await; + } + HeaderSyncEvent::StateFrontiersChanged(frontiers) => { + self.handle_state_frontiers_changed(frontiers).await; + } + HeaderSyncEvent::HeaderRangeCommitted { + start_height, + tip_height, + tip_hash, + } => { + self.handle_header_range_committed(start_height, tip_height, tip_hash) + .await + } + HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind, + } => { + self.handle_header_range_commit_failed(peer, start_height, count, kind) + .await + } + HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height, + requested_count, + returned_count, + } => self.handle_header_range_response_finished( + peer, + start_height, + requested_count, + returned_count, + ), + } + } + + async fn handle_peer_connected(&mut self, peer: ZakuraPeerId) { + self.state.peers.entry(peer.clone()).or_insert_with(|| { + PeerHeaderState::new( + self.state.anchor.0, + self.startup.config.advertised_max_headers_per_response(), + self.startup.config.advertised_max_inflight_requests(), + self.startup.status_refresh_interval, + DEFAULT_HS_INBOUND_STATUS_MIN_INTERVAL, + DEFAULT_HS_INBOUND_NEW_BLOCK_MIN_INTERVAL, + ) + }); + self.send_status(peer).await; + self.schedule().await; + } + + fn handle_peer_disconnected(&mut self, peer: ZakuraPeerId) { + self.state.peers.remove(&peer); + self.state.schedule.forget_peer(&peer); + } + + async fn handle_full_block_committed(&mut self, height: block::Height, hash: block::Hash) { + self.state.pending_new_blocks.remove(&hash); + let _ = self.state.seen.insert(hash); + self.state.verified_block_tip = self.state.verified_block_tip.max(height); + self.state.schedule.mark_height_covered(height); + self.cancel_covered_outstanding(); + if height > self.state.best_header_tip { + self.publish_best_tip(height, hash).await; + } + self.schedule().await; + } + + async fn handle_new_block_accepted( + &mut self, + peer: ZakuraPeerId, + height: block::Height, + hash: block::Hash, + block: Arc, + ) { + self.state.pending_new_blocks.remove(&hash); + let inserted = self.state.seen.insert(hash); + if !inserted { + metrics::counter!("sync.header.tip.new_block.deduped").increment(1); + self.trace_new_block_deduped(&peer, height, hash, "seen_cache"); + return; + } + + self.state.verified_block_tip = self.state.verified_block_tip.max(height); + self.state.schedule.mark_height_covered(height); + self.cancel_covered_outstanding(); + if height > self.state.best_header_tip { + self.publish_best_tip(height, hash).await; + } + + let destinations = self.eligible_tip_destinations(&peer, height); + let destination_count = destinations.len(); + for destination in destinations { + metrics::counter!("sync.header.tip.new_block.forwarded").increment(1); + self.trace_new_block_forwarded(&peer, &destination, height, hash, destination_count); + let _ = self + .actions + .send(HeaderSyncAction::ForwardNewBlock { + source: Some(peer.clone()), + peer: destination, + height, + hash, + block: block.clone(), + }) + .await; + } + self.schedule().await; + } + + fn handle_new_block_duplicate( + &mut self, + peer: ZakuraPeerId, + height: block::Height, + hash: block::Hash, + ) { + self.state.pending_new_blocks.remove(&hash); + let _ = self.state.seen.insert(hash); + metrics::counter!("sync.header.tip.new_block.deduped").increment(1); + self.trace_new_block_deduped(&peer, height, hash, "already_in_chain"); + } + + async fn handle_new_block_rejected(&mut self, peer: ZakuraPeerId, hash: block::Hash) { + self.state.pending_new_blocks.remove(&hash); + metrics::counter!("sync.header.tip.new_block.rejected").increment(1); + debug!( + ?peer, + ?hash, + "Zakura header-sync NewBlock rejected by block pipeline" + ); + self.report_misbehavior(peer, HeaderSyncMisbehavior::InvalidNewBlock) + .await; + } + + async fn handle_wire_decode_failed( + &mut self, + peer: ZakuraPeerId, + error: Arc, + ) { + self.trace_peer_violation(&peer, HeaderSyncMisbehavior::MalformedMessage); + tracing::debug!(?peer, ?error, "malformed Zakura header-sync frame"); + self.report_misbehavior(peer, HeaderSyncMisbehavior::MalformedMessage) + .await; + } + + async fn handle_state_frontiers_changed(&mut self, frontiers: HeaderSyncFrontiers) { + self.state.finalized_height = frontiers.finalized_height; + self.state.verified_block_tip = frontiers.verified_block_tip; + self.schedule().await; + } + + async fn handle_header_range_committed( + &mut self, + start_height: block::Height, + tip_height: block::Height, + tip_hash: block::Hash, + ) { + metrics::counter!("sync.header.range.committed").increment(1); + self.trace_range_event( + hs_trace::HEADER_RANGE_COMMITTED, + start_height, + count_between(start_height, tip_height), + None, + None, + ); + self.state + .pending_commits + .retain(|_, range| !range.is_within(start_height, tip_height)); + self.state + .schedule + .mark_range_covered(start_height, tip_height); + // The zebrad driver also uses this event to reload the durable best header tip at + // startup. In that path start==tip, so covered-range side effects are bounded. + self.cancel_covered_outstanding(); + if tip_height > self.state.best_header_tip { + self.publish_best_tip(tip_height, tip_hash).await; + } + self.notify_body_gaps().await; + self.schedule().await; + } + + async fn handle_header_range_commit_failed( + &mut self, + peer: ZakuraPeerId, + start_height: block::Height, + count: u32, + kind: HeaderSyncCommitFailureKind, + ) { + metrics::counter!("sync.header.range.rejected").increment(1); + self.trace_range_event( + hs_trace::HEADER_RANGE_REJECTED, + start_height, + count, + Some(&peer), + Some(commit_failure_reason_label(kind)), + ); + if kind == HeaderSyncCommitFailureKind::InvalidPeerRange { + self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) + .await; + } + let key = PendingCommitKey { + peer, + start_height, + count, + }; + if let Some(range) = self.state.pending_commits.remove(&key) { + if kind == HeaderSyncCommitFailureKind::Local { + self.state.schedule.clear_assignment(range); + } + self.state.schedule.retry(range); + } + self.schedule().await; + } + + fn handle_header_range_response_finished( + &mut self, + peer: ZakuraPeerId, + start_height: block::Height, + requested_count: u32, + returned_count: u32, + ) { + self.trace_headers_served(&peer, start_height, requested_count, returned_count); + if let Some(peer_state) = self.state.peers.get_mut(&peer) { + peer_state.finish_serving_headers(); + } + } + + async fn handle_wire_frame(&mut self, peer: ZakuraPeerId, frame: Frame) { + if u8::try_from(frame.message_type).ok() != Some(MSG_HS_HEADERS) { + match HeaderSyncMessage::decode_frame(frame, HeaderSyncDecodeContext::control()) { + Ok(msg) => self.handle_wire_message(peer, msg).await, + Err(error) => { + self.trace_peer_violation(&peer, HeaderSyncMisbehavior::MalformedMessage); + tracing::debug!(?peer, ?error, "malformed Zakura header-sync frame"); + self.report_misbehavior(peer, HeaderSyncMisbehavior::MalformedMessage) + .await; + } + } + return; + } + + // `Headers` response decode still depends on this actor's per-peer + // outstanding-request state. The per-peer concurrency epic moves that + // contract into the Sink task and removes this residual raw-frame hop. + metrics::counter!("sync.header.response.received").increment(1); + let Some(peer_state) = self.state.peers.get_mut(&peer) else { + self.report_misbehavior(peer, HeaderSyncMisbehavior::UnsolicitedHeaders) + .await; + return; + }; + let Some(outstanding) = peer_state.pop_oldest_outstanding() else { + if peer_state.take_late_covered_response() { + return; + } + self.report_misbehavior(peer, HeaderSyncMisbehavior::UnsolicitedHeaders) + .await; + return; + }; + let peer_max_headers_per_response = peer_state.max_headers_per_response; + let in_flight_count = peer_state.outstanding.len(); + + let decode_context = HeaderSyncDecodeContext::for_headers_response( + HeaderSyncRequestContract::new( + outstanding.range.start_height, + outstanding.expected_max_count, + ) + .expect("outstanding range uses a non-zero bounded count"), + peer_max_headers_per_response, + ); + + let headers = match HeaderSyncMessage::decode_frame(frame, decode_context) { + Ok(HeaderSyncMessage::Headers(headers)) => headers, + Ok(_) => { + self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::MalformedMessage) + .await; + self.state.schedule.retry(outstanding.range); + self.schedule().await; + return; + } + Err(error) => { + self.trace_peer_violation(&peer, HeaderSyncMisbehavior::MalformedMessage); + tracing::debug!(?peer, ?error, "malformed Zakura header-sync frame"); + self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::MalformedMessage) + .await; + self.state.schedule.retry(outstanding.range); + self.schedule().await; + return; + } + }; + + self.handle_headers_for_outstanding( + peer, + headers, + outstanding, + peer_max_headers_per_response, + in_flight_count, + ) + .await; + } + + async fn handle_wire_message(&mut self, peer: ZakuraPeerId, msg: HeaderSyncMessage) { + match msg { + HeaderSyncMessage::Status(status) => { + metrics::counter!("sync.header.peer.status.received").increment(1); + if status.anchor_height > status.tip_height { + self.report_misbehavior(peer, HeaderSyncMisbehavior::InvalidStatus) + .await; + return; + } + + let Some(peer_state) = self.state.peers.get_mut(&peer) else { + return; + }; + if !peer_state.inbound_status.try_take(Instant::now()) { + self.report_misbehavior(peer, HeaderSyncMisbehavior::StatusSpam) + .await; + return; + } + peer_state.advertised_tip = status.tip_height; + peer_state.anchor = status.anchor_height; + peer_state.max_headers_per_response = + clamp_advertised_range(status.max_headers_per_response); + peer_state.max_inflight_requests = status + .max_inflight_requests + .clamp(1, LOCAL_MAX_HS_INFLIGHT_PER_PEER); + peer_state.received_status = true; + self.trace_status_received(&peer, status); + self.schedule().await; + } + HeaderSyncMessage::Headers(headers) => { + self.handle_headers(peer, headers).await; + } + HeaderSyncMessage::GetHeaders { + start_height, + count, + } => { + self.handle_get_headers(peer, start_height, count).await; + } + HeaderSyncMessage::NewBlock(block) => { + self.handle_new_block(peer, block).await; + } + } + } + + async fn handle_get_headers( + &mut self, + peer: ZakuraPeerId, + start_height: block::Height, + count: u32, + ) { + let local_inflight_cap = self.startup.config.advertised_max_inflight_requests(); + let Some(peer_state) = self.state.peers.get_mut(&peer) else { + self.report_misbehavior(peer, HeaderSyncMisbehavior::GetHeadersSpam) + .await; + return; + }; + + if !peer_state.received_status { + self.report_misbehavior(peer, HeaderSyncMisbehavior::GetHeadersSpam) + .await; + return; + } + + let allowed_count = inbound_get_headers_count_limit( + &self.startup.config, + &self.startup.network, + self.startup.max_frame_bytes, + ); + if count == 0 || count > allowed_count { + self.report_misbehavior(peer, HeaderSyncMisbehavior::GetHeadersTooLong) + .await; + return; + } + + if !peer_state.try_start_serving_headers(local_inflight_cap) { + self.report_misbehavior(peer, HeaderSyncMisbehavior::GetHeadersSpam) + .await; + return; + } + + if self + .actions + .send(HeaderSyncAction::QueryHeadersByHeightRange { + peer: peer.clone(), + start: start_height, + count, + }) + .await + .is_err() + { + if let Some(peer_state) = self.state.peers.get_mut(&peer) { + peer_state.finish_serving_headers(); + } + } + } + + #[tracing::instrument(skip(self, block))] + async fn handle_new_block(&mut self, peer: ZakuraPeerId, block: Arc) { + metrics::counter!("sync.header.tip.new_block.received").increment(1); + + if !self.state.peers.contains_key(&peer) { + self.report_misbehavior(peer, HeaderSyncMisbehavior::UnknownPeer) + .await; + return; + } + + let hash = block.hash(); + let Some(height) = block.coinbase_height() else { + self.report_misbehavior(peer, HeaderSyncMisbehavior::MalformedMessage) + .await; + return; + }; + self.trace_new_block_received(&peer, height, hash); + + if self.state.seen.contains(&hash) { + metrics::counter!("sync.header.tip.new_block.deduped").increment(1); + self.trace_new_block_deduped(&peer, height, hash, "seen_cache"); + return; + } + if self.state.pending_new_blocks.contains(&hash) { + metrics::counter!("sync.header.tip.new_block.deduped").increment(1); + self.trace_new_block_deduped(&peer, height, hash, "pending_acceptance"); + return; + } + + if !self + .state + .peers + .get_mut(&peer) + .expect("peer exists because it was checked before validation") + .inbound_new_block + .try_take(Instant::now()) + { + self.report_misbehavior(peer, HeaderSyncMisbehavior::NewBlockSpam) + .await; + return; + } + + if validate_new_block_stateless(block.clone(), &self.startup.network, Utc::now(), height) + .await + .is_err() + { + self.report_misbehavior(peer, HeaderSyncMisbehavior::InvalidNewBlock) + .await; + return; + } + + if !self.startup.inbound_new_block_acceptance_enabled { + metrics::counter!("sync.header.tip.new_block.acceptance_unavailable").increment(1); + debug!( + ?peer, + ?hash, + "Zakura header-sync NewBlock body suppressed until block acceptance is wired" + ); + return; + } + + let inserted = self.state.pending_new_blocks.insert(hash); + debug_assert!(inserted, "pending acceptance was checked before insert"); + + if self + .actions + .send(HeaderSyncAction::NewBlockReceived { + peer, + height, + hash, + block, + }) + .await + .is_err() + { + self.state.pending_new_blocks.remove(&hash); + } + } + + fn eligible_tip_destinations( + &self, + source: &ZakuraPeerId, + height: block::Height, + ) -> Vec { + let mut peers: Vec<_> = self + .state + .peers + .iter() + .filter(|(peer_id, peer)| { + *peer_id != source && (!peer.received_status || peer.advertised_tip < height) + }) + .map(|(peer_id, _)| peer_id.clone()) + .collect(); + peers.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes())); + peers + } + + #[tracing::instrument(skip(self, headers))] + async fn handle_headers(&mut self, peer: ZakuraPeerId, headers: Vec>) { + metrics::counter!("sync.header.response.received").increment(1); + let Some(peer_state) = self.state.peers.get_mut(&peer) else { + self.report_misbehavior(peer, HeaderSyncMisbehavior::UnsolicitedHeaders) + .await; + return; + }; + let Some(outstanding) = peer_state.pop_oldest_outstanding() else { + if peer_state.take_late_covered_response() { + return; + } + self.report_misbehavior(peer, HeaderSyncMisbehavior::UnsolicitedHeaders) + .await; + return; + }; + let peer_max_headers_per_response = peer_state.max_headers_per_response; + let in_flight_count = peer_state.outstanding.len(); + + self.handle_headers_for_outstanding( + peer, + headers, + outstanding, + peer_max_headers_per_response, + in_flight_count, + ) + .await; + } + + async fn handle_headers_for_outstanding( + &mut self, + peer: ZakuraPeerId, + headers: Vec>, + outstanding: OutstandingRange, + peer_max_headers_per_response: u32, + in_flight_count: usize, + ) { + if headers.is_empty() { + let deadline = Instant::now() + self.empty_headers_retry_delay(); + self.trace_headers_received( + &peer, + outstanding.range.start_height, + 0, + outstanding.expected_max_count, + peer_max_headers_per_response, + in_flight_count, + ); + if let Some(peer_state) = self.state.peers.get_mut(&peer) { + peer_state.outstanding.push(OutstandingRange { + deadline, + clear_assignment_on_timeout: true, + ..outstanding + }); + } + return; + } + + let header_count = + u32::try_from(headers.len()).expect("decoded Headers length is capped by u32"); + self.trace_headers_received( + &peer, + outstanding.range.start_height, + header_count, + outstanding.expected_max_count, + peer_max_headers_per_response, + in_flight_count, + ); + if header_count > outstanding.expected_max_count || header_count > outstanding.range.count { + self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::ResponseTooLong) + .await; + self.state.schedule.retry(outstanding.range); + self.schedule().await; + return; + } + + let validation_context = HeaderSyncValidationContext { + network: &self.startup.network, + now: Utc::now(), + start_height: outstanding.range.start_height, + decode_context: HeaderSyncDecodeContext::for_headers_response( + HeaderSyncRequestContract::new( + outstanding.range.start_height, + outstanding.expected_max_count, + ) + .expect("outstanding range uses a non-zero bounded count"), + outstanding.expected_max_count, + ), + }; + if validate_header_range_links(outstanding.range.anchor_hash, &headers).is_err() { + self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) + .await; + self.state.schedule.retry(outstanding.range); + self.schedule().await; + return; + } + if validate_headers_stateless(headers.clone(), validation_context) + .await + .is_err() + { + self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) + .await; + self.state.schedule.retry(outstanding.range); + self.schedule().await; + return; + } + + let end_height = height_after_count(outstanding.range.start_height, header_count) + .and_then(previous_height) + .expect("non-empty bounded range has an end height"); + if outstanding.range.finalized { + let last_hash = headers + .last() + .map(|header| block::Hash::from(header.as_ref())) + .expect("headers is non-empty"); + if end_height != outstanding.range.end_height() + || self.startup.network.checkpoint_list().hash(end_height) != Some(last_hash) + { + self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) + .await; + self.state.schedule.retry(outstanding.range); + self.schedule().await; + return; + } + } + + self.state.pending_commits.insert( + PendingCommitKey { + peer: peer.clone(), + start_height: outstanding.range.start_height, + count: header_count, + }, + outstanding.range, + ); + let _ = self + .actions + .send(HeaderSyncAction::CommitHeaderRange { + peer, + anchor: outstanding.range.anchor_hash, + start_height: outstanding.range.start_height, + headers, + finalized: outstanding.range.finalized, + }) + .await; + } + + async fn handle_timeouts(&mut self) { + let now = Instant::now(); + let mut timed_out = Vec::new(); + for peer in self.state.peers.values_mut() { + let mut index = 0; + while index < peer.outstanding.len() { + if peer.outstanding[index].deadline <= now { + let outstanding = peer.outstanding.remove(index); + timed_out.push((outstanding.range, outstanding.clear_assignment_on_timeout)); + } else { + index += 1; + } + } + } + for (range, clear_assignment) in timed_out { + if clear_assignment { + self.state.schedule.clear_assignment(range); + } + self.state.schedule.retry(range); + } + self.schedule().await; + } + + fn empty_headers_retry_delay(&self) -> Duration { + self.startup.request_timeout.min(EMPTY_HEADERS_RETRY_DELAY) + } + + async fn schedule(&mut self) { + if !self.startup.range_state_actions_enabled { + return; + } + + self.state.refresh_forward_range(&self.startup); + self.state.refresh_backward_range(&self.startup); + + let mut peer_ids: Vec = self.state.peers.keys().cloned().collect(); + peer_ids.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes())); + + for peer_id in peer_ids { + let Some(peer) = self.state.peers.get(&peer_id) else { + continue; + }; + if !peer.received_status || peer.available_slots() == 0 { + continue; + } + + let Some(mut range) = self.state.schedule.next_for_peer(&peer_id, peer) else { + continue; + }; + let original_range = range; + + let count = clamp_header_sync_request_count( + range.count, + peer.max_headers_per_response, + &self.startup.network, + self.startup.max_frame_bytes, + ); + if range.finalized && count < range.count { + self.state.schedule.retry(range); + continue; + } + range.count = count; + self.state + .schedule + .narrow_queued_range(original_range, range); + + let deadline = Instant::now() + self.startup.request_timeout; + let outstanding = OutstandingRange { + range, + deadline, + expected_max_count: count, + clear_assignment_on_timeout: false, + }; + let peer_cap = peer.max_headers_per_response; + if let Some(peer) = self.state.peers.get_mut(&peer_id) { + peer.outstanding.push(outstanding); + } + self.state.schedule.mark_assigned(peer_id.clone(), range); + let destination = peer_id.clone(); + let _ = self + .actions + .send(HeaderSyncAction::SendMessage { + peer: peer_id, + msg: HeaderSyncMessage::GetHeaders { + start_height: range.start_height, + count, + }, + }) + .await; + metrics::counter!("sync.header.request.sent").increment(1); + self.trace_get_headers_sent(&destination, range.start_height, count, peer_cap); + } + } + + async fn send_status(&self, peer: ZakuraPeerId) { + metrics::counter!("sync.header.peer.status.sent").increment(1); + self.trace_status_sent(&peer, self.local_status()); + let _ = self + .actions + .send(HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::Status(self.local_status()), + }) + .await; + } + + async fn publish_best_tip(&mut self, height: block::Height, hash: block::Hash) { + self.state.best_header_tip = height; + self.state.best_header_hash = hash; + metrics::gauge!("sync.header.best_tip.height").set(height.0 as f64); + self.trace_frontier_advanced(height, hash); + let _ = self.tip.send((height, hash)); + self.broadcast_status_refresh().await; + } + + async fn broadcast_status_refresh(&mut self) { + let now = Instant::now(); + let status = self.local_status(); + let peer_ids: Vec<_> = self + .state + .peers + .iter_mut() + .filter_map(|(peer_id, peer)| peer.unsolicited.try_take(now).then(|| peer_id.clone())) + .collect(); + + for peer in peer_ids { + let _ = self + .actions + .send(HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::Status(status), + }) + .await; + } + } + + async fn notify_body_gaps(&self) { + if !self.startup.range_state_actions_enabled { + return; + } + + if self.state.best_header_tip > self.state.verified_block_tip { + let from = + next_height(self.state.verified_block_tip).unwrap_or(self.state.verified_block_tip); + metrics::gauge!("sync.header.missing_bodies") + .set(count_between(from, self.state.best_header_tip) as f64); + self.trace_missing_bodies(from, self.state.best_header_tip); + let _ = self + .actions + .send(HeaderSyncAction::BodyGaps { + from, + to: self.state.best_header_tip, + }) + .await; + } + } + + async fn report_misbehavior(&mut self, peer: ZakuraPeerId, reason: HeaderSyncMisbehavior) { + if let Some(peer_state) = self.state.peers.get_mut(&peer) { + peer_state.misbehavior = peer_state.misbehavior.saturating_add(1); + } + metrics::counter!("sync.header.peer.disconnect").increment(1); + self.trace_peer_violation(&peer, reason); + self.trace_peer_disconnect_requested(&peer, reason); + let _ = self + .actions + .send(HeaderSyncAction::Misbehavior { peer, reason }) + .await; + } + + fn trace_status_sent(&self, peer: &ZakuraPeerId, status: HeaderSyncStatus) { + self.emit_trace(hs_trace::HEADER_STATUS_SENT, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, status.tip_height); + insert_hash(row, hs_trace::HASH, status.tip_hash); + insert_height(row, hs_trace::RANGE_START, status.anchor_height); + insert_u64( + row, + hs_trace::ADVERTISED_CAP, + u64::from(status.max_headers_per_response), + ); + insert_u64( + row, + hs_trace::IN_FLIGHT_COUNT, + u64::from(status.max_inflight_requests), + ); + }); + } + + fn trace_status_received(&self, peer: &ZakuraPeerId, status: HeaderSyncStatus) { + self.emit_trace(hs_trace::HEADER_STATUS_RECEIVED, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, status.tip_height); + insert_hash(row, hs_trace::HASH, status.tip_hash); + insert_height(row, hs_trace::RANGE_START, status.anchor_height); + insert_u64( + row, + hs_trace::ADVERTISED_CAP, + u64::from(status.max_headers_per_response), + ); + insert_u64( + row, + hs_trace::IN_FLIGHT_COUNT, + u64::from(status.max_inflight_requests), + ); + }); + } + + fn trace_get_headers_sent( + &self, + peer: &ZakuraPeerId, + start_height: block::Height, + count: u32, + advertised_cap: u32, + ) { + self.emit_trace(hs_trace::HEADER_GET_HEADERS_SENT, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(count)); + insert_u64(row, hs_trace::ADVERTISED_CAP, u64::from(advertised_cap)); + }); + } + + fn trace_headers_received( + &self, + peer: &ZakuraPeerId, + start_height: block::Height, + count: u32, + expected_max_count: u32, + advertised_cap: u32, + in_flight_count: usize, + ) { + self.emit_trace(hs_trace::HEADER_HEADERS_RECEIVED, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(count)); + insert_u64(row, hs_trace::ADVERTISED_CAP, u64::from(advertised_cap)); + insert_u64(row, hs_trace::EXPECTED_COUNT, u64::from(expected_max_count)); + insert_u64(row, hs_trace::IN_FLIGHT_COUNT, in_flight_count as u64); + }); + } + + fn trace_headers_served( + &self, + peer: &ZakuraPeerId, + start_height: block::Height, + requested_count: u32, + returned_count: u32, + ) { + self.emit_trace(hs_trace::HEADER_HEADERS_SERVED, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(returned_count)); + insert_u64(row, hs_trace::EXPECTED_COUNT, u64::from(requested_count)); + }); + } + + fn trace_range_event( + &self, + event: &'static str, + start_height: block::Height, + count: u32, + peer: Option<&ZakuraPeerId>, + reason: Option<&'static str>, + ) { + self.emit_trace(event, |row| { + if let Some(peer) = peer { + insert_peer(row, hs_trace::PEER, peer); + } + insert_height(row, hs_trace::RANGE_START, start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(count)); + insert_optional_str(row, hs_trace::REASON, reason); + }); + } + + fn trace_new_block_received( + &self, + peer: &ZakuraPeerId, + height: block::Height, + hash: block::Hash, + ) { + self.emit_trace(hs_trace::HEADER_NEW_BLOCK_RECEIVED, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, height); + insert_hash(row, hs_trace::HASH, hash); + }); + } + + fn trace_new_block_forwarded( + &self, + source: &ZakuraPeerId, + destination: &ZakuraPeerId, + height: block::Height, + hash: block::Hash, + destination_count: usize, + ) { + self.emit_trace(hs_trace::HEADER_NEW_BLOCK_FORWARDED, |row| { + insert_peer(row, hs_trace::SOURCE_PEER, source); + insert_peer(row, hs_trace::PEER, destination); + insert_height(row, hs_trace::HEIGHT, height); + insert_hash(row, hs_trace::HASH, hash); + insert_u64( + row, + hs_trace::DESTINATION_PEER_COUNT, + destination_count as u64, + ); + }); + } + + fn trace_new_block_deduped( + &self, + peer: &ZakuraPeerId, + height: block::Height, + hash: block::Hash, + reason: &'static str, + ) { + self.emit_trace(hs_trace::HEADER_NEW_BLOCK_DEDUPED, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, height); + insert_hash(row, hs_trace::HASH, hash); + insert_optional_str(row, hs_trace::REASON, Some(reason)); + }); + } + + fn trace_peer_violation(&self, peer: &ZakuraPeerId, reason: HeaderSyncMisbehavior) { + self.emit_trace(hs_trace::HEADER_PEER_VIOLATION, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_optional_str( + row, + hs_trace::REASON, + Some(misbehavior_reason_label(reason)), + ); + }); + } + + fn trace_peer_disconnect_requested(&self, peer: &ZakuraPeerId, reason: HeaderSyncMisbehavior) { + self.emit_trace(hs_trace::HEADER_PEER_DISCONNECT_REQUESTED, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_optional_str( + row, + hs_trace::REASON, + Some(misbehavior_reason_label(reason)), + ); + }); + } + + fn trace_frontier_advanced(&self, height: block::Height, hash: block::Hash) { + self.emit_trace(hs_trace::HEADER_FRONTIER_ADVANCED, |row| { + insert_height(row, hs_trace::HEIGHT, height); + insert_hash(row, hs_trace::HASH, hash); + }); + } + + fn trace_missing_bodies(&self, from: block::Height, to: block::Height) { + self.emit_trace(hs_trace::HEADER_MISSING_BODIES_REPORTED, |row| { + insert_height(row, hs_trace::RANGE_START, from); + insert_u64( + row, + hs_trace::RANGE_COUNT, + u64::from(count_between(from, to)), + ); + }); + } + + fn emit_trace( + &self, + event: &'static str, + build: impl FnOnce(&mut serde_json::Map), + ) { + self.startup.trace.emit_with(HEADER_SYNC_TABLE, |row| { + row.insert( + hs_trace::EVENT.to_string(), + Value::String(event.to_string()), + ); + build(row); + }); + } + + fn local_status(&self) -> HeaderSyncStatus { + HeaderSyncStatus { + tip_height: self.state.best_header_tip, + tip_hash: self.state.best_header_hash, + anchor_height: self.state.anchor.0, + max_headers_per_response: self.startup.config.advertised_max_headers_per_response(), + max_inflight_requests: self.startup.config.advertised_max_inflight_requests(), + } + } + + fn cancel_covered_outstanding(&mut self) { + for peer in self.state.peers.values_mut() { + let mut index = 0; + while index < peer.outstanding.len() { + if self + .state + .schedule + .is_covered(peer.outstanding[index].range) + { + peer.outstanding.remove(index); + peer.late_covered_responses = peer.late_covered_responses.saturating_add(1); + } else { + index += 1; + } + } + } + } +} diff --git a/zebra-network/src/zakura/header_sync/scheduler.rs b/zebra-network/src/zakura/header_sync/scheduler.rs new file mode 100644 index 00000000000..fd6e4e47d7c --- /dev/null +++ b/zebra-network/src/zakura/header_sync/scheduler.rs @@ -0,0 +1,246 @@ +use super::{state::*, wire::*, *}; + +#[derive(Clone, Debug, Default)] +pub(super) struct HeaderHashDedup { + pub(super) hashes: HashSet, + pub(super) order: VecDeque, +} + +impl HeaderHashDedup { + pub(super) fn contains(&self, hash: &block::Hash) -> bool { + self.hashes.contains(hash) + } + + pub(super) fn insert(&mut self, hash: block::Hash) -> bool { + if !self.hashes.insert(hash) { + return false; + } + self.order.push_back(hash); + while self.order.len() > HEADER_SYNC_SEEN_HASH_CAPACITY { + if let Some(oldest) = self.order.pop_front() { + self.hashes.remove(&oldest); + } + } + true + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(super) struct PendingCommitKey { + pub(super) peer: ZakuraPeerId, + pub(super) start_height: block::Height, + pub(super) count: u32, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(super) struct CoveredRange { + pub(super) start: block::Height, + pub(super) end: block::Height, +} + +#[derive(Clone, Debug)] +pub(super) struct RangeScheduler { + pub(super) forward: VecDeque, + pub(super) backward: VecDeque, + pub(super) assigned: HashMap>, + pub(super) covered: Vec, +} + +impl RangeScheduler { + pub(super) fn new() -> Self { + Self { + forward: VecDeque::new(), + backward: VecDeque::new(), + assigned: HashMap::new(), + covered: Vec::new(), + } + } + + pub(super) fn ensure_forward(&mut self, range: RangeRequest) { + self.ensure(range, RangePriority::Forward); + } + + pub(super) fn ensure_backward(&mut self, range: RangeRequest) { + self.ensure(range, RangePriority::Backward); + } + + pub(super) fn ensure(&mut self, range: RangeRequest, priority: RangePriority) { + if self.is_covered(range) + || self.assigned.contains_key(&range) + || self.assigned.keys().any(|assigned| { + assigned.start_height == range.start_height && assigned.priority == priority + }) + { + return; + } + let queue = match priority { + RangePriority::Forward => &mut self.forward, + RangePriority::Backward => &mut self.backward, + }; + if !queue.contains(&range) + && !queue.iter().any(|queued| { + queued.start_height == range.start_height && queued.priority == priority + }) + { + queue.push_back(range); + } + } + + pub(super) fn next_for_peer( + &mut self, + peer_id: &ZakuraPeerId, + peer: &PeerHeaderState, + ) -> Option { + Self::pop_assignable(&mut self.forward, &self.assigned, peer_id, peer) + .or_else(|| Self::pop_assignable(&mut self.backward, &self.assigned, peer_id, peer)) + } + + pub(super) fn pop_assignable( + queue: &mut VecDeque, + assigned: &HashMap>, + peer_id: &ZakuraPeerId, + peer: &PeerHeaderState, + ) -> Option { + let index = queue.iter().position(|range| { + range.end_height() <= peer.advertised_tip + && assigned.get(range).is_none_or(|peers| { + peers.len() < HEADER_SYNC_FANOUT && !peers.contains(peer_id) + }) + })?; + let range = queue[index]; + if assigned + .get(&range) + .is_some_and(|peers| peers.len() + 1 >= HEADER_SYNC_FANOUT) + { + queue.remove(index); + } + Some(range) + } + + pub(super) fn mark_assigned(&mut self, peer: ZakuraPeerId, range: RangeRequest) { + self.assigned.entry(range).or_default().insert(peer); + } + + pub(super) fn narrow_queued_range(&mut self, original: RangeRequest, narrowed: RangeRequest) { + if original == narrowed { + return; + } + + let queue = match original.priority { + RangePriority::Forward => &mut self.forward, + RangePriority::Backward => &mut self.backward, + }; + for queued in queue { + if *queued == original { + *queued = narrowed; + break; + } + } + if let Some(peers) = self.assigned.remove(&original) { + self.assigned.entry(narrowed).or_default().extend(peers); + } + } + + pub(super) fn retry(&mut self, range: RangeRequest) { + if self.is_covered(range) { + return; + } + match range.priority { + RangePriority::Forward => self.forward.push_front(range), + RangePriority::Backward => self.backward.push_front(range), + } + } + + pub(super) fn forget_peer(&mut self, peer: &ZakuraPeerId) { + for peers in self.assigned.values_mut() { + peers.remove(peer); + } + } + + pub(super) fn clear_assignment(&mut self, range: RangeRequest) { + self.assigned.remove(&range); + } + + pub(super) fn mark_height_covered(&mut self, height: block::Height) { + self.mark_covered_interval(CoveredRange { + start: height, + end: height, + }); + self.prune_covered(); + } + + pub(super) fn mark_range_covered(&mut self, start: block::Height, end: block::Height) { + self.mark_covered_interval(CoveredRange { start, end }); + self.prune_covered(); + } + + pub(super) fn is_covered(&self, range: RangeRequest) -> bool { + let end = range.end_height(); + self.covered + .iter() + .any(|covered| covered.start <= range.start_height && covered.end >= end) + } + + pub(super) fn mark_covered_interval(&mut self, mut interval: CoveredRange) { + if interval.end < interval.start { + return; + } + + let mut merged = Vec::with_capacity(self.covered.len().saturating_add(1)); + let mut inserted = false; + for covered in self.covered.drain(..) { + if covered.end.0.saturating_add(1) < interval.start.0 { + merged.push(covered); + } else if interval.end.0.saturating_add(1) < covered.start.0 { + if !inserted { + merged.push(interval); + inserted = true; + } + merged.push(covered); + } else { + interval.start = interval.start.min(covered.start); + interval.end = interval.end.max(covered.end); + } + } + if !inserted { + merged.push(interval); + } + self.covered = merged; + } + + pub(super) fn prune_covered(&mut self) { + let covered = self.covered.clone(); + let is_covered = |range: &RangeRequest| { + let end = range.end_height(); + covered + .iter() + .any(|covered| covered.start <= range.start_height && covered.end >= end) + }; + self.forward.retain(|range| !is_covered(range)); + self.backward.retain(|range| !is_covered(range)); + self.assigned.retain(|range, _| !is_covered(range)); + } +} + +#[derive(Clone, Debug)] +pub(super) struct RateMeter { + pub(super) next_allowed: Instant, + pub(super) interval: Duration, +} + +impl RateMeter { + pub(super) fn new(interval: Duration) -> Self { + Self { + next_allowed: Instant::now(), + interval, + } + } + + pub(super) fn try_take(&mut self, now: Instant) -> bool { + if now < self.next_allowed { + return false; + } + self.next_allowed = now + self.interval; + true + } +} diff --git a/zebra-network/src/zakura/header_sync/service.rs b/zebra-network/src/zakura/header_sync/service.rs new file mode 100644 index 00000000000..0f1cf7db07d --- /dev/null +++ b/zebra-network/src/zakura/header_sync/service.rs @@ -0,0 +1,509 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex as StdMutex, OnceLock}, +}; + +use tokio::{sync::mpsc, task}; +use tokio_util::sync::CancellationToken; + +use super::{events::*, validation::*, wire::*, *}; +use crate::{ + zakura::{ + BoxRunFuture, Frame, FramedRecv, FramedSend, Peer, Service, Sink, SinkReject, Stream, + StreamMode, ZakuraPeerId, ZakuraSupervisorHandle, ZAKURA_CAP_HEADER_SYNC, + }, + BoxError, +}; + +const HEADER_SYNC_SERVICE_STREAMS: [Stream; 1] = [Stream { + kind: ZAKURA_STREAM_HEADER_SYNC, + version: ZAKURA_HEADER_SYNC_STREAM_VERSION, + // Advisory until the transport wires Stream::frame_cap end-to-end; the + // authoritative inbound cap is app_frame_cap_for_stream_kind. The cast is + // safe because both terms are small protocol constants checked against the + // local message cap in header_sync::wire. + frame_cap: (MAX_HS_MESSAGE_BYTES + FRAME_HEADER_BYTES) as u32, + capability: ZAKURA_CAP_HEADER_SYNC, + mode: StreamMode::Ordered, +}]; + +/// Service-declared streams for native header sync. +pub(crate) fn header_sync_streams() -> &'static [Stream] { + &HEADER_SYNC_SERVICE_STREAMS +} + +/// Stream-5 outbound sources keyed by peer. +#[derive(Clone, Debug, Default)] +pub(crate) struct HeaderSyncOutbound { + senders: Arc>>, +} + +impl HeaderSyncOutbound { + pub(crate) async fn send(&self, peer: &ZakuraPeerId, frame: Frame) -> Result<(), BoxError> { + let sender = { + let senders = self + .senders + .lock() + .expect("header-sync outbound mutex is never poisoned"); + senders.get(peer).cloned() + }; + let Some(sender) = sender else { + return Err("no ready Zakura header-sync source for peer".into()); + }; + + sender + .send(frame) + .await + .map_err(|_| -> BoxError { "Zakura header-sync source queue closed".into() }) + } + + fn insert(&self, peer: ZakuraPeerId, sender: FramedSend) { + self.senders + .lock() + .expect("header-sync outbound mutex is never poisoned") + .insert(peer, sender); + } + + fn remove(&self, peer: &ZakuraPeerId) { + self.senders + .lock() + .expect("header-sync outbound mutex is never poisoned") + .remove(peer); + } +} + +static HEADER_SYNC_OUTBOUND_BY_SUPERVISOR: OnceLock>> = + OnceLock::new(); + +pub(crate) fn header_sync_outbound_for_supervisor( + supervisor: &ZakuraSupervisorHandle, +) -> HeaderSyncOutbound { + let registry = HEADER_SYNC_OUTBOUND_BY_SUPERVISOR.get_or_init(Default::default); + let mut registry = registry + .lock() + .expect("header-sync outbound registry mutex is never poisoned"); + registry.entry(supervisor.id()).or_default().clone() +} + +/// Send a stream-5 message through the service-owned source for `peer`. +pub(crate) async fn send_header_sync_message( + supervisor: &ZakuraSupervisorHandle, + peer: &ZakuraPeerId, + msg: HeaderSyncMessage, +) { + let frame = match msg.encode_frame() { + Ok(frame) => frame, + Err(error) => { + tracing::debug!(?error, "failed to encode Zakura header-sync frame"); + return; + } + }; + + let outbound = header_sync_outbound_for_supervisor(supervisor); + if let Err(error) = outbound.send(peer, frame).await { + tracing::debug!(?error, ?peer, "failed to send Zakura header-sync frame"); + } +} + +/// Pump actor actions that can be satisfied at the transport/service seam. +pub(crate) async fn drive_header_sync_actions( + mut actions: mpsc::Receiver, + handle: HeaderSyncHandle, + supervisor: ZakuraSupervisorHandle, + shutdown: CancellationToken, +) { + loop { + let action = tokio::select! { + _ = shutdown.cancelled() => return, + action = actions.recv() => { + let Some(action) = action else { + return; + }; + action + } + }; + + match action { + HeaderSyncAction::SendMessage { peer, msg } => { + send_header_sync_message(&supervisor, &peer, msg).await; + } + HeaderSyncAction::ForwardNewBlock { peer, block, .. } => { + send_header_sync_message(&supervisor, &peer, HeaderSyncMessage::NewBlock(block)) + .await; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + tracing::debug!( + ?peer, + ?reason, + "disconnecting peer for Zakura header-sync violation" + ); + let _ = supervisor.disconnect_peer(&peer).await; + } + HeaderSyncAction::NewBlockReceived { + peer, + hash, + block: _, + .. + } => { + tracing::debug!( + ?peer, + ?hash, + "Zakura header-sync NewBlock body arrived before block-acceptance hook is wired" + ); + } + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + let _ = handle + .send(HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + .. + } => { + tracing::debug!( + ?peer, + ?start_height, + count = headers.len(), + "suppressing Zakura header range commit until state driver is wired" + ); + } + HeaderSyncAction::QueryBestHeaderTip + | HeaderSyncAction::QueryMissingBlockBodies { .. } + | HeaderSyncAction::BodyGaps { .. } => {} + } + } +} + +/// Native stream-5 header-sync service. +#[derive(Debug)] +pub(crate) struct HeaderSyncService { + header_sync: HeaderSyncHandle, + outbound: HeaderSyncOutbound, +} + +impl HeaderSyncService { + pub(crate) fn new(header_sync: HeaderSyncHandle, outbound: HeaderSyncOutbound) -> Self { + Self { + header_sync, + outbound, + } + } +} + +impl Service for HeaderSyncService { + fn name(&self) -> &'static str { + "header-sync" + } + + fn streams(&self) -> &[Stream] { + header_sync_streams() + } + + fn add_peer(&self, mut peer: Peer) { + let Some((recv, send)) = peer.take_stream(ZAKURA_STREAM_HEADER_SYNC) else { + return; + }; + + let peer_id = peer.id.clone(); + let cancel_token = peer.cancel_token(); + + self.outbound.insert(peer_id.clone(), send); + let _ = self + .header_sync + .send_lifecycle(HeaderSyncEvent::PeerConnected(peer_id.clone())); + + spawn_header_sync_sink( + peer_id, + recv, + self.header_sync.clone(), + self.outbound.clone(), + cancel_token, + ); + } + + fn remove_peer(&self, peer: &ZakuraPeerId) { + self.outbound.remove(peer); + let _ = self + .header_sync + .send_lifecycle(HeaderSyncEvent::PeerDisconnected(peer.clone())); + } + + fn deliver_frame( + &self, + peer_id: ZakuraPeerId, + stream_kind: u16, + frame: Frame, + ) -> Result<(), SinkReject> { + if stream_kind != ZAKURA_STREAM_HEADER_SYNC { + return Ok(()); + } + + deliver_header_sync_frame(&self.header_sync, peer_id, frame) + } + + fn request_frame<'a>( + &'a self, + _peer_id: ZakuraPeerId, + _stream_kind: u16, + _request_id: u64, + _max_frame_bytes: u32, + _frame: Frame, + ) -> BoxRunFuture<'a, Result, SinkReject>> { + Box::pin(async { + Err(SinkReject::protocol( + "header-sync request streams are not supported", + )) + }) + } +} + +/// Testkit/no-reactor mode records stream-5 inbound frames without running header sync. +#[derive(Debug)] +pub(crate) struct HeaderSyncPassthroughService { + inner: Arc, +} + +impl HeaderSyncPassthroughService { + pub(crate) fn new(inner: Arc) -> Self { + Self { inner } + } +} + +impl Service for HeaderSyncPassthroughService { + fn name(&self) -> &'static str { + "header-sync-passthrough" + } + + fn streams(&self) -> &[Stream] { + header_sync_streams() + } + + fn add_peer(&self, mut peer: Peer) { + let Some((recv, _send)) = peer.take_stream(ZAKURA_STREAM_HEADER_SYNC) else { + return; + }; + + let inner = self.inner.clone(); + let peer_id = peer.id.clone(); + let cancel_token = peer.cancel_token(); + + task::spawn(async move { + let sink = Box::new(HeaderSyncPassthroughSink { + peer_id: peer_id.clone(), + inner, + cancel_token: cancel_token.clone(), + }); + + match sink.run(recv).await { + Ok(()) => {} + Err(SinkReject::Protocol(error)) => { + tracing::debug!( + ?error, + ?peer_id, + "header-sync passthrough rejected protocol-invalid frame" + ); + cancel_token.cancel(); + } + Err(SinkReject::Local(error)) => { + tracing::debug!( + ?error, + ?peer_id, + "header-sync passthrough could not deliver frame locally" + ); + } + } + }); + } + + fn remove_peer(&self, _peer: &ZakuraPeerId) {} + + fn deliver_frame( + &self, + peer_id: ZakuraPeerId, + stream_kind: u16, + frame: Frame, + ) -> Result<(), SinkReject> { + self.inner.deliver_frame(peer_id, stream_kind, frame) + } + + fn request_frame<'a>( + &'a self, + _peer_id: ZakuraPeerId, + _stream_kind: u16, + _request_id: u64, + _max_frame_bytes: u32, + _frame: Frame, + ) -> BoxRunFuture<'a, Result, SinkReject>> { + Box::pin(async { + Err(SinkReject::protocol( + "header-sync request streams are not supported", + )) + }) + } +} + +fn spawn_header_sync_sink( + peer_id: ZakuraPeerId, + recv: FramedRecv, + header_sync: HeaderSyncHandle, + outbound: HeaderSyncOutbound, + cancel_token: CancellationToken, +) { + task::spawn(async move { + let sink = Box::new(HeaderSyncSink { + peer_id: peer_id.clone(), + header_sync, + cancel_token: cancel_token.clone(), + }); + + let result = sink.run(recv).await; + outbound.remove(&peer_id); + + match result { + Ok(()) => {} + Err(SinkReject::Protocol(error)) => { + tracing::debug!( + ?error, + ?peer_id, + "header-sync stream rejected protocol-invalid frame" + ); + cancel_token.cancel(); + } + Err(SinkReject::Local(error)) => { + tracing::debug!( + ?error, + ?peer_id, + "header-sync stream could not deliver frame locally" + ); + } + } + }); +} + +#[derive(Debug)] +struct HeaderSyncSink { + peer_id: ZakuraPeerId, + header_sync: HeaderSyncHandle, + cancel_token: CancellationToken, +} + +impl Sink for HeaderSyncSink { + fn run(self: Box, mut recv: FramedRecv) -> BoxRunFuture<'static, Result<(), SinkReject>> { + Box::pin(async move { + loop { + let frame = tokio::select! { + _ = self.cancel_token.cancelled() => return Ok(()), + frame = recv.recv() => { + let Some(frame) = frame else { + return Ok(()); + }; + frame + } + }; + + match deliver_header_sync_frame(&self.header_sync, self.peer_id.clone(), frame) { + Ok(()) => {} + Err(SinkReject::Protocol(error)) => return Err(SinkReject::Protocol(error)), + Err(SinkReject::Local(error)) => { + tracing::debug!( + ?error, + peer_id = ?self.peer_id, + "header-sync stream could not deliver frame locally" + ); + } + } + } + }) + } +} + +#[derive(Debug)] +struct HeaderSyncPassthroughSink { + peer_id: ZakuraPeerId, + inner: Arc, + cancel_token: CancellationToken, +} + +impl Sink for HeaderSyncPassthroughSink { + fn run(self: Box, mut recv: FramedRecv) -> BoxRunFuture<'static, Result<(), SinkReject>> { + Box::pin(async move { + loop { + let frame = tokio::select! { + _ = self.cancel_token.cancelled() => return Ok(()), + frame = recv.recv() => { + let Some(frame) = frame else { + return Ok(()); + }; + frame + } + }; + + match self.inner.deliver_frame( + self.peer_id.clone(), + ZAKURA_STREAM_HEADER_SYNC, + frame, + ) { + Ok(()) => {} + Err(SinkReject::Protocol(error)) => return Err(SinkReject::Protocol(error)), + Err(SinkReject::Local(error)) => { + tracing::debug!( + ?error, + peer_id = ?self.peer_id, + "header-sync passthrough could not deliver frame locally" + ); + } + } + } + }) + } +} + +fn deliver_header_sync_frame( + header_sync: &HeaderSyncHandle, + peer_id: ZakuraPeerId, + frame: Frame, +) -> Result<(), SinkReject> { + if u8::try_from(frame.message_type).ok() == Some(MSG_HS_HEADERS) { + // `Headers` response decode still needs the actor's per-peer + // outstanding-request state; the per-peer concurrency epic moves that + // contract into this sink and removes the residual raw-frame hop. + return header_sync + .try_send(HeaderSyncEvent::WireFrame { + peer: peer_id, + frame, + }) + .map_err(|error| SinkReject::local(format!("header-sync queue closed: {error}"))); + } + + let msg = match decode_header_sync_frame(frame) { + Ok(msg) => msg, + Err(error) => { + let protocol_error = + std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); + let _ = header_sync.try_send(HeaderSyncEvent::WireDecodeFailed { + peer: peer_id, + error: Arc::new(error), + }); + return Err(SinkReject::protocol(protocol_error)); + } + }; + + header_sync + .try_send(HeaderSyncEvent::WireMessage { peer: peer_id, msg }) + .map_err(|error| SinkReject::local(format!("header-sync queue closed: {error}"))) +} + +fn decode_header_sync_frame(frame: Frame) -> Result { + if u8::try_from(frame.message_type).ok() == Some(MSG_HS_HEADERS) { + return Err(HeaderSyncWireError::UnsolicitedHeaders); + } + + HeaderSyncMessage::decode_frame(frame, HeaderSyncDecodeContext::control()) +} diff --git a/zebra-network/src/zakura/header_sync/state.rs b/zebra-network/src/zakura/header_sync/state.rs new file mode 100644 index 00000000000..b47e6f02089 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/state.rs @@ -0,0 +1,212 @@ +use super::{error::*, events::*, scheduler::*, validation::*, wire::*, *}; + +#[derive(Clone, Debug)] +pub(super) struct HeaderSyncState { + pub(super) anchor: (block::Height, block::Hash), + pub(super) finalized_height: block::Height, + pub(super) verified_block_tip: block::Height, + pub(super) best_header_tip: block::Height, + pub(super) best_header_hash: block::Hash, + pub(super) peers: HashMap, + pub(super) seen: HeaderHashDedup, + pub(super) pending_new_blocks: HashSet, + pub(super) schedule: RangeScheduler, + pub(super) pending_commits: HashMap, +} + +impl HeaderSyncState { + pub(super) fn new(startup: &HeaderSyncStartup) -> Result { + validate_anchor(&startup.network, startup.anchor)?; + let (best_header_tip, best_header_hash) = startup.best_header_tip.unwrap_or(startup.anchor); + + Ok(Self { + anchor: startup.anchor, + finalized_height: startup.frontiers.finalized_height, + verified_block_tip: startup.frontiers.verified_block_tip, + best_header_tip, + best_header_hash, + peers: HashMap::new(), + seen: HeaderHashDedup::default(), + pending_new_blocks: HashSet::new(), + schedule: RangeScheduler::new(), + pending_commits: HashMap::new(), + }) + } + + pub(super) fn refresh_forward_range(&mut self, startup: &HeaderSyncStartup) { + let best_peer_tip = self + .peers + .values() + .filter(|peer| peer.received_status) + .map(|peer| peer.advertised_tip) + .max() + .unwrap_or(self.best_header_tip); + if best_peer_tip <= self.best_header_tip { + return; + } + + let checkpoints = startup.network.checkpoint_list(); + let Some(start) = next_height(self.best_header_tip) else { + return; + }; + let mut end = best_peer_tip; + let mut finalized = false; + if let Some(first_checkpoint) = checkpoints.min_height_in_range(block::Height(1)..) { + if self.best_header_tip < first_checkpoint { + if best_peer_tip < first_checkpoint { + return; + } + end = first_checkpoint; + finalized = true; + } + } + + let count = count_between(start, end); + if count == 0 { + return; + } + self.schedule.ensure_forward(RangeRequest { + start_height: start, + count, + anchor_hash: self.best_header_hash, + finalized, + priority: RangePriority::Forward, + }); + } + + pub(super) fn refresh_backward_range(&mut self, startup: &HeaderSyncStartup) { + if self.anchor.0 == block::Height(0) { + return; + } + let checkpoints = startup.network.checkpoint_list(); + // v1 backfill schedules one checkpoint bracket below the configured anchor. + // Iterating all deeper brackets is left to final node wiring/backfill policy. + let Some(previous_checkpoint) = checkpoints.max_height_in_range(..self.anchor.0) else { + return; + }; + let Some(previous_hash) = checkpoints.hash(previous_checkpoint) else { + return; + }; + let Some(start) = next_height(previous_checkpoint) else { + return; + }; + let count = count_between(start, self.anchor.0); + if count == 0 { + return; + } + self.schedule.ensure_backward(RangeRequest { + start_height: start, + count, + anchor_hash: previous_hash, + finalized: true, + priority: RangePriority::Backward, + }); + } +} + +#[derive(Clone, Debug)] +pub(super) struct PeerHeaderState { + pub(super) advertised_tip: block::Height, + pub(super) anchor: block::Height, + pub(super) max_headers_per_response: u32, + pub(super) max_inflight_requests: u16, + pub(super) received_status: bool, + pub(super) outstanding: Vec, + pub(super) late_covered_responses: usize, + pub(super) unsolicited: RateMeter, + pub(super) inbound_status: RateMeter, + pub(super) inbound_new_block: RateMeter, + pub(super) served_headers_inflight: u16, + pub(super) misbehavior: u32, +} + +impl PeerHeaderState { + pub(super) fn new( + anchor: block::Height, + local_range: u32, + local_inflight: u16, + status_refresh_interval: Duration, + inbound_status_min_interval: Duration, + inbound_new_block_min_interval: Duration, + ) -> Self { + Self { + advertised_tip: anchor, + anchor, + max_headers_per_response: clamp_advertised_range(local_range), + max_inflight_requests: local_inflight.clamp(1, LOCAL_MAX_HS_INFLIGHT_PER_PEER), + received_status: false, + outstanding: Vec::new(), + late_covered_responses: 0, + unsolicited: RateMeter::new(status_refresh_interval), + inbound_status: RateMeter::new(inbound_status_min_interval), + inbound_new_block: RateMeter::new(inbound_new_block_min_interval), + served_headers_inflight: 0, + misbehavior: 0, + } + } + + pub(super) fn available_slots(&self) -> usize { + usize::from(self.max_inflight_requests) + .min(EFFECTIVE_HS_OUTBOUND_INFLIGHT_PER_PEER) + .saturating_sub(self.outstanding.len()) + } + + pub(super) fn pop_oldest_outstanding(&mut self) -> Option { + (!self.outstanding.is_empty()).then(|| self.outstanding.remove(0)) + } + + pub(super) fn take_late_covered_response(&mut self) -> bool { + if self.late_covered_responses == 0 { + return false; + } + self.late_covered_responses -= 1; + true + } + + pub(super) fn try_start_serving_headers(&mut self, local_inflight_cap: u16) -> bool { + if self.served_headers_inflight >= local_inflight_cap { + return false; + } + self.served_headers_inflight = self.served_headers_inflight.saturating_add(1); + true + } + + pub(super) fn finish_serving_headers(&mut self) { + self.served_headers_inflight = self.served_headers_inflight.saturating_sub(1); + } +} + +#[derive(Copy, Clone, Debug)] +pub(super) struct OutstandingRange { + pub(super) range: RangeRequest, + pub(super) deadline: Instant, + pub(super) expected_max_count: u32, + pub(super) clear_assignment_on_timeout: bool, +} + +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] +pub(super) struct RangeRequest { + pub(super) start_height: block::Height, + pub(super) count: u32, + pub(super) anchor_hash: block::Hash, + pub(super) finalized: bool, + pub(super) priority: RangePriority, +} + +impl RangeRequest { + pub(super) fn end_height(self) -> block::Height { + height_after_count(self.start_height, self.count) + .and_then(previous_height) + .expect("range request count is non-zero") + } + + pub(super) fn is_within(self, start: block::Height, end: block::Height) -> bool { + self.start_height >= start && self.end_height() <= end + } +} + +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] +pub(super) enum RangePriority { + Forward, + Backward, +} diff --git a/zebra-network/src/zakura/header_sync/tests.rs b/zebra-network/src/zakura/header_sync/tests.rs new file mode 100644 index 00000000000..8baf79d5b9f --- /dev/null +++ b/zebra-network/src/zakura/header_sync/tests.rs @@ -0,0 +1,3164 @@ +use super::*; +use super::{config::*, error::*, events::*, reactor::*, validation::*, wire::*}; +use crate::zakura::testkit::TraceCapture; +use chrono::Duration; +use metrics::{ + Counter, CounterFn, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit, +}; +use std::{ + collections::BTreeMap, + sync::{Mutex, OnceLock}, +}; +use zebra_chain::{ + parameters::{ + testnet::{ + ConfiguredActivationHeights, ConfiguredCheckpoints, Parameters, RegtestParameters, + }, + Network, + }, + serialization::{ZcashDeserializeInto, ZcashSerialize}, + work::{difficulty::CompactDifficulty, equihash::Solution}, +}; +use zebra_test::vectors::{ + BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES, BLOCK_MAINNET_3_BYTES, BLOCK_MAINNET_4_BYTES, + BLOCK_MAINNET_GENESIS_BYTES, BLOCK_TESTNET_GENESIS_BYTES, +}; + +#[derive(Default)] +struct HeaderSyncMetricsRecorder { + counters: Mutex>, +} + +struct RecordedCounter { + name: String, + recorder: &'static HeaderSyncMetricsRecorder, +} + +impl CounterFn for RecordedCounter { + fn increment(&self, value: u64) { + let mut counters = self.recorder.counters.lock().expect("metrics mutex ok"); + let counter = counters.entry(self.name.clone()).or_default(); + *counter = counter.saturating_add(value); + } + + fn absolute(&self, value: u64) { + let mut counters = self.recorder.counters.lock().expect("metrics mutex ok"); + counters.insert(self.name.clone(), value); + } +} + +impl Recorder for HeaderSyncMetricsRecorder { + fn describe_counter(&self, _key: KeyName, _unit: Option, _description: SharedString) {} + + fn describe_gauge(&self, _key: KeyName, _unit: Option, _description: SharedString) {} + + fn describe_histogram(&self, _key: KeyName, _unit: Option, _description: SharedString) {} + + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { + Counter::from_arc(Arc::new(RecordedCounter { + name: key.name().to_string(), + recorder: header_sync_metrics_recorder(), + })) + } + + fn register_gauge(&self, _key: &Key, _metadata: &Metadata<'_>) -> Gauge { + Gauge::noop() + } + + fn register_histogram(&self, _key: &Key, _metadata: &Metadata<'_>) -> Histogram { + Histogram::noop() + } +} + +fn header_sync_metrics_recorder() -> &'static HeaderSyncMetricsRecorder { + static RECORDER: OnceLock = OnceLock::new(); + let recorder = RECORDER.get_or_init(HeaderSyncMetricsRecorder::default); + let _ = metrics::set_global_recorder(recorder); + recorder +} + +fn metric_value(name: &str) -> u64 { + let recorder = header_sync_metrics_recorder(); + recorder + .counters + .lock() + .expect("metrics mutex ok") + .get(name) + .copied() + .unwrap_or_default() +} + +fn metric_snapshot(names: &[&'static str]) -> BTreeMap<&'static str, u64> { + names + .iter() + .copied() + .map(|name| (name, metric_value(name))) + .collect() +} + +fn assert_metric_incremented(snapshot: &BTreeMap<&'static str, u64>, name: &'static str) { + assert!( + metric_value(name) > snapshot.get(name).copied().unwrap_or_default(), + "expected metric {name} to increment" + ); +} + +fn mainnet_block(bytes: &[u8]) -> Arc { + Arc::new(bytes.zcash_deserialize_into().expect("block vector parses")) +} + +fn mainnet_header(bytes: &[u8]) -> Arc { + mainnet_block(bytes).header.clone() +} + +async fn validate_headers_stateless_after_equihash_acceptance( + headers: Vec>, + context: HeaderSyncValidationContext<'_>, +) -> Result<(), HeaderSyncWireError> { + validate_header_count(headers.len(), context.decode_context)?; + validate_internal_continuity(&headers)?; + validate_header_times(&headers, context.now, context.start_height)?; + validate_solution_sizes(&headers, context.network)?; + tokio::task::spawn_blocking(move || { + for header in headers { + let hash = block::Hash::from(header.as_ref()); + validate_difficulty_filter(hash, header.difficulty_threshold)?; + } + Ok(()) + }) + .await? +} + +fn headers_context(count: u32, peer_cap: u32) -> HeaderSyncDecodeContext { + HeaderSyncDecodeContext::for_headers_response( + HeaderSyncRequestContract::new(block::Height(1), count).unwrap(), + peer_cap, + ) +} + +struct ReactorFixture { + handle: HeaderSyncHandle, + actions: mpsc::Receiver, + task: JoinHandle<()>, +} + +impl Drop for ReactorFixture { + fn drop(&mut self) { + self.task.abort(); + } +} + +fn peer(byte: u8) -> ZakuraPeerId { + ZakuraPeerId::new(vec![byte; 32]).expect("test peer id is within bounds") +} + +fn regtest_network() -> Network { + Network::new_regtest(Default::default()) +} + +fn checkpoint_testnet_with_hash( + checkpoint_height: block::Height, + checkpoint_hash: block::Hash, +) -> (Network, block::Hash) { + let mainnet = Network::Mainnet; + let network = Parameters::build() + .with_network_name("HeadersyncCheckpointTest") + .expect("custom network name is valid") + .with_genesis_hash(mainnet.genesis_hash()) + .expect("mainnet genesis hash is valid") + .with_activation_heights(ConfiguredActivationHeights { + overwinter: Some(1), + sapling: Some(2), + blossom: Some(3), + heartwood: Some(4), + canopy: Some(4), + ..Default::default() + }) + .expect("custom activation heights are in order") + .clear_funding_streams() + .with_checkpoints(ConfiguredCheckpoints::HeightsAndHashes(vec![ + (block::Height(0), mainnet.genesis_hash()), + (checkpoint_height, checkpoint_hash), + ])) + .expect("custom checkpoints are valid") + .to_network() + .expect("custom testnet parameters are valid"); + + (network, checkpoint_hash) +} + +fn checkpoint_regtest(checkpoint_height: block::Height) -> (Network, block::Hash) { + let checkpoint_hash = block::Hash::from(mainnet_header(&BLOCK_MAINNET_1_BYTES).as_ref()); + checkpoint_regtest_with_hash(checkpoint_height, checkpoint_hash) +} + +fn checkpoint_regtest_with_hash( + checkpoint_height: block::Height, + checkpoint_hash: block::Hash, +) -> (Network, block::Hash) { + let default_regtest = regtest_network(); + let params = RegtestParameters { + checkpoints: Some(ConfiguredCheckpoints::HeightsAndHashes(vec![ + (block::Height(0), default_regtest.genesis_hash()), + (checkpoint_height, checkpoint_hash), + ])), + ..Default::default() + }; + + (Network::new_regtest(params), checkpoint_hash) +} + +fn startup_for( + network: Network, + anchor: (block::Height, block::Hash), + best_header_tip: Option<(block::Height, block::Hash)>, +) -> HeaderSyncStartup { + let mut startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: anchor.0, + verified_block_tip: anchor.0, + }, + best_header_tip, + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.range_state_actions_enabled = true; + startup.inbound_new_block_acceptance_enabled = true; + startup +} + +#[test] +fn startup_new_is_passive_until_local_hooks_are_wired() { + let network = Network::Mainnet; + let anchor = (block::Height(0), network.genesis_hash()); + let startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: anchor.0, + verified_block_tip: anchor.0, + }, + Some(anchor), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + + assert!(!startup.range_state_actions_enabled); + assert!(!startup.inbound_new_block_acceptance_enabled); +} + +fn startup_with_timeout( + network: Network, + anchor: (block::Height, block::Hash), + request_timeout: std::time::Duration, +) -> HeaderSyncStartup { + let mut startup = startup_for(network, anchor, None); + startup.request_timeout = request_timeout; + startup +} + +fn spawn_test_reactor(startup: HeaderSyncStartup) -> ReactorFixture { + let (handle, actions, task) = spawn_header_sync_reactor(startup).unwrap(); + ReactorFixture { + handle, + actions, + task, + } +} + +async fn next_action(actions: &mut mpsc::Receiver) -> HeaderSyncAction { + tokio::time::timeout(std::time::Duration::from_secs(5), actions.recv()) + .await + .expect("action arrives before timeout") + .expect("reactor action channel stays open") +} + +async fn next_non_query_action(actions: &mut mpsc::Receiver) -> HeaderSyncAction { + loop { + let action = next_action(actions).await; + if !matches!( + action, + HeaderSyncAction::QueryBestHeaderTip + | HeaderSyncAction::QueryMissingBlockBodies { .. } + | HeaderSyncAction::QueryHeadersByHeightRange { .. } + ) { + return action; + } + } +} + +async fn next_query_headers_action( + actions: &mut mpsc::Receiver, +) -> HeaderSyncAction { + loop { + let action = next_action(actions).await; + if matches!(action, HeaderSyncAction::QueryHeadersByHeightRange { .. }) { + return action; + } + } +} + +async fn assert_no_commit_or_misbehavior(actions: &mut mpsc::Receiver) { + while let Ok(Some(action)) = + tokio::time::timeout(std::time::Duration::from_millis(50), actions.recv()).await + { + assert!( + !matches!( + action, + HeaderSyncAction::CommitHeaderRange { .. } | HeaderSyncAction::Misbehavior { .. } + ), + "unexpected commit or misbehavior action: {action:?}" + ); + } +} + +async fn connect_peer(fixture: &ReactorFixture, peer_id: ZakuraPeerId) { + fixture + .handle + .send(HeaderSyncEvent::PeerConnected(peer_id)) + .await + .unwrap(); +} + +async fn advertise_tip( + fixture: &ReactorFixture, + peer_id: ZakuraPeerId, + anchor_height: block::Height, + tip_height: block::Height, + max_headers_per_response: u32, + max_inflight_requests: u16, +) { + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id, + msg: HeaderSyncMessage::Status(HeaderSyncStatus { + tip_height, + tip_hash: block::Hash([9; 32]), + anchor_height, + max_headers_per_response, + max_inflight_requests, + }), + }) + .await + .unwrap(); +} + +#[test] +fn codec_round_trips_status() { + let status = HeaderSyncStatus { + tip_height: block::Height(10), + tip_hash: block::Hash([9; 32]), + anchor_height: block::Height(1), + max_headers_per_response: DEFAULT_HS_RANGE, + max_inflight_requests: DEFAULT_HS_MAX_INFLIGHT, + }; + let message = HeaderSyncMessage::Status(status); + + let encoded = message.encode().unwrap(); + let decoded = HeaderSyncMessage::decode(&encoded, HeaderSyncDecodeContext::control()).unwrap(); + + assert_eq!(decoded, message); +} + +#[test] +fn codec_round_trips_get_headers() { + let message = HeaderSyncMessage::GetHeaders { + start_height: block::Height(42), + count: DEFAULT_HS_RANGE, + }; + + let encoded = message.encode().unwrap(); + let decoded = HeaderSyncMessage::decode(&encoded, HeaderSyncDecodeContext::control()).unwrap(); + + assert_eq!(decoded, message); +} + +#[test] +fn codec_round_trips_headers_with_bounded_vector() { + let headers = vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]; + let message = HeaderSyncMessage::Headers(headers); + + let encoded = message.encode().unwrap(); + let decoded = HeaderSyncMessage::decode(&encoded, headers_context(1, 1)).unwrap(); + + assert_eq!(decoded, message); +} + +#[test] +fn codec_round_trips_new_block() { + let message = HeaderSyncMessage::NewBlock(mainnet_block(&BLOCK_MAINNET_1_BYTES)); + + let encoded = message.encode().unwrap(); + let decoded = HeaderSyncMessage::decode(&encoded, HeaderSyncDecodeContext::control()).unwrap(); + + assert_eq!(decoded, message); +} + +#[test] +fn codec_rejects_unknown_message_types_and_trailing_bytes() { + assert!(matches!( + HeaderSyncMessage::decode(&[99], HeaderSyncDecodeContext::control()), + Err(HeaderSyncWireError::UnknownMessageType(99)) + )); + + let mut encoded = HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 1, + } + .encode() + .unwrap(); + encoded.push(0); + + assert!(matches!( + HeaderSyncMessage::decode(&encoded, HeaderSyncDecodeContext::control()), + Err(HeaderSyncWireError::TrailingBytes) + )); +} + +#[test] +fn frame_decode_rejects_oversized_payload_length_before_allocating() { + let mut bytes = Vec::new(); + bytes + .write_u16::(u16::from(MSG_HS_STATUS)) + .unwrap(); + bytes.write_u16::(0).unwrap(); + bytes + .write_u32::(MAX_HS_MESSAGE_BYTES as u32 + 1) + .unwrap(); + + assert!(Frame::decode(&bytes, MAX_HS_MESSAGE_BYTES as u32).is_err()); +} + +#[test] +fn decode_rejects_header_counts_over_contract_caps() { + let mut encoded = vec![MSG_HS_HEADERS]; + encoded.write_u32::(MAX_HS_RANGE + 1).unwrap(); + assert!(matches!( + HeaderSyncMessage::decode(&encoded, headers_context(MAX_HS_RANGE, MAX_HS_RANGE)), + Err(HeaderSyncWireError::HeaderCountLimit { .. }) + )); + + let mut encoded = vec![MSG_HS_HEADERS]; + encoded.write_u32::(2).unwrap(); + assert!(matches!( + HeaderSyncMessage::decode(&encoded, headers_context(1, MAX_HS_RANGE)), + Err(HeaderSyncWireError::HeaderCountLimit { actual: 2, max: 1 }) + )); + + let mut encoded = vec![MSG_HS_HEADERS]; + encoded.write_u32::(2).unwrap(); + assert!(matches!( + HeaderSyncMessage::decode(&encoded, headers_context(MAX_HS_RANGE, 1)), + Err(HeaderSyncWireError::HeaderCountLimit { actual: 2, max: 1 }) + )); +} + +#[test] +fn headers_codec_does_not_use_legacy_160_header_cap() { + let header = mainnet_header(&BLOCK_MAINNET_1_BYTES); + let headers = vec![header; 161]; + let message = HeaderSyncMessage::Headers(headers); + + let encoded = message.encode().unwrap(); + let decoded = HeaderSyncMessage::decode(&encoded, headers_context(161, 161)).unwrap(); + + match decoded { + HeaderSyncMessage::Headers(headers) => assert_eq!(headers.len(), 161), + _ => panic!("decoded message must be Headers"), + } +} + +#[test] +fn get_headers_rejects_invalid_counts() { + assert!(HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 0, + } + .encode() + .is_err()); + + assert!(HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: MAX_HS_RANGE + 1, + } + .encode() + .is_err()); +} + +#[test] +fn advertised_defaults_and_clamping_match_design() { + let config = ZakuraHeaderSyncConfig::default(); + assert_eq!(config.max_headers_per_response, DEFAULT_HS_RANGE); + assert_eq!(config.max_inflight_requests, DEFAULT_HS_MAX_INFLIGHT); + assert_eq!( + ZakuraHeaderSyncConfig { + max_inflight_requests: u16::MAX, + ..ZakuraHeaderSyncConfig::default() + } + .advertised_max_inflight_requests(), + LOCAL_MAX_HS_INFLIGHT_PER_PEER + ); + + let status = HeaderSyncStatus { + max_headers_per_response: MAX_HS_RANGE + 10, + ..HeaderSyncStatus::default() + }; + let encoded = HeaderSyncMessage::Status(status).encode().unwrap(); + let decoded = HeaderSyncMessage::decode(&encoded, HeaderSyncDecodeContext::control()).unwrap(); + match decoded { + HeaderSyncMessage::Status(status) => { + assert_eq!(status.max_headers_per_response, MAX_HS_RANGE); + } + _ => panic!("decoded message must be Status"), + } +} + +#[test] +fn header_serialized_sizes_are_exact_and_message_cap_has_headroom() { + let mainnet = mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES); + let mut mainnet_bytes = Vec::new(); + mainnet.zcash_serialize(&mut mainnet_bytes).unwrap(); + assert_eq!(mainnet_bytes.len(), COMMON_HEADER_BYTES); + + let testnet = mainnet_header(&BLOCK_TESTNET_GENESIS_BYTES); + let mut testnet_bytes = Vec::new(); + testnet.zcash_serialize(&mut testnet_bytes).unwrap(); + assert_eq!(testnet_bytes.len(), COMMON_HEADER_BYTES); + + let mut regtest = *mainnet; + regtest.solution = Solution::Regtest([0; 36]); + let mut regtest_bytes = Vec::new(); + regtest.zcash_serialize(&mut regtest_bytes).unwrap(); + assert_eq!(regtest_bytes.len(), REGTEST_HEADER_BYTES); + + let default_response_bytes = HEADER_SYNC_MESSAGE_TYPE_BYTES + + HEADER_SYNC_COUNT_BYTES + + COMMON_HEADER_BYTES * DEFAULT_HS_RANGE as usize; + assert!(default_response_bytes < MAX_HS_MESSAGE_BYTES); + assert!(MAX_HS_MESSAGE_BYTES < LOCAL_MAX_MESSAGE_BYTES as usize); +} + +#[test] +fn request_and_serving_counts_are_clamped_by_byte_budget() { + let count = clamp_header_sync_request_count( + MAX_HS_RANGE, + MAX_HS_RANGE, + &Network::Mainnet, + LOCAL_MAX_MESSAGE_BYTES, + ); + + assert!(count < MAX_HS_RANGE); + let headers = + vec![mainnet_header(&BLOCK_MAINNET_1_BYTES); usize::try_from(count).unwrap() + 100]; + let headers = + truncate_headers_to_byte_budget(headers, &Network::Mainnet, LOCAL_MAX_MESSAGE_BYTES); + let encoded = HeaderSyncMessage::Headers(headers).encode().unwrap(); + + assert!(encoded.len() <= MAX_HS_MESSAGE_BYTES); + assert!(encoded.len() + FRAME_HEADER_BYTES <= LOCAL_MAX_MESSAGE_BYTES as usize); +} + +#[tokio::test(flavor = "current_thread")] +async fn reactor_starts_from_storage_frontiers_and_publishes_watch() { + let network = regtest_network(); + let best = (block::Height(7), block::Hash([7; 32])); + let startup = HeaderSyncStartup::new( + network.clone(), + (block::Height(0), network.genesis_hash()), + HeaderSyncFrontiers { + finalized_height: block::Height(2), + verified_block_tip: block::Height(5), + }, + Some(best), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + let fixture = spawn_test_reactor(startup); + + assert_eq!(fixture.handle.best_header_tip(), best); + assert_eq!(*fixture.handle.subscribe_tip().borrow(), best); +} + +#[tokio::test(flavor = "current_thread")] +async fn restart_rebuilds_schedule_from_durable_best_tip_and_peer_status() { + let network = regtest_network(); + let best = (block::Height(4), block::Hash([4; 32])); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some(best), + )); + let peer_id = peer(41); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(8), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + loop { + if let HeaderSyncAction::SendMessage { + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + .. + } = next_non_query_action(&mut fixture.actions).await + { + assert_eq!(start_height, block::Height(5)); + assert_eq!(count, 4); + break; + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn handle_sends_events_and_peer_connect_sends_status_first() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(1); + + connect_peer(&fixture, peer_id.clone()).await; + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::SendMessage { peer, msg } => { + assert_eq!(peer, peer_id); + assert!(matches!(msg, HeaderSyncMessage::Status(_))); + } + action => panic!("unexpected action: {action:?}"), + } +} + +#[tokio::test(flavor = "current_thread")] +async fn status_updates_peer_caps_and_scheduler_respects_them() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(2); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(10), + 2, + u16::MAX, + ) + .await; + + let mut saw_get_headers = false; + for _ in 0..4 { + if let HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + } = next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, peer_id); + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 2); + saw_get_headers = true; + break; + } + } + assert!(saw_get_headers); +} + +#[tokio::test(flavor = "current_thread")] +async fn scheduler_limits_v1_to_one_outstanding_request_per_peer() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(31); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(20), + 2, + u16::MAX, + ) + .await; + + let mut get_headers_count = 0; + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(100), + fixture.actions.recv(), + ) + .await + { + if matches!( + action, + HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::GetHeaders { .. }, + } if peer == peer_id + ) { + get_headers_count += 1; + } + } + + assert_eq!(get_headers_count, 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn scheduler_fans_out_same_forward_range_to_three_peers() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peers = [peer(3), peer(4), peer(5)]; + + for peer_id in peers.clone() { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip(&fixture, peer_id, block::Height(0), block::Height(5), 5, 1).await; + } + + let mut requested = HashSet::new(); + while requested.len() < HEADER_SYNC_FANOUT { + if let HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + } = next_non_query_action(&mut fixture.actions).await + { + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 5); + requested.insert(peer); + } + } + + assert_eq!(requested.len(), HEADER_SYNC_FANOUT); +} + +#[tokio::test(flavor = "current_thread")] +async fn scheduler_narrows_large_ranges_before_tracking_fanout() { + let network = Network::Mainnet; + let first_checkpoint = network + .checkpoint_list() + .min_height_in_range(block::Height(1)..) + .expect("mainnet has a checkpoint above genesis"); + let best_header_hash = block::Hash([3; 32]); + let start = next_height(first_checkpoint).expect("checkpoint height has successor"); + let unclamped_tip = block::Height( + start + .0 + .checked_add(MAX_HS_RANGE) + .expect("test range fits in height"), + ); + let clamped_count = clamp_header_sync_request_count( + MAX_HS_RANGE, + MAX_HS_RANGE, + &network, + LOCAL_MAX_MESSAGE_BYTES, + ); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((first_checkpoint, best_header_hash)), + )); + let peers = [peer(37), peer(38), peer(39), peer(40)]; + + for peer_id in peers.clone() { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + unclamped_tip, + MAX_HS_RANGE, + 1, + ) + .await; + } + + let mut requested = HashSet::new(); + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(100), + fixture.actions.recv(), + ) + .await + { + if let HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + } = action + { + assert_eq!(start_height, start); + assert_eq!(count, clamped_count); + assert!( + requested.insert(peer), + "scheduler must not duplicate a clamped chunk for one peer" + ); + } + } + assert_eq!(requested.len(), HEADER_SYNC_FANOUT); + + let chunk_tip = height_after_count(start, clamped_count) + .and_then(previous_height) + .expect("clamped range has a tip"); + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: start, + tip_height: chunk_tip, + tip_hash: block::Hash([4; 32]), + }) + .await + .unwrap(); + + loop { + if let HeaderSyncAction::SendMessage { + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + .. + } = next_non_query_action(&mut fixture.actions).await + { + assert_eq!( + start_height, + next_height(chunk_tip).expect("committed chunk tip has successor") + ); + assert_eq!(count, clamped_count); + break; + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn scheduler_creates_checkpoint_forward_before_backward_ranges() { + let (network, checkpoint_hash) = checkpoint_regtest(block::Height(3)); + let mut fixture = spawn_test_reactor(startup_for( + network, + (block::Height(3), checkpoint_hash), + Some((block::Height(3), checkpoint_hash)), + )); + let peer_id = peer(6); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(8), + DEFAULT_HS_RANGE, + 10, + ) + .await; + + loop { + if let HeaderSyncAction::SendMessage { + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + .. + } = next_non_query_action(&mut fixture.actions).await + { + assert_eq!(start_height, block::Height(4)); + assert_eq!(count, 5); + break; + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn scheduler_creates_backward_checkpoint_terminating_ranges() { + let (network, checkpoint_hash) = checkpoint_regtest(block::Height(3)); + let mut fixture = spawn_test_reactor(startup_for( + network, + (block::Height(3), checkpoint_hash), + Some((block::Height(3), checkpoint_hash)), + )); + let peer_id = peer(7); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(3), + DEFAULT_HS_RANGE, + 10, + ) + .await; + + loop { + if let HeaderSyncAction::SendMessage { + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + .. + } = next_non_query_action(&mut fixture.actions).await + { + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 3); + break; + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn incoming_headers_match_outstanding_before_commit() { + let checkpoint_hash = block::Hash::from(mainnet_header(&BLOCK_MAINNET_3_BYTES).as_ref()); + let (network, _) = checkpoint_testnet_with_hash(block::Height(3), checkpoint_hash); + let first_checkpoint = block::Height(3); + let start = block::Height(4); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((first_checkpoint, checkpoint_hash)), + )); + let peer_id = peer(8); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip(&fixture, peer_id.clone(), block::Height(0), start, 1, 1).await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + finalized, + .. + } => { + assert_eq!(peer, peer_id); + assert_eq!(start_height, start); + assert!(!finalized); + } + action => panic!("unexpected action: {action:?}"), + } +} + +#[tokio::test(flavor = "current_thread")] +async fn headers_over_outstanding_contract_reports_response_too_long_without_flooding() { + let network = Network::Mainnet; + let first_checkpoint = network + .checkpoint_list() + .min_height_in_range(block::Height(1)..) + .expect("mainnet has a checkpoint above genesis"); + let previous_hash = block::Hash([1; 32]); + let start = next_height(first_checkpoint).expect("checkpoint height has successor"); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((first_checkpoint, previous_hash)), + )); + let peer_id = peer(61); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(start.0 + 1), + 1, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { count: 1, .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(vec![ + mainnet_header(&BLOCK_MAINNET_1_BYTES), + mainnet_header(&BLOCK_MAINNET_2_BYTES), + ]), + }) + .await + .unwrap(); + + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::ResponseTooLong); + break; + } + HeaderSyncAction::ForwardNewBlock { .. } => { + panic!("backfill Headers must never produce tip-flood forwarding") + } + _ => {} + } + } + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn matching_headers_are_statelessly_validated_before_commit() { + let network = Network::Mainnet; + let first_checkpoint = network + .checkpoint_list() + .min_height_in_range(block::Height(1)..) + .expect("mainnet has a checkpoint above genesis"); + let two_before_checkpoint = block::Height( + first_checkpoint + .0 + .checked_sub(2) + .expect("mainnet first checkpoint has two predecessors"), + ); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((two_before_checkpoint, block::Hash([1; 32]))), + )); + let peer_id = peer(32); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + first_checkpoint, + 2, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + let mut bad_second = *mainnet_header(&BLOCK_MAINNET_2_BYTES); + bad_second.previous_block_hash = block::Hash([7; 32]); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(vec![ + mainnet_header(&BLOCK_MAINNET_1_BYTES), + Arc::new(bad_second), + ]), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + } + action => panic!("unexpected action: {action:?}"), + } + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn invalid_async_header_commit_failure_reports_peer_disconnect() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(62); + + connect_peer(&fixture, peer_id.clone()).await; + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer: peer_id.clone(), + start_height: block::Height(1), + count: 1, + kind: HeaderSyncCommitFailureKind::InvalidPeerRange, + }) + .await + .unwrap(); + + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + break; + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn peer_disconnect_removes_outstanding_requests_for_that_peer() { + let network = Network::Mainnet; + let first_checkpoint = network + .checkpoint_list() + .min_height_in_range(block::Height(1)..) + .expect("mainnet has a checkpoint above genesis"); + let previous_checkpoint_height = + previous_height(first_checkpoint).expect("checkpoint above genesis has predecessor"); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((previous_checkpoint_height, block::Hash([1; 32]))), + )); + let peer_id = peer(11); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + first_checkpoint, + 1, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::PeerDisconnected(peer_id.clone())) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::UnsolicitedHeaders); + } + action => panic!("unexpected action: {action:?}"), + } +} + +#[tokio::test(flavor = "current_thread")] +async fn timed_out_range_retries_with_another_peer() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_with_timeout( + network.clone(), + (block::Height(0), network.genesis_hash()), + std::time::Duration::from_millis(1), + )); + let first_peer = peer(12); + let second_peer = peer(13); + + connect_peer(&fixture, first_peer.clone()).await; + advertise_tip( + &fixture, + first_peer, + block::Height(0), + block::Height(2), + 2, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + connect_peer(&fixture, second_peer.clone()).await; + advertise_tip( + &fixture, + second_peer.clone(), + block::Height(0), + block::Height(2), + 2, + 1, + ) + .await; + + loop { + if let HeaderSyncAction::SendMessage { peer, msg } = + next_non_query_action(&mut fixture.actions).await + { + if matches!(msg, HeaderSyncMessage::GetHeaders { .. }) { + assert_eq!(peer, second_peer); + break; + } + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn covered_hedged_outstanding_ranges_do_not_commit_twice() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let first_peer = peer(33); + let second_peer = peer(34); + + for peer_id in [first_peer.clone(), second_peer.clone()] { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip(&fixture, peer_id, block::Height(0), block::Height(2), 2, 1).await; + } + + let mut requested = HashSet::new(); + while requested.len() < 2 { + if let HeaderSyncAction::SendMessage { peer, msg } = + next_non_query_action(&mut fixture.actions).await + { + if matches!(msg, HeaderSyncMessage::GetHeaders { .. }) { + requested.insert(peer); + } + } + } + + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: block::Height(1), + tip_height: block::Height(2), + tip_hash: block::Hash([2; 32]), + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: second_peer, + msg: HeaderSyncMessage::Headers(Vec::new()), + }) + .await + .unwrap(); + + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn local_commit_failure_retries_without_peer_misbehavior() { + let checkpoint_hash = block::Hash::from(mainnet_header(&BLOCK_MAINNET_3_BYTES).as_ref()); + let (network, _) = checkpoint_testnet_with_hash(block::Height(3), checkpoint_hash); + let start = block::Height(4); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((block::Height(3), checkpoint_hash)), + )); + let first_peer = peer(35); + let second_peer = peer(36); + + for peer_id in [first_peer.clone(), second_peer.clone()] { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip(&fixture, peer_id, block::Height(0), start, 1, 1).await; + } + + loop { + if let HeaderSyncAction::SendMessage { peer, msg } = + next_non_query_action(&mut fixture.actions).await + { + if matches!(msg, HeaderSyncMessage::GetHeaders { .. }) && peer == first_peer { + break; + } + } + } + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: first_peer.clone(), + msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), + }) + .await + .unwrap(); + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { .. } => { + panic!("valid headers must not be scored before local commit failure") + } + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + .. + } => { + assert_eq!(peer, first_peer); + assert_eq!(start_height, start); + assert_eq!(headers.len(), 1); + break; + } + _ => {} + } + } + + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer: first_peer.clone(), + start_height: start, + count: 1, + kind: HeaderSyncCommitFailureKind::Local, + }) + .await + .unwrap(); + + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { .. } => { + panic!("local commit failure must not score peer") + } + HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + } if peer == first_peer || peer == second_peer => { + assert_eq!(start_height, start); + assert_eq!(count, 1); + break; + } + _ => {} + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn material_tip_advance_sends_rate_limited_unsolicited_status() { + let network = regtest_network(); + let mut startup = startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + ); + startup.status_refresh_interval = std::time::Duration::from_secs(60); + let mut fixture = spawn_test_reactor(startup); + let peer_id = peer(14); + + connect_peer(&fixture, peer_id.clone()).await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + ) { + break; + } + } + + for height in [block::Height(1), block::Height(2)] { + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: height, + tip_height: height, + tip_hash: block::Hash( + [u8::try_from(height.0).expect("test heights fit in u8"); 32], + ), + }) + .await + .unwrap(); + } + + let mut status_count = 0; + while let Ok(Some(action)) = + tokio::time::timeout(std::time::Duration::from_millis(20), fixture.actions.recv()).await + { + if matches!( + action, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + ) { + status_count += 1; + } + } + + assert_eq!(status_count, 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn full_block_committed_covers_outstanding_height() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(42); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(1), + 1, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::FullBlockCommitted { + height: block::Height(1), + hash: block::Hash([1; 32]), + header: mainnet_header(&BLOCK_MAINNET_1_BYTES), + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id, + msg: HeaderSyncMessage::Headers(Vec::new()), + }) + .await + .unwrap(); + + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn inbound_unseen_valid_new_block_is_seen_and_forwarded_to_eligible_peers() { + let network = Network::Mainnet; + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let hash = block.hash(); + let height = block.coinbase_height().expect("test block has height"); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let source = peer(46); + let eligible = peer(47); + let redundant = peer(48); + + for peer_id in [source.clone(), eligible.clone(), redundant.clone()] { + connect_peer(&fixture, peer_id).await; + } + advertise_tip( + &fixture, + source.clone(), + block::Height(0), + block::Height(0), + DEFAULT_HS_RANGE, + 1, + ) + .await; + advertise_tip( + &fixture, + eligible.clone(), + block::Height(0), + block::Height(0), + DEFAULT_HS_RANGE, + 1, + ) + .await; + advertise_tip( + &fixture, + redundant.clone(), + block::Height(0), + height, + DEFAULT_HS_RANGE, + 1, + ) + .await; + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: source.clone(), + msg: HeaderSyncMessage::NewBlock(block.clone()), + }) + .await + .unwrap(); + + let mut saw_pipeline_fact = false; + let mut forwarded = Vec::new(); + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(200), + fixture.actions.recv(), + ) + .await + { + match action { + HeaderSyncAction::NewBlockReceived { + peer, + height: action_height, + hash: action_hash, + .. + } => { + assert_eq!(peer, source); + assert_eq!(action_height, height); + assert_eq!(action_hash, hash); + saw_pipeline_fact = true; + fixture + .handle + .send(HeaderSyncEvent::NewBlockAccepted { + peer: source.clone(), + height, + hash, + block: block.clone(), + }) + .await + .unwrap(); + } + HeaderSyncAction::ForwardNewBlock { + source: action_source, + peer, + height: action_height, + hash: action_hash, + .. + } => { + assert_eq!(action_source, Some(source.clone())); + assert_eq!(action_height, height); + assert_eq!(action_hash, hash); + forwarded.push(peer); + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("valid NewBlock must not score {peer:?}: {reason:?}"); + } + _ => {} + } + } + + assert!(saw_pipeline_fact); + assert_eq!(forwarded, vec![eligible]); + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: source.clone(), + msg: HeaderSyncMessage::NewBlock(block), + }) + .await + .unwrap(); + + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(100), + fixture.actions.recv(), + ) + .await + { + if matches!( + action, + HeaderSyncAction::ForwardNewBlock { .. } + | HeaderSyncAction::NewBlockReceived { .. } + | HeaderSyncAction::Misbehavior { .. } + ) { + panic!("duplicate NewBlock must be cheap-deduped without scoring: {action:?}"); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_duplicate_new_block_dedups_pending_acceptance_without_scoring() { + let network = Network::Mainnet; + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let hash = block.hash(); + let height = block.coinbase_height().expect("test block has height"); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let first_peer = peer(52); + let duplicate_peer = peer(53); + let eligible_peer = peer(54); + + for peer_id in [ + first_peer.clone(), + duplicate_peer.clone(), + eligible_peer.clone(), + ] { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(0), + DEFAULT_HS_RANGE, + 1, + ) + .await; + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: first_peer.clone(), + msg: HeaderSyncMessage::NewBlock(block.clone()), + }) + .await + .unwrap(); + + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::NewBlockReceived { + peer, + height: action_height, + hash: action_hash, + .. + } => { + assert_eq!(peer, first_peer); + assert_eq!(action_height, height); + assert_eq!(action_hash, hash); + break; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("first valid NewBlock must not score {peer:?}: {reason:?}"); + } + _ => {} + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: duplicate_peer.clone(), + msg: HeaderSyncMessage::NewBlock(block.clone()), + }) + .await + .unwrap(); + + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(100), + fixture.actions.recv(), + ) + .await + { + if matches!( + action, + HeaderSyncAction::NewBlockReceived { .. } | HeaderSyncAction::Misbehavior { .. } + ) { + panic!("pending duplicate NewBlock must not re-enter acceptance or score: {action:?}"); + } + } + + fixture + .handle + .send(HeaderSyncEvent::NewBlockAccepted { + peer: first_peer, + height, + hash, + block, + }) + .await + .unwrap(); + + let mut forwarded = HashSet::new(); + while forwarded.len() < 2 { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::ForwardNewBlock { + peer, + height: action_height, + hash: action_hash, + .. + } => { + assert_eq!(action_height, height); + assert_eq!(action_hash, hash); + forwarded.insert(peer); + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("accepted duplicate flow must not score {peer:?}: {reason:?}"); + } + _ => {} + } + } + assert_eq!(forwarded, HashSet::from([duplicate_peer, eligible_peer])); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn local_full_block_commit_prevents_later_new_block_regossip() { + let network = Network::Mainnet; + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let height = block.coinbase_height().expect("test block has height"); + let hash = block.hash(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let source = peer(49); + let destination = peer(50); + + for peer_id in [source.clone(), destination] { + connect_peer(&fixture, peer_id).await; + } + fixture + .handle + .send(HeaderSyncEvent::FullBlockCommitted { + height, + hash, + header: block.header.clone(), + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: source, + msg: HeaderSyncMessage::NewBlock(block), + }) + .await + .unwrap(); + + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(100), + fixture.actions.recv(), + ) + .await + { + if matches!(action, HeaderSyncAction::ForwardNewBlock { .. }) { + panic!("locally committed block must not be gossiped twice: {action:?}"); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_and_malformed_new_block_report_disconnect() { + let network = Network::Mainnet; + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let unknown_peer = peer(63); + let invalid_peer = peer(51); + let malformed_peer = peer(52); + connect_peer(&fixture, invalid_peer.clone()).await; + connect_peer(&fixture, malformed_peer.clone()).await; + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: unknown_peer.clone(), + msg: HeaderSyncMessage::NewBlock(mainnet_block(&BLOCK_MAINNET_1_BYTES)), + }) + .await + .unwrap(); + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, unknown_peer); + assert_eq!(reason, HeaderSyncMisbehavior::UnknownPeer); + break; + } + } + + let mut bad_block = (*mainnet_block(&BLOCK_MAINNET_1_BYTES)).clone(); + let mut bad_header = *bad_block.header; + bad_header.nonce[0] ^= 1; + bad_block.header = Arc::new(bad_header); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: invalid_peer.clone(), + msg: HeaderSyncMessage::NewBlock(Arc::new(bad_block)), + }) + .await + .unwrap(); + + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, invalid_peer); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidNewBlock); + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireDecodeFailed { + peer: malformed_peer.clone(), + error: Arc::new(HeaderSyncWireError::UnknownMessageType(MSG_HS_NEW_BLOCK)), + }) + .await + .unwrap(); + + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, malformed_peer); + assert_eq!(reason, HeaderSyncMisbehavior::MalformedMessage); + break; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rapid_status_updates_and_new_block_spam_report_disconnect() { + let network = Network::Mainnet; + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let status_peer = peer(53); + let block_peer = peer(54); + connect_peer(&fixture, status_peer.clone()).await; + connect_peer(&fixture, block_peer.clone()).await; + + for _ in 0..2 { + advertise_tip( + &fixture, + status_peer.clone(), + block::Height(0), + block::Height(1), + DEFAULT_HS_RANGE, + 1, + ) + .await; + } + + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, status_peer); + assert_eq!(reason, HeaderSyncMisbehavior::StatusSpam); + break; + } + } + + for bytes in [ + BLOCK_MAINNET_1_BYTES.as_slice(), + BLOCK_MAINNET_2_BYTES.as_slice(), + ] { + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: block_peer.clone(), + msg: HeaderSyncMessage::NewBlock(mainnet_block(bytes)), + }) + .await + .unwrap(); + } + + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, block_peer); + assert_eq!(reason, HeaderSyncMisbehavior::NewBlockSpam); + break; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn new_block_spam_does_not_poison_seen_cache() { + let network = Network::Mainnet; + let first_block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let second_block = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let second_hash = second_block.hash(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let spam_peer = peer(56); + let honest_peer = peer(57); + let destination = peer(58); + + for peer_id in [spam_peer.clone(), honest_peer.clone(), destination] { + connect_peer(&fixture, peer_id).await; + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: spam_peer.clone(), + msg: HeaderSyncMessage::NewBlock(first_block), + }) + .await + .unwrap(); + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::NewBlockReceived { hash, .. } if hash != second_hash + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: spam_peer.clone(), + msg: HeaderSyncMessage::NewBlock(second_block.clone()), + }) + .await + .unwrap(); + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, spam_peer); + assert_eq!(reason, HeaderSyncMisbehavior::NewBlockSpam); + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: honest_peer.clone(), + msg: HeaderSyncMessage::NewBlock(second_block), + }) + .await + .unwrap(); + + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::NewBlockReceived { peer, hash, .. } if hash == second_hash => { + assert_eq!(peer, honest_peer); + break; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("honest retry must not be deduped or scored: {peer:?} {reason:?}"); + } + _ => {} + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rejected_new_block_does_not_forward_or_poison_seen_cache() { + let network = Network::Mainnet; + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let hash = block.hash(); + let height = block.coinbase_height().expect("test block has height"); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let source = peer(59); + let retry_peer = peer(60); + let destination = peer(61); + + for peer_id in [source.clone(), retry_peer.clone(), destination] { + connect_peer(&fixture, peer_id).await; + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: source.clone(), + msg: HeaderSyncMessage::NewBlock(block.clone()), + }) + .await + .unwrap(); + + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::NewBlockReceived { peer, hash: action_hash, .. } + if peer == source && action_hash == hash + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::NewBlockRejected { + peer: source.clone(), + hash, + }) + .await + .unwrap(); + + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, source); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidNewBlock); + break; + } + HeaderSyncAction::ForwardNewBlock { .. } => { + panic!("rejected NewBlock must not be forwarded"); + } + _ => {} + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: retry_peer.clone(), + msg: HeaderSyncMessage::NewBlock(block), + }) + .await + .unwrap(); + + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::NewBlockReceived { + peer, + height: action_height, + hash: action_hash, + .. + } => { + assert_eq!(peer, retry_peer); + assert_eq!(action_height, height); + assert_eq!(action_hash, hash); + break; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("retry after rejection must not be scored: {peer:?} {reason:?}"); + } + _ => {} + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn inbound_get_headers_requires_status_and_respects_serving_cap() { + let network = regtest_network(); + let mut startup = startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + ); + startup.config.max_headers_per_response = 3; + startup.config.max_inflight_requests = 2; + let mut fixture = spawn_test_reactor(startup); + let no_status_peer = peer(59); + let requester = peer(60); + + connect_peer(&fixture, no_status_peer.clone()).await; + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: no_status_peer.clone(), + msg: HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 1, + }, + }) + .await + .unwrap(); + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, no_status_peer); + assert_eq!(reason, HeaderSyncMisbehavior::GetHeadersSpam); + break; + } + } + + connect_peer(&fixture, requester.clone()).await; + advertise_tip( + &fixture, + requester.clone(), + block::Height(0), + block::Height(0), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + for start in [block::Height(1), block::Height(4)] { + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: requester.clone(), + msg: HeaderSyncMessage::GetHeaders { + start_height: start, + count: 3, + }, + }) + .await + .unwrap(); + match next_query_headers_action(&mut fixture.actions).await { + HeaderSyncAction::QueryHeadersByHeightRange { + peer, + start: action_start, + count, + } => { + assert_eq!(peer, requester); + assert_eq!(action_start, start); + assert_eq!(count, 3); + } + action => panic!("unexpected action: {action:?}"), + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: requester.clone(), + msg: HeaderSyncMessage::GetHeaders { + start_height: block::Height(7), + count: 1, + }, + }) + .await + .unwrap(); + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, requester); + assert_eq!(reason, HeaderSyncMisbehavior::GetHeadersSpam); + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeResponseFinished { + peer: requester.clone(), + start_height: block::Height(1), + requested_count: 1, + returned_count: 0, + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: requester.clone(), + msg: HeaderSyncMessage::GetHeaders { + start_height: block::Height(8), + count: 1, + }, + }) + .await + .unwrap(); + match next_query_headers_action(&mut fixture.actions).await { + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + assert_eq!(peer, requester); + assert_eq!(start, block::Height(8)); + assert_eq!(count, 1); + } + action => panic!("unexpected action: {action:?}"), + } +} + +#[tokio::test(flavor = "current_thread")] +async fn inbound_get_headers_over_cap_disconnects_without_state_read() { + let network = regtest_network(); + let mut startup = startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + ); + startup.config.max_headers_per_response = 3; + let mut fixture = spawn_test_reactor(startup); + let requester = peer(61); + + connect_peer(&fixture, requester.clone()).await; + advertise_tip( + &fixture, + requester.clone(), + block::Height(0), + block::Height(0), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: requester.clone(), + msg: HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 4, + }, + }) + .await + .unwrap(); + + loop { + match next_action(&mut fixture.actions).await { + HeaderSyncAction::QueryHeadersByHeightRange { .. } => { + panic!("over-cap GetHeaders must not query state"); + } + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, requester); + assert_eq!(reason, HeaderSyncMisbehavior::GetHeadersTooLong); + break; + } + _ => {} + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn header_sync_jsonl_trace_captures_status_range_dedup_and_disconnect() { + let network = Network::Mainnet; + let mut capture = TraceCapture::for_test( + "header_sync_jsonl_trace_captures_status_range_dedup_and_disconnect", + ) + .unwrap(); + let first_checkpoint = network + .checkpoint_list() + .min_height_in_range(block::Height(1)..) + .expect("mainnet has a checkpoint above genesis"); + let checkpoint_hash = network + .checkpoint_list() + .hash(first_checkpoint) + .expect("checkpoint height has a hash"); + let mut startup = startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((first_checkpoint, checkpoint_hash)), + ); + startup.trace = ZakuraTrace::new(capture.tracer(), "01"); + let fixture = spawn_test_reactor(startup); + let peer_id = peer(55); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + next_height(first_checkpoint).expect("checkpoint has a successor"), + DEFAULT_HS_RANGE, + 1, + ) + .await; + fixture + .handle + .send(HeaderSyncEvent::FullBlockCommitted { + height: block::Height(1), + hash: mainnet_block(&BLOCK_MAINNET_1_BYTES).hash(), + header: mainnet_header(&BLOCK_MAINNET_1_BYTES), + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::NewBlock(mainnet_block(&BLOCK_MAINNET_1_BYTES)), + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireDecodeFailed { + peer: peer_id, + error: Arc::new(HeaderSyncWireError::UnknownMessageType(99)), + }) + .await + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + capture.flush().await; + let reader = capture.reader().unwrap(); + let header_sync = reader.table(HEADER_SYNC_TABLE.table()); + + assert!(header_sync.count(hs_trace::HEADER_STATUS_SENT) >= 1); + assert!(header_sync.count(hs_trace::HEADER_STATUS_RECEIVED) >= 1); + assert!(header_sync.count(hs_trace::HEADER_GET_HEADERS_SENT) >= 1); + assert!(header_sync.count(hs_trace::HEADER_NEW_BLOCK_DEDUPED) >= 1); + assert!(header_sync.count(hs_trace::HEADER_PEER_DISCONNECT_REQUESTED) >= 1); + + for row in header_sync.rows() { + assert!( + row.get("block").is_none() && row.get("headers").is_none(), + "header-sync trace rows must not contain full payloads: {row:?}" + ); + } + + let _ = capture.finish().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn header_sync_metrics_record_status_range_new_block_dedup_and_disconnect() { + let metrics = [ + "sync.header.peer.status.sent", + "sync.header.peer.status.received", + "sync.header.request.sent", + "sync.header.response.received", + "sync.header.range.committed", + "sync.header.tip.new_block.received", + "sync.header.tip.new_block.deduped", + "sync.header.peer.disconnect", + ]; + let before = metric_snapshot(&metrics); + + let first_checkpoint = block::Height(3); + let checkpoint_hash = block::Hash::from(mainnet_header(&BLOCK_MAINNET_3_BYTES).as_ref()); + let (network, _) = checkpoint_testnet_with_hash(first_checkpoint, checkpoint_hash); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + Some((first_checkpoint, checkpoint_hash)), + )); + let peer_id = peer(56); + + connect_peer(&fixture, peer_id.clone()).await; + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } => {} + action => panic!("unexpected action: {action:?}"), + } + + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), + }) + .await + .unwrap(); + let committed_hash = match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::CommitHeaderRange { + start_height, + headers, + .. + } => { + assert_eq!( + start_height, + next_height(first_checkpoint).expect("checkpoint has a successor") + ); + block::Hash::from(headers.last().expect("one header").as_ref()) + } + action => panic!("unexpected action: {action:?}"), + }; + + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: next_height(first_checkpoint).expect("checkpoint has a successor"), + tip_height: next_height(first_checkpoint).expect("checkpoint has a successor"), + tip_hash: committed_hash, + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::FullBlockCommitted { + height: block::Height(1), + hash: mainnet_block(&BLOCK_MAINNET_1_BYTES).hash(), + header: mainnet_header(&BLOCK_MAINNET_1_BYTES), + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::NewBlock(mainnet_block(&BLOCK_MAINNET_1_BYTES)), + }) + .await + .unwrap(); + fixture + .handle + .send(HeaderSyncEvent::WireDecodeFailed { + peer: peer_id, + error: Arc::new(HeaderSyncWireError::UnknownMessageType(99)), + }) + .await + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + for metric in metrics { + assert_metric_incremented(&before, metric); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn unsolicited_headers_are_misbehavior_but_empty_headers_retry() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_with_timeout( + network.clone(), + (block::Height(0), network.genesis_hash()), + std::time::Duration::from_millis(20), + )); + let peer_id = peer(9); + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(Vec::new()), + }) + .await + .unwrap(); + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::UnsolicitedHeaders); + } + action => panic!("unexpected action: {action:?}"), + } + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(1), + 1, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(Vec::new()), + }) + .await + .unwrap(); + assert!( + matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ), + "empty Headers for an outstanding range should retry without disconnecting" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn committed_range_updates_best_tip_watch_and_does_not_advance_finality() { + let network = regtest_network(); + let fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let mut tip = fixture.handle.subscribe_tip(); + let tip_hash = block::Hash([12; 32]); + + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: block::Height(1), + tip_height: block::Height(1), + tip_hash, + }) + .await + .unwrap(); + + tip.changed().await.unwrap(); + assert_eq!(*tip.borrow(), (block::Height(1), tip_hash)); + assert_ne!(fixture.handle.best_header_tip().0, block::Height(0)); +} + +#[tokio::test(flavor = "current_thread")] +async fn forward_genesis_backfill_reaches_checkpoint_before_finalized_commit() { + let headers = [ + mainnet_header(&BLOCK_MAINNET_1_BYTES), + mainnet_header(&BLOCK_MAINNET_2_BYTES), + mainnet_header(&BLOCK_MAINNET_3_BYTES), + ]; + let checkpoint_hash = block::Hash::from(headers[2].as_ref()); + let (network, _) = checkpoint_testnet_with_hash(block::Height(3), checkpoint_hash); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(43); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(3), + DEFAULT_HS_RANGE, + 1, + ) + .await; + loop { + if let HeaderSyncAction::SendMessage { + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + .. + } = next_non_query_action(&mut fixture.actions).await + { + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 3); + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(headers.to_vec()), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + finalized, + .. + } => { + assert_eq!(peer, peer_id); + assert_eq!(start_height, block::Height(1)); + assert_eq!(headers.len(), 3); + assert!(finalized); + } + action => panic!("unexpected action: {action:?}"), + } +} + +#[tokio::test(flavor = "current_thread")] +async fn truncated_finalized_backfill_is_rejected_before_commit() { + let headers = [ + mainnet_header(&BLOCK_MAINNET_1_BYTES), + mainnet_header(&BLOCK_MAINNET_2_BYTES), + mainnet_header(&BLOCK_MAINNET_3_BYTES), + ]; + let checkpoint_hash = block::Hash::from(headers[2].as_ref()); + let (network, _) = checkpoint_testnet_with_hash(block::Height(3), checkpoint_hash); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(44); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(3), + DEFAULT_HS_RANGE, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(headers[..2].to_vec()), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + } + action => panic!("unexpected action: {action:?}"), + } + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn backward_checkpoint_backfill_accepts_linking_run_as_finalized() { + let headers = [ + mainnet_header(&BLOCK_MAINNET_1_BYTES), + mainnet_header(&BLOCK_MAINNET_2_BYTES), + mainnet_header(&BLOCK_MAINNET_3_BYTES), + ]; + let checkpoint_hash = block::Hash::from(headers[2].as_ref()); + let (network, _) = checkpoint_testnet_with_hash(block::Height(3), checkpoint_hash); + let mut fixture = spawn_test_reactor(startup_for( + network, + (block::Height(3), checkpoint_hash), + Some((block::Height(3), checkpoint_hash)), + )); + let peer_id = peer(45); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(3), + DEFAULT_HS_RANGE, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(headers.to_vec()), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + finalized, + .. + } => { + assert_eq!(peer, peer_id); + assert_eq!(start_height, block::Height(1)); + assert_eq!(headers.len(), 3); + assert!(finalized); + } + action => panic!("unexpected action: {action:?}"), + } +} + +#[tokio::test(flavor = "current_thread")] +async fn checkpoint_backfill_rejects_non_contiguous_run_before_commit() { + let (network, checkpoint_hash) = checkpoint_regtest(block::Height(3)); + let mut fixture = spawn_test_reactor(startup_for( + network, + (block::Height(3), checkpoint_hash), + Some((block::Height(3), checkpoint_hash)), + )); + let peer_id = peer(10); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(3), + DEFAULT_HS_RANGE, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(vec![ + mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES), + mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES), + mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES), + ]), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + } + action => panic!("unexpected action: {action:?}"), + } +} + +#[tokio::test(flavor = "current_thread")] +async fn header_response_that_does_not_link_to_anchor_is_misbehavior_before_commit() { + let checkpoint_hash = block::Hash::from(mainnet_header(&BLOCK_MAINNET_3_BYTES).as_ref()); + let (network, _) = checkpoint_testnet_with_hash(block::Height(3), checkpoint_hash); + let anchor = (block::Height(0), network.genesis_hash()); + let mut fixture = spawn_test_reactor(startup_for(network, anchor, Some(anchor))); + let peer_id = peer(46); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_2_BYTES)]), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + } + action => panic!("unexpected action: {action:?}"), + } + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn checkpoint_backfill_rejects_checkpoint_hash_mismatch_before_commit() { + let headers = [ + mainnet_header(&BLOCK_MAINNET_1_BYTES), + mainnet_header(&BLOCK_MAINNET_2_BYTES), + mainnet_header(&BLOCK_MAINNET_3_BYTES), + ]; + let divergent_checkpoint_hash = block::Hash::from(headers[0].as_ref()); + let (network, _) = checkpoint_testnet_with_hash(block::Height(3), divergent_checkpoint_hash); + let mut fixture = spawn_test_reactor(startup_for( + network, + (block::Height(3), divergent_checkpoint_hash), + Some((block::Height(3), divergent_checkpoint_hash)), + )); + let peer_id = peer(46); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(3), + DEFAULT_HS_RANGE, + 1, + ) + .await; + loop { + if matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + ) { + break; + } + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: HeaderSyncMessage::Headers(headers.to_vec()), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + } + action => panic!("unexpected action: {action:?}"), + } + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stateless_validation_accepts_valid_contiguous_headers() { + let headers = vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]; + let context = HeaderSyncValidationContext { + network: &Network::Mainnet, + now: Utc::now(), + start_height: block::Height(1), + decode_context: headers_context(1, DEFAULT_HS_RANGE), + }; + + validate_headers_stateless(headers, context).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stateless_validation_rejects_non_contiguous_and_future_headers() { + let mut second = *mainnet_header(&BLOCK_MAINNET_1_BYTES); + second.previous_block_hash = block::Hash([1; 32]); + let headers = vec![ + mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES), + Arc::new(second), + ]; + let context = HeaderSyncValidationContext { + network: &Network::Mainnet, + now: Utc::now(), + start_height: block::Height(0), + decode_context: headers_context(2, DEFAULT_HS_RANGE), + }; + assert!(matches!( + validate_headers_stateless(headers, context).await, + Err(HeaderSyncWireError::NonContiguousHeaders) + )); + + let mut future = *mainnet_header(&BLOCK_MAINNET_1_BYTES); + future.time = Utc::now() + Duration::hours(3); + let context = HeaderSyncValidationContext { + network: &Network::Mainnet, + now: Utc::now(), + start_height: block::Height(1), + decode_context: headers_context(1, DEFAULT_HS_RANGE), + }; + assert!(matches!( + validate_headers_stateless(vec![Arc::new(future)], context).await, + Err(HeaderSyncWireError::Time(_)) + )); +} + +#[test] +fn range_link_validation_rejects_non_linking_headers() { + let genesis = mainnet_block(&BLOCK_MAINNET_GENESIS_BYTES); + let block1 = mainnet_header(&BLOCK_MAINNET_1_BYTES); + let block2 = mainnet_header(&BLOCK_MAINNET_2_BYTES); + + let mut bad_first = *block1; + bad_first.previous_block_hash = block::Hash([1; 32]); + assert!(matches!( + validate_header_range_links(genesis.hash(), &[Arc::new(bad_first)]), + Err(HeaderSyncWireError::FirstHeaderDoesNotLink) + )); + + let mut bad_second = *block2; + bad_second.previous_block_hash = block::Hash([2; 32]); + assert!(matches!( + validate_header_range_links(genesis.hash(), &[block1, Arc::new(bad_second)]), + Err(HeaderSyncWireError::NonContiguousHeaders) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stateless_validation_rejects_bad_pow() { + let mut bad_solution = *mainnet_header(&BLOCK_MAINNET_1_BYTES); + bad_solution.nonce[0] ^= 1; + let context = HeaderSyncValidationContext { + network: &Network::Mainnet, + now: Utc::now(), + start_height: block::Height(1), + decode_context: headers_context(1, DEFAULT_HS_RANGE), + }; + assert!(matches!( + validate_headers_stateless(vec![Arc::new(bad_solution)], context).await, + Err(HeaderSyncWireError::Equihash(_)) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn new_block_stateless_validation_accepts_valid_mainnet_block() { + validate_new_block_stateless( + mainnet_block(&BLOCK_MAINNET_1_BYTES), + &Network::Mainnet, + Utc::now(), + block::Height(1), + ) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn new_block_stateless_validation_rejects_wrong_solution_size_and_bad_pow() { + let mut wrong_solution_size = (*mainnet_block(&BLOCK_MAINNET_1_BYTES)).clone(); + let mut header = *wrong_solution_size.header; + header.solution = Solution::Regtest([0; 36]); + wrong_solution_size.header = Arc::new(header); + + assert!(matches!( + validate_new_block_stateless( + Arc::new(wrong_solution_size), + &Network::Mainnet, + Utc::now(), + block::Height(1), + ) + .await, + Err(HeaderSyncWireError::WrongEquihashSolutionSize) + )); + + let mut bad_pow = (*mainnet_block(&BLOCK_MAINNET_1_BYTES)).clone(); + let mut header = *bad_pow.header; + header.nonce[0] ^= 1; + bad_pow.header = Arc::new(header); + + assert!(matches!( + validate_new_block_stateless( + Arc::new(bad_pow), + &Network::Mainnet, + Utc::now(), + block::Height(1), + ) + .await, + Err(HeaderSyncWireError::Equihash(_)) + )); +} + +#[test] +fn difficulty_filter_rejects_hash_above_threshold() { + let threshold = + CompactDifficulty::from_bytes_in_display_order(&[0x01, 0x01, 0x00, 0x00]).unwrap(); + + assert!(matches!( + validate_difficulty_filter(block::Hash([0xff; 32]), threshold), + Err(HeaderSyncWireError::DifficultyFilter { .. }) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stateless_header_validation_surfaces_difficulty_filter_after_equihash_acceptance() { + let mut header = *mainnet_header(&BLOCK_MAINNET_1_BYTES); + header.difficulty_threshold = + CompactDifficulty::from_bytes_in_display_order(&[0x01, 0x01, 0x00, 0x00]).unwrap(); + let context = HeaderSyncValidationContext { + network: &Network::Mainnet, + now: Utc::now(), + start_height: block::Height(1), + decode_context: headers_context(1, DEFAULT_HS_RANGE), + }; + + assert!(matches!( + validate_headers_stateless_after_equihash_acceptance(vec![Arc::new(header)], context).await, + Err(HeaderSyncWireError::DifficultyFilter { .. }) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stateless_validation_rejects_wrong_solution_size_for_network() { + let mut regtest_sized = *mainnet_header(&BLOCK_MAINNET_1_BYTES); + regtest_sized.solution = Solution::Regtest([0; 36]); + let context = HeaderSyncValidationContext { + network: &Network::Mainnet, + now: Utc::now(), + start_height: block::Height(1), + decode_context: headers_context(1, DEFAULT_HS_RANGE), + }; + + assert!(matches!( + validate_headers_stateless(vec![Arc::new(regtest_sized)], context).await, + Err(HeaderSyncWireError::WrongEquihashSolutionSize) + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn pow_validation_does_not_monopolize_the_runtime_thread() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let headers = vec![ + mainnet_header(&BLOCK_MAINNET_1_BYTES), + mainnet_header(&BLOCK_MAINNET_2_BYTES), + mainnet_header(&BLOCK_MAINNET_3_BYTES), + mainnet_header(&BLOCK_MAINNET_4_BYTES), + ]; + let context = HeaderSyncValidationContext { + network: &Network::Mainnet, + now: Utc::now(), + start_height: block::Height(1), + decode_context: headers_context(4, DEFAULT_HS_RANGE), + }; + + let ticks = Arc::new(AtomicUsize::new(0)); + let ticker_ticks = ticks.clone(); + let ticker = tokio::spawn(async move { + loop { + ticker_ticks.fetch_add(1, Ordering::SeqCst); + tokio::task::yield_now().await; + } + }); + + validate_headers_stateless(headers, context).await.unwrap(); + let progressed = ticks.load(Ordering::SeqCst); + ticker.abort(); + + assert!( + progressed > 0, + "reactor thread was blocked during PoW validation" + ); +} + +#[test] +fn hostile_vectors_are_rejected_for_allocation_and_unsolicited_headers() { + let mut encoded = vec![MSG_HS_HEADERS]; + encoded.write_u32::(u32::MAX).unwrap(); + assert!(matches!( + HeaderSyncMessage::decode(&encoded, headers_context(MAX_HS_RANGE, MAX_HS_RANGE)), + Err(HeaderSyncWireError::HeaderCountLimit { .. }) + )); + + let mut encoded = vec![MSG_HS_HEADERS]; + encoded.write_u32::(1).unwrap(); + assert!(matches!( + HeaderSyncMessage::decode(&encoded, HeaderSyncDecodeContext::control()), + Err(HeaderSyncWireError::UnsolicitedHeaders) + )); +} diff --git a/zebra-network/src/zakura/header_sync/validation.rs b/zebra-network/src/zakura/header_sync/validation.rs new file mode 100644 index 00000000000..c2a095b81b3 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/validation.rs @@ -0,0 +1,349 @@ +use super::{error::*, events::*, wire::*, *}; + +pub(super) fn validate_anchor( + network: &Network, + anchor: (block::Height, block::Hash), +) -> Result<(), HeaderSyncStartError> { + let expected = if anchor.0 == block::Height(0) { + Some(network.genesis_hash()) + } else { + network.checkpoint_list().hash(anchor.0) + }; + match expected { + Some(hash) if hash == anchor.1 => Ok(()), + _ => Err(HeaderSyncStartError::InvalidAnchor { anchor }), + } +} + +pub(super) fn next_height(height: block::Height) -> Option { + height.0.checked_add(1).map(block::Height) +} + +pub(super) fn previous_height(height: block::Height) -> Option { + height.0.checked_sub(1).map(block::Height) +} + +pub(super) fn height_after_count(start: block::Height, count: u32) -> Option { + start.0.checked_add(count).map(block::Height) +} + +pub(super) fn count_between(start: block::Height, end: block::Height) -> u32 { + end.0 + .checked_sub(start.0) + .and_then(|diff| diff.checked_add(1)) + .unwrap_or(0) +} + +pub(super) fn insert_peer( + row: &mut serde_json::Map, + key: &'static str, + peer: &ZakuraPeerId, +) { + row.insert(key.to_string(), Value::String(trace_peer_label(peer))); +} + +pub(super) fn insert_height( + row: &mut serde_json::Map, + key: &'static str, + height: block::Height, +) { + insert_u64(row, key, u64::from(height.0)); +} + +pub(super) fn insert_hash( + row: &mut serde_json::Map, + key: &'static str, + hash: block::Hash, +) { + row.insert(key.to_string(), Value::String(format!("{hash}"))); +} + +pub(super) fn insert_u64(row: &mut serde_json::Map, key: &'static str, value: u64) { + row.insert(key.to_string(), Value::Number(Number::from(value))); +} + +pub(super) fn insert_optional_str( + row: &mut serde_json::Map, + key: &'static str, + value: Option<&'static str>, +) { + row.insert( + key.to_string(), + value.map_or(Value::Null, |value| Value::String(value.to_string())), + ); +} + +pub(super) fn misbehavior_reason_label(reason: HeaderSyncMisbehavior) -> &'static str { + match reason { + HeaderSyncMisbehavior::InvalidStatus => "invalid_status", + HeaderSyncMisbehavior::UnsolicitedHeaders => "unsolicited_headers", + HeaderSyncMisbehavior::EmptyHeaders => "empty_headers", + HeaderSyncMisbehavior::ResponseTooLong => "response_too_long", + HeaderSyncMisbehavior::InvalidRange => "invalid_range", + HeaderSyncMisbehavior::MalformedMessage => "malformed_message", + HeaderSyncMisbehavior::StatusSpam => "status_spam", + HeaderSyncMisbehavior::NewBlockSpam => "new_block_spam", + HeaderSyncMisbehavior::GetHeadersSpam => "get_headers_spam", + HeaderSyncMisbehavior::GetHeadersTooLong => "get_headers_too_long", + HeaderSyncMisbehavior::UnknownPeer => "unknown_peer", + HeaderSyncMisbehavior::InvalidNewBlock => "invalid_new_block", + } +} + +pub(super) fn commit_failure_reason_label(kind: HeaderSyncCommitFailureKind) -> &'static str { + match kind { + HeaderSyncCommitFailureKind::InvalidPeerRange => "invalid_peer_range", + HeaderSyncCommitFailureKind::Local => "local", + } +} + +/// Peer/request bounds required to decode a header response without over-allocation. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct HeaderSyncDecodeContext { + /// The matching in-flight request, when a `Headers` response is expected. + pub requested: Option, + /// Peer's advertised response cap. + pub peer_max_headers_per_response: u32, +} + +impl HeaderSyncDecodeContext { + /// Context for messages that are not `Headers` responses. + pub fn control() -> Self { + Self { + requested: None, + peer_max_headers_per_response: DEFAULT_HS_RANGE, + } + } + + /// Context for a `Headers` response to `requested`. + pub fn for_headers_response( + requested: HeaderSyncRequestContract, + peer_max_headers_per_response: u32, + ) -> Self { + Self { + requested: Some(requested), + peer_max_headers_per_response: clamp_advertised_range(peer_max_headers_per_response), + } + } + + pub(super) fn headers_response_limit(self) -> Result, HeaderSyncWireError> { + let Some(requested) = self.requested else { + return Ok(None); + }; + let cap = min( + min(requested.count, self.peer_max_headers_per_response), + MAX_HS_RANGE, + ); + Some(usize_from_u32(cap, "headers response limit")).transpose() + } +} + +/// Stateless header validation context. +#[derive(Copy, Clone, Debug)] +pub struct HeaderSyncValidationContext<'a> { + /// Active network, used for Equihash solution-size policy. + pub network: &'a Network, + /// Wall clock used for future-time checks. + pub now: DateTime, + /// First height in the received run. + pub start_height: block::Height, + /// Matching request and peer cap for count validation. + pub decode_context: HeaderSyncDecodeContext, +} + +/// Run all context-free validation checks for an inbound `Headers` response. +#[tracing::instrument(skip(headers, context))] +pub async fn validate_headers_stateless( + headers: Vec>, + context: HeaderSyncValidationContext<'_>, +) -> Result<(), HeaderSyncWireError> { + validate_header_count(headers.len(), context.decode_context)?; + validate_internal_continuity(&headers)?; + validate_header_times(&headers, context.now, context.start_height)?; + validate_solution_sizes(&headers, context.network)?; + validate_pow_spawn_blocking(headers).await +} + +/// Check that a header range links to its anchor and is internally contiguous. +pub fn validate_header_range_links( + anchor: block::Hash, + headers: &[Arc], +) -> Result<(), HeaderSyncWireError> { + let Some(first) = headers.first() else { + return Ok(()); + }; + + if first.previous_block_hash != anchor { + return Err(HeaderSyncWireError::FirstHeaderDoesNotLink); + } + + validate_internal_continuity(headers) +} + +/// Run all context-free validation checks for an inbound full-block tip flood. +#[tracing::instrument(skip(block, network))] +pub async fn validate_new_block_stateless( + block: Arc, + network: &Network, + now: DateTime, + height: block::Height, +) -> Result<(), HeaderSyncWireError> { + let header = block.header.clone(); + validate_header_times(std::slice::from_ref(&header), now, height)?; + validate_solution_sizes(std::slice::from_ref(&header), network)?; + validate_pow_spawn_blocking(vec![header]).await +} + +pub(super) fn validate_header_count( + len: usize, + context: HeaderSyncDecodeContext, +) -> Result<(), HeaderSyncWireError> { + let Some(max_headers) = context.headers_response_limit()? else { + return Err(HeaderSyncWireError::UnsolicitedHeaders); + }; + validate_headers_len(len, max_headers) +} + +pub(super) fn validate_internal_continuity( + headers: &[Arc], +) -> Result<(), HeaderSyncWireError> { + for adjacent in headers.windows(2) { + let previous_hash = block::Hash::from(adjacent[0].as_ref()); + if previous_hash != adjacent[1].previous_block_hash { + return Err(HeaderSyncWireError::NonContiguousHeaders); + } + } + Ok(()) +} + +pub(super) fn validate_header_times( + headers: &[Arc], + now: DateTime, + start_height: block::Height, +) -> Result<(), HeaderSyncWireError> { + for (offset, header) in headers.iter().enumerate() { + let offset = u32::try_from(offset) + .map_err(|_| HeaderSyncWireError::NumericOverflow("header height offset"))?; + let height = block::Height( + start_height + .0 + .checked_add(offset) + .ok_or(HeaderSyncWireError::NumericOverflow("header height"))?, + ); + let hash = block::Hash::from(header.as_ref()); + header.time_is_valid_at(now, &height, &hash)?; + } + Ok(()) +} + +pub(super) fn validate_solution_sizes( + headers: &[Arc], + network: &Network, +) -> Result<(), HeaderSyncWireError> { + let expect_regtest = network + .parameters() + .is_some_and(|parameters| parameters.is_regtest()); + for header in headers { + match (expect_regtest, header.solution) { + (true, equihash::Solution::Regtest(_)) | (false, equihash::Solution::Common(_)) => {} + _ => return Err(HeaderSyncWireError::WrongEquihashSolutionSize), + } + } + Ok(()) +} + +pub(super) async fn validate_pow_spawn_blocking( + headers: Vec>, +) -> Result<(), HeaderSyncWireError> { + tokio::task::spawn_blocking(move || validate_pow_blocking(&headers)).await? +} + +pub(super) fn validate_pow_blocking( + headers: &[Arc], +) -> Result<(), HeaderSyncWireError> { + for header in headers { + header.solution.check(header)?; + let hash = block::Hash::from(header.as_ref()); + validate_difficulty_filter(hash, header.difficulty_threshold)?; + } + Ok(()) +} + +pub(super) fn validate_difficulty_filter( + hash: block::Hash, + difficulty_threshold: CompactDifficulty, +) -> Result<(), HeaderSyncWireError> { + let threshold = difficulty_threshold + .to_expanded() + .ok_or(HeaderSyncWireError::InvalidDifficultyThreshold)?; + if hash > threshold { + return Err(HeaderSyncWireError::DifficultyFilter { hash, threshold }); + } + Ok(()) +} + +pub(super) fn validate_get_headers_count(count: u32) -> Result<(), HeaderSyncWireError> { + if count == 0 { + return Err(HeaderSyncWireError::ZeroHeaderRequestCount); + } + if count > MAX_HS_RANGE { + return Err(HeaderSyncWireError::HeaderCountLimit { + actual: usize_from_u32(count, "headers count")?, + max: usize_from_u32(MAX_HS_RANGE, "headers cap")?, + }); + } + Ok(()) +} + +pub(super) fn validate_headers_len(len: usize, max: usize) -> Result<(), HeaderSyncWireError> { + if len > max { + return Err(HeaderSyncWireError::HeaderCountLimit { actual: len, max }); + } + Ok(()) +} + +pub(super) fn clamp_advertised_range(value: u32) -> u32 { + value.clamp(1, MAX_HS_RANGE) +} + +pub(super) fn write_height( + writer: &mut W, + height: block::Height, +) -> Result<(), HeaderSyncWireError> { + writer.write_u32::(height.0)?; + Ok(()) +} + +pub(super) fn read_height(reader: &mut R) -> Result { + let height = block::Height(reader.read_u32::()?); + if height > block::Height::MAX { + return Err(HeaderSyncWireError::HeightOutOfRange(height.0)); + } + Ok(height) +} + +pub(super) fn reject_trailing( + bytes: &[u8], + reader: &Cursor<&[u8]>, +) -> Result<(), HeaderSyncWireError> { + let consumed = usize::try_from(reader.position()) + .map_err(|_| HeaderSyncWireError::NumericOverflow("cursor position"))?; + if consumed != bytes.len() { + return Err(HeaderSyncWireError::TrailingBytes); + } + Ok(()) +} + +pub(super) fn usize_from_u32( + value: u32, + field: &'static str, +) -> Result { + usize::try_from(value).map_err(|_| HeaderSyncWireError::NumericOverflow(field)) +} + +pub(super) fn u32_from_usize( + value: usize, + field: &'static str, +) -> Result { + u32::try_from(value).map_err(|_| HeaderSyncWireError::NumericOverflow(field)) +} diff --git a/zebra-network/src/zakura/header_sync/wire.rs b/zebra-network/src/zakura/header_sync/wire.rs new file mode 100644 index 00000000000..cadbd977ab6 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/wire.rs @@ -0,0 +1,187 @@ +use super::{config::*, error::*, validation::*, *}; + +/// Zakura stream kind reserved for native header sync. +pub const ZAKURA_STREAM_HEADER_SYNC: u16 = 5; +/// Version of the native header-sync stream. +pub const ZAKURA_HEADER_SYNC_STREAM_VERSION: u16 = 1; + +/// Peer status advertisement. +pub const MSG_HS_STATUS: u8 = 1; +/// Request a contiguous range of headers by height. +pub const MSG_HS_GET_HEADERS: u8 = 2; +/// Respond with a contiguous run of headers. +pub const MSG_HS_HEADERS: u8 = 3; +/// Flood a newly seen tip block, including its full body. +pub const MSG_HS_NEW_BLOCK: u8 = 4; + +/// Maximum encoded stream-5 message bytes. +pub const MAX_HS_MESSAGE_BYTES: usize = 2 * 1024 * 1024; +/// Default number of headers advertised per response. +pub const DEFAULT_HS_RANGE: u32 = 1000; +/// Maximum number of headers ever honored by stream 5. +pub const MAX_HS_RANGE: u32 = 4000; +/// Default number of in-flight header requests advertised per peer. +pub const DEFAULT_HS_MAX_INFLIGHT: u16 = 10; + +pub(super) const HEADER_SYNC_MESSAGE_TYPE_BYTES: usize = 1; +pub(super) const HEADER_SYNC_COUNT_BYTES: usize = 4; +pub(super) const COMMON_HEADER_BYTES: usize = 1_487; +pub(super) const REGTEST_HEADER_BYTES: usize = 177; +pub(super) const HEADER_SYNC_FANOUT: usize = 3; +pub(super) const LOCAL_MAX_HS_INFLIGHT_PER_PEER: u16 = 16; +pub(super) const EFFECTIVE_HS_OUTBOUND_INFLIGHT_PER_PEER: usize = 1; +pub(super) const HEADER_SYNC_SEEN_HASH_CAPACITY: usize = 4096; +pub(super) const DEFAULT_HS_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +pub(super) const EMPTY_HEADERS_RETRY_DELAY: Duration = Duration::from_secs(1); +pub(super) const DEFAULT_HS_STATUS_REFRESH_INTERVAL: Duration = Duration::from_secs(30); +// v1 semantic meters intentionally use strict spacing for unsolicited status refreshes and +// distinct unseen full-block floods. Cheap duplicate `NewBlock` echoes are deduped before this +// meter, so they do not consume tokens or cause honest peers to be scored. +pub(super) const DEFAULT_HS_INBOUND_STATUS_MIN_INTERVAL: Duration = Duration::from_secs(5); +pub(super) const DEFAULT_HS_INBOUND_NEW_BLOCK_MIN_INTERVAL: Duration = Duration::from_secs(5); + +const _: () = assert!(MAX_HS_MESSAGE_BYTES < LOCAL_MAX_MESSAGE_BYTES as usize); +const _: () = assert!( + HEADER_SYNC_MESSAGE_TYPE_BYTES + + HEADER_SYNC_COUNT_BYTES + + COMMON_HEADER_BYTES * (DEFAULT_HS_RANGE as usize) + < MAX_HS_MESSAGE_BYTES +); + +/// Native stream-5 header-sync message. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum HeaderSyncMessage { + /// Peer tip, anchor, and served-range advertisement. + Status(HeaderSyncStatus), + /// Request `count` headers starting at `start_height`. + GetHeaders { + /// First requested height. + start_height: block::Height, + /// Requested header count. + count: u32, + }, + /// A bounded contiguous header run. + Headers(Vec>), + /// Full block tip-flood payload. + NewBlock(Arc), +} + +impl HeaderSyncMessage { + /// Returns this message's stream-5 discriminator. + pub fn message_type(&self) -> u8 { + match self { + Self::Status(_) => MSG_HS_STATUS, + Self::GetHeaders { .. } => MSG_HS_GET_HEADERS, + Self::Headers(_) => MSG_HS_HEADERS, + Self::NewBlock(_) => MSG_HS_NEW_BLOCK, + } + } + + /// Encode this message as `[u8 message_type][bounded fields...]`. + pub fn encode(&self) -> Result, HeaderSyncWireError> { + let mut bytes = Vec::new(); + bytes.write_u8(self.message_type())?; + match self { + Self::Status(status) => status.encode_to(&mut bytes)?, + Self::GetHeaders { + start_height, + count, + } => { + validate_get_headers_count(*count)?; + write_height(&mut bytes, *start_height)?; + bytes.write_u32::(*count)?; + } + Self::Headers(headers) => { + validate_headers_len(headers.len(), usize_from_u32(MAX_HS_RANGE, "headers cap")?)?; + bytes.write_u32::(u32_from_usize(headers.len(), "headers count")?)?; + for header in headers { + header.zcash_serialize(&mut bytes)?; + } + } + Self::NewBlock(block) => { + block.zcash_serialize(&mut bytes)?; + } + } + if bytes.len() > MAX_HS_MESSAGE_BYTES { + return Err(HeaderSyncWireError::OversizedPayload { + actual: bytes.len(), + max: MAX_HS_MESSAGE_BYTES, + }); + } + Ok(bytes) + } + + /// Decode a stream-5 message using the peer/request bounds in `context`. + pub fn decode( + bytes: &[u8], + context: HeaderSyncDecodeContext, + ) -> Result { + if bytes.len() > MAX_HS_MESSAGE_BYTES { + return Err(HeaderSyncWireError::OversizedPayload { + actual: bytes.len(), + max: MAX_HS_MESSAGE_BYTES, + }); + } + let mut reader = Cursor::new(bytes); + let message_type = reader.read_u8()?; + let message = match message_type { + MSG_HS_STATUS => Self::Status(HeaderSyncStatus::decode_from(&mut reader)?), + MSG_HS_GET_HEADERS => { + let start_height = read_height(&mut reader)?; + let count = reader.read_u32::()?; + validate_get_headers_count(count)?; + Self::GetHeaders { + start_height, + count, + } + } + MSG_HS_HEADERS => { + let count = usize_from_u32(reader.read_u32::()?, "headers count")?; + let Some(max_headers) = context.headers_response_limit()? else { + return Err(HeaderSyncWireError::UnsolicitedHeaders); + }; + validate_headers_len(count, max_headers)?; + let mut headers = Vec::with_capacity(count); + for _ in 0..count { + headers.push(Arc::new(block::Header::zcash_deserialize(&mut reader)?)); + } + Self::Headers(headers) + } + MSG_HS_NEW_BLOCK => { + Self::NewBlock(Arc::new(block::Block::zcash_deserialize(&mut reader)?)) + } + value => return Err(HeaderSyncWireError::UnknownMessageType(value)), + }; + reject_trailing(bytes, &reader)?; + Ok(message) + } + + /// Convert this message into a bounded Zakura frame. + pub fn encode_frame(&self) -> Result { + Ok(Frame { + message_type: u16::from(self.message_type()), + flags: 0, + payload: self.encode()?, + }) + } + + /// Decode this message from a Zakura frame after checking flags and type agreement. + pub fn decode_frame( + frame: Frame, + context: HeaderSyncDecodeContext, + ) -> Result { + if frame.flags != 0 { + return Err(HeaderSyncWireError::UnsupportedFlags(frame.flags)); + } + let message = Self::decode(&frame.payload, context)?; + let frame_message_type = u8::try_from(frame.message_type) + .map_err(|_| HeaderSyncWireError::UnknownFrameMessageType(frame.message_type))?; + if frame_message_type != message.message_type() { + return Err(HeaderSyncWireError::MismatchedFrameMessageType { + frame: frame.message_type, + payload: message.message_type(), + }); + } + Ok(message) + } +} diff --git a/zebra-network/src/zakura/testkit/cluster.rs b/zebra-network/src/zakura/testkit/cluster.rs index 391c8ae17fb..82239293582 100644 --- a/zebra-network/src/zakura/testkit/cluster.rs +++ b/zebra-network/src/zakura/testkit/cluster.rs @@ -133,18 +133,43 @@ fn contains_peer(peers: &[ZakuraPeerId], expected: &[u8]) -> bool { #[cfg(test)] mod tests { - use super::super::HostilePeer; + use super::super::{trace_reader::TraceValue, HostilePeer, WaitError}; use super::*; use crate::{ + zakura::trace::header_sync_trace as hs_trace, zakura::{ - DiscoveryMessage, Frame, FramedSend, Peer, Service, Stream, ZakuraLocalLimits, - ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_LEGACY_GOSSIP, ZAKURA_STREAM_DISCOVERY, - ZAKURA_STREAM_GOSSIP, + spawn_header_sync_reactor, DiscoveryMessage, Frame, FramedSend, HeaderSyncAction, + HeaderSyncCommitFailureKind, HeaderSyncEvent, HeaderSyncFrontiers, HeaderSyncHandle, + HeaderSyncMessage, HeaderSyncMisbehavior, HeaderSyncStartup, HeaderSyncStatus, Peer, + Service, Stream, ZakuraHeaderSyncConfig, ZakuraLocalLimits, ZakuraTrace, + ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_HEADER_SYNC, ZAKURA_CAP_LEGACY_GOSSIP, + ZAKURA_STREAM_DISCOVERY, ZAKURA_STREAM_GOSSIP, ZAKURA_STREAM_HEADER_SYNC, }, Config, }; - use std::{collections::HashMap, sync::Arc}; - use tokio::sync::{mpsc, Mutex}; + use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::{Arc, Mutex as StdMutex}, + }; + use tokio::{ + sync::{mpsc, Mutex}, + task::JoinHandle, + }; + use tokio_util::sync::CancellationToken; + use zebra_chain::{ + block, + parameters::{ + testnet::{ + ConfiguredActivationHeights, ConfiguredCheckpoints, Parameters as TestnetParameters, + }, + Network, + }, + serialization::ZcashDeserializeInto, + }; + use zebra_test::vectors::{ + BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES, BLOCK_MAINNET_3_BYTES, BLOCK_MAINNET_4_BYTES, + BLOCK_MAINNET_5_BYTES, BLOCK_MAINNET_GENESIS_BYTES, + }; #[derive(Debug, Default)] struct OrderedSourceProbeService { @@ -308,6 +333,849 @@ mod tests { .map_err(|_| -> BoxError { format!("timed out waiting for {label}").into() })? } + #[derive(Debug)] + struct E2eHeaderStore { + headers: BTreeMap)>, + bodies: HashSet, + finalized_height: block::Height, + verified_block_tip: block::Height, + reject_next_commit: Option, + } + + impl E2eHeaderStore { + fn genesis_only() -> Self { + let genesis = mainnet_block(&BLOCK_MAINNET_GENESIS_BYTES); + let mut headers = BTreeMap::new(); + headers.insert(block::Height(0), (genesis.hash(), genesis.header.clone())); + Self { + headers, + bodies: HashSet::from([genesis.hash()]), + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + reject_next_commit: None, + } + } + + fn with_headers(up_to: u32) -> Self { + let mut store = Self::genesis_only(); + for height in 1..=up_to { + let block = mainnet_block(block_bytes(height)); + store + .headers + .insert(block::Height(height), (block.hash(), block.header.clone())); + } + store + } + + fn with_checkpoint_anchor(height: u32) -> Self { + let mut store = Self::genesis_only(); + let block = mainnet_block(block_bytes(height)); + store + .headers + .insert(block::Height(height), (block.hash(), block.header.clone())); + store.finalized_height = block::Height(height); + store.verified_block_tip = block::Height(0); + store + } + + fn best_header_tip(&self) -> (block::Height, block::Hash) { + self.headers + .last_key_value() + .map(|(height, (hash, _))| (*height, *hash)) + .expect("test stores always contain genesis") + } + + fn frontiers(&self) -> HeaderSyncFrontiers { + HeaderSyncFrontiers { + finalized_height: self.finalized_height, + verified_block_tip: self.verified_block_tip, + } + } + + fn headers_by_range(&self, start: block::Height, count: u32) -> Vec> { + let mut headers = Vec::new(); + for offset in 0..count { + let Some(height) = start + i64::from(offset) else { + break; + }; + let Some((_hash, header)) = self.headers.get(&height) else { + break; + }; + headers.push(header.clone()); + } + headers + } + + fn commit_headers( + &mut self, + anchor: block::Hash, + start: block::Height, + headers: Vec>, + finalized: bool, + ) -> Result<(block::Height, block::Hash), HeaderSyncCommitFailureKind> { + if let Some(kind) = self.reject_next_commit.take() { + return Err(kind); + } + + let mut expected_previous = anchor; + for (offset, header) in headers.iter().enumerate() { + if header.previous_block_hash != expected_previous { + return Err(HeaderSyncCommitFailureKind::InvalidPeerRange); + } + let height = (start + + i64::try_from(offset) + .map_err(|_| HeaderSyncCommitFailureKind::InvalidPeerRange)?) + .ok_or(HeaderSyncCommitFailureKind::InvalidPeerRange)?; + let hash = block::Hash::from(header.as_ref()); + self.headers.insert(height, (hash, header.clone())); + expected_previous = hash; + } + + let count = u32::try_from(headers.len()).unwrap_or(u32::MAX); + let tip_height = block::Height(start.0.saturating_add(count.saturating_sub(1))); + let tip_hash = expected_previous; + if finalized { + self.finalized_height = self.finalized_height.max(tip_height); + } + + Ok((tip_height, tip_hash)) + } + + fn commit_body(&mut self, block: Arc) { + let height = block.coinbase_height().expect("test block has height"); + let hash = block.hash(); + self.headers.insert(height, (hash, block.header.clone())); + self.bodies.insert(hash); + self.verified_block_tip = self.verified_block_tip.max(height); + } + + fn has_body(&self, hash: block::Hash) -> bool { + self.bodies.contains(&hash) + } + + fn missing_bodies(&self, from: block::Height, limit: u32) -> Vec { + let (tip, _) = self.best_header_tip(); + let start = self + .verified_block_tip + .next() + .unwrap_or(self.verified_block_tip) + .max(from); + + (start.0..=tip.0) + .map(block::Height) + .filter(|height| { + self.headers + .get(height) + .is_some_and(|(hash, _)| !self.bodies.contains(hash)) + }) + .take(limit as usize) + .collect() + } + } + + #[derive(Clone, Debug)] + struct E2eNodeView { + peer_id: ZakuraPeerId, + handle: HeaderSyncHandle, + store: Arc>, + observed_gaps: Arc>>, + disconnects: Arc>>, + sent: Arc>>, + } + + #[derive(Debug)] + struct E2eNode { + view: E2eNodeView, + actions: Option>, + task: JoinHandle<()>, + shutdown: CancellationToken, + } + + #[derive(Debug)] + struct HeaderSyncE2eCluster { + nodes: Vec, + drivers: Vec>, + } + + impl HeaderSyncE2eCluster { + fn new() -> Self { + Self { + nodes: Vec::new(), + drivers: Vec::new(), + } + } + + fn spawn_node( + &mut self, + seed: u8, + network: Network, + anchor: (block::Height, block::Hash), + store: E2eHeaderStore, + trace: ZakuraTrace, + ) -> Result { + let store = Arc::new(StdMutex::new(store)); + let startup_store = store + .lock() + .map_err(|_| std::io::Error::other("test store mutex is poisoned"))?; + let mut startup = HeaderSyncStartup::new( + network, + anchor, + startup_store.frontiers(), + Some(startup_store.best_header_tip()), + ZakuraHeaderSyncConfig::default(), + 4 * 1024 * 1024, + ); + drop(startup_store); + startup.trace = trace; + startup.range_state_actions_enabled = true; + startup.inbound_new_block_acceptance_enabled = true; + startup.status_refresh_interval = Duration::from_millis(200); + startup.request_timeout = Duration::from_millis(500); + let shutdown = CancellationToken::new(); + startup.shutdown = shutdown.clone(); + + let (handle, actions, task) = spawn_header_sync_reactor(startup)?; + let view = E2eNodeView { + peer_id: e2e_peer(seed), + handle, + store, + observed_gaps: Arc::new(Mutex::new(Vec::new())), + disconnects: Arc::new(Mutex::new(Vec::new())), + sent: Arc::new(Mutex::new(Vec::new())), + }; + self.nodes.push(E2eNode { + view, + actions: Some(actions), + task, + shutdown, + }); + Ok(self.nodes.len() - 1) + } + + fn start_drivers(&mut self) { + let views: Vec<_> = self.nodes.iter().map(|node| node.view.clone()).collect(); + let peer_to_index: HashMap<_, _> = views + .iter() + .enumerate() + .map(|(index, view)| (view.peer_id.clone(), index)) + .collect(); + + for index in 0..self.nodes.len() { + let Some(actions) = self.nodes[index].actions.take() else { + continue; + }; + self.drivers.push(tokio::spawn(drive_e2e_node( + index, + actions, + views.clone(), + peer_to_index.clone(), + ))); + } + } + + async fn connect_all(&self) { + for left in 0..self.nodes.len() { + for right in 0..self.nodes.len() { + if left == right { + continue; + } + self.nodes[left] + .view + .handle + .send(HeaderSyncEvent::PeerConnected( + self.nodes[right].view.peer_id.clone(), + )) + .await + .unwrap(); + } + } + } + + async fn connect_peer(&self, node: usize, peer: ZakuraPeerId) { + self.nodes[node] + .view + .handle + .send(HeaderSyncEvent::PeerConnected(peer)) + .await + .unwrap(); + } + + async fn inject(&self, node: usize, peer: ZakuraPeerId, msg: HeaderSyncMessage) { + self.nodes[node] + .view + .handle + .send(HeaderSyncEvent::WireMessage { peer, msg }) + .await + .unwrap(); + } + + async fn commit_body(&self, node: usize, block: Arc) { + self.nodes[node] + .view + .store + .lock() + .expect("test store mutex is not poisoned") + .commit_body(block.clone()); + let height = block.coinbase_height().expect("test block has height"); + self.nodes[node] + .view + .handle + .send(HeaderSyncEvent::FullBlockCommitted { + height, + hash: block.hash(), + header: block.header.clone(), + }) + .await + .unwrap(); + } + + async fn missing_bodies(&self, node: usize) -> Vec { + self.nodes[node] + .view + .store + .lock() + .expect("test store mutex is not poisoned") + .missing_bodies(block::Height(1), 100) + } + + async fn finalized_height(&self, node: usize) -> block::Height { + self.nodes[node] + .view + .store + .lock() + .expect("test store mutex is not poisoned") + .finalized_height + } + + async fn reject_next_commit(&self, node: usize, kind: HeaderSyncCommitFailureKind) { + self.nodes[node] + .view + .store + .lock() + .expect("test store mutex is not poisoned") + .reject_next_commit = Some(kind); + } + + async fn wait_for_tip(&self, node: usize, height: block::Height) -> Result<(), BoxError> { + let handle = self.nodes[node].view.handle.clone(); + await_until("header-sync e2e best tip", Duration::from_secs(5), || { + handle.best_header_tip().0 >= height + }) + .await + .map_err(Into::into) + } + + async fn wait_for_body(&self, node: usize, hash: block::Hash) -> Result<(), BoxError> { + let store = self.nodes[node].view.store.clone(); + await_until( + "header-sync e2e body commit", + Duration::from_secs(5), + || { + store + .lock() + .expect("test store mutex is not poisoned") + .has_body(hash) + }, + ) + .await + .map_err(Into::into) + } + + async fn disconnect_reasons(&self, node: usize) -> Vec { + self.nodes[node] + .view + .disconnects + .lock() + .await + .iter() + .map(|(_, reason)| *reason) + .collect() + } + + async fn wait_for_disconnect_reason( + &self, + node: usize, + reason: HeaderSyncMisbehavior, + ) -> Result<(), BoxError> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + if self.disconnect_reasons(node).await.contains(&reason) { + tokio::time::sleep(Duration::from_millis(50)).await; + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(Box::new(WaitError::new( + "header-sync e2e disconnect reason", + Duration::from_secs(5), + ))); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + async fn wait_for_get_headers( + &self, + node: usize, + peer: &ZakuraPeerId, + start_height: block::Height, + count: u32, + ) -> Result<(), BoxError> { + let sent = self.nodes[node].view.sent.clone(); + let peer = peer.clone(); + await_until( + format!( + "header-sync e2e outbound getheaders peer={peer:?} start={start_height:?} count={count}" + ), + Duration::from_secs(5), + || { + sent.try_lock().is_ok_and(|sent| { + sent.iter().any(|(sent_peer, msg)| { + sent_peer == &peer + && matches!( + msg, + HeaderSyncMessage::GetHeaders { + start_height: actual_start, + count: actual_count, + } if *actual_start == start_height && *actual_count == count + ) + }) + }) + }, + ) + .await + .map_err(Into::into) + } + + async fn observed_gaps(&self, node: usize) -> Vec<(block::Height, block::Height)> { + self.nodes[node].view.observed_gaps.lock().await.clone() + } + + async fn shutdown(&mut self) { + for node in &self.nodes { + node.shutdown.cancel(); + node.task.abort(); + } + for driver in self.drivers.drain(..) { + driver.abort(); + } + } + } + + async fn drive_e2e_node( + index: usize, + mut actions: mpsc::Receiver, + nodes: Vec, + peer_to_index: HashMap, + ) { + let local = nodes[index].clone(); + while let Some(action) = actions.recv().await { + match action { + HeaderSyncAction::SendMessage { peer, msg } => { + if let Some(target) = peer_to_index.get(&peer) { + let _ = nodes[*target] + .handle + .send(HeaderSyncEvent::WireMessage { + peer: local.peer_id.clone(), + msg, + }) + .await; + } else { + local.sent.lock().await.push((peer, msg)); + } + } + HeaderSyncAction::ForwardNewBlock { peer, block, .. } => { + if let Some(target) = peer_to_index.get(&peer) { + let _ = nodes[*target] + .handle + .send(HeaderSyncEvent::WireMessage { + peer: local.peer_id.clone(), + msg: HeaderSyncMessage::NewBlock(block), + }) + .await; + } else { + local + .sent + .lock() + .await + .push((peer, HeaderSyncMessage::NewBlock(block))); + } + } + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + let headers = local + .store + .lock() + .expect("test store mutex is not poisoned") + .headers_by_range(start, count); + let returned_count = u32::try_from(headers.len()).unwrap_or(u32::MAX); + if let Some(target) = peer_to_index.get(&peer) { + let _ = nodes[*target] + .handle + .send(HeaderSyncEvent::WireMessage { + peer: local.peer_id.clone(), + msg: HeaderSyncMessage::Headers(headers), + }) + .await; + let _ = local + .handle + .send(HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count, + }) + .await; + } + } + HeaderSyncAction::CommitHeaderRange { + peer, + anchor, + start_height, + headers, + finalized, + } => { + let count = u32::try_from(headers.len()).unwrap_or(u32::MAX); + let result = local + .store + .lock() + .expect("test store mutex is not poisoned") + .commit_headers(anchor, start_height, headers, finalized); + match result { + Ok((tip_height, tip_hash)) => { + let frontiers = local + .store + .lock() + .expect("test store mutex is not poisoned") + .frontiers(); + let _ = local + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height, + tip_height, + tip_hash, + }) + .await; + let _ = local + .handle + .send(HeaderSyncEvent::StateFrontiersChanged(frontiers)) + .await; + } + Err(kind) => { + let _ = local + .handle + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind, + }) + .await; + } + } + } + HeaderSyncAction::QueryBestHeaderTip => { + let (tip_height, tip_hash) = local + .store + .lock() + .expect("test store mutex is not poisoned") + .best_header_tip(); + let _ = local + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: tip_height, + tip_height, + tip_hash, + }) + .await; + } + HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { + let heights = local + .store + .lock() + .expect("test store mutex is not poisoned") + .missing_bodies(from, limit); + if let (Some(first), Some(last)) = + (heights.first().copied(), heights.last().copied()) + { + local.observed_gaps.lock().await.push((first, last)); + } + } + HeaderSyncAction::BodyGaps { from, to } => { + local.observed_gaps.lock().await.push((from, to)); + } + HeaderSyncAction::NewBlockReceived { + peer, + height, + hash, + block, + } => { + if local + .store + .lock() + .expect("test store mutex is not poisoned") + .has_body(hash) + { + let _ = local + .handle + .send(HeaderSyncEvent::NewBlockDuplicate { peer, height, hash }) + .await; + } else { + local + .store + .lock() + .expect("test store mutex is not poisoned") + .commit_body(block.clone()); + let _ = local + .handle + .send(HeaderSyncEvent::NewBlockAccepted { + peer, + height, + hash, + block, + }) + .await; + } + } + HeaderSyncAction::Misbehavior { peer, reason } => { + local.disconnects.lock().await.push((peer.clone(), reason)); + let _ = local + .handle + .send(HeaderSyncEvent::PeerDisconnected(peer)) + .await; + } + } + } + } + + fn e2e_peer(byte: u8) -> ZakuraPeerId { + ZakuraPeerId::new(vec![byte; 32]).expect("test peer id is within bounds") + } + + fn mainnet_block(bytes: &[u8]) -> Arc { + Arc::new(bytes.zcash_deserialize_into().expect("block vector parses")) + } + + fn block_bytes(height: u32) -> &'static [u8] { + match height { + 1 => &BLOCK_MAINNET_1_BYTES, + 2 => &BLOCK_MAINNET_2_BYTES, + 3 => &BLOCK_MAINNET_3_BYTES, + 4 => &BLOCK_MAINNET_4_BYTES, + 5 => &BLOCK_MAINNET_5_BYTES, + _ => panic!("missing test vector for height {height}"), + } + } + + fn mainnet_genesis_hash() -> block::Hash { + mainnet_block(&BLOCK_MAINNET_GENESIS_BYTES).hash() + } + + fn status_for_tip( + height: u32, + max_headers_per_response: u32, + max_inflight_requests: u16, + ) -> HeaderSyncMessage { + let tip_hash = if height == 0 { + mainnet_genesis_hash() + } else { + mainnet_block(block_bytes(height)).hash() + }; + + HeaderSyncMessage::Status(HeaderSyncStatus { + tip_height: block::Height(height), + tip_hash, + anchor_height: block::Height(0), + max_headers_per_response, + max_inflight_requests, + }) + } + + fn e2e_network(checkpoints: impl IntoIterator) -> Network { + let checkpoints = std::iter::once((block::Height(0), mainnet_genesis_hash())) + .chain(checkpoints.into_iter().map(|height| { + ( + block::Height(height), + mainnet_block(block_bytes(height)).hash(), + ) + })) + .collect(); + + TestnetParameters::build() + .with_genesis_hash(mainnet_genesis_hash()) + .expect("mainnet genesis vector hash parses") + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: None, + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: None, + nu6: None, + nu6_1: None, + nu7: None, + #[cfg(zcash_unstable = "zfuture")] + zfuture: None, + }) + .expect("height-1 activation set is valid") + .with_funding_streams(Vec::new()) + .with_checkpoints(ConfiguredCheckpoints::HeightsAndHashes(checkpoints)) + .expect("e2e checkpoints use valid header hashes") + .to_network() + .expect("e2e network has enough checkpoint coverage") + } + + fn e2e_network_with_checkpoint_hash(height: u32, hash: block::Hash) -> Network { + let checkpoints = vec![ + (block::Height(0), mainnet_genesis_hash()), + (block::Height(height), hash), + ]; + + TestnetParameters::build() + .with_genesis_hash(mainnet_genesis_hash()) + .expect("mainnet genesis vector hash parses") + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: None, + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: None, + nu6: None, + nu6_1: None, + nu7: None, + #[cfg(zcash_unstable = "zfuture")] + zfuture: None, + }) + .expect("height-1 activation set is valid") + .with_funding_streams(Vec::new()) + .with_checkpoints(ConfiguredCheckpoints::HeightsAndHashes(checkpoints)) + .expect("e2e checkpoints use valid header hashes") + .to_network() + .expect("e2e network has enough checkpoint coverage") + } + + fn checkpoint_network(checkpoint_height: u32) -> (Network, block::Hash) { + let checkpoint_hash = mainnet_block(block_bytes(checkpoint_height)).hash(); + (e2e_network([checkpoint_height]), checkpoint_hash) + } + + fn header_sync_test_builder( + seed: u64, + network: Network, + trace: &mut TraceCapture, + ) -> super::super::ZakuraTestNodeBuilder { + let anchor = (block::Height(0), mainnet_genesis_hash()); + ZakuraTestNode::builder(seed) + .tracer(trace.tracer_for_node(seed)) + .header_sync_driver( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + }, + Some(anchor), + ) + } + + async fn drive_native_header_sync_actions( + node: &ZakuraTestNode, + disconnects: Arc>>, + ) -> JoinHandle<()> { + let endpoint = node.endpoint(); + let supervisor = node.supervisor(); + let mut actions = node + .take_header_sync_actions() + .await + .expect("header-sync action receiver is enabled"); + + tokio::spawn(async move { + while let Some(action) = actions.recv().await { + match action { + HeaderSyncAction::SendMessage { peer, msg } => { + endpoint.send_header_sync_message(&peer, msg).await; + } + HeaderSyncAction::ForwardNewBlock { peer, block, .. } => { + endpoint + .send_header_sync_message(&peer, HeaderSyncMessage::NewBlock(block)) + .await; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + disconnects + .lock() + .expect("disconnect list mutex is not poisoned") + .push(reason); + let _ = supervisor.disconnect_peer(&peer).await; + } + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + let Some(handle) = endpoint.header_sync() else { + continue; + }; + let _ = handle + .send(HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + .. + } => { + let Some(handle) = endpoint.header_sync() else { + continue; + }; + let count = u32::try_from(headers.len()).unwrap_or(u32::MAX); + let _ = handle + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind: HeaderSyncCommitFailureKind::Local, + }) + .await; + } + HeaderSyncAction::NewBlockReceived { + peer, height, hash, .. + } => { + let Some(handle) = endpoint.header_sync() else { + continue; + }; + let _ = handle + .send(HeaderSyncEvent::NewBlockDuplicate { peer, height, hash }) + .await; + } + HeaderSyncAction::QueryBestHeaderTip + | HeaderSyncAction::QueryMissingBlockBodies { .. } + | HeaderSyncAction::BodyGaps { .. } => {} + } + } + }) + } + + async fn wait_for_native_disconnect( + disconnects: Arc>>, + reason: HeaderSyncMisbehavior, + ) -> Result<(), BoxError> { + await_until( + "native header-sync disconnect reason", + Duration::from_secs(5), + || { + disconnects + .lock() + .expect("disconnect list mutex is not poisoned") + .contains(&reason) + }, + ) + .await + .map_err(Into::into) + } + #[tokio::test] #[ignore = "native handler mesh smoke is exercised by the zakura-integration nextest profile once dial scheduling is made deterministic"] async fn cluster_forms_native_two_node_mesh() -> Result<(), BoxError> { @@ -452,6 +1320,98 @@ mod tests { } } + #[tokio::test] + async fn recorder_transport_survives_malformed_header_sync_frame() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut cluster = ZakuraTestCluster::new(); + let victim_idx = cluster.spawn_node(5).await?; + let victim = cluster.node(victim_idx); + let recorder = victim.recorder(); + let hostile = HostilePeer::connect_native(victim, 6).await?; + + let before = b"before-header-sync-error".to_vec(); + let bad_header_sync_payload = vec![99]; + hostile + .send_frame(ZAKURA_STREAM_HEADER_SYNC, bad_header_sync_payload.clone()) + .await?; + hostile.send_frame(2, before.clone()).await?; + await_until("pre-error gossip delivered", Duration::from_secs(5), || { + recorder.contains_payload(2, &before) + }) + .await?; + + let after = b"after-header-sync-error".to_vec(); + hostile.send_frame(2, after.clone()).await?; + await_until( + "post-header-sync gossip delivered", + Duration::from_secs(5), + || recorder.contains_payload(2, &after), + ) + .await?; + + let delivered = recorder.drain(); + assert!( + delivered + .iter() + .any(|m| m.stream_kind == 2 && m.frame.payload == before), + "pre-error gossip frame must be delivered" + ); + assert!( + delivered + .iter() + .any(|m| m.stream_kind == ZAKURA_STREAM_HEADER_SYNC + && m.frame.payload == bad_header_sync_payload), + "recorder nodes assert transport routing only; production header-sync owners decode stream-5 frames and reject malformed payloads, got {delivered:?}" + ); + assert!( + delivered.iter().any(|m| m.frame.payload == after), + "generic transport must not close on header-sync payload decode, got {delivered:?}" + ); + + hostile.shutdown().await; + cluster.shutdown().await; + Ok(()) + } + + #[tokio::test] + async fn unnegotiated_header_sync_stream_is_rejected_before_delivery() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut cluster = ZakuraTestCluster::new(); + let victim_idx = cluster.spawn_node(7).await?; + let victim = cluster.node(victim_idx); + let recorder = victim.recorder(); + + let zero_cap_peer = HostilePeer::connect_native_with_capabilities(victim, 8, 0).await?; + let rejected_payload = b"unnegotiated-header-sync".to_vec(); + zero_cap_peer + .send_frame(ZAKURA_STREAM_HEADER_SYNC, rejected_payload.clone()) + .await?; + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + !recorder.contains_payload(ZAKURA_STREAM_HEADER_SYNC, &rejected_payload), + "stream 5 from a zero-capability peer must be rejected before delivery" + ); + + let header_cap_peer = + HostilePeer::connect_native_with_capabilities(victim, 9, ZAKURA_CAP_HEADER_SYNC) + .await?; + let admitted_payload = b"negotiated-header-sync".to_vec(); + header_cap_peer + .send_frame(ZAKURA_STREAM_HEADER_SYNC, admitted_payload.clone()) + .await?; + await_until( + "negotiated header-sync stream delivered", + Duration::from_secs(5), + || recorder.contains_payload(ZAKURA_STREAM_HEADER_SYNC, &admitted_payload), + ) + .await?; + + zero_cap_peer.shutdown().await; + header_cap_peer.shutdown().await; + cluster.shutdown().await; + Ok(()) + } + #[tokio::test] async fn discovery_stream_requires_negotiated_capability_and_responds() -> Result<(), BoxError> { @@ -939,4 +1899,936 @@ mod tests { victim.shutdown().await; Ok(()) } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn native_stream5_status_exchange_uses_handler_wire_path() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "native_stream5_status_exchange_uses_handler_wire_path", + false, + )?; + let network = e2e_network([1]); + let mut cluster = ZakuraTestCluster::new(); + let node1 = header_sync_test_builder(1, network.clone(), &mut capture) + .spawn() + .await?; + let node2 = header_sync_test_builder(2, network, &mut capture) + .spawn() + .await?; + let disconnects1 = Arc::new(StdMutex::new(Vec::new())); + let disconnects2 = Arc::new(StdMutex::new(Vec::new())); + let driver1 = drive_native_header_sync_actions(&node1, disconnects1.clone()).await; + let driver2 = drive_native_header_sync_actions(&node2, disconnects2.clone()).await; + cluster.nodes.push(node1); + cluster.nodes.push(node2); + + cluster.connect_full_mesh(Duration::from_secs(5)).await?; + cluster.await_all_connected(Duration::from_secs(5)).await?; + await_until( + "native stream-5 status received", + Duration::from_secs(5), + || { + capture.reader().is_ok_and(|reader| { + reader + .node("02") + .table("header_sync") + .count(hs_trace::HEADER_STATUS_RECEIVED) + >= 1 + }) + }, + ) + .await?; + + capture.flush().await; + let reader = capture.reader()?; + reader.node("01").table("stream").assert_row( + "accepted", + &[("stream_kind", TraceValue::Str("header_sync"))], + ); + reader.node("02").table("stream").assert_row( + "accepted", + &[("stream_kind", TraceValue::Str("header_sync"))], + ); + reader + .node("01") + .table("header_sync") + .assert_event(hs_trace::HEADER_STATUS_SENT); + reader + .node("02") + .table("header_sync") + .assert_event(hs_trace::HEADER_STATUS_RECEIVED); + assert!(disconnects1.lock().unwrap().is_empty()); + assert!(disconnects2.lock().unwrap().is_empty()); + + cluster.shutdown().await; + driver1.abort(); + driver2.abort(); + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn native_stream5_hostile_bytes_disconnect_with_traceable_reasons() -> Result<(), BoxError> + { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "native_stream5_hostile_bytes_disconnect_with_traceable_reasons", + false, + )?; + let victim = header_sync_test_builder(11, e2e_network([1]), &mut capture) + .spawn() + .await?; + let disconnects = Arc::new(StdMutex::new(Vec::new())); + let driver = drive_native_header_sync_actions(&victim, disconnects.clone()).await; + + let malformed = + HostilePeer::connect_native_with_capabilities(&victim, 12, ZAKURA_CAP_HEADER_SYNC) + .await?; + malformed + .send_raw_frame( + ZAKURA_STREAM_HEADER_SYNC, + Frame { + message_type: 99, + flags: 0, + payload: Vec::new(), + }, + ) + .await?; + wait_for_native_disconnect(disconnects.clone(), HeaderSyncMisbehavior::MalformedMessage) + .await?; + malformed.shutdown().await; + + let unsolicited = + HostilePeer::connect_native_with_capabilities(&victim, 13, ZAKURA_CAP_HEADER_SYNC) + .await?; + let unsolicited_headers = + HeaderSyncMessage::Headers(vec![mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone()]) + .encode_frame()?; + unsolicited + .send_raw_frame(ZAKURA_STREAM_HEADER_SYNC, unsolicited_headers) + .await?; + wait_for_native_disconnect( + disconnects.clone(), + HeaderSyncMisbehavior::UnsolicitedHeaders, + ) + .await?; + unsolicited.shutdown().await; + + let oversized = + HostilePeer::connect_native_with_capabilities(&victim, 14, ZAKURA_CAP_HEADER_SYNC) + .await?; + let oversized_peer = oversized.id()?; + let peer_set = victim.supervisor().subscribe(); + await_until("oversized peer registered", Duration::from_secs(5), || { + peer_set.borrow().contains(&oversized_peer) + }) + .await?; + oversized + .oversize_frame_declared_len(ZAKURA_STREAM_HEADER_SYNC) + .await?; + await_until( + "native stream-5 oversize trace", + Duration::from_secs(5), + || { + capture.reader().is_ok_and(|reader| { + reader + .node("11") + .table("ratelimit") + .rows() + .iter() + .any(|row| { + row.get("event").and_then(serde_json::Value::as_str) + == Some("frame.oversize") + && row.get("stream_kind").and_then(serde_json::Value::as_str) + == Some("header_sync") + }) + }) + }, + ) + .await?; + await_until( + "oversized persistent stream disconnects peer", + Duration::from_secs(5), + || !peer_set.borrow().contains(&oversized_peer), + ) + .await?; + oversized.shutdown().await; + + let truncated = + HostilePeer::connect_native_with_capabilities(&victim, 15, ZAKURA_CAP_HEADER_SYNC) + .await?; + truncated + .send_truncated_frame(ZAKURA_STREAM_HEADER_SYNC) + .await?; + truncated.shutdown().await; + + capture.flush().await; + let reader = capture.reader()?; + let header_sync = reader.node("11").table("header_sync"); + header_sync.assert_header_disconnect("malformed_message"); + header_sync.assert_header_disconnect("unsolicited_headers"); + reader.node("11").table("stream").assert_row( + "accepted", + &[("stream_kind", TraceValue::Str("header_sync"))], + ); + reader.node("11").table("ratelimit").assert_row( + "frame.oversize", + &[("stream_kind", TraceValue::Str("header_sync"))], + ); + + victim.shutdown().await; + driver.abort(); + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn header_sync_e2e_status_trace_smoke_uses_real_emitted_rows() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "header_sync_e2e_status_trace_smoke_uses_real_emitted_rows", + false, + )?; + let mut cluster = HeaderSyncE2eCluster::new(); + let network = e2e_network([4]); + let anchor = (block::Height(0), mainnet_genesis_hash()); + + cluster.spawn_node( + 1, + network.clone(), + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(1), "01"), + )?; + cluster.spawn_node( + 2, + network, + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(2), "02"), + )?; + cluster.start_drivers(); + cluster.connect_all().await; + + tokio::time::sleep(Duration::from_millis(300)).await; + capture.flush().await; + let reader = capture.reader()?; + reader + .node("01") + .table("header_sync") + .assert_event(hs_trace::HEADER_STATUS_SENT); + reader + .node("02") + .table("header_sync") + .assert_event(hs_trace::HEADER_STATUS_RECEIVED); + + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn header_sync_e2e_genesis_converges_and_body_gap_api_shrinks() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "header_sync_e2e_genesis_converges_and_body_gap_api_shrinks", + false, + )?; + let mut cluster = HeaderSyncE2eCluster::new(); + let network = e2e_network([4]); + let anchor = (block::Height(0), mainnet_genesis_hash()); + let source = cluster.spawn_node( + 1, + network.clone(), + anchor, + E2eHeaderStore::with_headers(4), + ZakuraTrace::new(capture.tracer_for_node(1), "01"), + )?; + let empty = cluster.spawn_node( + 2, + network, + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(2), "02"), + )?; + assert_eq!(source, 0); + cluster.start_drivers(); + cluster.connect_all().await; + cluster.wait_for_tip(empty, block::Height(4)).await?; + + let missing = cluster.missing_bodies(empty).await; + assert_eq!( + missing, + vec![ + block::Height(1), + block::Height(2), + block::Height(3), + block::Height(4) + ] + ); + assert!( + !cluster.observed_gaps(empty).await.is_empty(), + "body-gap watch/API surface must be observable without header-sync body commands" + ); + + for height in 1..=4 { + cluster + .commit_body(empty, mainnet_block(block_bytes(height))) + .await; + } + await_until("missing body gap shrinks", Duration::from_secs(5), || { + cluster + .nodes + .get(empty) + .expect("node exists") + .view + .store + .lock() + .expect("test store mutex is not poisoned") + .missing_bodies(block::Height(1), 100) + .is_empty() + }) + .await?; + + capture.flush().await; + let reader = capture.reader()?; + let target_trace = reader.node("02").table("header_sync"); + target_trace.assert_event(hs_trace::HEADER_STATUS_RECEIVED); + target_trace.assert_header_range_request(1, 4); + target_trace.assert_header_range_response(1, 4); + target_trace.assert_header_range_commit(1, 4); + target_trace.assert_row( + hs_trace::HEADER_MISSING_BODIES_REPORTED, + &[ + (hs_trace::RANGE_START, TraceValue::U64(1)), + (hs_trace::RANGE_COUNT, TraceValue::U64(4)), + ], + ); + + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn header_sync_e2e_checkpoint_forward_then_backward_finalizes_backfill( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "header_sync_e2e_checkpoint_forward_then_backward_finalizes_backfill", + false, + )?; + let (network, checkpoint_hash) = checkpoint_network(3); + let mut cluster = HeaderSyncE2eCluster::new(); + let source = cluster.spawn_node( + 1, + network.clone(), + (block::Height(0), mainnet_genesis_hash()), + E2eHeaderStore::with_headers(4), + ZakuraTrace::new(capture.tracer_for_node(1), "01"), + )?; + let checkpointed = cluster.spawn_node( + 2, + network, + (block::Height(3), checkpoint_hash), + E2eHeaderStore::with_checkpoint_anchor(3), + ZakuraTrace::new(capture.tracer_for_node(2), "02"), + )?; + assert_eq!(source, 0); + cluster.start_drivers(); + cluster.connect_all().await; + cluster.wait_for_tip(checkpointed, block::Height(4)).await?; + await_until( + "checkpoint backfill finalized", + Duration::from_secs(5), + || { + cluster + .nodes + .get(checkpointed) + .expect("node exists") + .view + .store + .lock() + .expect("test store mutex is not poisoned") + .finalized_height + >= block::Height(3) + }, + ) + .await?; + + capture.flush().await; + let reader = capture.reader()?; + let target_trace = reader.node("02").table("header_sync"); + target_trace.assert_header_range_request(4, 1); + target_trace.assert_header_range_request(1, 3); + target_trace.assert_header_range_commit(1, 3); + assert_eq!( + cluster.finalized_height(checkpointed).await, + block::Height(3) + ); + + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn header_sync_e2e_tip_flood_and_no_double_gossip_cover_both_orderings( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "header_sync_e2e_tip_flood_and_no_double_gossip_cover_both_orderings", + false, + )?; + let mut cluster = HeaderSyncE2eCluster::new(); + let network = e2e_network([]); + let anchor = (block::Height(0), mainnet_genesis_hash()); + for seed in 1..=3 { + cluster.spawn_node( + seed, + network.clone(), + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new( + capture.tracer_for_node(u64::from(seed)), + format!("{seed:02}"), + ), + )?; + } + cluster.start_drivers(); + cluster.connect_all().await; + + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let hash1 = block1.hash(); + cluster + .inject( + 0, + cluster.nodes[1].view.peer_id.clone(), + HeaderSyncMessage::NewBlock(block1.clone()), + ) + .await; + cluster.wait_for_body(0, hash1).await?; + cluster.wait_for_body(2, hash1).await?; + + cluster.commit_body(0, block1.clone()).await; + cluster + .inject( + 0, + cluster.nodes[1].view.peer_id.clone(), + HeaderSyncMessage::NewBlock(block1), + ) + .await; + + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + cluster.disconnect_reasons(0).await.is_empty(), + "honest tip-flood and duplicate paths must not disconnect peers" + ); + + capture.flush().await; + let reader = capture.reader()?; + let node1_trace = reader.node("01").table("header_sync"); + node1_trace.assert_event(hs_trace::HEADER_NEW_BLOCK_RECEIVED); + node1_trace.assert_event(hs_trace::HEADER_NEW_BLOCK_FORWARDED); + node1_trace.assert_header_new_block_deduped("seen_cache"); + assert_eq!( + node1_trace.count(hs_trace::HEADER_GET_HEADERS_SENT), + 0, + "tip full-block flood must not use header advertise/body-pull" + ); + + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn header_sync_e2e_concurrent_duplicate_new_block_does_not_disconnect( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "header_sync_e2e_concurrent_duplicate_new_block_does_not_disconnect", + false, + )?; + let mut cluster = HeaderSyncE2eCluster::new(); + let network = e2e_network([]); + let anchor = (block::Height(0), mainnet_genesis_hash()); + for seed in 1..=3 { + cluster.spawn_node( + seed, + network.clone(), + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new( + capture.tracer_for_node(u64::from(seed)), + format!("{seed:02}"), + ), + )?; + } + cluster.connect_all().await; + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let hash1 = block1.hash(); + cluster + .inject( + 0, + cluster.nodes[1].view.peer_id.clone(), + HeaderSyncMessage::NewBlock(block1.clone()), + ) + .await; + cluster + .inject( + 0, + cluster.nodes[2].view.peer_id.clone(), + HeaderSyncMessage::NewBlock(block1), + ) + .await; + cluster.start_drivers(); + cluster.wait_for_body(0, hash1).await?; + tokio::time::sleep(Duration::from_millis(300)).await; + assert!(cluster.disconnect_reasons(0).await.is_empty()); + + capture.flush().await; + capture + .reader()? + .node("01") + .table("header_sync") + .assert_header_new_block_deduped("pending_acceptance"); + + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn header_sync_e2e_hostile_peer_disconnect_matrix_has_traceable_reasons( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "header_sync_e2e_hostile_peer_disconnect_matrix_has_traceable_reasons", + false, + )?; + let mut cluster = HeaderSyncE2eCluster::new(); + let network = e2e_network([]); + let anchor = (block::Height(0), mainnet_genesis_hash()); + let victim = cluster.spawn_node( + 1, + network.clone(), + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(1), "01"), + )?; + cluster.start_drivers(); + + let unsolicited = e2e_peer(90); + cluster.connect_peer(victim, unsolicited.clone()).await; + cluster + .inject( + victim, + unsolicited, + HeaderSyncMessage::Headers(vec![mainnet_block(&BLOCK_MAINNET_1_BYTES) + .header + .clone()]), + ) + .await; + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::UnsolicitedHeaders) + .await?; + + let out_of_range = e2e_peer(95); + cluster.connect_peer(victim, out_of_range.clone()).await; + cluster + .inject(victim, out_of_range.clone(), status_for_tip(4, 4, 1)) + .await; + cluster + .wait_for_get_headers(victim, &out_of_range, block::Height(1), 4) + .await?; + cluster + .inject( + victim, + out_of_range, + HeaderSyncMessage::Headers(vec![mainnet_block(&BLOCK_MAINNET_2_BYTES) + .header + .clone()]), + ) + .await; + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::InvalidRange) + .await?; + + let over_in_flight = e2e_peer(96); + cluster.connect_peer(victim, over_in_flight.clone()).await; + cluster + .inject(victim, over_in_flight.clone(), status_for_tip(0, 4, 1)) + .await; + for start in 1..=17 { + cluster + .inject( + victim, + over_in_flight.clone(), + HeaderSyncMessage::GetHeaders { + start_height: block::Height(start), + count: 1, + }, + ) + .await; + } + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::GetHeadersSpam) + .await?; + + let response_too_long = e2e_peer(97); + cluster + .connect_peer(victim, response_too_long.clone()) + .await; + cluster + .inject(victim, response_too_long.clone(), status_for_tip(4, 1, 1)) + .await; + cluster + .wait_for_get_headers(victim, &response_too_long, block::Height(1), 1) + .await?; + cluster + .inject( + victim, + response_too_long, + HeaderSyncMessage::Headers(vec![ + mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), + mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone(), + ]), + ) + .await; + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::ResponseTooLong) + .await?; + + let bad_continuity_victim = cluster.spawn_node( + 3, + network.clone(), + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(3), "03"), + )?; + cluster.start_drivers(); + let bad_continuity = e2e_peer(98); + cluster + .connect_peer(bad_continuity_victim, bad_continuity.clone()) + .await; + cluster + .inject( + bad_continuity_victim, + bad_continuity.clone(), + status_for_tip(4, 4, 1), + ) + .await; + cluster + .wait_for_get_headers(bad_continuity_victim, &bad_continuity, block::Height(1), 4) + .await?; + let mut non_contiguous = *mainnet_block(&BLOCK_MAINNET_2_BYTES).header; + non_contiguous.previous_block_hash = block::Hash([7; 32]); + cluster + .inject( + bad_continuity_victim, + bad_continuity, + HeaderSyncMessage::Headers(vec![ + mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), + Arc::new(non_contiguous), + ]), + ) + .await; + cluster + .wait_for_disconnect_reason(bad_continuity_victim, HeaderSyncMisbehavior::InvalidRange) + .await?; + + let bad_pow_victim = cluster.spawn_node( + 4, + network.clone(), + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(4), "04"), + )?; + cluster.start_drivers(); + let bad_pow = e2e_peer(99); + cluster.connect_peer(bad_pow_victim, bad_pow.clone()).await; + cluster + .inject(bad_pow_victim, bad_pow.clone(), status_for_tip(4, 4, 1)) + .await; + cluster + .wait_for_get_headers(bad_pow_victim, &bad_pow, block::Height(1), 4) + .await?; + let mut bad_pow_header = *mainnet_block(&BLOCK_MAINNET_1_BYTES).header; + bad_pow_header.nonce = [1; 32].into(); + cluster + .inject( + bad_pow_victim, + bad_pow, + HeaderSyncMessage::Headers(vec![Arc::new(bad_pow_header)]), + ) + .await; + cluster + .wait_for_disconnect_reason(bad_pow_victim, HeaderSyncMisbehavior::InvalidRange) + .await?; + + let bad_daa_victim = cluster.spawn_node( + 5, + network, + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(5), "05"), + )?; + cluster.start_drivers(); + let bad_daa = e2e_peer(100); + cluster.connect_peer(bad_daa_victim, bad_daa.clone()).await; + cluster + .inject(bad_daa_victim, bad_daa.clone(), status_for_tip(4, 4, 1)) + .await; + cluster + .wait_for_get_headers(bad_daa_victim, &bad_daa, block::Height(1), 4) + .await?; + cluster + .reject_next_commit( + bad_daa_victim, + HeaderSyncCommitFailureKind::InvalidPeerRange, + ) + .await; + cluster + .inject( + bad_daa_victim, + bad_daa, + HeaderSyncMessage::Headers(vec![ + mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), + mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone(), + mainnet_block(&BLOCK_MAINNET_3_BYTES).header.clone(), + mainnet_block(&BLOCK_MAINNET_4_BYTES).header.clone(), + ]), + ) + .await; + cluster + .wait_for_disconnect_reason(bad_daa_victim, HeaderSyncMisbehavior::InvalidRange) + .await?; + + let bad_checkpoint_backfill = e2e_peer(101); + let checkpoint_hash = mainnet_block(&BLOCK_MAINNET_1_BYTES).hash(); + let checkpoint_network = e2e_network_with_checkpoint_hash(3, checkpoint_hash); + let checkpointed = cluster.spawn_node( + 6, + checkpoint_network, + (block::Height(3), checkpoint_hash), + E2eHeaderStore::with_checkpoint_anchor(3), + ZakuraTrace::new(capture.tracer_for_node(6), "06"), + )?; + cluster.start_drivers(); + cluster + .connect_peer(checkpointed, bad_checkpoint_backfill.clone()) + .await; + cluster + .inject( + checkpointed, + bad_checkpoint_backfill.clone(), + status_for_tip(3, 4, 1), + ) + .await; + cluster + .wait_for_get_headers(checkpointed, &bad_checkpoint_backfill, block::Height(1), 3) + .await?; + cluster + .inject( + checkpointed, + bad_checkpoint_backfill, + HeaderSyncMessage::Headers(vec![ + mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), + mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone(), + mainnet_block(&BLOCK_MAINNET_3_BYTES).header.clone(), + ]), + ) + .await; + cluster + .wait_for_disconnect_reason(checkpointed, HeaderSyncMisbehavior::InvalidRange) + .await?; + + let over_cap = e2e_peer(91); + cluster.connect_peer(victim, over_cap.clone()).await; + cluster + .inject( + victim, + over_cap.clone(), + HeaderSyncMessage::Status(Default::default()), + ) + .await; + cluster + .inject( + victim, + over_cap, + HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 4_001, + }, + ) + .await; + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::GetHeadersTooLong) + .await?; + + let status_spam = e2e_peer(92); + cluster.connect_peer(victim, status_spam.clone()).await; + cluster + .inject( + victim, + status_spam.clone(), + HeaderSyncMessage::Status(Default::default()), + ) + .await; + cluster + .inject( + victim, + status_spam, + HeaderSyncMessage::Status(Default::default()), + ) + .await; + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::StatusSpam) + .await?; + + let new_block_spam = e2e_peer(93); + cluster.connect_peer(victim, new_block_spam.clone()).await; + cluster + .inject( + victim, + new_block_spam.clone(), + HeaderSyncMessage::NewBlock(mainnet_block(&BLOCK_MAINNET_1_BYTES)), + ) + .await; + cluster + .inject( + victim, + new_block_spam, + HeaderSyncMessage::NewBlock(mainnet_block(&BLOCK_MAINNET_2_BYTES)), + ) + .await; + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::NewBlockSpam) + .await?; + + let invalid_block = e2e_peer(94); + cluster.connect_peer(victim, invalid_block.clone()).await; + let mut bad_block = (*mainnet_block(&BLOCK_MAINNET_1_BYTES)).clone(); + bad_block.transactions.clear(); + cluster + .inject( + victim, + invalid_block, + HeaderSyncMessage::NewBlock(Arc::new(bad_block)), + ) + .await; + cluster + .wait_for_disconnect_reason(victim, HeaderSyncMisbehavior::MalformedMessage) + .await?; + + capture.flush().await; + let reader = capture.reader()?; + let trace = reader.node("01").table("header_sync"); + trace.assert_header_disconnect("unsolicited_headers"); + trace.assert_header_disconnect("invalid_range"); + trace.assert_header_disconnect("get_headers_spam"); + trace.assert_header_disconnect("response_too_long"); + trace.assert_header_disconnect("get_headers_too_long"); + trace.assert_header_disconnect("status_spam"); + trace.assert_header_disconnect("new_block_spam"); + trace.assert_header_disconnect("malformed_message"); + for node in ["03", "04", "05", "06"] { + reader + .node(node) + .table("header_sync") + .assert_header_disconnect("invalid_range"); + } + + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn header_sync_e2e_restart_reloads_durable_tip_and_rebuilds_scheduler( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "header_sync_e2e_restart_reloads_durable_tip_and_rebuilds_scheduler", + false, + )?; + let network = e2e_network([4]); + let anchor = (block::Height(0), mainnet_genesis_hash()); + + let mut first = HeaderSyncE2eCluster::new(); + let seeded = first.spawn_node( + 1, + network.clone(), + anchor, + E2eHeaderStore::with_headers(4), + ZakuraTrace::new(capture.tracer_for_node(1), "01"), + )?; + let syncing = first.spawn_node( + 2, + network.clone(), + anchor, + E2eHeaderStore::genesis_only(), + ZakuraTrace::new(capture.tracer_for_node(2), "02"), + )?; + first.start_drivers(); + first.connect_all().await; + first.wait_for_tip(syncing, block::Height(4)).await?; + let durable_store = first.nodes[syncing].view.store.clone(); + first.shutdown().await; + + let restart_store = { + let store = durable_store + .lock() + .expect("test store mutex is not poisoned"); + E2eHeaderStore { + headers: store.headers.clone(), + bodies: store.bodies.clone(), + finalized_height: store.finalized_height, + verified_block_tip: store.verified_block_tip, + reject_next_commit: None, + } + }; + + let mut restarted = HeaderSyncE2eCluster::new(); + restarted.spawn_node( + 1, + network.clone(), + anchor, + E2eHeaderStore::with_headers(5), + ZakuraTrace::new(capture.tracer_for_node(3), "03"), + )?; + let restarted_idx = restarted.spawn_node( + 2, + network, + anchor, + restart_store, + ZakuraTrace::new(capture.tracer_for_node(4), "04"), + )?; + restarted.start_drivers(); + restarted.connect_all().await; + restarted + .wait_for_tip(restarted_idx, block::Height(5)) + .await?; + + capture.flush().await; + let reader = capture.reader()?; + reader + .node("04") + .table("header_sync") + .assert_header_range_request(5, 1); + assert_eq!( + restarted.nodes[restarted_idx] + .view + .handle + .best_header_tip() + .0, + block::Height(5) + ); + assert_eq!(seeded, 0); + + restarted.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } } diff --git a/zebra-network/src/zakura/testkit/hostile.rs b/zebra-network/src/zakura/testkit/hostile.rs index cc9be82056c..ff71516f4e4 100644 --- a/zebra-network/src/zakura/testkit/hostile.rs +++ b/zebra-network/src/zakura/testkit/hostile.rs @@ -11,8 +11,8 @@ use crate::{ zakura::{ legacy_gossip::ZAKURA_STREAM_GOSSIP, run_native_initiator_handshake, Frame, StreamPrelude, ZakuraHandshakeConfig, ZakuraLocalLimits, ZakuraPeerId, FRAME_HEADER_BYTES, P2P_V2_ALPN, - STREAM_PRELUDE_MAGIC, ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_LEGACY_GOSSIP, - ZAKURA_STREAM_DISCOVERY, + STREAM_PRELUDE_MAGIC, ZAKURA_CAP_HEADER_SYNC, ZAKURA_CAP_LEGACY_GOSSIP, + ZAKURA_STREAM_DISCOVERY, ZAKURA_STREAM_HEADER_SYNC, }, BoxError, Config, }; @@ -33,7 +33,7 @@ impl HostilePeer { Self::connect_native_with_capabilities( victim, seed, - ZAKURA_CAP_LEGACY_GOSSIP | ZAKURA_CAP_DISCOVERY, + ZAKURA_CAP_LEGACY_GOSSIP | ZAKURA_CAP_HEADER_SYNC, ) .await } @@ -87,7 +87,10 @@ impl HostilePeer { /// Open one stream and send a valid prelude followed by `frame`. pub async fn send_raw_frame(&self, stream_kind: u16, frame: Frame) -> Result<(), BoxError> { - if matches!(stream_kind, ZAKURA_STREAM_GOSSIP | ZAKURA_STREAM_DISCOVERY) { + if matches!( + stream_kind, + ZAKURA_STREAM_GOSSIP | ZAKURA_STREAM_DISCOVERY | ZAKURA_STREAM_HEADER_SYNC + ) { return self.send_ordered_raw_frame(stream_kind, frame).await; } diff --git a/zebra-network/src/zakura/testkit/node.rs b/zebra-network/src/zakura/testkit/node.rs index 5a3ea8a93f5..e0df3eb3fa7 100644 --- a/zebra-network/src/zakura/testkit/node.rs +++ b/zebra-network/src/zakura/testkit/node.rs @@ -3,19 +3,25 @@ use std::{fmt, net::SocketAddr, sync::Arc, time::Duration}; use iroh::{endpoint::TransportConfig, protocol::Router, NodeAddr, NodeId}; -use tokio::{sync::Mutex, task::JoinHandle}; +use tokio::{ + sync::{mpsc, Mutex}, + task::JoinHandle, +}; +use tokio_util::sync::CancellationToken; use zebra_jsonl_trace::JsonlTracer; use super::{InboundRecorder, LocalEndpointFactory, WaitError}; use crate::{ zakura::{ - discovery::build_discovery_handle, service_registry, DiscoveryService, Service, - ZakuraDiscoveryHandle, ZakuraEndpoint, ZakuraHandshakeConfig, ZakuraLocalLimits, - ZakuraPeerId, ZakuraProtocolHandler, ZakuraServiceId, ZakuraSupervisorHandle, ZakuraTrace, - P2P_V2_ALPN, + discovery::build_discovery_handle, service_registry, spawn_header_sync_reactor, + DiscoveryService, HeaderSyncAction, HeaderSyncFrontiers, HeaderSyncHandle, + HeaderSyncStartup, Service, ZakuraDiscoveryHandle, ZakuraEndpoint, ZakuraHandshakeConfig, + ZakuraHeaderSyncConfig, ZakuraLocalLimits, ZakuraPeerId, ZakuraProtocolHandler, + ZakuraServiceId, ZakuraSupervisorHandle, ZakuraTrace, P2P_V2_ALPN, }, BoxError, Config, }; +use zebra_chain::{block, parameters::Network}; /// A running in-process Zakura node for integration tests. #[derive(Debug)] @@ -50,6 +56,32 @@ impl ZakuraTestNode { self.endpoint.supervisor() } + /// Clone the underlying endpoint for test-only external drivers. + pub(crate) fn endpoint(&self) -> ZakuraEndpoint { + self.endpoint.clone() + } + + /// Active header-sync handle, if this test node was spawned with stream-5 + /// header sync enabled. + pub fn header_sync(&self) -> Option { + self.endpoint.header_sync() + } + + /// Take the stream-5 header-sync action receiver for an externally driven + /// test node. + pub async fn take_header_sync_actions(&self) -> Option> { + self.endpoint.take_header_sync_actions().await + } + + /// Send one header-sync message through the production stream-5 sender. + pub async fn send_header_sync_message( + &self, + peer: &ZakuraPeerId, + msg: crate::zakura::HeaderSyncMessage, + ) { + self.endpoint.send_header_sync_message(peer, msg).await; + } + /// Local limits used by this node. pub fn limits(&self) -> &ZakuraLocalLimits { &self.limits @@ -172,6 +204,15 @@ pub struct ZakuraTestNodeBuilder { service_factory: Option Arc + Send>>, discovery_direct_addrs: Vec, extra_advertised_services: Vec, + header_sync: Option, +} + +#[derive(Clone, Debug)] +struct TestHeaderSyncStartup { + network: Network, + anchor: (block::Height, block::Hash), + frontiers: HeaderSyncFrontiers, + best_header_tip: Option<(block::Height, block::Hash)>, } impl fmt::Debug for ZakuraTestNodeBuilder { @@ -186,6 +227,7 @@ impl fmt::Debug for ZakuraTestNodeBuilder { "service", &(self.service.is_some() || self.service_factory.is_some()), ) + .field("header_sync", &self.header_sync.is_some()) .finish() } } @@ -208,6 +250,7 @@ impl ZakuraTestNodeBuilder { service_factory: None, discovery_direct_addrs: Vec::new(), extra_advertised_services: Vec::new(), + header_sync: None, } } @@ -264,6 +307,24 @@ impl ZakuraTestNodeBuilder { self } + /// Enable the production stream-5 header-sync adapter on this test node and + /// expose its action receiver for an external test driver. + pub fn header_sync_driver( + mut self, + network: Network, + anchor: (block::Height, block::Hash), + frontiers: HeaderSyncFrontiers, + best_header_tip: Option<(block::Height, block::Hash)>, + ) -> Self { + self.header_sync = Some(TestHeaderSyncStartup { + network, + anchor, + frontiers, + best_header_tip, + }); + self + } + /// Spawn the node. pub async fn spawn(self) -> Result { if self.legacy_upgrade { @@ -301,7 +362,37 @@ impl ZakuraTestNodeBuilder { )?; let discovery_service = Arc::new(DiscoveryService::new(discovery.clone())) as Arc; - let registry = service_registry(&supervisor, base_service, discovery_service)?; + + let mut header_sync_handle = None; + let mut header_sync_actions = None; + let mut header_sync_tasks = Vec::new(); + let header_sync = if let Some(header_sync) = self.header_sync { + let mut startup = HeaderSyncStartup::new( + header_sync.network, + header_sync.anchor, + header_sync.frontiers, + header_sync.best_header_tip, + ZakuraHeaderSyncConfig::default(), + self.limits.max_frame_bytes, + ); + startup.range_state_actions_enabled = true; + startup.inbound_new_block_acceptance_enabled = true; + startup.status_refresh_interval = Duration::from_millis(200); + let shutdown = CancellationToken::new(); + startup.shutdown = shutdown.clone(); + startup.trace = ZakuraTrace::new(self.tracer.clone(), seed_label(self.seed)); + + let (handle, actions, task) = spawn_header_sync_reactor(startup)?; + header_sync_tasks.push(task); + header_sync_actions = Some((shutdown, actions)); + header_sync_handle = Some(handle.clone()); + Some(handle) + } else { + // Recorder-only nodes use the stream-5 passthrough so tests can + // inspect header-sync frames without spawning the reactor. + None + }; + let registry = service_registry(&supervisor, header_sync, base_service, discovery_service)?; let handler = ZakuraProtocolHandler::new_with_registry_and_trace( supervisor.clone(), network.clone(), @@ -313,7 +404,21 @@ impl ZakuraTestNodeBuilder { let router = Router::builder(endpoint) .accept(P2P_V2_ALPN, handler.clone()) .spawn(); - let endpoint = ZakuraEndpoint::from_parts(router, supervisor, handler); + let endpoint = if let (Some(handle), Some((shutdown, actions))) = + (header_sync_handle, header_sync_actions) + { + ZakuraEndpoint::from_parts_with_header_sync( + router, + supervisor, + handler, + handle, + shutdown, + header_sync_tasks, + Some(actions), + ) + } else { + ZakuraEndpoint::from_parts(router, supervisor, handler) + }; Ok(ZakuraTestNode { seed: self.seed, diff --git a/zebra-network/src/zakura/testkit/trace_reader.rs b/zebra-network/src/zakura/testkit/trace_reader.rs index 347db1393a8..985bdbaa072 100644 --- a/zebra-network/src/zakura/testkit/trace_reader.rs +++ b/zebra-network/src/zakura/testkit/trace_reader.rs @@ -8,6 +8,8 @@ use std::{ use serde_json::Value; +use crate::zakura::trace::header_sync_trace as hs_trace; + /// Loaded Zakura trace tables. #[derive(Clone, Debug, Default)] pub struct TraceReader { @@ -219,6 +221,65 @@ impl<'a> TraceQuery<'a> { self.rows() ); } + + /// Assert a `header_get_headers_sent` range request row. + pub fn assert_header_range_request(&self, start_height: u32, count: u32) { + self.assert_header_range(hs_trace::HEADER_GET_HEADERS_SENT, start_height, count); + } + + /// Assert a `header_headers_received` response row. + pub fn assert_header_range_response(&self, start_height: u32, count: u32) { + self.assert_header_range(hs_trace::HEADER_HEADERS_RECEIVED, start_height, count); + } + + /// Assert a committed header range row. + pub fn assert_header_range_commit(&self, start_height: u32, count: u32) { + self.assert_header_range(hs_trace::HEADER_RANGE_COMMITTED, start_height, count); + } + + /// Assert a rejected header range row with a bounded reason label. + pub fn assert_header_range_rejected(&self, start_height: u32, count: u32, reason: &str) { + self.assert_row( + hs_trace::HEADER_RANGE_REJECTED, + &[ + ( + hs_trace::RANGE_START, + TraceValue::U64(u64::from(start_height)), + ), + (hs_trace::RANGE_COUNT, TraceValue::U64(u64::from(count))), + (hs_trace::REASON, TraceValue::Str(reason)), + ], + ); + } + + /// Assert a `NewBlock` dedup row with its bounded reason label. + pub fn assert_header_new_block_deduped(&self, reason: &str) { + self.assert_row( + hs_trace::HEADER_NEW_BLOCK_DEDUPED, + &[(hs_trace::REASON, TraceValue::Str(reason))], + ); + } + + /// Assert a requested disconnect row with its bounded reason label. + pub fn assert_header_disconnect(&self, reason: &str) { + self.assert_row( + hs_trace::HEADER_PEER_DISCONNECT_REQUESTED, + &[(hs_trace::REASON, TraceValue::Str(reason))], + ); + } + + fn assert_header_range(&self, event: &str, start_height: u32, count: u32) { + self.assert_row( + event, + &[ + ( + hs_trace::RANGE_START, + TraceValue::U64(u64::from(start_height)), + ), + (hs_trace::RANGE_COUNT, TraceValue::U64(u64::from(count))), + ], + ); + } } fn trace_value_matches(actual: Option<&Value>, expected: TraceValue<'_>) -> bool { @@ -326,4 +387,39 @@ mod tests { assert_eq!(events, ["from-a", "from-b"]); } + + #[test] + fn reader_asserts_header_sync_rows_without_ordering() { + let dir = tempfile::tempdir().expect("tempdir"); + let node_dir = dir.path().join("node-01"); + fs::create_dir_all(&node_dir).expect("node dir"); + fs::write( + node_dir.join("header_sync.jsonl"), + r#"{"node":"01","event":"header_peer_disconnect_requested","reason":"invalid_range"}"# + .to_string() + + "\n" + + r#"{"node":"01","event":"header_new_block_deduped","reason":"seen_cache"}"# + + "\n" + + r#"{"node":"01","event":"header_range_committed","range_start":4,"range_count":2,"reason":null}"# + + "\n" + + r#"{"node":"01","event":"header_headers_received","range_start":4,"range_count":2}"# + + "\n" + + r#"{"node":"01","event":"header_get_headers_sent","range_start":4,"range_count":2}"# + + "\n", + ) + .expect("trace file"); + + let reader = TraceReader::load(dir.path()).expect("reader"); + let header_sync = reader.node("01").table("header_sync"); + + header_sync.assert_header_range_request(4, 2); + header_sync.assert_header_range_response(4, 2); + header_sync.assert_header_range_commit(4, 2); + header_sync.assert_header_new_block_deduped("seen_cache"); + header_sync.assert_header_disconnect("invalid_range"); + header_sync.assert_row( + hs_trace::HEADER_RANGE_COMMITTED, + &[(hs_trace::REASON, TraceValue::Null)], + ); + } } diff --git a/zebra-network/src/zakura/trace.rs b/zebra-network/src/zakura/trace.rs index ddfda5f44d8..83933ecb85a 100644 --- a/zebra-network/src/zakura/trace.rs +++ b/zebra-network/src/zakura/trace.rs @@ -60,12 +60,75 @@ pub const RATELIMIT_TABLE: ZakuraTraceTable = ZakuraTraceTable { file_name: "ratelimit.jsonl", }; +/// Header-sync policy, accounting, and frontier events. +pub const HEADER_SYNC_TABLE: ZakuraTraceTable = ZakuraTraceTable { + table: "header_sync", + file_name: "header_sync.jsonl", +}; + /// Legacy compatibility request/response events. pub const LEGACY_REQUEST_TABLE: ZakuraTraceTable = ZakuraTraceTable { table: "legacy_request", file_name: "legacy_request.jsonl", }; +/// Shared header-sync trace event names and field keys. +pub mod header_sync_trace { + /// Trace row event field. + pub const EVENT: &str = "event"; + /// Peer field. + pub const PEER: &str = "peer"; + /// Source peer field for forwarded full-block floods. + pub const SOURCE_PEER: &str = "source_peer"; + /// Height field. + pub const HEIGHT: &str = "height"; + /// Hash field. + pub const HASH: &str = "hash"; + /// Range start height field. + pub const RANGE_START: &str = "range_start"; + /// Range count field. + pub const RANGE_COUNT: &str = "range_count"; + /// Advertised peer range cap field. + pub const ADVERTISED_CAP: &str = "advertised_cap"; + /// Expected header count field. + pub const EXPECTED_COUNT: &str = "expected_count"; + /// In-flight request count field. + pub const IN_FLIGHT_COUNT: &str = "in_flight_count"; + /// Destination peer count field. + pub const DESTINATION_PEER_COUNT: &str = "destination_peer_count"; + /// Bounded reason field. + pub const REASON: &str = "reason"; + + /// Local status sent to a peer. + pub const HEADER_STATUS_SENT: &str = "header_status_sent"; + /// Peer status received. + pub const HEADER_STATUS_RECEIVED: &str = "header_status_received"; + /// Header range request sent. + pub const HEADER_GET_HEADERS_SENT: &str = "header_get_headers_sent"; + /// Header range response received. + pub const HEADER_HEADERS_RECEIVED: &str = "header_headers_received"; + /// Header range response served from local state. + pub const HEADER_HEADERS_SERVED: &str = "header_headers_served"; + /// Header range committed. + pub const HEADER_RANGE_COMMITTED: &str = "header_range_committed"; + /// Header range rejected. + pub const HEADER_RANGE_REJECTED: &str = "header_range_rejected"; + /// NewBlock tip flood received. + pub const HEADER_NEW_BLOCK_RECEIVED: &str = "header_new_block_received"; + /// NewBlock tip flood forwarded. + pub const HEADER_NEW_BLOCK_FORWARDED: &str = "header_new_block_forwarded"; + /// NewBlock tip flood deduped. + pub const HEADER_NEW_BLOCK_DEDUPED: &str = "header_new_block_deduped"; + /// Peer violation observed. + pub const HEADER_PEER_VIOLATION: &str = "header_peer_violation"; + /// Peer disconnect requested. + pub const HEADER_PEER_DISCONNECT_REQUESTED: &str = "header_peer_disconnect_requested"; + /// Header frontier advanced. + pub const HEADER_FRONTIER_ADVANCED: &str = "header_frontier_advanced"; + /// Missing block bodies reported. + pub const HEADER_MISSING_BODIES_REPORTED: &str = "header_missing_bodies_reported"; +} + /// Cloneable Zakura trace emitter. #[derive(Clone, Debug)] pub struct ZakuraTrace { diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs index b81a8e238b3..dc69023999e 100644 --- a/zebra-state/src/constants.rs +++ b/zebra-state/src/constants.rs @@ -145,6 +145,12 @@ pub const MAX_FIND_BLOCK_HASHES_RESULTS: u32 = 500; /// The maximum number of block headers allowed in `getheaders` responses in the Zcash network protocol. pub const MAX_FIND_BLOCK_HEADERS_RESULTS: u32 = 160; +/// The maximum number of headers returned by native Zakura header-sync range reads. +/// +/// This must match `zebra-network`'s stream-5 hard cap, but lives here to avoid +/// an upward dependency from `zebra-state` to `zebra-network`. +pub const MAX_HEADER_SYNC_HEIGHT_RANGE: u32 = 4000; + /// The maximum number of invalidated block records. /// /// Each record can hold a chain of invalidated descendants up to the rollback diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 76f880b5000..15da13f4c5d 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -15,7 +15,10 @@ use zebra_chain::{ work::difficulty::CompactDifficulty, }; -use crate::{constants::MIN_TRANSPARENT_COINBASE_MATURITY, HashOrHeight, KnownBlock}; +use crate::{ + constants::{MAX_HEADER_SYNC_HEIGHT_RANGE, MIN_TRANSPARENT_COINBASE_MATURITY}, + HashOrHeight, KnownBlock, +}; /// A wrapper for type erased errors that is itself clonable and implements the /// Error trait @@ -58,6 +61,10 @@ pub enum CommitBlockError { #[error("could not contextually validate semantically verified block")] ValidateContextError(#[from] Box), + /// Header-only commit validation failed. + #[error("could not commit header range")] + HeaderCommitError(#[from] Box), + /// The write task exited (likely during shutdown). #[error("block commit task exited. Is Zebra shutting down?")] #[non_exhaustive] @@ -77,6 +84,12 @@ impl CommitBlockError { } } +impl From for CommitBlockError { + fn from(value: CommitHeaderRangeError) -> Self { + Self::HeaderCommitError(Box::new(value)) + } +} + /// An error describing why a `CommitSemanticallyVerified` request failed. #[derive(Debug, Error, Clone, PartialEq, Eq)] #[error("could not commit semantically-verified block")] @@ -88,6 +101,12 @@ impl From for CommitSemanticallyVerifiedError { } } +impl From for CommitSemanticallyVerifiedError { + fn from(value: CommitHeaderRangeError) -> Self { + Self(CommitBlockError::HeaderCommitError(Box::new(value))) + } +} + #[derive(Debug, Error)] pub enum LayeredStateError { #[error("{0}")] @@ -116,6 +135,104 @@ impl From for CommitCheckpointVerifiedError { } } +impl From for CommitCheckpointVerifiedError { + fn from(value: CommitHeaderRangeError) -> Self { + Self(CommitBlockError::HeaderCommitError(Box::new(value))) + } +} + +/// An error describing why a header-only range could not be committed. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum CommitHeaderRangeError { + /// The request did not contain any headers. + #[error("header range is empty")] + EmptyRange, + + /// The request exceeded the native header-sync range cap. + #[error( + "header range contains {actual} headers, exceeding the maximum {MAX_HEADER_SYNC_HEIGHT_RANGE}" + )] + RangeTooLong { + /// Number of headers in the request. + actual: usize, + }, + + /// The supplied anchor is not known to state. + #[error("header range anchor {anchor} is not known")] + UnknownAnchor { + /// The supplied anchor hash. + anchor: block::Hash, + }, + + /// The inferred header height overflowed the valid block height range. + #[error("header height overflow")] + HeightOverflow, + + /// A committed immutable header conflicts with the requested header. + #[error("header at finalized height {height:?} conflicts with an existing header")] + ImmutableConflict { + /// The conflicting height. + height: block::Height, + }, + + /// A provisional reorg tried to overwrite too far behind the best header tip. + #[error( + "header reorg at {height:?} is deeper than the maximum reorg window from best header tip {best_header_tip:?}" + )] + ReorgTooDeep { + /// Height of the conflicting header. + height: block::Height, + /// Current best header tip. + best_header_tip: block::Height, + }, + + /// A header conflicts with a trusted checkpoint hash. + #[error("checkpoint conflict at {height:?}: expected {expected}, got {actual}")] + CheckpointConflict { + /// Checkpoint height. + height: block::Height, + /// Expected checkpoint hash. + expected: block::Hash, + /// Actual header hash. + actual: block::Hash, + }, + + /// The requested header conflicts with a full block already stored at the same height. + #[error("header at height {height:?} conflicts with an already stored full block")] + ConflictingFullBlockHeader { + /// The conflicting height. + height: block::Height, + }, + + /// Contextual header validation failed. + #[error("could not contextually validate header")] + ValidateContextError(#[from] Box), + + /// Local storage failed while writing a validated header range. + /// + /// This is a local resource/storage failure, not a peer validation failure. + #[error("failed to write validated header range to disk: {error}")] + StorageWriteError { + /// RocksDB error details. + error: String, + }, + + /// Sending the commit request to the write task failed. + #[error("failed to send header range commit request to block write task")] + SendCommitRequestFailed, + + /// The commit request was dropped before processing. + #[error("header range commit request was unexpectedly dropped")] + CommitResponseDropped, +} + +impl From for CommitHeaderRangeError { + fn from(value: ValidateContextError) -> Self { + Self::ValidateContextError(Box::new(value)) + } +} + /// An error describing why a `InvalidateBlock` request failed. #[derive(Debug, Error)] #[non_exhaustive] diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 0b29dd2b7e7..d6cddbf8a49 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -42,8 +42,8 @@ pub use config::{ }; pub use constants::{state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT}; pub use error::{ - BoxError, CloneError, CommitBlockError, CommitSemanticallyVerifiedError, - DuplicateNullifierError, ValidateContextError, + BoxError, CloneError, CommitBlockError, CommitHeaderRangeError, + CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError, }; pub use request::{ CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest, diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 865a0380f47..4df145237d8 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -836,6 +836,18 @@ pub enum Request { /// [0]: (crate::error::CommitCheckpointVerifiedError) CommitCheckpointVerifiedBlock(CheckpointVerifiedBlock), + /// Persist a validated, contiguous run of Zakura headers that links to `anchor`. + /// + /// Header-only commits write separate header-sync indexes, not finalized block + /// indexes. They do not write transaction/body rows and therefore do not make + /// the block known to body-serving APIs. + CommitHeaderRange { + /// Hash of the held parent header for the first header in `headers`. + anchor: block::Hash, + /// Contiguous headers in ascending height order. + headers: Vec>, + }, + /// Computes the depth in the current best chain of the block identified by the given hash. /// /// Returns @@ -1047,6 +1059,7 @@ impl Request { match self { Request::CommitSemanticallyVerifiedBlock(_) => "commit_semantically_verified_block", Request::CommitCheckpointVerifiedBlock(_) => "commit_checkpoint_verified_block", + Request::CommitHeaderRange { .. } => "commit_header_range", Request::AwaitUtxo(_) => "await_utxo", Request::Depth(_) => "depth", Request::Tip => "tip", @@ -1099,6 +1112,10 @@ pub enum ReadRequest { /// with the current best chain tip. Tip, + /// Returns [`ReadResponse::FinalizedTip(Option<(Height, block::Hash)>)`](ReadResponse::FinalizedTip) + /// with the durable finalized chain tip. + FinalizedTip, + /// Returns [`ReadResponse::TipPoolValues(Option<(Height, block::Hash, ValueBalance)>)`](ReadResponse::TipPoolValues) /// with the pool values of the current best chain tip. TipPoolValues, @@ -1275,6 +1292,28 @@ pub enum ReadRequest { stop: Option, }, + /// Returns contiguous headers by height, in ascending order. + /// + /// The response stops before the first missing height and is capped by + /// [`MAX_HEADER_SYNC_HEIGHT_RANGE`](crate::constants::MAX_HEADER_SYNC_HEIGHT_RANGE). + HeadersByHeightRange { + /// First height to read. + start: block::Height, + /// Maximum number of headers to return. + count: u32, + }, + + /// Returns the highest header held on disk. + BestHeaderTip, + + /// Returns header-known, body-missing heights in `(verified_block_tip, best_header_tip]`. + MissingBlockBodies { + /// First height to consider. + from: block::Height, + /// Maximum number of heights to return. + limit: u32, + }, + /// Looks up a Sapling note commitment tree either by a hash or height. /// /// Returns @@ -1423,6 +1462,7 @@ impl ReadRequest { ReadRequest::UsageInfo => "usage_info", ReadRequest::IsPruned => "is_pruned", ReadRequest::Tip => "tip", + ReadRequest::FinalizedTip => "finalized_tip", ReadRequest::TipPoolValues => "tip_pool_values", ReadRequest::BlockInfo(_) => "block_info", ReadRequest::Depth(_) => "depth", @@ -1439,6 +1479,9 @@ impl ReadRequest { ReadRequest::BlockLocator => "block_locator", ReadRequest::FindBlockHashes { .. } => "find_block_hashes", ReadRequest::FindBlockHeaders { .. } => "find_block_headers", + ReadRequest::HeadersByHeightRange { .. } => "headers_by_height_range", + ReadRequest::BestHeaderTip => "best_header_tip", + ReadRequest::MissingBlockBodies { .. } => "missing_block_bodies", ReadRequest::SaplingTree { .. } => "sapling_tree", ReadRequest::OrchardTree { .. } => "orchard_tree", ReadRequest::SaplingSubtrees { .. } => "sapling_subtrees", @@ -1512,6 +1555,7 @@ impl TryFrom for ReadRequest { Request::CommitSemanticallyVerifiedBlock(_) | Request::CommitCheckpointVerifiedBlock(_) + | Request::CommitHeaderRange { .. } | Request::InvalidateBlock(_) | Request::ReconsiderBlock(_) => Err("ReadService does not write blocks"), diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 9bacfabed25..762c44f7663 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -314,6 +314,9 @@ pub enum ReadResponse { /// Response to [`ReadRequest::Tip`] with the current best chain tip. Tip(Option<(block::Height, block::Hash)>), + /// Response to [`ReadRequest::FinalizedTip`] with the durable finalized chain tip. + FinalizedTip(Option<(block::Height, block::Hash)>), + /// Response to [`ReadRequest::TipPoolValues`] with /// the current best chain tip and its [`ValueBalance`]. TipPoolValues { @@ -382,6 +385,15 @@ pub enum ReadResponse { /// The response to a `FindBlockHeaders` request. BlockHeaders(Vec), + /// Response to [`ReadRequest::HeadersByHeightRange`]. + Headers(Vec<(block::Height, block::Hash, Arc)>), + + /// Response to [`ReadRequest::BestHeaderTip`]. + BestHeaderTip(Option<(block::Height, block::Hash)>), + + /// Response to [`ReadRequest::MissingBlockBodies`]. + MissingBlockBodies(Vec), + /// The response to a `UnspentBestChainUtxo` request, from verified blocks in the /// _best_ non-finalized chain, or the finalized chain. UnspentBestChainUtxo(Option), @@ -505,6 +517,9 @@ impl TryFrom for Response { fn try_from(response: ReadResponse) -> Result { match response { ReadResponse::Tip(height_and_hash) => Ok(Response::Tip(height_and_hash)), + ReadResponse::FinalizedTip(_) => { + Err("there is no corresponding Response for this ReadResponse") + } ReadResponse::Depth(depth) => Ok(Response::Depth(depth)), ReadResponse::BestChainNextMedianTimePast(median_time_past) => Ok(Response::BestChainNextMedianTimePast(median_time_past)), ReadResponse::BlockHash(hash) => Ok(Response::BlockHash(hash)), @@ -552,6 +567,9 @@ impl TryFrom for Response { | ReadResponse::AddressesTransactionIds(_) | ReadResponse::AddressUtxos(_) | ReadResponse::ChainInfo(_) + | ReadResponse::Headers(_) + | ReadResponse::BestHeaderTip(_) + | ReadResponse::MissingBlockBodies(_) | ReadResponse::NonFinalizedBlocksListener(_) | ReadResponse::IsTransparentOutputSpent(_) => { Err("there is no corresponding Response for this ReadResponse") diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 3f586181977..4a0af44d65e 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -41,7 +41,8 @@ use zebra_chain::{ use crate::{ constants::{ - MAX_FIND_BLOCK_HASHES_RESULTS, MAX_FIND_BLOCK_HEADERS_RESULTS, MAX_LEGACY_CHAIN_BLOCKS, + MAX_FIND_BLOCK_HASHES_RESULTS, MAX_FIND_BLOCK_HEADERS_RESULTS, + MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_LEGACY_CHAIN_BLOCKS, }, error::{CommitBlockError, CommitCheckpointVerifiedError, InvalidateError, ReconsiderError}, request::TimedSpan, @@ -56,8 +57,8 @@ use crate::{ read::find, watch_receiver::WatchReceiver, }, - BoxError, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, Config, KnownBlock, - ReadRequest, ReadResponse, Request, Response, SemanticallyVerifiedBlock, + BoxError, CheckpointVerifiedBlock, CommitHeaderRangeError, CommitSemanticallyVerifiedError, + Config, KnownBlock, ReadRequest, ReadResponse, Request, Response, SemanticallyVerifiedBlock, }; pub mod block_iter; @@ -807,7 +808,7 @@ impl StateService { if self .read_service .db - .contains_height(semantically_verified.height) + .contains_body_at_height(semantically_verified.height) { let (rsp_tx, rsp_rx) = oneshot::channel(); let _ = rsp_tx.send(Err(CommitBlockError::new_duplicate( @@ -979,6 +980,35 @@ impl StateService { rsp_rx } + fn send_header_range( + &self, + anchor: block::Hash, + headers: Vec>, + ) -> oneshot::Receiver> { + let (rsp_tx, rsp_rx) = oneshot::channel(); + + let Some(sender) = &self.block_write_sender.non_finalized else { + let _ = rsp_tx.send(Err(CommitHeaderRangeError::SendCommitRequestFailed)); + return rsp_rx; + }; + + if let Err(tokio::sync::mpsc::error::SendError(error)) = + sender.send(NonFinalizedWriteMessage::CommitHeaderRange { + anchor, + headers, + rsp_tx, + }) + { + let NonFinalizedWriteMessage::CommitHeaderRange { rsp_tx, .. } = error else { + unreachable!("should return the same CommitHeaderRange message could not be sent"); + }; + + let _ = rsp_tx.send(Err(CommitHeaderRangeError::SendCommitRequestFailed)); + } + + rsp_rx + } + /// Assert some assumptions about the semantically verified `block` before it is queued. fn assert_block_can_be_validated(&self, block: &SemanticallyVerifiedBlock) { // required by `Request::CommitSemanticallyVerifiedBlock` call @@ -1200,6 +1230,24 @@ impl Service for StateService { .boxed() } + Request::CommitHeaderRange { anchor, headers } => { + let rsp_rx = tokio::task::block_in_place(move || { + span.in_scope(|| self.send_header_range(anchor, headers)) + }); + + let span = Span::current(); + async move { + rsp_rx + .await + .map_err(|_recv_error| CommitHeaderRangeError::CommitResponseDropped) + .and_then(|result| result) + .map_err(BoxError::from) + .map(Response::Committed) + } + .instrument(span) + .boxed() + } + // Uses pending_utxos and non_finalized_state_queued_blocks in the StateService. // If the UTXO isn't in the queued blocks, runs concurrently using the ReadStateService. Request::AwaitUtxo(outpoint) => { @@ -1381,6 +1429,44 @@ impl Service for StateService { } } +fn headers_by_height_range( + chain: Option, + db: &ZebraDb, + start: block::Height, + count: u32, +) -> Vec<(block::Height, block::Hash, Arc)> +where + C: AsRef + Clone, +{ + let capped_count = count.min(MAX_HEADER_SYNC_HEIGHT_RANGE); + let mut headers = Vec::with_capacity( + usize::try_from(capped_count).expect("capped header count fits in usize"), + ); + let mut height = start; + + for _ in 0..capped_count { + let next_header = read::hash_by_height(chain.clone(), db, height) + .and_then(|hash| { + read::block_header(chain.clone(), db, height.into()) + .map(|header| (height, hash, header)) + }) + .or_else(|| db.headers_by_height_range(height, 1).into_iter().next()); + + let Some(header) = next_header else { + break; + }; + + headers.push(header); + + let Ok(next_height) = height.next() else { + break; + }; + height = next_height; + } + + headers +} + impl Service for ReadStateService { type Response = ReadResponse; type Error = BoxError; @@ -1449,6 +1535,8 @@ impl Service for ReadStateService { &state.db, ))), + ReadRequest::FinalizedTip => Ok(ReadResponse::FinalizedTip(state.db.tip())), + // Used by `getblockchaininfo` RPC method. ReadRequest::TipPoolValues => { let (tip_height, tip_hash, value_balance) = @@ -1608,6 +1696,40 @@ impl Service for ReadStateService { .collect(), )), + ReadRequest::HeadersByHeightRange { start, count } => Ok(ReadResponse::Headers( + headers_by_height_range(state.latest_best_chain(), &state.db, start, count), + )), + + ReadRequest::BestHeaderTip => { + let best_disk_header_tip = state.db.best_header_tip(); + let verified_block_tip = read::tip(state.latest_best_chain(), &state.db); + + Ok(ReadResponse::BestHeaderTip( + match (best_disk_header_tip, verified_block_tip) { + (Some(header_tip), Some(block_tip)) if block_tip.0 > header_tip.0 => { + Some(block_tip) + } + (Some(header_tip), _) => Some(header_tip), + (None, block_tip) => block_tip, + }, + )) + } + + ReadRequest::MissingBlockBodies { from, limit } => { + let verified_block_tip = read::tip_height(state.latest_best_chain(), &state.db); + let best_header_tip = state + .db + .best_header_tip() + .map(|(height, _)| height) + .max(verified_block_tip); + + Ok(ReadResponse::MissingBlockBodies( + state + .db + .missing_block_bodies(verified_block_tip, best_header_tip, from, limit), + )) + } + ReadRequest::SaplingTree(hash_or_height) => Ok(ReadResponse::SaplingTree( read::sapling_tree(state.latest_best_chain(), &state.db, hash_or_height), )), diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs index bc590689c01..34c87d4ff72 100644 --- a/zebra-state/src/service/check.rs +++ b/zebra-state/src/service/check.rs @@ -131,6 +131,38 @@ where Ok(()) } +/// Check that a header is contextually valid for `network`, using the previous +/// header height and recent held header data. +#[tracing::instrument(skip(candidate_header, network, relevant_headers))] +pub(crate) fn header_is_valid_for_recent_chain( + candidate_header: &block::Header, + previous_block_height: block::Height, + network: &Network, + relevant_headers: C, +) -> Result<(), ValidateContextError> +where + C: IntoIterator)>, +{ + let relevant_headers: Vec<_> = relevant_headers + .into_iter() + .take(POW_ADJUSTMENT_BLOCK_SPAN) + .collect(); + + let difficulty_adjustment = AdjustedDifficulty::new_from_header_time( + candidate_header.time, + previous_block_height, + network, + relevant_headers, + ); + + check::difficulty_threshold_and_time_are_valid( + candidate_header.difficulty_threshold, + difficulty_adjustment, + )?; + + Ok(()) +} + /// Check that `block` is contextually valid for `network`, using /// the `history_tree` up to and including the previous block. #[tracing::instrument(skip(block, history_tree))] @@ -264,7 +296,7 @@ fn height_one_more_than_parent_height( /// /// These checks are performed together, because the time field is used to /// calculate the expected difficulty adjustment. -fn difficulty_threshold_and_time_are_valid( +pub(crate) fn difficulty_threshold_and_time_are_valid( difficulty_threshold: CompactDifficulty, difficulty_adjustment: AdjustedDifficulty, ) -> Result<(), ValidateContextError> { @@ -300,7 +332,16 @@ fn difficulty_threshold_and_time_are_valid( // of that block plus 90*60 seconds. // // https://zips.z.cash/protocol/protocol.pdf#blockheader - if network.is_max_block_time_enforced(candidate_height) && candidate_time > block_time_max { + // + // Mainnet height 1 is outside this rule, and Testnet does not enforce it + // until height 653_606. Zebra's full-block contextual validation on Mainnet + // and Testnet starts after the mandatory checkpoint, so this early-chain + // branch is only reachable through header sync and non-checkpointed test + // networks, where it preserves the spec exception. + if candidate_height.0 >= 2 + && network.is_max_block_time_enforced(candidate_height) + && candidate_time > block_time_max + { Err(ValidateContextError::TimeTooLate { candidate_time, block_time_max, diff --git a/zebra-state/src/service/check/difficulty.rs b/zebra-state/src/service/check/difficulty.rs index 7369914f26a..684bab99dd2 100644 --- a/zebra-state/src/service/check/difficulty.rs +++ b/zebra-state/src/service/check/difficulty.rs @@ -211,6 +211,23 @@ impl AdjustedDifficulty { /// Implements `ThresholdBits` from the Zcash specification. (Which excludes the /// Testnet minimum difficulty adjustment.) fn threshold_bits(&self) -> CompactDifficulty { + let averaging_window_height = u32::try_from(POW_AVERAGING_WINDOW) + .expect("averaging window is much smaller than u32::MAX"); + + if self.relevant_difficulty_thresholds.len() < POW_AVERAGING_WINDOW + && self.candidate_height.0 <= averaging_window_height + { + // # Consensus + // + // `ThresholdBits(height)` is `PoWLimit` for `height <= PoWAveragingWindow`. + // Zebra's full-block contextual validation on Mainnet and Testnet + // starts after the mandatory checkpoint, so this short-context path + // is only reachable through header sync and non-checkpointed test + // networks, where it implements the Zcash specification's early-chain + // `PoWLimit` rule. + return self.network.target_difficulty_limit().to_compact(); + } + let averaging_window_timespan = NetworkUpgrade::averaging_window_timespan_for_height( &self.network, self.candidate_height, diff --git a/zebra-state/src/service/check/tests/vectors.rs b/zebra-state/src/service/check/tests/vectors.rs index 5cab1aa018b..83326a03dbd 100644 --- a/zebra-state/src/service/check/tests/vectors.rs +++ b/zebra-state/src/service/check/tests/vectors.rs @@ -1,6 +1,12 @@ //! Fixed test vectors for state contextual validation checks. -use zebra_chain::serialization::ZcashDeserializeInto; +use chrono::{DateTime, Duration}; + +use zebra_chain::{ + parameters::{Network, NetworkUpgrade}, + serialization::ZcashDeserializeInto, + work::difficulty::ParameterDifficulty, +}; use super::super::*; @@ -46,3 +52,139 @@ fn test_sequential_height_check() { height_one_more_than_parent_height(block::Height(500000), height) .expect_err("parent height is way more, should panic"); } + +#[test] +fn header_daa_accepts_valid_threshold_with_full_context() { + let _init_guard = zebra_test::init(); + + let network = Network::Mainnet; + let previous_block_height = block::Height(99); + let candidate_time = DateTime::from_timestamp(15_000, 0).expect("test timestamp is in-range"); + let relevant_headers = daa_context(&network, previous_block_height, candidate_time); + let expected = AdjustedDifficulty::new_from_header_time( + candidate_time, + previous_block_height, + &network, + relevant_headers.clone(), + ) + .expected_difficulty_threshold(); + let mut candidate = *zebra_test::vectors::BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into::>() + .expect("block 1 deserializes") + .header + .as_ref(); + candidate.time = candidate_time; + candidate.difficulty_threshold = expected; + + header_is_valid_for_recent_chain( + &candidate, + previous_block_height, + &network, + relevant_headers, + ) + .expect("expected DAA threshold is accepted"); +} + +#[test] +fn header_daa_rejects_bad_threshold_with_full_context() { + let _init_guard = zebra_test::init(); + + let network = Network::Mainnet; + let previous_block_height = block::Height(99); + let candidate_time = DateTime::from_timestamp(15_000, 0).expect("test timestamp is in-range"); + let relevant_headers = daa_context(&network, previous_block_height, candidate_time); + let mut candidate = *zebra_test::vectors::BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into::>() + .expect("block 1 deserializes") + .header + .as_ref(); + candidate.time = candidate_time; + candidate.difficulty_threshold = network.target_difficulty_limit().to_compact(); + + header_is_valid_for_recent_chain( + &candidate, + previous_block_height, + &network, + relevant_headers, + ) + .expect_err("unexpected DAA threshold is rejected"); +} + +#[test] +fn height_one_header_skips_max_time_limit_but_later_mainnet_headers_do_not() { + let _init_guard = zebra_test::init(); + + let network = Network::Mainnet; + let genesis = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into::>() + .expect("genesis block deserializes"); + let block1 = zebra_test::vectors::BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into::>() + .expect("block 1 deserializes"); + let mut candidate = *block1.header; + candidate.time = genesis.header.time + Duration::hours(24); + let context = [(genesis.header.difficulty_threshold, genesis.header.time)]; + + header_is_valid_for_recent_chain(&candidate, block::Height(0), &network, context) + .expect("height 1 is outside the Mainnet max-time consensus rule"); + + let block2 = zebra_test::vectors::BLOCK_MAINNET_2_BYTES + .zcash_deserialize_into::>() + .expect("block 2 deserializes"); + let mut candidate = *block2.header; + candidate.time = block1.header.time + Duration::hours(24); + let context = [ + (block1.header.difficulty_threshold, block1.header.time), + (genesis.header.difficulty_threshold, genesis.header.time), + ]; + + assert!(matches!( + header_is_valid_for_recent_chain(&candidate, block::Height(1), &network, context), + Err(ValidateContextError::TimeTooLate { .. }) + )); +} + +#[test] +fn short_context_early_height_uses_pow_limit_threshold() { + let _init_guard = zebra_test::init(); + + let network = Network::Mainnet; + let genesis = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into::>() + .expect("genesis block deserializes"); + let candidate_time = + genesis.header.time + NetworkUpgrade::target_spacing_for_height(&network, block::Height(1)); + let context = [(genesis.header.difficulty_threshold, genesis.header.time)]; + + let expected = difficulty::AdjustedDifficulty::new_from_header_time( + candidate_time, + block::Height(0), + &network, + context, + ) + .expected_difficulty_threshold(); + + assert_eq!(expected, network.target_difficulty_limit().to_compact()); +} + +fn daa_context( + network: &Network, + previous_block_height: block::Height, + candidate_time: DateTime, +) -> Vec<( + zebra_chain::work::difficulty::CompactDifficulty, + DateTime, +)> { + let candidate_height = previous_block_height + .next() + .expect("test candidate height is valid"); + let target_spacing = NetworkUpgrade::target_spacing_for_height(network, candidate_height); + let difficulty = network.target_difficulty_limit().to_compact(); + + (0..difficulty::POW_ADJUSTMENT_BLOCK_SPAN) + .map(|offset| { + let offset = i32::try_from(offset + 1).expect("test offset fits in i32"); + (difficulty, candidate_time - target_spacing * offset) + }) + .collect() +} diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 98ad7a980ce..f610229cb09 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -77,6 +77,10 @@ pub const STATE_COLUMN_FAMILIES_IN_CODE: &[&str] = &[ "hash_by_height", "height_by_hash", "block_header_by_height", + // Header sync + "zakura_header_hash_by_height", + "zakura_header_height_by_hash", + "zakura_header_by_height", // Transactions "tx_by_loc", "hash_by_tx_loc", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap index 258dea1cd81..b5926f73e64 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap @@ -30,4 +30,7 @@ expression: cf_names "tx_loc_by_transparent_addr_loc", "utxo_by_out_loc", "utxo_loc_by_transparent_addr_loc", + "zakura_header_by_height", + "zakura_header_hash_by_height", + "zakura_header_height_by_hash", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap index cc95cf50109..3f48bc4c88a 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap @@ -15,4 +15,7 @@ expression: empty_column_families "tx_loc_by_transparent_addr_loc: no entries", "utxo_by_out_loc: no entries", "utxo_loc_by_transparent_addr_loc: no entries", + "zakura_header_by_height: no entries", + "zakura_header_hash_by_height: no entries", + "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap index 828873f3dee..446df58d472 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap @@ -11,4 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_by_height: no entries", + "zakura_header_hash_by_height: no entries", + "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap index 828873f3dee..446df58d472 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap @@ -11,4 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_by_height: no entries", + "zakura_header_hash_by_height: no entries", + "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap index 9fcf1615420..03a0f42d1ff 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap @@ -29,4 +29,7 @@ expression: empty_column_families "tx_loc_by_transparent_addr_loc: no entries", "utxo_by_out_loc: no entries", "utxo_loc_by_transparent_addr_loc: no entries", + "zakura_header_by_height: no entries", + "zakura_header_hash_by_height: no entries", + "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap index cc95cf50109..3f48bc4c88a 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap @@ -15,4 +15,7 @@ expression: empty_column_families "tx_loc_by_transparent_addr_loc: no entries", "utxo_by_out_loc: no entries", "utxo_loc_by_transparent_addr_loc: no entries", + "zakura_header_by_height: no entries", + "zakura_header_hash_by_height: no entries", + "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap index 828873f3dee..446df58d472 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap @@ -11,4 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_by_height: no entries", + "zakura_header_hash_by_height: no entries", + "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap index 828873f3dee..446df58d472 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap @@ -11,4 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_by_height: no entries", + "zakura_header_hash_by_height: no entries", + "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 073f3d4528e..3f5b6a1cab8 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -32,9 +32,12 @@ use zebra_chain::{ }; use crate::{ - constants::MAX_PRUNE_HEIGHTS_PER_COMMIT, - error::CommitCheckpointVerifiedError, + constants::{ + MAX_BLOCK_REORG_HEIGHT, MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_PRUNE_HEIGHTS_PER_COMMIT, + }, + error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, request::FinalizedBlock, + service::check, service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, disk_format::{ @@ -53,6 +56,10 @@ use crate::request::Spend; #[cfg(test)] mod tests; +const ZAKURA_HEADER_HASH_BY_HEIGHT: &str = "zakura_header_hash_by_height"; +const ZAKURA_HEADER_HEIGHT_BY_HASH: &str = "zakura_header_height_by_hash"; +const ZAKURA_HEADER_BY_HEIGHT: &str = "zakura_header_by_height"; + impl ZebraDb { // Read block methods @@ -60,8 +67,7 @@ impl ZebraDb { // // TODO: move this method to the tip section pub fn is_empty(&self) -> bool { - let hash_by_height = self.db.cf_handle("hash_by_height").unwrap(); - self.db.zs_is_empty(&hash_by_height) + self.tip().is_none() } /// Returns the tip height and hash, if there is one. @@ -70,8 +76,12 @@ impl ZebraDb { // move this method to the tip section #[allow(clippy::unwrap_in_result)] pub fn tip(&self) -> Option<(block::Height, block::Hash)> { - let hash_by_height = self.db.cf_handle("hash_by_height").unwrap(); - self.db.zs_last_key_value(&hash_by_height) + let tx_by_loc = self.db.cf_handle("tx_by_loc").unwrap(); + let (last_tx_loc, _tx): (TransactionLocation, Transaction) = + self.db.zs_last_key_value(&tx_by_loc)?; + + self.hash(last_tx_loc.height) + .map(|hash| (last_tx_loc.height, hash)) } /// Returns `true` if `height` is present in the finalized state. @@ -82,6 +92,19 @@ impl ZebraDb { self.db.zs_contains(&hash_by_height, &height) } + /// Returns `true` if a full block body is present at `height`. + /// + /// Header-only commits never write transaction rows. Since valid Zcash + /// blocks always have at least a coinbase transaction, the first + /// transaction location is the body-availability marker. + #[allow(clippy::unwrap_in_result)] + pub fn contains_body_at_height(&self, height: block::Height) -> bool { + let tx_by_loc = self.db.cf_handle("tx_by_loc").unwrap(); + let first_tx = TransactionLocation::min_for_height(height); + + self.db.zs_contains(&tx_by_loc, &first_tx) + } + /// Returns the finalized hash for a given `block::Height` if it is present. #[allow(clippy::unwrap_in_result)] pub fn hash(&self, height: block::Height) -> Option { @@ -89,12 +112,18 @@ impl ZebraDb { self.db.zs_get(&hash_by_height, &height) } + /// Returns the hash for `height` only if a full block body is present. + pub fn body_hash(&self, height: block::Height) -> Option { + self.contains_body_at_height(height) + .then(|| self.hash(height)) + .flatten() + } + /// Returns `true` if `hash` is present in the finalized state. #[allow(clippy::unwrap_in_result)] pub fn contains_hash(&self, hash: block::Hash) -> bool { - let height_by_hash = self.db.cf_handle("height_by_hash").unwrap(); - - self.db.zs_contains(&height_by_hash, &hash) + self.height(hash) + .is_some_and(|height| self.contains_body_at_height(height)) } /// Returns the height of the given block if it exists. @@ -222,6 +251,10 @@ impl ZebraDb { fn raw_block(&self, hash_or_height: HashOrHeight) -> Option<(RawBytes, Vec)> { // Block let height = hash_or_height.height_or_else(|hash| self.height(hash))?; + if !self.contains_body_at_height(height) { + return None; + } + let header = self.raw_block_header(height.into())?; // Transactions @@ -295,6 +328,157 @@ impl ZebraDb { self.block(height.into()) } + /// Returns the highest header stored on disk. + #[allow(clippy::unwrap_in_result)] + pub fn best_header_tip(&self) -> Option<(block::Height, block::Hash)> { + let body_tip = self.tip(); + let zakura_header_tip = self.zakura_header_tip(); + + match (body_tip, zakura_header_tip) { + (Some(body_tip), Some(header_tip)) if body_tip.0 >= header_tip.0 => Some(body_tip), + (Some(_), Some(header_tip)) => Some(header_tip), + (Some(body_tip), None) => Some(body_tip), + (None, Some(header_tip)) => Some(header_tip), + (None, None) => None, + } + } + + /// Returns a contiguous ascending header range from full blocks and Zakura header rows. + pub fn headers_by_height_range( + &self, + start: block::Height, + count: u32, + ) -> Vec<(block::Height, block::Hash, Arc)> { + let capped_count = count.min(MAX_HEADER_SYNC_HEIGHT_RANGE); + + let mut headers = Vec::with_capacity( + usize::try_from(capped_count).expect("capped header count fits in usize"), + ); + let mut height = start; + + for _ in 0..capped_count { + let Some((hash, header)) = self.header_by_height(height) else { + break; + }; + + headers.push((height, hash, header)); + + let Ok(next_height) = height.next() else { + break; + }; + height = next_height; + } + + headers + } + + /// Returns recent header difficulty/time context in reverse height order, + /// starting at `height`. + pub fn recent_header_context( + &self, + height: block::Height, + ) -> Vec<( + zebra_chain::work::difficulty::CompactDifficulty, + DateTime, + )> { + let mut context = Vec::with_capacity(check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); + let mut current_height = Some(height); + + while let Some(height) = current_height { + let Some((_hash, header)) = self.header_by_height(height) else { + break; + }; + + context.push((header.difficulty_threshold, header.time)); + if context.len() == check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN { + break; + } + + current_height = height.previous().ok(); + } + + context + } + + /// Returns header-known, body-missing heights. + pub fn missing_block_bodies( + &self, + verified_block_tip: Option, + best_header_tip: Option, + from: block::Height, + limit: u32, + ) -> Vec { + let Some(best_header_tip) = best_header_tip else { + return Vec::new(); + }; + + let start = verified_block_tip + .and_then(|tip| tip.next().ok()) + .map_or(from, |first_missing| first_missing.max(from)); + + if start > best_header_tip { + return Vec::new(); + } + + let count = limit.min(best_header_tip.0.saturating_sub(start.0).saturating_add(1)); + + self.headers_by_height_range(start, count) + .into_iter() + .map(|(height, _, _)| height) + .filter(|height| !self.contains_body_at_height(*height)) + .take(limit as usize) + .collect() + } + + #[allow(clippy::unwrap_in_result)] + fn zakura_header_tip(&self) -> Option<(block::Height, block::Hash)> { + let hash_by_height = self.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + self.db.zs_last_key_value(&hash_by_height) + } + + #[allow(clippy::unwrap_in_result)] + fn zakura_header_hash(&self, height: block::Height) -> Option { + let hash_by_height = self.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + self.db.zs_get(&hash_by_height, &height) + } + + #[allow(clippy::unwrap_in_result)] + fn zakura_header_height(&self, hash: block::Hash) -> Option { + let height_by_hash = self.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + self.db.zs_get(&height_by_hash, &hash) + } + + #[allow(clippy::unwrap_in_result)] + fn zakura_header(&self, height: block::Height) -> Option> { + let header_by_height = self.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + self.db.zs_get(&header_by_height, &height) + } + + fn header_hash(&self, height: block::Height) -> Option { + self.body_hash(height) + .or_else(|| self.zakura_header_hash(height)) + } + + fn header_height(&self, hash: block::Hash) -> Option { + self.contains_hash(hash) + .then(|| self.height(hash)) + .flatten() + .or_else(|| self.zakura_header_height(hash)) + } + + fn header_by_height(&self, height: block::Height) -> Option<(block::Hash, Arc)> { + if let Some(hash) = self.body_hash(height) { + return self + .block_header(height.into()) + .map(|header| (hash, header)); + } + + let hash = self.zakura_header_hash(height)?; + let header = self.zakura_header(height)?; + + Some((hash, header)) + } + // Read transaction methods /// Returns the [`Transaction`] with [`transaction::Hash`], and its [`Height`], @@ -414,6 +598,9 @@ impl ZebraDb { ) -> Option> { // Block let height = hash_or_height.height_or_else(|hash| self.height(hash))?; + if !self.contains_body_at_height(height) { + return None; + } // Transaction hashes let hash_by_tx_loc = self.db.cf_handle("hash_by_tx_loc").unwrap(); @@ -798,7 +985,7 @@ impl DiskWriteBatch { let db = &zebra_db.db; // Commit block, transaction, and note commitment tree data. - self.prepare_block_header_and_transaction_data_batch(db, finalized); + self.prepare_block_header_and_transaction_data_batch(db, finalized)?; // The consensus rules are silent on shielded transactions in the genesis block, // because there aren't any in the mainnet or testnet genesis blocks. @@ -915,11 +1102,14 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, - ) { + ) -> Result<(), CommitCheckpointVerifiedError> { // Blocks let block_header_by_height = db.cf_handle("block_header_by_height").unwrap(); let hash_by_height = db.cf_handle("hash_by_height").unwrap(); let height_by_hash = db.cf_handle("height_by_hash").unwrap(); + let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); // Transactions let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); @@ -934,10 +1124,54 @@ impl DiskWriteBatch { .. } = finalized; - // Commit block header data - self.zs_insert(&block_header_by_height, height, &block.header); + // Commit block header data. Full block verification is authoritative: + // it may replace conflicting header-only provisional rows at this + // height and truncate their header-only descendants. + let existing_body_header: Option> = + db.zs_get(&block_header_by_height, height); + if existing_body_header.is_some_and(|existing_header| existing_header != block.header) { + return Err( + CommitHeaderRangeError::ConflictingFullBlockHeader { height: *height }.into(), + ); + } + + let existing_zakura_header: Option> = + db.zs_get(&zakura_header_by_height, height); + if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { + let best_header_tip: Option<(block::Height, block::Hash)> = + db.zs_last_key_value(&zakura_hash_by_height); + + if let Some((best_header_tip, _)) = best_header_tip { + for old_height in height.0..=best_header_tip.0 { + let old_height = block::Height(old_height); + + if old_height != *height + && db.zs_contains( + &tx_by_loc, + &TransactionLocation::min_for_height(old_height), + ) + { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { + height: old_height, + } + .into()); + } + + if let Some(old_hash) = + db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &old_height) + { + self.zs_delete(&zakura_height_by_hash, old_hash); + } + + self.zs_delete(&zakura_hash_by_height, old_height); + self.zs_delete(&zakura_header_by_height, old_height); + } + } + } - // Index the block hash and height + // Index the block header, hash, and height. This also restores the + // verified full block row after any provisional cleanup above. + self.zs_insert(&block_header_by_height, height, &block.header); self.zs_insert(&hash_by_height, height, hash); self.zs_insert(&height_by_hash, hash, height); @@ -956,6 +1190,137 @@ impl DiskWriteBatch { self.zs_insert(&hash_by_tx_loc, transaction_location, transaction_hash); self.zs_insert(&tx_loc_by_hash, transaction_hash, transaction_location); } + + Ok(()) + } + + /// Prepare a database batch containing a contextually validated header range. + pub fn prepare_header_range_batch( + &mut self, + zebra_db: &ZebraDb, + anchor: block::Hash, + headers: &[Arc], + ) -> Result { + if headers.is_empty() { + return Err(CommitHeaderRangeError::EmptyRange); + } + + if headers.len() > MAX_HEADER_SYNC_HEIGHT_RANGE as usize { + return Err(CommitHeaderRangeError::RangeTooLong { + actual: headers.len(), + }); + } + + let header_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_by_hash = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + + let anchor_height = zebra_db + .header_height(anchor) + .or_else(|| (anchor == zebra_db.network().genesis_hash()).then_some(block::Height(0))) + .ok_or(CommitHeaderRangeError::UnknownAnchor { anchor })?; + + if anchor != zebra_db.network().genesis_hash() + && zebra_db.header_hash(anchor_height) != Some(anchor) + { + return Err(CommitHeaderRangeError::UnknownAnchor { anchor }); + } + + let finalized_height = zebra_db.finalized_tip_height(); + let best_header_tip = zebra_db.best_header_tip().map(|(height, _)| height); + let checkpoints = zebra_db.network().checkpoint_list(); + + let mut recent_headers = zebra_db.recent_header_context(anchor_height); + if recent_headers.is_empty() { + return Err(CommitHeaderRangeError::UnknownAnchor { anchor }); + } + + let mut first_conflicting_height = None; + let mut validated_headers = Vec::with_capacity(headers.len()); + + for (index, header) in headers.iter().enumerate() { + let offset = + u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; + let height = (anchor_height + i64::from(offset)) + .ok_or(CommitHeaderRangeError::HeightOverflow)?; + let hash = block::Hash::from(&**header); + + if let Some(expected) = checkpoints.hash(height) { + if expected != hash { + return Err(CommitHeaderRangeError::CheckpointConflict { + height, + expected, + actual: hash, + }); + } + } + + if let Some((_existing_hash, existing_header)) = zebra_db.header_by_height(height) { + if existing_header != *header { + if finalized_height.is_some_and(|finalized_height| height <= finalized_height) { + return Err(CommitHeaderRangeError::ImmutableConflict { height }); + } + + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + + if let Some(best_header_tip) = best_header_tip { + if best_header_tip.0.saturating_sub(height.0) >= MAX_BLOCK_REORG_HEIGHT { + return Err(CommitHeaderRangeError::ReorgTooDeep { + height, + best_header_tip, + }); + } + } + + first_conflicting_height.get_or_insert(height); + } + } + + check::header_is_valid_for_recent_chain( + header, + height + .previous() + .map_err(|_| CommitHeaderRangeError::HeightOverflow)?, + &zebra_db.network(), + recent_headers.iter().copied(), + )?; + + recent_headers.insert(0, (header.difficulty_threshold, header.time)); + recent_headers.truncate(check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); + + validated_headers.push((height, hash, header)); + } + + if let (Some(first_conflicting_height), Some(best_header_tip)) = + (first_conflicting_height, best_header_tip) + { + for height in first_conflicting_height.0..=best_header_tip.0 { + let height = block::Height(height); + + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + + if let Some(old_hash) = zebra_db.zakura_header_hash(height) { + self.zs_delete(&height_by_hash, old_hash); + } + + self.zs_delete(&hash_by_height, height); + self.zs_delete(&header_by_height, height); + } + } + + for (height, hash, header) in validated_headers { + self.zs_insert(&header_by_height, height, header); + self.zs_insert(&hash_by_height, height, hash); + self.zs_insert(&height_by_hash, hash, height); + } + + Ok(block::Hash::from( + &**headers.last().expect("headers is non-empty"), + )) } /// Deletes the block header at `height`. diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index e3d5e9a4234..546c3b520dc 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -10,27 +10,42 @@ //! //! Check transparent address indexes, UTXOs, etc. -use std::{iter, sync::Arc}; +use std::{iter, path::Path, sync::Arc}; use zebra_chain::{ block::{ + self, tests::generate::{ large_multi_transaction_block, large_single_transaction_block_many_inputs, large_single_transaction_block_many_outputs, }, Block, Height, }, - parameters::Network::{self, *}, + parameters::{ + testnet, + Network::{self, *}, + NetworkUpgrade, + }, serialization::{ZcashDeserializeInto, ZcashSerialize}, transparent::new_ordered_outputs_with_height, + work::difficulty::ParameterDifficulty, }; use zebra_test::vectors::{MAINNET_BLOCKS, TESTNET_BLOCKS}; use crate::{ - constants::{state_database_format_version_in_code, STATE_DATABASE_KIND}, + constants::{ + state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT, + MAX_HEADER_SYNC_HEIGHT_RANGE, STATE_DATABASE_KIND, + }, + error::CommitHeaderRangeError, request::{FinalizedBlock, Treestate}, - service::finalized_state::{ - disk_db::DiskWriteBatch, ZebraDb, PRUNING_METADATA, STATE_COLUMN_FAMILIES_IN_CODE, + service::{ + check::difficulty::AdjustedDifficulty, + finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, PRUNING_METADATA, STATE_COLUMN_FAMILIES_IN_CODE, + }, + read, }, CheckpointVerifiedBlock, Config, SemanticallyVerifiedBlock, }; @@ -76,6 +91,674 @@ fn test_block_db_round_trip() { test_block_db_round_trip_with(&Mainnet, iter::once(block)); } +#[test] +fn header_range_commit_keeps_body_availability_separate() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + + let mut batch = DiskWriteBatch::new(); + let committed_hash = batch + .prepare_header_range_batch(&state, genesis.hash(), std::slice::from_ref(&block1.header)) + .expect("block 1 header links to genesis and has valid context"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + + assert_eq!(committed_hash, block1.hash()); + assert_eq!(state.best_header_tip(), Some((Height(1), block1.hash()))); + assert_eq!(state.finalized_tip_height(), Some(Height(0))); + assert_eq!(state.tip(), Some((Height(0), genesis.hash()))); + + assert_eq!( + state.headers_by_height_range(Height(1), 1), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); + assert_eq!( + state.missing_block_bodies(Some(Height(0)), Some(Height(1)), Height(0), 10), + vec![Height(1)], + ); + + assert!(!state.contains_hash(block1.hash())); + assert!(!state.contains_body_at_height(Height(1))); + assert!(state.block(Height(1).into()).is_none()); + assert!(state + .transaction_hashes_for_block(Height(1).into()) + .is_none()); +} + +#[test] +fn header_range_read_is_contiguous_capped_and_stops_at_first_gap() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + + let mut batch = DiskWriteBatch::new(); + batch + .prepare_header_range_batch(&state, genesis.hash(), std::slice::from_ref(&block1.header)) + .expect("block 1 header is valid"); + state.write_batch(batch).expect("header batch writes"); + + assert_eq!(state.headers_by_height_range(Height(0), 3).len(), 2); + assert_eq!( + state.headers_by_height_range(Height(1), u32::MAX), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); + assert!(state.headers_by_height_range(Height(2), 10).is_empty()); +} + +#[test] +fn header_range_read_enforces_max_range_cap() { + let _init_guard = zebra_test::init(); + let (state, _genesis, block1) = mainnet_state_with_genesis(); + let header_by_height = state.db.cf_handle("zakura_header_by_height").unwrap(); + let hash_by_height = state.db.cf_handle("zakura_header_hash_by_height").unwrap(); + + let mut batch = DiskWriteBatch::new(); + for height in 1..=MAX_HEADER_SYNC_HEIGHT_RANGE + 1 { + batch.zs_insert(&header_by_height, Height(height), &block1.header); + batch.zs_insert(&hash_by_height, Height(height), block1.hash()); + } + state.db.write(batch).expect("header test rows write"); + + assert_eq!( + state.headers_by_height_range(Height(1), u32::MAX).len(), + usize::try_from(MAX_HEADER_SYNC_HEIGHT_RANGE).expect("range cap fits in usize"), + ); +} + +#[test] +fn missing_block_bodies_respects_from_limit_and_empty_body_gap() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + let block3 = mainnet_block(3); + + assert!(state + .missing_block_bodies(Some(Height(0)), Some(Height(0)), Height(0), 10) + .is_empty()); + + commit_header_range( + &state, + genesis.hash(), + &[ + block1.header.clone(), + block2.header.clone(), + block3.header.clone(), + ], + ); + + assert_eq!( + state.missing_block_bodies(Some(Height(0)), Some(Height(3)), Height(2), 1), + vec![Height(2)], + ); + assert_eq!( + state.missing_block_bodies(Some(Height(0)), Some(Height(3)), Height(2), 10), + vec![Height(2), Height(3)], + ); +} + +#[test] +fn header_range_commit_rejects_finalized_or_body_conflicts() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + write_full_block_header_and_transactions(&state, block1.clone()); + + let mut conflicting = *block1.header; + conflicting.nonce.0[0] ^= 1; + + let mut batch = DiskWriteBatch::new(); + assert!(matches!( + batch.prepare_header_range_batch(&state, genesis.hash(), &[Arc::new(conflicting)]), + Err(CommitHeaderRangeError::ImmutableConflict { height: Height(1) }) + | Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height: Height(1) }) + )); +} + +#[test] +fn header_range_commit_rejects_checkpoint_conflicts() { + let _init_guard = zebra_test::init(); + let genesis = mainnet_block(0); + let block1 = mainnet_block(1); + let network = checkpoint_test_network(genesis.hash(), block1.hash()); + let state = state_with_genesis(&network, genesis.clone()); + + let mut forged = *block1.header; + forged.nonce.0[0] ^= 1; + + let mut batch = DiskWriteBatch::new(); + assert!(matches!( + batch.prepare_header_range_batch(&state, genesis.hash(), &[Arc::new(forged)]), + Err(CommitHeaderRangeError::CheckpointConflict { + height: Height(1), + expected, + actual, + }) if expected == block1.hash() && actual != expected + )); + + assert_eq!(state.best_header_tip(), Some((Height(0), genesis.hash()))); +} + +#[test] +fn header_range_commit_accepts_matching_checkpoint_hash() { + let _init_guard = zebra_test::init(); + let genesis = mainnet_block(0); + let block1 = mainnet_block(1); + let network = checkpoint_test_network(genesis.hash(), block1.hash()); + let state = state_with_genesis(&network, genesis.clone()); + + let committed_hash = + commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header)); + + assert_eq!(committed_hash, block1.hash()); + assert_eq!(state.best_header_tip(), Some((Height(1), block1.hash()))); + assert_eq!( + state.headers_by_height_range(Height(1), 1), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); +} + +#[test] +fn header_range_reorg_replaces_shorter_range_without_stale_indexes() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + let block3 = mainnet_block(3); + + commit_header_range( + &state, + genesis.hash(), + &[ + block1.header.clone(), + block2.header.clone(), + block3.header.clone(), + ], + ); + + let old_block2_hash = block2.hash(); + let old_block3_hash = block3.hash(); + let alternate_block2 = alternate_header(block1.hash(), &block2.header, 1); + + let alternate_block2_hash = commit_header_range( + &state, + block1.hash(), + std::slice::from_ref(&alternate_block2), + ); + + assert_eq!(alternate_block2_hash, block::Hash::from(&*alternate_block2)); + assert_eq!( + state.best_header_tip(), + Some((Height(2), alternate_block2_hash)) + ); + assert_eq!(state.height(old_block2_hash), None); + assert_eq!(state.height(old_block3_hash), None); + assert!(!state.contains_hash(old_block2_hash)); + assert!(state.block(old_block2_hash.into()).is_none()); + + assert_eq!( + state.headers_by_height_range(Height(1), 3), + vec![ + (Height(1), block1.hash(), block1.header.clone()), + (Height(2), alternate_block2_hash, alternate_block2), + ], + ); +} + +#[test] +fn header_range_reorg_replaces_longer_range_without_stale_indexes() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + let block3 = mainnet_block(3); + + commit_header_range( + &state, + genesis.hash(), + &[block1.header.clone(), block2.header.clone()], + ); + + let old_block2_hash = block2.hash(); + let alternate_block2 = alternate_header(block1.hash(), &block2.header, 1); + let alternate_block2_hash = block::Hash::from(&*alternate_block2); + let alternate_block3 = alternate_header(alternate_block2_hash, &block3.header, 2); + let alternate_block3_hash = block::Hash::from(&*alternate_block3); + + let committed_hash = commit_header_range( + &state, + block1.hash(), + &[alternate_block2.clone(), alternate_block3.clone()], + ); + + assert_eq!(committed_hash, alternate_block3_hash); + assert_eq!( + state.best_header_tip(), + Some((Height(3), alternate_block3_hash)) + ); + assert_eq!(state.height(old_block2_hash), None); + assert!(state.block(old_block2_hash.into()).is_none()); + + assert_eq!( + state.headers_by_height_range(Height(1), 4), + vec![ + (Height(1), block1.hash(), block1.header.clone()), + (Height(2), alternate_block2_hash, alternate_block2), + (Height(3), alternate_block3_hash, alternate_block3), + ], + ); +} + +#[test] +fn header_range_reorg_rejects_too_deep_overwrite() { + let _init_guard = zebra_test::init(); + let genesis = mainnet_block(0); + let network = no_extra_checkpoint_test_network(genesis.hash()); + let state = state_with_genesis(&network, genesis.clone()); + let tip_height = Height(MAX_BLOCK_REORG_HEIGHT + 1); + let original_headers = + synthetic_headers_from_state(&state, Height(0), genesis.hash(), tip_height.0, 1); + + commit_header_range(&state, genesis.hash(), &original_headers); + + let conflict_height = Height(1); + let conflict_anchor = genesis.hash(); + let conflicting_header = synthetic_headers_from_state(&state, Height(0), conflict_anchor, 1, 2) + .pop() + .expect("one synthetic conflicting header was generated"); + + let mut batch = DiskWriteBatch::new(); + assert!(matches!( + batch.prepare_header_range_batch(&state, conflict_anchor, &[conflicting_header]), + Err(CommitHeaderRangeError::ReorgTooDeep { + height, + best_header_tip, + }) if height == conflict_height && best_header_tip == tip_height + )); + + assert_eq!( + state.best_header_tip(), + Some(( + tip_height, + block::Hash::from(&**original_headers.last().unwrap()) + )), + ); +} + +#[test] +fn header_range_reorg_accepts_boundary_adjacent_overwrite() { + let _init_guard = zebra_test::init(); + let genesis = mainnet_block(0); + let network = no_extra_checkpoint_test_network(genesis.hash()); + let state = state_with_genesis(&network, genesis.clone()); + let tip_height = Height(MAX_BLOCK_REORG_HEIGHT); + let original_headers = + synthetic_headers_from_state(&state, Height(0), genesis.hash(), tip_height.0, 1); + + commit_header_range(&state, genesis.hash(), &original_headers); + + let conflict_height = Height(1); + let old_conflict_hash = block::Hash::from(&*original_headers[0]); + let conflicting_header = synthetic_headers_from_state(&state, Height(0), genesis.hash(), 1, 2) + .pop() + .expect("one synthetic conflicting header was generated"); + let conflicting_hash = block::Hash::from(&*conflicting_header); + + let committed_hash = commit_header_range( + &state, + genesis.hash(), + std::slice::from_ref(&conflicting_header), + ); + + assert_eq!(committed_hash, conflicting_hash); + assert_eq!( + state.best_header_tip(), + Some((conflict_height, conflicting_hash)), + ); + assert_eq!(state.height(old_conflict_hash), None); + assert_eq!(state.hash(Height(2)), None); + assert_eq!( + state.headers_by_height_range(Height(1), 2), + vec![(Height(1), conflicting_hash, conflicting_header)], + ); +} + +#[test] +fn header_range_commit_rejects_non_current_anchor_hash() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let stale_anchor = block::Hash([0x42; 32]); + let height_by_hash = state.db.cf_handle("height_by_hash").unwrap(); + + commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header)); + + let mut stale_index_batch = DiskWriteBatch::new(); + stale_index_batch.zs_insert(&height_by_hash, stale_anchor, Height(1)); + state + .db + .write(stale_index_batch) + .expect("stale test index writes"); + + let block2 = mainnet_block(2); + let alternate_block2 = alternate_header(stale_anchor, &block2.header, 1); + + let mut batch = DiskWriteBatch::new(); + assert!(matches!( + batch.prepare_header_range_batch(&state, stale_anchor, &[alternate_block2]), + Err(CommitHeaderRangeError::UnknownAnchor { anchor }) if anchor == stale_anchor + )); + + assert_eq!(state.hash(Height(1)), None); + assert_eq!( + state.headers_by_height_range(Height(1), 1), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); +} + +#[test] +fn header_range_rows_and_tip_survive_reopen_without_body_availability() { + let _init_guard = zebra_test::init(); + let tempdir = tempfile::tempdir().expect("temporary cache directory is created"); + let cache_dir = tempdir.path().to_owned(); + let config = persistent_config(&cache_dir); + let genesis = mainnet_block(0); + let block1 = mainnet_block(1); + let block2 = mainnet_block(2); + + { + let state = persistent_state(&config, &Mainnet); + write_full_block_header_and_transactions(&state, genesis.clone()); + commit_header_range( + &state, + genesis.hash(), + &[block1.header.clone(), block2.header.clone()], + ); + + assert_eq!(state.best_header_tip(), Some((Height(2), block2.hash()))); + let mut state = state; + state.shutdown(true); + } + + let reopened = persistent_state(&config, &Mainnet); + + assert_eq!(reopened.tip(), Some((Height(0), genesis.hash()))); + assert_eq!(reopened.best_header_tip(), Some((Height(2), block2.hash()))); + assert_eq!( + reopened.headers_by_height_range(Height(1), 2), + vec![ + (Height(1), block1.hash(), block1.header.clone()), + (Height(2), block2.hash(), block2.header.clone()), + ], + ); + assert!(!reopened.contains_hash(block2.hash())); + assert_eq!(reopened.body_hash(Height(2)), None); + assert!(reopened.block(Height(2).into()).is_none()); +} + +#[test] +fn block_facing_hash_by_height_requires_body_availability() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + + commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header)); + + assert_eq!(state.hash(Height(1)), None); + assert_eq!(state.body_hash(Height(1)), None); + assert_eq!( + read::hash_by_height( + Option::>::None, + &state, + Height(1), + ), + None, + ); + assert_eq!( + state.headers_by_height_range(Height(1), 1), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); +} + +#[test] +fn full_block_commit_over_identical_header_only_row_is_noop_for_header_indexes() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + + commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header)); + + assert_eq!(state.hash(Height(1)), None); + assert_eq!(state.height(block1.hash()), None); + assert!(!state.contains_hash(block1.hash())); + assert_eq!( + state.headers_by_height_range(Height(1), 2), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); + + write_full_block_header_and_transactions(&state, block1.clone()); + + assert_eq!(state.hash(Height(1)), Some(block1.hash())); + assert_eq!(state.height(block1.hash()), Some(Height(1))); + assert!(state.contains_hash(block1.hash())); + assert_eq!( + state.headers_by_height_range(Height(1), 2), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); +} + +#[test] +fn full_block_commit_overwrites_conflicting_header_only_rows() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + let block3 = mainnet_block(3); + + commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header)); + + let alternate_block2 = alternate_header(block1.hash(), &block2.header, 1); + let alternate_block2_hash = block::Hash::from(&*alternate_block2); + let alternate_block3 = alternate_header(alternate_block2_hash, &block3.header, 2); + let alternate_block3_hash = block::Hash::from(&*alternate_block3); + commit_header_range(&state, block1.hash(), &[alternate_block2, alternate_block3]); + + write_full_block_header_and_transactions(&state, block2.clone()); + + assert_eq!(state.hash(Height(2)), Some(block2.hash())); + assert_eq!(state.height(alternate_block2_hash), None); + assert_eq!(state.height(alternate_block3_hash), None); + assert_eq!(state.best_header_tip(), Some((Height(2), block2.hash()))); + assert!(state.contains_hash(block2.hash())); + assert!(state.block(block2.hash().into()).is_some()); + assert!(state.block(alternate_block2_hash.into()).is_none()); +} + +fn mainnet_state_with_genesis() -> (ZebraDb, Arc, Arc) { + let genesis = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into::>() + .expect("genesis block deserializes"); + let block1 = zebra_test::vectors::BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into::>() + .expect("block 1 deserializes"); + let state = state_with_genesis(&Mainnet, genesis.clone()); + + (state, genesis, block1) +} + +fn state_with_genesis(network: &Network, genesis: Arc) -> ZebraDb { + let state = ZebraDb::new( + &Config::ephemeral(), + STATE_DATABASE_KIND, + &state_database_format_version_in_code(), + network, + true, + STATE_COLUMN_FAMILIES_IN_CODE + .iter() + .map(ToString::to_string), + false, + ); + + write_full_block_header_and_transactions(&state, genesis.clone()); + + state +} + +fn persistent_config(cache_dir: &Path) -> Config { + Config { + cache_dir: cache_dir.to_owned(), + ephemeral: false, + debug_skip_non_finalized_state_backup_task: true, + ..Config::default() + } +} + +fn persistent_state(config: &Config, network: &Network) -> ZebraDb { + ZebraDb::new( + config, + STATE_DATABASE_KIND, + &state_database_format_version_in_code(), + network, + true, + STATE_COLUMN_FAMILIES_IN_CODE + .iter() + .map(ToString::to_string), + false, + ) +} + +fn checkpoint_test_network(genesis_hash: block::Hash, checkpoint_hash: block::Hash) -> Network { + testnet::Parameters::build() + .with_network_name("HeaderCheckpointTest") + .expect("test network name is valid") + .with_genesis_hash(genesis_hash) + .expect("test genesis hash is valid") + .with_target_difficulty_limit(Mainnet.target_difficulty_limit()) + .expect("mainnet difficulty limit is valid for test network") + .with_activation_heights(testnet::ConfiguredActivationHeights { + canopy: Some(2), + ..Default::default() + }) + .expect("test activation heights are valid") + .clear_funding_streams() + .with_checkpoints(testnet::ConfiguredCheckpoints::HeightsAndHashes(vec![ + (Height(0), genesis_hash), + (Height(1), checkpoint_hash), + ])) + .expect("test checkpoints are valid") + .to_network() + .expect("test network is valid") +} + +fn no_extra_checkpoint_test_network(genesis_hash: block::Hash) -> Network { + testnet::Parameters::build() + .with_network_name("HeaderReorgTest") + .expect("test network name is valid") + .with_genesis_hash(genesis_hash) + .expect("test genesis hash is valid") + .with_target_difficulty_limit(Mainnet.target_difficulty_limit()) + .expect("mainnet difficulty limit is valid for test network") + .with_activation_heights(testnet::ConfiguredActivationHeights { + canopy: Some(1), + ..Default::default() + }) + .expect("test activation heights are valid") + .clear_funding_streams() + .clear_checkpoints() + .expect("genesis-only checkpoints are valid") + .to_network() + .expect("test network is valid") +} + +fn mainnet_block(height: u32) -> Arc { + MAINNET_BLOCKS + .get(&height) + .expect("test vector exists") + .zcash_deserialize_into::>() + .expect("mainnet test block deserializes") +} + +fn synthetic_headers_from_state( + state: &ZebraDb, + anchor_height: Height, + anchor_hash: block::Hash, + count: u32, + nonce_seed: u8, +) -> Vec> { + let network = state.network(); + let template = mainnet_block(1); + let mut context = state.recent_header_context(anchor_height); + let mut previous_hash = anchor_hash; + let mut previous_height = anchor_height; + let mut nonce_tag = nonce_seed; + + (0..count) + .map(|_| { + let candidate_height = previous_height + .next() + .expect("test header height remains in range"); + let previous_time = context + .first() + .expect("anchor header context is available") + .1; + let target_spacing = + NetworkUpgrade::target_spacing_for_height(&network, candidate_height); + let candidate_time = previous_time + target_spacing; + let expected_difficulty = AdjustedDifficulty::new_from_header_time( + candidate_time, + previous_height, + &network, + context.iter().copied(), + ) + .expected_difficulty_threshold(); + + let mut header = *template.header; + header.previous_block_hash = previous_hash; + header.time = candidate_time; + header.difficulty_threshold = expected_difficulty; + header.nonce.0[0] = header.nonce.0[0].wrapping_add(nonce_tag); + nonce_tag = nonce_tag.wrapping_add(1); + + let header = Arc::new(header); + previous_hash = block::Hash::from(&*header); + previous_height = candidate_height; + context.insert(0, (header.difficulty_threshold, header.time)); + context.truncate(crate::service::check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); + header + }) + .collect() +} + +fn alternate_header( + previous_block_hash: block::Hash, + template: &Arc, + nonce_tag: u8, +) -> Arc { + let mut header = **template; + header.previous_block_hash = previous_block_hash; + header.nonce.0[0] ^= nonce_tag; + Arc::new(header) +} + +fn commit_header_range( + state: &ZebraDb, + anchor: block::Hash, + headers: &[Arc], +) -> block::Hash { + let mut batch = DiskWriteBatch::new(); + let committed_hash = batch + .prepare_header_range_batch(state, anchor, headers) + .expect("header range is valid"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + committed_hash +} + +fn write_full_block_header_and_transactions(state: &ZebraDb, block: Arc) { + let checkpoint_verified = CheckpointVerifiedBlock::from(block); + let finalized = + FinalizedBlock::from_checkpoint_verified(checkpoint_verified, Treestate::default()); + + let mut batch = DiskWriteBatch::new(); + batch + .prepare_block_header_and_transaction_data_batch(&state.db, &finalized) + .expect("full block header and transaction batch is valid"); + state.db.write(batch).expect("full block batch writes"); +} + fn test_block_db_round_trip_with( network: &Network, block_test_cases: impl IntoIterator, @@ -148,7 +831,9 @@ fn test_block_db_round_trip_with( // Skip validation by writing the block directly to the database let mut batch = DiskWriteBatch::new(); - batch.prepare_block_header_and_transaction_data_batch(&state.db, &finalized); + batch + .prepare_block_header_and_transaction_data_batch(&state.db, &finalized) + .expect("test block header and transaction batch is valid"); state.db.write(batch).expect("block is valid for writing"); // Now read it back from the state diff --git a/zebra-state/src/service/read/find.rs b/zebra-state/src/service/read/find.rs index d3f6052ba59..dfcd31327c5 100644 --- a/zebra-state/src/service/read/find.rs +++ b/zebra-state/src/service/read/find.rs @@ -149,9 +149,11 @@ where // can only add overlapping blocks, and hashes are unique. let tip = tip_height(chain, db)?; - let height = height_by_hash(chain, db, hash)?; + let height = chain + .and_then(|chain| chain.as_ref().height_by_hash(hash)) + .or_else(|| db.contains_hash(hash).then(|| db.height(hash)).flatten())?; - Some(tip.0 - height.0) + tip.0.checked_sub(height.0) } /// Returns the location of the block if present in the non-finalized state. @@ -210,7 +212,7 @@ where chain .and_then(|chain| chain.as_ref().hash_by_height(height)) - .or_else(|| db.hash(height)) + .or_else(|| db.body_hash(height)) } /// Return true if `hash` is in `chain` or `db`. diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index a0d2406a9c3..fb176793304 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -7,7 +7,7 @@ use std::{env, sync::Arc, time::Duration}; use tokio::runtime::Runtime; -use tower::{buffer::Buffer, util::BoxService}; +use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt}; use zebra_chain::{ block::{self, Block, CountedHeader, Height}, @@ -24,9 +24,16 @@ use zebra_test::{prelude::*, transcript::Transcript}; use crate::{ arbitrary::Prepare, init_test, - service::{arbitrary::populated_state, chain_tip::TipAction, StateService}, - tests::setup::{partial_nu5_chain_strategy, transaction_v4_from_coinbase}, - BoxError, CheckpointVerifiedBlock, Config, Request, Response, SemanticallyVerifiedBlock, + service::{ + arbitrary::populated_state, chain_tip::TipAction, headers_by_height_range, + non_finalized_state::Chain, StateService, + }, + tests::{ + setup::{partial_nu5_chain_strategy, transaction_v4_from_coinbase}, + FakeChainHelper, + }, + BoxError, CheckpointVerifiedBlock, Config, ReadRequest, ReadResponse, Request, Response, + SemanticallyVerifiedBlock, }; const LAST_BLOCK_HEIGHT: u32 = 10; @@ -437,6 +444,200 @@ async fn handoff_trigger_microbench() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn header_only_service_requests_preserve_body_boundary() -> std::result::Result<(), BoxError> +{ + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let (mut state_service, read_state, _, _) = + StateService::new(Config::ephemeral(), &network, Height::MAX, 0).await; + let genesis = + zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES.zcash_deserialize_into::>()?; + let block1 = + zebra_test::vectors::BLOCK_MAINNET_1_BYTES.zcash_deserialize_into::>()?; + let block2 = + zebra_test::vectors::BLOCK_MAINNET_2_BYTES.zcash_deserialize_into::>()?; + let block1_hash = block1.hash(); + let block2_hash = block2.hash(); + + assert_eq!( + state_service + .ready() + .await? + .call(Request::CommitCheckpointVerifiedBlock( + CheckpointVerifiedBlock::from(genesis.clone()), + )) + .await?, + Response::Committed(genesis.hash()), + ); + + state_service.block_write_sender.finalized = None; + let state = Buffer::new(BoxService::new(state_service), 1); + + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::FinalizedTip) + .await?, + ReadResponse::FinalizedTip(Some((Height(0), genesis.hash()))), + ); + + assert_eq!( + state + .clone() + .oneshot(Request::CommitHeaderRange { + anchor: genesis.hash(), + headers: vec![block1.header.clone(), block2.header.clone()], + }) + .await?, + Response::Committed(block2_hash), + ); + + assert_eq!( + state.clone().oneshot(Request::Depth(block1_hash)).await?, + Response::Depth(None), + ); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::Depth(block1_hash)) + .await?, + ReadResponse::Depth(None), + ); + assert_eq!( + state + .clone() + .oneshot(Request::KnownBlock(block1_hash)) + .await?, + Response::KnownBlock(None), + ); + assert_eq!( + state + .clone() + .oneshot(Request::KnownBlock(block2_hash)) + .await?, + Response::KnownBlock(None), + ); + assert_eq!( + state + .clone() + .oneshot(Request::Block(Height(1).into())) + .await?, + Response::Block(None), + ); + assert_eq!( + state + .clone() + .oneshot(Request::Block(Height(2).into())) + .await?, + Response::Block(None), + ); + assert_eq!( + state + .oneshot(Request::AnyChainBlock(block1_hash.into())) + .await?, + Response::Block(None), + ); + + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::BestChainBlockHash(Height(1))) + .await?, + ReadResponse::BlockHash(None), + ); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::TransactionIdsForBlock(Height(1).into())) + .await?, + ReadResponse::TransactionIdsForBlock(None), + ); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::HeadersByHeightRange { + start: Height(1), + count: 2, + }) + .await?, + ReadResponse::Headers(vec![ + (Height(1), block1_hash, block1.header.clone()), + (Height(2), block2_hash, block2.header.clone()), + ]), + ); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::BestHeaderTip) + .await?, + ReadResponse::BestHeaderTip(Some((Height(2), block2_hash))), + ); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::MissingBlockBodies { + from: Height(1), + limit: 10, + }) + .await?, + ReadResponse::MissingBlockBodies(vec![Height(1), Height(2)]), + ); + assert_eq!( + read_state.oneshot(ReadRequest::FinalizedTip).await?, + ReadResponse::FinalizedTip(Some((Height(0), genesis.hash()))), + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn header_range_reads_include_non_finalized_best_chain_blocks() -> Result<()> { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let (state_service, _read_state, _, _) = + StateService::new(Config::ephemeral(), &network, Height::MAX, 0).await; + let block1 = Arc::new( + network + .test_block(653599, 583999) + .expect("fake test block can be built for a post-Canopy height"), + ); + let block2 = block1.make_fake_child(); + let start = block1.coinbase_height().unwrap(); + let block1_hash = block1.hash(); + let block2_hash = block2.hash(); + let mut chain = Chain::new( + &network, + (start - 1).unwrap(), + Default::default(), + Default::default(), + Default::default(), + Default::default(), + ValueBalance::fake_populated_pool(), + ); + chain = chain.push(block1.clone().prepare().test_with_zero_spent_utxos())?; + chain = chain.push(block2.clone().prepare().test_with_zero_spent_utxos())?; + + assert_eq!( + headers_by_height_range( + Some(Arc::new(chain)), + &state_service.read_service.db, + start, + 2, + ), + vec![ + (start, block1_hash, block1.header.clone()), + (start.next().unwrap(), block2_hash, block2.header.clone()), + ], + ); + assert_eq!( + headers_by_height_range(None::>, &state_service.read_service.db, start, 2), + Vec::new(), + ); + + Ok(()) +} + #[test] fn state_behaves_when_blocks_are_committed_in_order() -> Result<()> { let _init_guard = zebra_test::init(); diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 530b1886b1f..d9145d82300 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -16,6 +16,7 @@ use zebra_chain::block::{self, Height}; use crate::{ constants::MAX_BLOCK_REORG_HEIGHT, + error::CommitHeaderRangeError, service::{ check, finalized_state::{FinalizedState, ZebraDb}, @@ -146,6 +147,13 @@ pub enum NonFinalizedWriteMessage { /// A newly downloaded and semantically verified block prepared for /// contextual validation and insertion into the non-finalized state. Commit(QueuedSemanticallyVerified), + /// A validated header range prepared for contextual storage checks and + /// insertion into the durable header store. + CommitHeaderRange { + anchor: block::Hash, + headers: Vec>, + rsp_tx: oneshot::Sender>, + }, /// The hash of a block that should be invalidated and removed from /// the non-finalized state, if present. Invalidate { @@ -357,6 +365,34 @@ impl WriteBlockWorkerTask { while let Some(msg) = non_finalized_block_write_receiver.blocking_recv() { let queued_child_and_rsp_tx = match msg { NonFinalizedWriteMessage::Commit(queued_child) => Some(queued_child), + NonFinalizedWriteMessage::CommitHeaderRange { + anchor, + headers, + rsp_tx, + } => { + let mut batch = crate::service::finalized_state::DiskWriteBatch::new(); + let result = batch + .prepare_header_range_batch(&finalized_state.db, anchor, &headers) + .and_then(|hash| { + finalized_state + .db + .write_batch(batch) + .map(|()| hash) + .map_err(|error| { + tracing::error!( + ?error, + "failed to write validated header range" + ); + + CommitHeaderRangeError::StorageWriteError { + error: error.to_string(), + } + }) + }); + + let _ = rsp_tx.send(result); + continue; + } NonFinalizedWriteMessage::Invalidate { hash, rsp_tx } => { tracing::info!(?hash, "invalidating a block in the non-finalized state"); let _ = rsp_tx.send(non_finalized_state.invalidate_block(hash)); diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 08c35e1623b..e1386b4060a 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -73,21 +73,25 @@ //! //! Some of the diagnostic features are optional, and need to be enabled at compile-time. -use std::{net::SocketAddr, path::Path, sync::Arc}; +use std::{future::Future, net::SocketAddr, path::Path, sync::Arc}; use abscissa_core::{config, Command, FrameworkError}; use color_eyre::eyre::{eyre, Report}; use futures::FutureExt; use tokio::{ pin, select, - sync::{oneshot, watch}, + sync::{mpsc, oneshot, watch}, }; -use tower::{builder::ServiceBuilder, util::BoxService, ServiceExt}; +use tower::{builder::ServiceBuilder, util::BoxService, Service, ServiceExt}; use tracing_futures::Instrument; -use zebra_chain::block::genesis::regtest_genesis_block; +use zebra_chain::block::{self, genesis::regtest_genesis_block}; use zebra_consensus::router::BackgroundTaskHandles; use zebra_network::types::PeerServices; +use zebra_network::zakura::{ + HeaderSyncAction, HeaderSyncCommitFailureKind, HeaderSyncEvent, HeaderSyncFrontiers, + HeaderSyncMessage, ZakuraEndpoint, ZakuraHeaderSyncDriverStartup, DEFAULT_HS_RANGE, +}; use zebra_rpc::{methods::RpcImpl, server::RpcServer, SubmitBlockChannel}; use crate::{ @@ -157,6 +161,464 @@ fn check_tcp_slow_start_after_idle() { ); } +async fn zakura_header_sync_driver_startup( + read_state: zebra_state::ReadStateService, + network: &zebra_chain::parameters::Network, +) -> Result { + let best_header_tip = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::BestHeaderTip) + .await + .map_err(|error| eyre!("{error}"))? + { + zebra_state::ReadResponse::BestHeaderTip(tip) => tip, + response => Err(eyre!("unexpected BestHeaderTip response: {response:?}"))?, + }; + + let finalized_tip = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::FinalizedTip) + .await + .map_err(|error| eyre!("{error}"))? + { + zebra_state::ReadResponse::FinalizedTip(tip) => tip, + response => Err(eyre!("unexpected FinalizedTip response: {response:?}"))?, + }; + + let verified_block_tip = match read_state + .oneshot(zebra_state::ReadRequest::Tip) + .await + .map_err(|error| eyre!("{error}"))? + { + zebra_state::ReadResponse::Tip(tip) => tip, + response => Err(eyre!("unexpected Tip response: {response:?}"))?, + }; + + let empty_state_tip = (block::Height(0), network.genesis_hash()); + Ok(ZakuraHeaderSyncDriverStartup { + frontiers: HeaderSyncFrontiers { + finalized_height: finalized_tip.map_or(block::Height(0), |(height, _)| height), + verified_block_tip: verified_block_tip.map_or(block::Height(0), |(height, _)| height), + }, + best_header_tip: Some(best_header_tip.unwrap_or(empty_state_tip)), + }) +} + +async fn drive_zakura_header_sync_actions( + mut actions: mpsc::Receiver, + endpoint: ZakuraEndpoint, + header_sync: zebra_network::zakura::HeaderSyncHandle, + state: State, + read_state: ReadState, + block_verifier: BlockVerifier, + shutdown: impl Future + Send + 'static, +) where + State: Service< + zebra_state::Request, + Response = zebra_state::Response, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + State::Future: Send + 'static, + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, + BlockVerifier: + Service + Clone + Send + 'static, + BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, + BlockVerifier::Future: Send + 'static, +{ + pin!(shutdown); + loop { + let action = select! { + _ = &mut shutdown => return, + action = actions.recv() => { + let Some(action) = action else { + return; + }; + action + } + }; + + match action { + HeaderSyncAction::SendMessage { peer, msg } => { + endpoint.send_header_sync_message(&peer, msg).await; + } + HeaderSyncAction::ForwardNewBlock { peer, block, .. } => { + endpoint + .send_header_sync_message(&peer, HeaderSyncMessage::NewBlock(block)) + .await; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + debug!( + ?peer, + ?reason, + "disconnecting peer for Zakura header-sync violation" + ); + let _ = endpoint.supervisor().disconnect_peer(&peer).await; + } + HeaderSyncAction::NewBlockReceived { + peer, + height, + hash, + block, + } => { + match block_verifier + .clone() + .oneshot(zebra_consensus::Request::Commit(block.clone())) + .await + { + Ok(committed_hash) if committed_hash == hash => { + let _ = header_sync + .send(HeaderSyncEvent::NewBlockAccepted { + peer, + height, + hash, + block, + }) + .await; + } + Ok(committed_hash) => { + warn!( + ?peer, + ?hash, + ?committed_hash, + "Zakura NewBlock verifier returned an unexpected hash" + ); + let _ = header_sync + .send(HeaderSyncEvent::NewBlockRejected { peer, hash }) + .await; + } + Err(error) => { + if block_verify_error_is_duplicate(&error) { + debug!( + ?peer, + ?height, + ?hash, + ?error, + "Zakura NewBlock was already known by the block verifier" + ); + let _ = header_sync + .send(HeaderSyncEvent::NewBlockDuplicate { peer, height, hash }) + .await; + continue; + } + + debug!( + ?peer, + ?hash, + ?error, + "Zakura NewBlock rejected by block verifier" + ); + let _ = header_sync + .send(HeaderSyncEvent::NewBlockRejected { peer, hash }) + .await; + } + } + } + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + let mut returned_count = 0u32; + match read_state + .clone() + .oneshot(zebra_state::ReadRequest::HeadersByHeightRange { start, count }) + .await + { + Ok(zebra_state::ReadResponse::Headers(headers)) => { + returned_count = u32::try_from(headers.len()).unwrap_or(u32::MAX); + let headers = headers + .into_iter() + .map(|(_height, _hash, header)| header) + .collect(); + endpoint + .send_header_sync_message(&peer, HeaderSyncMessage::Headers(headers)) + .await; + } + Ok(response) => { + warn!(?peer, ?response, "unexpected HeadersByHeightRange response"); + } + Err(error) => { + warn!( + ?peer, + ?error, + "failed to read Zakura Headers response from state" + ); + } + } + let _ = header_sync + .send(HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count, + }) + .await; + } + HeaderSyncAction::CommitHeaderRange { + peer, + anchor, + start_height, + headers, + finalized: _finalized, + } => { + let count = u32::try_from(headers.len()).unwrap_or(u32::MAX); + match state + .clone() + .oneshot(zebra_state::Request::CommitHeaderRange { anchor, headers }) + .await + { + Ok(zebra_state::Response::Committed(tip_hash)) => { + let tip_height = + block::Height(start_height.0.saturating_add(count.saturating_sub(1))); + let _ = header_sync + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height, + tip_height, + tip_hash, + }) + .await; + } + Ok(response) => { + warn!(?peer, ?response, "unexpected CommitHeaderRange response"); + let _ = header_sync + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind: HeaderSyncCommitFailureKind::Local, + }) + .await; + } + Err(error) => { + let kind = header_range_commit_failure_kind(error.as_ref()); + debug!( + ?peer, + ?start_height, + ?count, + ?kind, + ?error, + "Zakura header range commit failed" + ); + let _ = header_sync + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind, + }) + .await; + } + } + } + HeaderSyncAction::QueryBestHeaderTip => { + match read_state + .clone() + .oneshot(zebra_state::ReadRequest::BestHeaderTip) + .await + { + Ok(zebra_state::ReadResponse::BestHeaderTip(Some((tip_height, tip_hash)))) => { + // Reuse the range-commit event as a startup/tip-refresh fact: a + // single-height covered mark is harmless and refreshes the reactor tip. + let _ = header_sync + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: tip_height, + tip_height, + tip_hash, + }) + .await; + } + Ok(zebra_state::ReadResponse::BestHeaderTip(None)) => {} + Ok(response) => warn!(?response, "unexpected BestHeaderTip response"), + Err(error) => warn!(?error, "failed to query Zakura best header tip"), + } + } + HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { + log_missing_block_bodies(read_state.clone(), from, limit).await; + } + HeaderSyncAction::BodyGaps { from, to } => { + let limit = + to.0.saturating_sub(from.0) + .saturating_add(1) + .min(DEFAULT_HS_RANGE); + log_missing_block_bodies(read_state.clone(), from, limit).await; + } + } + } +} + +fn block_verify_error_is_duplicate(error: &Error) -> bool +where + Error: std::fmt::Debug + Send + Sync + 'static, +{ + let error = error as &dyn std::any::Any; + + error + .downcast_ref::() + .is_some_and(zebra_consensus::RouterError::is_duplicate_request) + || error + .downcast_ref::() + .is_some_and(zebra_consensus::VerifyBlockError::is_duplicate_request) + || error + .downcast_ref::() + .is_some_and(|error| { + error + .downcast_ref::() + .is_some_and(zebra_consensus::RouterError::is_duplicate_request) + || error + .downcast_ref::() + .is_some_and(zebra_consensus::VerifyBlockError::is_duplicate_request) + }) +} + +async fn log_missing_block_bodies(read_state: ReadState, from: block::Height, limit: u32) +where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Send + + 'static, + ReadState::Future: Send + 'static, +{ + // This driver boundary is observational only: header sync exposes bounded body gaps + // from state, but block download must consume them through its own policy/watches. + match read_state + .oneshot(zebra_state::ReadRequest::MissingBlockBodies { from, limit }) + .await + { + Ok(zebra_state::ReadResponse::MissingBlockBodies(heights)) => { + let first = heights.first().copied(); + let last = heights.last().copied(); + let count = heights.len(); + debug!( + ?from, + ?limit, + ?count, + ?first, + ?last, + "Zakura header-known body gaps from state" + ); + } + Ok(response) => warn!(?response, "unexpected MissingBlockBodies response"), + Err(error) => warn!(?error, "failed to query Zakura missing block bodies"), + } +} + +fn header_range_commit_failure_kind( + error: &(dyn std::error::Error + Send + Sync + 'static), +) -> HeaderSyncCommitFailureKind { + let Some(error) = error.downcast_ref::() else { + return HeaderSyncCommitFailureKind::Local; + }; + + match error { + zebra_state::CommitHeaderRangeError::StorageWriteError { .. } + | zebra_state::CommitHeaderRangeError::SendCommitRequestFailed + | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { + HeaderSyncCommitFailureKind::Local + } + zebra_state::CommitHeaderRangeError::EmptyRange + | zebra_state::CommitHeaderRangeError::RangeTooLong { .. } + | zebra_state::CommitHeaderRangeError::UnknownAnchor { .. } + | zebra_state::CommitHeaderRangeError::HeightOverflow + | zebra_state::CommitHeaderRangeError::ImmutableConflict { .. } + | zebra_state::CommitHeaderRangeError::ReorgTooDeep { .. } + | zebra_state::CommitHeaderRangeError::CheckpointConflict { .. } + | zebra_state::CommitHeaderRangeError::ConflictingFullBlockHeader { .. } + | zebra_state::CommitHeaderRangeError::ValidateContextError(_) => { + HeaderSyncCommitFailureKind::InvalidPeerRange + } + _ => HeaderSyncCommitFailureKind::Local, + } +} + +async fn mirror_zakura_full_block_commits( + mut chain_tip_change: zebra_state::ChainTipChange, + read_state: ReadState, + header_sync: zebra_network::zakura::HeaderSyncHandle, + shutdown: impl Future + Send + 'static, +) where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + pin!(shutdown); + loop { + let action = select! { + _ = &mut shutdown => return, + action = chain_tip_change.wait_for_tip_change() => { + let Ok(action) = action else { + return; + }; + action + } + }; + let height = action.best_tip_height(); + let hash = action.best_tip_hash(); + + let finalized_height = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::FinalizedTip) + .await + { + Ok(zebra_state::ReadResponse::FinalizedTip(Some((height, _hash)))) => height, + Ok(zebra_state::ReadResponse::FinalizedTip(None)) => block::Height(0), + Ok(response) => { + warn!(?response, "unexpected FinalizedTip response"); + block::Height(0) + } + Err(error) => { + warn!(?error, "failed to query Zakura finalized frontier"); + block::Height(0) + } + }; + + let _ = header_sync + .send(HeaderSyncEvent::StateFrontiersChanged( + HeaderSyncFrontiers { + finalized_height, + verified_block_tip: height, + }, + )) + .await; + + match read_state + .clone() + .oneshot(zebra_state::ReadRequest::Block(hash.into())) + .await + { + Ok(zebra_state::ReadResponse::Block(Some(block))) => { + let _ = header_sync + .send(HeaderSyncEvent::FullBlockCommitted { + height, + hash, + header: block.header.clone(), + }) + .await; + } + Ok(zebra_state::ReadResponse::Block(None)) => { + debug!( + ?height, + ?hash, + "Zakura full-block mirror could not find committed tip block" + ); + } + Ok(response) => warn!(?response, "unexpected block lookup response"), + Err(error) => warn!(?error, "failed to mirror Zakura full-block commit"), + } + } +} + #[cfg(not(target_os = "linux"))] fn check_tcp_slow_start_after_idle() {} @@ -505,6 +967,18 @@ impl StartCmd { .buffer(Self::state_buffer_bound()) .service(state_service); + let zakura_header_sync_driver_startup = if config.network.v2_p2p { + Some( + zakura_header_sync_driver_startup( + read_only_state_service.clone(), + &config.network.network, + ) + .await?, + ) + } else { + None + }; + info!("initializing network"); // The service that our node uses to respond to requests by peers. The // load_shed middleware ensures that we reduce the size of the peer set @@ -534,14 +1008,16 @@ impl StartCmd { PeerServices::NODE_NETWORK }; - let (peer_set, address_book, misbehavior_sender) = zebra_network::init( - config.network.clone(), - inbound, - latest_chain_tip.clone(), - user_agent(), - advertised_services, - ) - .await; + let (peer_set, address_book, misbehavior_sender, zakura_endpoint) = + zebra_network::init_with_zakura_header_sync( + config.network.clone(), + inbound, + latest_chain_tip.clone(), + user_agent(), + advertised_services, + zakura_header_sync_driver_startup, + ) + .await; // Start health server if configured (after sync_status is available) @@ -556,6 +1032,39 @@ impl StartCmd { ) .await; + if let Some(endpoint) = zakura_endpoint.clone() { + if let (Some(header_sync), Some(shutdown), Some(actions)) = ( + endpoint.header_sync(), + endpoint.header_sync_shutdown(), + endpoint.take_header_sync_actions().await, + ) { + let driver_task = tokio::spawn( + drive_zakura_header_sync_actions( + actions, + endpoint.clone(), + header_sync.clone(), + state.clone(), + read_only_state_service.clone(), + block_verifier_router.clone(), + shutdown.clone().cancelled_owned(), + ) + .in_current_span(), + ); + endpoint.push_header_sync_task(driver_task).await; + + let full_block_task = tokio::spawn( + mirror_zakura_full_block_commits( + chain_tip_change.clone(), + read_only_state_service.clone(), + header_sync, + shutdown.cancelled_owned(), + ) + .in_current_span(), + ); + endpoint.push_header_sync_task(full_block_task).await; + } + } + info!("initializing syncer"); let (mut syncer, sync_status) = ChainSync::new( &config, @@ -1817,3 +2326,35 @@ mod tests { ))); } } + +#[cfg(test)] +mod zakura_header_sync_driver_tests { + use super::*; + + #[test] + fn block_verify_error_duplicate_classifier_detects_router_and_block_errors() { + let hash = block::Hash([1; 32]); + let duplicate_block_error = zebra_consensus::VerifyBlockError::Block { + source: zebra_consensus::BlockError::AlreadyInChain( + hash, + zebra_state::KnownBlock::BestChain, + ), + }; + assert!(block_verify_error_is_duplicate(&duplicate_block_error)); + + let duplicate_router_error = zebra_consensus::RouterError::Block { + source: Box::new(zebra_consensus::VerifyBlockError::Block { + source: zebra_consensus::BlockError::AlreadyInChain( + hash, + zebra_state::KnownBlock::BestChain, + ), + }), + }; + assert!(block_verify_error_is_duplicate(&duplicate_router_error)); + + let invalid_block_error = zebra_consensus::VerifyBlockError::Block { + source: zebra_consensus::BlockError::NoTransactions, + }; + assert!(!block_verify_error_is_duplicate(&invalid_block_error)); + } +} diff --git a/zebrad/tests/common/configs/v4.5.0-zakura-p2p.toml b/zebrad/tests/common/configs/v4.5.0-zakura-p2p.toml index b14022d25ca..3253c0bed2a 100644 --- a/zebrad/tests/common/configs/v4.5.0-zakura-p2p.toml +++ b/zebrad/tests/common/configs/v4.5.0-zakura-p2p.toml @@ -86,6 +86,11 @@ max_pending_handshakes = 8 message_rate_per_second = 128 stream_open_rate_per_second = 16 +[network.zakura.header_sync] +max_headers_per_response = 1000 +max_inflight_requests = 10 +status_refresh_interval = "30s" + [rpc] cookie_dir = "cache_dir" debug_force_finished_sync = false