From 52889a0d21988c0fb6715ca315b99f5fabcbcbc2 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 13:38:18 -0300 Subject: [PATCH 001/146] Add zaino-sync crate: type-level axis markers and trait hierarchy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scaffolds the DAG-driven sync engine with sealed marker types for the two classification axes (InputScope × CompositionType), compile-time-enforced extract/merge trait families, and stub modules for DAG, engine, backend, and provisioner. The erased layer will be replaced with a partially-erased IndexPipeline approach in the next commit. --- Cargo.lock | 8 + Cargo.toml | 2 + packages/zaino-sync/Cargo.toml | 17 ++ packages/zaino-sync/src/backend.rs | 63 ++++++++ packages/zaino-sync/src/dag.rs | 119 ++++++++++++++ packages/zaino-sync/src/descriptor.rs | 165 +++++++++++++++++++ packages/zaino-sync/src/engine.rs | 58 +++++++ packages/zaino-sync/src/erased.rs | 105 +++++++++++++ packages/zaino-sync/src/lib.rs | 16 ++ packages/zaino-sync/src/progress.rs | 14 ++ packages/zaino-sync/src/provisioner.rs | 53 +++++++ packages/zaino-sync/src/traits.rs | 209 +++++++++++++++++++++++++ 12 files changed, 829 insertions(+) create mode 100644 packages/zaino-sync/Cargo.toml create mode 100644 packages/zaino-sync/src/backend.rs create mode 100644 packages/zaino-sync/src/dag.rs create mode 100644 packages/zaino-sync/src/descriptor.rs create mode 100644 packages/zaino-sync/src/engine.rs create mode 100644 packages/zaino-sync/src/erased.rs create mode 100644 packages/zaino-sync/src/lib.rs create mode 100644 packages/zaino-sync/src/progress.rs create mode 100644 packages/zaino-sync/src/provisioner.rs create mode 100644 packages/zaino-sync/src/traits.rs diff --git a/Cargo.lock b/Cargo.lock index 6df8aaeed..adcc2354c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6092,6 +6092,14 @@ dependencies = [ "zebra-state 9.0.1", ] +[[package]] +name = "zaino-sync" +version = "0.1.0" +dependencies = [ + "bitflags 2.13.0", + "thiserror 2.0.18", +] + [[package]] name = "zaino-testutils" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 8f50df514..9c0e50422 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "packages/zaino-state", "packages/zaino-fetch", "packages/zaino-proto", + "packages/zaino-sync", # Live-test suite (docs/adr/0002, docs/adr/0003, docs/adr/0004). # zaino-testutils first: the e2e/clientless partitions depend on it. "live-tests/zaino-testutils", @@ -23,6 +24,7 @@ default-members = [ "packages/zaino-state", "packages/zaino-fetch", "packages/zaino-proto", + "packages/zaino-sync", ] # Use the edition 2021 dependency resolver in the workspace, to match the crates diff --git a/packages/zaino-sync/Cargo.toml b/packages/zaino-sync/Cargo.toml new file mode 100644 index 000000000..0c1b4af70 --- /dev/null +++ b/packages/zaino-sync/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "zaino-sync" +version = "0.1.0" +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +description = "DAG-driven parallel sync engine for blockchain index building." + +[dependencies] +thiserror.workspace = true +bitflags.workspace = true + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" diff --git a/packages/zaino-sync/src/backend.rs b/packages/zaino-sync/src/backend.rs new file mode 100644 index 000000000..d1b056070 --- /dev/null +++ b/packages/zaino-sync/src/backend.rs @@ -0,0 +1,63 @@ +//! Backend trait — the storage layer the engine commits to and reads from. + +use crate::traits::WriteOp; + +/// Errors from backend operations. +#[derive(Debug, thiserror::Error)] +pub enum BackendError { + /// A write/commit/flush operation failed. + #[error("backend write error: {0}")] + Write(String), + /// A read operation failed. + #[error("backend read error: {0}")] + Read(String), +} + +/// Whether the backend supports per-index writers or forces a shared writer. +/// +/// This affects how the engine parallelises the commit step: +/// - `SharedWriter`: all indexes' write ops go through a single serialised writer. +/// - `PerIndexWriter`: each index gets its own writer; commits parallelise. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WriterTopology { + /// All indexes share a single writer. Writes are serialised. + SharedWriter, + /// Each index has its own writer. Writes parallelise. + PerIndexWriter, +} + +/// The storage backend the engine commits to and reads from. +/// +/// Generic — no blockchain or LMDB knowledge. Concrete implementations +/// (e.g. LMDB, in-memory for tests) live outside this crate. +pub trait Backend: Send + Sync { + /// Reader handle type. + type Reader: BackendReader; + /// Writer handle type. + type Writer: BackendWriter; + + /// Obtain a read handle. May be called concurrently. + fn reader(&self) -> Result; + + /// Obtain a write handle. + fn writer(&self) -> Result; + + /// Force durability of all committed data. + fn flush(&self) -> Result<(), BackendError>; + + /// Whether this backend supports per-index writers. + fn topology(&self) -> WriterTopology; +} + +/// Write handle. The engine sends batches of [`WriteOp`]s through this. +pub trait BackendWriter: Send { + /// Commit a batch of write operations atomically. + fn commit(&mut self, ops: Vec) -> Result<(), BackendError>; +} + +/// Read handle. Used by [`DepsReader`](crate::traits::DepsReader) and +/// the engine's progress tracking. +pub trait BackendReader: Send { + /// Read a single key from a named index. + fn get(&self, index_name: &str, key: &[u8]) -> Result>, BackendError>; +} diff --git a/packages/zaino-sync/src/dag.rs b/packages/zaino-sync/src/dag.rs new file mode 100644 index 000000000..44d3e3bf5 --- /dev/null +++ b/packages/zaino-sync/src/dag.rs @@ -0,0 +1,119 @@ +//! Dependency DAG construction and phase assignment. + +use std::collections::HashMap; + +use crate::descriptor::{CompositionType, Descriptor, InputScope}; + +/// A node in the dependency DAG, holding the descriptor and computed +/// scheduling metadata. +#[derive(Debug)] +pub struct DagNode { + /// The index's declarative descriptor. + pub descriptor: Descriptor, + /// Topological phase (layer in the DAG). Phase 0 has no dependencies. + pub phase: u32, +} + +/// The dependency DAG over registered indexes. +/// +/// Built at startup from the set of descriptors. The engine uses the DAG +/// to determine phase assignment, per-edge firing rules, and batch +/// scheduling. +#[derive(Debug)] +pub struct DependencyDag { + nodes: HashMap<&'static str, DagNode>, + edges: Vec, +} + +/// A directed edge in the DAG: `from` must commit before `to` can extract. +#[derive(Debug)] +pub struct DagEdge { + /// Name of the dependency (upstream index). + pub from: &'static str, + /// Name of the dependent (downstream index). + pub to: &'static str, + /// Scheduling rule derived from the dependency's composition type + /// and the dependent's read pattern. + pub firing: FiringRule, +} + +/// Per-edge firing rule — when may the downstream index begin extracting +/// blocks that depend on the upstream index's output? +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FiringRule { + /// Downstream can begin as soon as the upstream commits a batch + /// containing the needed height (pipelined). + Pipelined, + /// Downstream must wait until the upstream has committed the entire + /// chain range (barrier). Required when the downstream reads + /// forward or globally from the upstream. + Barrier, +} + +/// Errors during DAG construction. +#[derive(Debug, thiserror::Error)] +pub enum DagError { + /// A dependency references an index name that was not registered. + #[error("unknown dependency: {from} -> {to}")] + UnknownDependency { + /// The index that declared the dependency. + from: &'static str, + /// The dependency name that was not found. + to: &'static str, + }, + /// The dependency graph contains a cycle. + #[error("cycle detected involving: {participants:?}")] + CycleDetected { + /// The index names involved in the cycle. + participants: Vec<&'static str>, + }, +} + +impl DependencyDag { + /// Build the DAG from a set of descriptors. + /// + /// Validates acyclicity and computes phase assignments. + pub fn build(_descriptors: Vec) -> Result { + todo!() + } + + /// Return indexes grouped by phase, in phase order. + pub fn phases(&self) -> Vec> { + todo!() + } + + /// Return the firing rule for the edge between two indexes. + pub fn firing_rule(&self, _from: &str, _to: &str) -> Option { + todo!() + } + + /// Compute the scheduling properties of a given cell (scope × composition). + pub fn cell_properties( + _scope: InputScope, + _composition: CompositionType, + ) -> CellSchedulingProps { + CellSchedulingProps { + parallel_extract: matches!(_scope, InputScope::BlockLocal), + parallel_merge: matches!(_composition, CompositionType::Monoidal), + sequential_merge: matches!(_composition, CompositionType::Fold), + requires_phase_gate: matches!(_scope, InputScope::CrossIndex), + requires_self_feedback: matches!(_scope, InputScope::SelfCumulative), + } + } +} + +/// Scheduling properties mechanically derived from a (scope, composition) cell. +/// No runtime state — these are static properties of the index type. +#[derive(Debug, Clone, Copy)] +pub struct CellSchedulingProps { + /// Can extraction run in parallel across blocks? + pub parallel_extract: bool, + /// Can the merge step run as a parallel reduce? + pub parallel_merge: bool, + /// Must the merge step apply deltas sequentially? + pub sequential_merge: bool, + /// Must this index wait for dependency phases to commit? + pub requires_phase_gate: bool, + /// Must this index read its own prior committed state per block? + pub requires_self_feedback: bool, +} diff --git a/packages/zaino-sync/src/descriptor.rs b/packages/zaino-sync/src/descriptor.rs new file mode 100644 index 000000000..ac84486bc --- /dev/null +++ b/packages/zaino-sync/src/descriptor.rs @@ -0,0 +1,165 @@ +//! Declarative index descriptors and type-level axis markers. + +use bitflags::bitflags; + +// --------------------------------------------------------------------------- +// Sealed marker traits — one per axis. +// Implementors live only in this module; downstream code selects but cannot +// extend the set of axis values. +// --------------------------------------------------------------------------- + +mod sealed { + pub trait Scope: Send + Sync + 'static {} + pub trait Composition: Send + Sync + 'static {} +} + +// --------------------------------------------------------------------------- +// Axis 1 — InputScope (what data the extractor needs beyond the block) +// --------------------------------------------------------------------------- + +/// Extraction uses only the current block's context. +/// No DepsReader, no prior state, no source handle. +pub struct BlockLocal; + +/// Extraction needs this index's own accumulated state from prior blocks. +pub struct SelfCumulative; + +/// Extraction needs committed output from other indexes (via DepsReader). +pub struct CrossIndex; + +impl sealed::Scope for BlockLocal {} +impl sealed::Scope for SelfCumulative {} +impl sealed::Scope for CrossIndex {} + +/// Runtime-inspectable mirror of the type-level scope marker. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum InputScope { + /// Only needs the current block's context. + BlockLocal, + /// Needs own accumulated state from prior blocks. + SelfCumulative, + /// Needs committed output from other indexes. + CrossIndex, +} + +/// Bridge from type-level marker to runtime enum. +pub trait Scope: sealed::Scope { + /// The runtime-inspectable value matching this marker type. + const VALUE: InputScope; +} + +impl Scope for BlockLocal { + const VALUE: InputScope = InputScope::BlockLocal; +} +impl Scope for SelfCumulative { + const VALUE: InputScope = InputScope::SelfCumulative; +} +impl Scope for CrossIndex { + const VALUE: InputScope = InputScope::CrossIndex; +} + +// --------------------------------------------------------------------------- +// Axis 2 — CompositionType (how per-block deltas are merged) +// --------------------------------------------------------------------------- + +/// Disjoint keys across blocks. Merge = collect. +pub struct Append; + +/// Overlapping keys with associative + commutative combine. +/// Merge can be parallelised with a reduce tree. +pub struct Monoidal; + +/// Order-dependent. Must apply in chain order. Merge is sequential. +pub struct Fold; + +impl sealed::Composition for Append {} +impl sealed::Composition for Monoidal {} +impl sealed::Composition for Fold {} + +/// Runtime-inspectable mirror of the type-level composition marker. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CompositionType { + /// Disjoint keys. Merge = collect. + Append, + /// Associative + commutative. Merge = parallel reduce. + Monoidal, + /// Order-dependent. Merge = sequential fold. + Fold, +} + +/// Bridge from type-level marker to runtime enum. +pub trait Composition: sealed::Composition { + /// The runtime-inspectable value matching this marker type. + const VALUE: CompositionType; +} + +impl Composition for Append { + const VALUE: CompositionType = CompositionType::Append; +} +impl Composition for Monoidal { + const VALUE: CompositionType = CompositionType::Monoidal; +} +impl Composition for Fold { + const VALUE: CompositionType = CompositionType::Fold; +} + +// --------------------------------------------------------------------------- +// Source access — whether extraction may reach the source for non-local data +// --------------------------------------------------------------------------- + +/// Whether an extractor needs a source handle beyond the block context. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SourceAccess { + /// Extraction is pure: BlockContext + (optional deps/prior state) only. + None, + /// Extraction may call the source for non-local data. + /// The engine provides a source handle and adjusts scheduling. + NonLocal, +} + +// --------------------------------------------------------------------------- +// Source requirements — what the provisioner must fetch per block +// --------------------------------------------------------------------------- + +bitflags! { + /// Declared per-index. The provisioner computes the union across all + /// registered indexes and fetches only what is needed. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct SourceRequirements: u32 { + /// Raw block data (always implied). + const BLOCK = 0b0001; + /// Sapling/Orchard commitment tree roots. + const TREE_ROOTS = 0b0010; + /// Cumulative tree sizes. + const TREE_SIZES = 0b0100; + /// Chainwork of parent block. + const PARENT_WORK = 0b1000; + } +} + +// --------------------------------------------------------------------------- +// Descriptor — the full declarative spec of an index +// --------------------------------------------------------------------------- + +/// Static, declarative properties of an index. No logic. +/// +/// Carries both runtime-inspectable enums (for the engine's DAG builder) +/// and is associated with type-level markers (via [`IndexDef`]) for +/// compile-time enforcement of valid operations. +/// +/// [`IndexDef`]: super::traits::IndexDef +#[derive(Debug, Clone)] +pub struct Descriptor { + /// Unique name, used as the key in the DAG and in WriteOps. + pub name: &'static str, + /// What data the extractor needs beyond the block. + pub scope: InputScope, + /// How per-block deltas are merged. + pub composition: CompositionType, + /// Names of indexes this one depends on (must form a DAG). + pub dependencies: &'static [&'static str], + /// What the provisioner must fetch for this index. + pub requirements: SourceRequirements, + /// Whether extraction may reach the source for non-local data. + pub source_access: SourceAccess, +} diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs new file mode 100644 index 000000000..f091cd3d2 --- /dev/null +++ b/packages/zaino-sync/src/engine.rs @@ -0,0 +1,58 @@ +//! Sync engine — the orchestrator. +//! +//! Owns the DAG, dispatches extraction to workers, drives the +//! merge → commit → flush pipeline, and enforces phase gates. +//! Contains no blockchain knowledge. + +use crate::backend::Backend; +use crate::dag::DependencyDag; +use crate::erased::ErasedIndex; + +/// Configuration for the sync engine. +#[derive(Debug, Clone)] +pub struct EngineConfig { + /// Number of blocks per persistence batch. + pub batch_size: u32, + /// Maximum number of concurrent extraction tasks. + pub extraction_parallelism: usize, +} + +/// The sync engine. +/// +/// Generic over the backend. Holds the DAG, the registered (erased) indexes, +/// and drives the pipeline. +pub struct SyncEngine { + dag: DependencyDag, + indexes: Vec>, + backend: B, + config: EngineConfig, +} + +impl SyncEngine { + /// Create a new engine from a built DAG, registered indexes, and backend. + pub fn new( + dag: DependencyDag, + indexes: Vec>, + backend: B, + config: EngineConfig, + ) -> Self { + Self { + dag, + indexes, + backend, + config, + } + } + + // NOTE: the main `sync_range` / `sync_to_height` method is omitted + // from this initial sketch. It will orchestrate: + // + // 1. Configure provisioner with union of SourceRequirements. + // 2. Spawn provisioner task (streams BlockContexts). + // 3. Per phase: + // a. Spawn extraction workers (respecting scope constraints). + // b. Accumulate deltas in bounded channel. + // c. On batch boundary: merge → commit → flush → advance watermark. + // d. Signal downstream phases via phase gate. + // 4. On completion: final flush, report progress. +} diff --git a/packages/zaino-sync/src/erased.rs b/packages/zaino-sync/src/erased.rs new file mode 100644 index 000000000..13dbfc8e2 --- /dev/null +++ b/packages/zaino-sync/src/erased.rs @@ -0,0 +1,105 @@ +//! Type-erased index interface for the engine. +//! +//! The trait hierarchy in [`crate::traits`] gives compile-time safety at the +//! index *definition* site. The engine, however, must handle all 9 cells of +//! the (Scope × Composition) grid uniformly through dynamic dispatch. This +//! module bridges the two: blanket impls on concrete (Scope, Composition) +//! pairs produce [`ErasedIndex`] trait objects that the engine stores and +//! dispatches through. +//! +//! The engine never inspects `Delta` or `Accumulator` — it shuttles opaque +//! values between extract → merge → write_ops. Type erasure loses nothing +//! the engine needs. + +use std::any::Any; + +use crate::descriptor::Descriptor; +use crate::traits::{ExtractError, WriteOp}; + +/// Opaque delta produced by extraction, carrying the concrete `Delta` type +/// inside a `Box`. +pub struct ErasedDelta(pub(crate) Box); + +/// Type-erased interface the engine dispatches through. +/// +/// One `Box` per registered index. The engine uses +/// [`Descriptor`] to decide scheduling, then calls the erased methods +/// without knowing the concrete types. +pub trait ErasedIndex: Send + Sync { + /// The declarative descriptor. + fn descriptor(&self) -> &Descriptor; + + /// Extract a delta for one block. + /// + /// The engine provides the right inputs based on `descriptor().scope`: + /// - `BlockLocal`: `ctx` only (prior_state and deps are None). + /// - `SelfCumulative`: `ctx` + `prior_state`. + /// - `CrossIndex`: `ctx` + `deps`. + /// + /// Passing the wrong combination is a bug in the engine, not the index. + fn extract_erased( + &self, + ctx: &dyn Any, + prior_state: Option<&dyn Any>, + deps: Option<&crate::traits::DepsReader>, + ) -> Result; + + /// Merge a batch of deltas into write operations. + /// + /// The engine calls this based on `descriptor().composition`: + /// - `Append`: deltas are simply flattened. + /// - `Monoidal`: deltas are reduced with the declared monoid. + /// - `Fold`: deltas are folded in chain order. + /// + /// `deltas` is in chain order regardless of composition type. + fn merge_erased(&self, deltas: Vec) -> Result, MergeError>; +} + +/// Errors during the merge step. +#[derive(Debug, thiserror::Error)] +pub enum MergeError { + /// A delta's concrete type didn't match what this index expected. + /// Indicates an engine bug (wrong delta routed to wrong index). + #[error("delta type mismatch: expected {expected}")] + DeltaTypeMismatch { + /// The index name that expected a different delta type. + expected: &'static str, + }, + /// The merge logic itself failed. + #[error("merge failed: {0}")] + Failed(String), +} + +// =========================================================================== +// Blanket implementations — one per (Scope × Composition) cell. +// +// These connect the typed trait hierarchy to the erased interface. +// The engine never sees the concrete types; it works entirely through +// `Box`. +// +// Only the cells that have a concrete index registered will have an +// ErasedIndex instance. The blanket impls ensure that any type +// satisfying the right trait bounds automatically becomes erasable. +// =========================================================================== + +// TODO: blanket impls for each (Scope, Composition) pair. +// +// Each blanket impl will: +// 1. Store a PhantomData for the concrete index type. +// 2. In extract_erased: downcast `ctx` to `I::Context`, call the +// scope-specific extract, box the resulting Delta. +// 3. In merge_erased: downcast each ErasedDelta to `I::Delta`, call +// the composition-specific merge, return WriteOps. +// +// Example shape (not yet implemented): +// +// struct ErasedLocalAppend(PhantomData); +// +// impl ErasedIndex for ErasedLocalAppend +// where +// I: ExtractLocal + MergeAppend, +// { +// fn descriptor(&self) -> &Descriptor { ... } +// fn extract_erased(...) -> ... { ... } +// fn merge_erased(...) -> ... { ... } +// } diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs new file mode 100644 index 000000000..93a7f409d --- /dev/null +++ b/packages/zaino-sync/src/lib.rs @@ -0,0 +1,16 @@ +//! DAG-driven parallel sync engine for blockchain index building. +//! +//! Generic — contains no blockchain-specific knowledge. Blockchain-specific +//! index implementations and provisioners live in downstream crates. + +#![forbid(unsafe_code)] +#![warn(missing_docs)] + +pub mod backend; +pub mod dag; +pub mod descriptor; +pub mod engine; +pub mod erased; +pub mod progress; +pub mod provisioner; +pub mod traits; diff --git a/packages/zaino-sync/src/progress.rs b/packages/zaino-sync/src/progress.rs new file mode 100644 index 000000000..86e49210d --- /dev/null +++ b/packages/zaino-sync/src/progress.rs @@ -0,0 +1,14 @@ +//! Progress tracking and crash recovery. +//! +//! The engine maintains a persistent watermark: the height of the last +//! fully-flushed batch. On restart, it resumes from the watermark. +//! Partially committed batches are discarded (the backend's atomic commit +//! guarantees no partial state). + +/// Persistent sync progress. +#[derive(Debug, Clone, Copy)] +pub struct SyncProgress { + /// The height of the last fully-flushed batch (inclusive). + /// `None` if no batch has been flushed yet. + pub watermark: Option, +} diff --git a/packages/zaino-sync/src/provisioner.rs b/packages/zaino-sync/src/provisioner.rs new file mode 100644 index 000000000..754cd4e5c --- /dev/null +++ b/packages/zaino-sync/src/provisioner.rs @@ -0,0 +1,53 @@ +//! Provisioner trait — source data acquisition. +//! +//! The provisioner owns all source access. Indexes never call the source +//! directly (except via the [`NonLocalSource`](crate::traits::NonLocalSource) +//! escape hatch). The engine tells the provisioner what data to fetch +//! (via [`SourceRequirements`](crate::descriptor::SourceRequirements)), +//! and the provisioner streams block contexts as they become ready. + +use crate::descriptor::SourceRequirements; + +/// Errors from provisioner operations. +#[derive(Debug, thiserror::Error)] +pub enum ProvisionError { + /// The source was unreachable or returned an error. + #[error("source error: {0}")] + Source(String), + /// The channel to the engine was closed. + #[error("channel closed")] + ChannelClosed, +} + +/// The provisioner: first-class owner of all source access. +/// +/// Generic — no blockchain knowledge. The `BlockContext` associated type +/// is opaque to the engine; extractors know its concrete type through +/// [`IndexDef::Context`](crate::traits::IndexDef::Context). +/// +/// The provisioner streams contexts as they become ready. It may +/// internally parallelise source fetches. Backpressure propagates +/// from the engine's bounded channel. +pub trait Provisioner: Send + Sync { + /// Opaque block context type. Matches `IndexDef::Context` for all + /// registered indexes. + type BlockContext: Send + Sync; + + /// Height type. + type Height: Copy + Ord + Send + Sync; + + /// Configure which source data to fetch, based on the union of all + /// registered indexes' requirements. + fn configure(&mut self, requirements: SourceRequirements); + + // NOTE: the streaming `provision` method is intentionally omitted + // from this initial sketch. It will depend on the async runtime + // choice (tokio channels, crossbeam, etc.) and the engine's pipeline + // design. The signature will look roughly like: + // + // async fn provision( + // &self, + // range: Range, + // tx: Sender<(Self::Height, Self::BlockContext)>, + // ) -> Result<(), ProvisionError>; +} diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs new file mode 100644 index 000000000..618e06165 --- /dev/null +++ b/packages/zaino-sync/src/traits.rs @@ -0,0 +1,209 @@ +//! Type-level trait hierarchy for index definitions. +//! +//! Each index implements [`IndexDef`] to declare its two-axis position, then +//! exactly one extract trait (scope axis) and exactly one merge trait +//! (composition axis). The trait bounds make it a compile error to, e.g., +//! access a `DepsReader` from a `BlockLocal` index or declare a monoidal +//! `combine` on an `Append` index. + +use crate::descriptor::{ + Append, BlockLocal, Composition, CrossIndex, Descriptor, Fold, Monoidal, Scope, + SelfCumulative, +}; + +// --------------------------------------------------------------------------- +// Placeholder types — will be fleshed out in their own modules +// --------------------------------------------------------------------------- + +/// Read handle over committed index state, restricted to earlier phases. +pub struct DepsReader; + +/// Handle for non-local source access (the escape hatch). +pub struct SourceHandle; + +/// A single write operation produced by the merge step. +#[derive(Debug)] +pub enum WriteOp { + /// Insert or overwrite a key-value pair. + Put { + /// Index (table) name. + index: &'static str, + /// Serialised key. + key: Vec, + /// Serialised value. + value: Vec, + }, + /// Remove a key. + Delete { + /// Index (table) name. + index: &'static str, + /// Serialised key. + key: Vec, + }, +} + +// --------------------------------------------------------------------------- +// IndexDef — the root trait that pins both axes +// --------------------------------------------------------------------------- + +/// Root declaration for any index. Pins the scope and composition axes as +/// associated types, which downstream extract/merge traits use as bounds. +/// +/// `S` and `C` are marker types from [`crate::descriptor`]. +pub trait IndexDef: Send + Sync + 'static { + /// Type-level scope marker (BlockLocal | SelfCumulative | CrossIndex). + type Scope: Scope; + + /// Type-level composition marker (Append | Monoidal | Fold). + type Composition: Composition; + + /// The per-block contribution produced by extraction. + type Delta: Send + Sync; + + /// The block context type (matches the provisioner's output). + type Context: Send + Sync; + + /// The full declarative descriptor. + fn descriptor() -> Descriptor; +} + +// =========================================================================== +// Axis 1: Extract traits — one per scope value. +// The bound `IndexDef` makes it a compile error to implement +// the wrong one. +// =========================================================================== + +/// Extraction for block-local indexes. +/// +/// The extract signature takes only the block context — no deps reader, +/// no prior state, no source handle. This is the only scope that permits +/// full parallelism across blocks. +pub trait ExtractLocal: IndexDef { + /// Produce this block's delta from the block context alone. + fn extract(ctx: &Self::Context) -> Result; +} + +/// Extraction for self-cumulative indexes. +/// +/// The extract signature includes a read handle to this index's own +/// committed prior state. Extraction for this index is sequential (each +/// block depends on the previous block's committed output), but the index +/// runs in parallel with other indexes that have no dependency on it. +pub trait ExtractCumulative: IndexDef { + /// Opaque type representing this index's accumulated state, as read + /// from the backend after the prior batch's commit. + type PriorState: Send + Sync; + + /// Produce this block's delta given the block context and this index's + /// own committed state up to (but not including) this block. + fn extract( + ctx: &Self::Context, + prior: &Self::PriorState, + ) -> Result; +} + +/// Extraction for cross-index indexes. +/// +/// The extract signature includes a [`DepsReader`] — a read handle +/// restricted to committed state from indexes declared in the dependency +/// set. This index runs in a later phase than its dependencies. +pub trait ExtractCross: IndexDef { + /// Produce this block's delta given the block context and committed + /// state from dependency indexes. + fn extract(ctx: &Self::Context, deps: &DepsReader) -> Result; +} + +// =========================================================================== +// Axis 2: Merge traits — one per composition value. +// The bound `IndexDef` makes it a compile error to +// implement the wrong one. +// =========================================================================== + +/// Merge for append-type indexes (disjoint keys). +/// +/// No actual merge logic — each delta's write ops are collected and applied. +/// The engine can batch writes from multiple blocks without any combine step. +pub trait MergeAppend: IndexDef { + /// Convert a single block's delta directly to write operations. + fn to_write_ops(delta: Self::Delta) -> Vec; +} + +/// Merge for monoidal-type indexes (associative + commutative combine). +/// +/// The engine may merge deltas from multiple blocks in any order using a +/// parallel reduce tree. The implementor must ensure `combine` is +/// associative and commutative — the type system cannot enforce these +/// algebraic properties, but the engine relies on them for correctness. +pub trait MergeMonoidal: IndexDef { + /// The intermediate type used during the reduce. + type Accumulator: Send + Sync; + + /// The identity element: `combine(identity(), x) == x`. + fn identity() -> Self::Accumulator; + + /// Lift a single delta into the accumulator space. + fn lift(delta: Self::Delta) -> Self::Accumulator; + + /// Associative, commutative combine of two accumulators. + fn combine(a: Self::Accumulator, b: Self::Accumulator) -> Self::Accumulator; + + /// Convert the fully-merged accumulator to write operations. + fn to_write_ops(merged: Self::Accumulator) -> Vec; +} + +/// Merge for fold-type indexes (order-dependent sequential application). +/// +/// Deltas must be folded in strict chain order. The engine cannot +/// parallelise the merge step for this composition type. +pub trait MergeFold: IndexDef { + /// The running state threaded through the fold. + type FoldState: Send + Sync; + + /// The initial state before the first block in a batch. + fn initial_state() -> Self::FoldState; + + /// Apply one block's delta to the running state. Called in chain order. + fn fold(state: &mut Self::FoldState, delta: Self::Delta); + + /// Convert the final fold state to write operations. + fn to_write_ops(state: Self::FoldState) -> Vec; +} + +// =========================================================================== +// Source-access overlay — orthogonal to both axes. +// =========================================================================== + +/// Implemented by indexes that declared `SourceAccess::NonLocal` in their +/// descriptor. Provides a source handle to the extract method. +/// +/// This is an escape hatch. The default path is pure extraction from +/// BlockContext + (prior state | deps). When an index needs non-local source +/// data and adding an intermediate index is disproportionate, it implements +/// this trait to receive a [`SourceHandle`] during extraction. +/// +/// The engine uses the presence of this impl (via the erased layer) to +/// adjust scheduling — non-local extractions are assumed I/O-bound and +/// may be given dedicated task slots. +pub trait NonLocalSource: IndexDef { + /// Provide the source handle. Called by the engine just before + /// extraction for each block. + fn with_source(source: &SourceHandle) -> SourceGuard<'_>; +} + +/// RAII guard holding a reference to the source handle for the duration of +/// one extraction call. +pub struct SourceGuard<'a> { + _handle: &'a SourceHandle, +} + +// =========================================================================== +// Error types (stub) +// =========================================================================== + +/// Errors during index extraction. +#[derive(Debug, thiserror::Error)] +pub enum ExtractError { + /// A generic extraction failure. + #[error("extraction failed: {0}")] + Failed(String), +} From cd04f0fe72547dc6e94d8068b57ffa8cead2cf2b Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 13:48:13 -0300 Subject: [PATCH 002/146] Replace erased layer with partially-erased IndexPipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delta types stay inside each index's pipeline boundary — the engine only sees Ctx in and Vec out. No dyn Any, no downcasting. SyncEngine is now generic over Ctx (the provisioner's block context). --- packages/zaino-sync/src/engine.rs | 32 ++++----- packages/zaino-sync/src/erased.rs | 105 ---------------------------- packages/zaino-sync/src/lib.rs | 2 +- packages/zaino-sync/src/pipeline.rs | 98 ++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 125 deletions(-) delete mode 100644 packages/zaino-sync/src/erased.rs create mode 100644 packages/zaino-sync/src/pipeline.rs diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index f091cd3d2..3af5cb80f 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -6,7 +6,7 @@ use crate::backend::Backend; use crate::dag::DependencyDag; -use crate::erased::ErasedIndex; +use crate::pipeline::IndexPipeline; /// Configuration for the sync engine. #[derive(Debug, Clone)] @@ -19,20 +19,26 @@ pub struct EngineConfig { /// The sync engine. /// -/// Generic over the backend. Holds the DAG, the registered (erased) indexes, -/// and drives the pipeline. -pub struct SyncEngine { +/// Generic over: +/// - `Ctx`: the provisioner's block context type (concrete, shared across +/// all indexes — no type erasure). +/// - `B`: the storage backend. +/// +/// Holds the DAG and a heterogeneous collection of indexes via +/// `dyn IndexPipeline`. Delta types stay inside each index's +/// pipeline — the engine only sees `Ctx` in and `Vec` out. +pub struct SyncEngine { dag: DependencyDag, - indexes: Vec>, + indexes: Vec>>, backend: B, config: EngineConfig, } -impl SyncEngine { +impl SyncEngine { /// Create a new engine from a built DAG, registered indexes, and backend. pub fn new( dag: DependencyDag, - indexes: Vec>, + indexes: Vec>>, backend: B, config: EngineConfig, ) -> Self { @@ -43,16 +49,4 @@ impl SyncEngine { config, } } - - // NOTE: the main `sync_range` / `sync_to_height` method is omitted - // from this initial sketch. It will orchestrate: - // - // 1. Configure provisioner with union of SourceRequirements. - // 2. Spawn provisioner task (streams BlockContexts). - // 3. Per phase: - // a. Spawn extraction workers (respecting scope constraints). - // b. Accumulate deltas in bounded channel. - // c. On batch boundary: merge → commit → flush → advance watermark. - // d. Signal downstream phases via phase gate. - // 4. On completion: final flush, report progress. } diff --git a/packages/zaino-sync/src/erased.rs b/packages/zaino-sync/src/erased.rs deleted file mode 100644 index 13dbfc8e2..000000000 --- a/packages/zaino-sync/src/erased.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! Type-erased index interface for the engine. -//! -//! The trait hierarchy in [`crate::traits`] gives compile-time safety at the -//! index *definition* site. The engine, however, must handle all 9 cells of -//! the (Scope × Composition) grid uniformly through dynamic dispatch. This -//! module bridges the two: blanket impls on concrete (Scope, Composition) -//! pairs produce [`ErasedIndex`] trait objects that the engine stores and -//! dispatches through. -//! -//! The engine never inspects `Delta` or `Accumulator` — it shuttles opaque -//! values between extract → merge → write_ops. Type erasure loses nothing -//! the engine needs. - -use std::any::Any; - -use crate::descriptor::Descriptor; -use crate::traits::{ExtractError, WriteOp}; - -/// Opaque delta produced by extraction, carrying the concrete `Delta` type -/// inside a `Box`. -pub struct ErasedDelta(pub(crate) Box); - -/// Type-erased interface the engine dispatches through. -/// -/// One `Box` per registered index. The engine uses -/// [`Descriptor`] to decide scheduling, then calls the erased methods -/// without knowing the concrete types. -pub trait ErasedIndex: Send + Sync { - /// The declarative descriptor. - fn descriptor(&self) -> &Descriptor; - - /// Extract a delta for one block. - /// - /// The engine provides the right inputs based on `descriptor().scope`: - /// - `BlockLocal`: `ctx` only (prior_state and deps are None). - /// - `SelfCumulative`: `ctx` + `prior_state`. - /// - `CrossIndex`: `ctx` + `deps`. - /// - /// Passing the wrong combination is a bug in the engine, not the index. - fn extract_erased( - &self, - ctx: &dyn Any, - prior_state: Option<&dyn Any>, - deps: Option<&crate::traits::DepsReader>, - ) -> Result; - - /// Merge a batch of deltas into write operations. - /// - /// The engine calls this based on `descriptor().composition`: - /// - `Append`: deltas are simply flattened. - /// - `Monoidal`: deltas are reduced with the declared monoid. - /// - `Fold`: deltas are folded in chain order. - /// - /// `deltas` is in chain order regardless of composition type. - fn merge_erased(&self, deltas: Vec) -> Result, MergeError>; -} - -/// Errors during the merge step. -#[derive(Debug, thiserror::Error)] -pub enum MergeError { - /// A delta's concrete type didn't match what this index expected. - /// Indicates an engine bug (wrong delta routed to wrong index). - #[error("delta type mismatch: expected {expected}")] - DeltaTypeMismatch { - /// The index name that expected a different delta type. - expected: &'static str, - }, - /// The merge logic itself failed. - #[error("merge failed: {0}")] - Failed(String), -} - -// =========================================================================== -// Blanket implementations — one per (Scope × Composition) cell. -// -// These connect the typed trait hierarchy to the erased interface. -// The engine never sees the concrete types; it works entirely through -// `Box`. -// -// Only the cells that have a concrete index registered will have an -// ErasedIndex instance. The blanket impls ensure that any type -// satisfying the right trait bounds automatically becomes erasable. -// =========================================================================== - -// TODO: blanket impls for each (Scope, Composition) pair. -// -// Each blanket impl will: -// 1. Store a PhantomData for the concrete index type. -// 2. In extract_erased: downcast `ctx` to `I::Context`, call the -// scope-specific extract, box the resulting Delta. -// 3. In merge_erased: downcast each ErasedDelta to `I::Delta`, call -// the composition-specific merge, return WriteOps. -// -// Example shape (not yet implemented): -// -// struct ErasedLocalAppend(PhantomData); -// -// impl ErasedIndex for ErasedLocalAppend -// where -// I: ExtractLocal + MergeAppend, -// { -// fn descriptor(&self) -> &Descriptor { ... } -// fn extract_erased(...) -> ... { ... } -// fn merge_erased(...) -> ... { ... } -// } diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index 93a7f409d..3d6320ef0 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -10,7 +10,7 @@ pub mod backend; pub mod dag; pub mod descriptor; pub mod engine; -pub mod erased; +pub mod pipeline; pub mod progress; pub mod provisioner; pub mod traits; diff --git a/packages/zaino-sync/src/pipeline.rs b/packages/zaino-sync/src/pipeline.rs new file mode 100644 index 000000000..3f331c874 --- /dev/null +++ b/packages/zaino-sync/src/pipeline.rs @@ -0,0 +1,98 @@ +//! Partially-erased index pipeline. +//! +//! The typed trait hierarchy in [`crate::traits`] enforces correct +//! implementations at the index definition site. The engine, however, +//! needs to hold heterogeneous indexes in a single collection and +//! dispatch uniformly. +//! +//! The solution: erase only what must cross the trait-object boundary. +//! `Delta`, `Accumulator`, and `FoldState` stay *inside* the index — +//! they never leave its pipeline methods. The engine sees only: +//! +//! - `Ctx` in (the provisioner's block context — shared, concrete type) +//! - `Vec` out +//! +//! No `dyn Any`, no downcasting, no runtime type mismatches. + +use crate::descriptor::Descriptor; +use crate::traits::{DepsReader, ExtractError, WriteOp}; + +/// Per-block opaque contribution from one index's extraction. +/// +/// The engine holds these between extract and merge. The concrete +/// content is only meaningful to the index that produced it — the +/// engine never inspects it. +/// +/// Implemented as an index-specific closure over the delta, avoiding +/// `dyn Any`. The merge step calls back into the index to consume it. +pub struct BlockContribution { + write_ops: Vec, +} + +impl BlockContribution { + /// Create a contribution from pre-computed write ops (Append path). + pub fn from_write_ops(ops: Vec) -> Self { + Self { write_ops: ops } + } + + /// Consume into write operations. + pub fn into_write_ops(self) -> Vec { + self.write_ops + } +} + +/// Errors during pipeline operations. +#[derive(Debug, thiserror::Error)] +pub enum PipelineError { + /// Extraction failed. + #[error(transparent)] + Extract(#[from] ExtractError), + /// Merge/write-ops conversion failed. + #[error("merge failed: {0}")] + Merge(String), +} + +/// The trait-object-safe interface the engine dispatches through. +/// +/// `Ctx` is the provisioner's block context type — shared across all +/// indexes, kept concrete (not erased). The engine is generic over +/// `Ctx` once, not per-index. +/// +/// Each method on this trait encapsulates a full extract-or-merge step. +/// The `Delta` type never crosses this boundary — it lives and dies +/// inside the index's implementation of these methods. +pub trait IndexPipeline: Send + Sync { + /// The declarative descriptor. + fn descriptor(&self) -> &Descriptor; + + /// Extract one block's contribution. + /// + /// For `BlockLocal` indexes: `deps` is `None`. + /// For `SelfCumulative` indexes: the index reads its own prior state + /// from the backend internally (the pipeline impl holds a reader). + /// For `CrossIndex` indexes: `deps` provides committed state from + /// dependency indexes. + /// + /// Returns a [`BlockContribution`] — opaque to the engine, consumed + /// by [`merge_batch`](Self::merge_batch). + fn extract_block( + &self, + ctx: &Ctx, + deps: Option<&DepsReader>, + ) -> Result; + + /// Merge a batch of block contributions into final write operations. + /// + /// `contributions` is in chain order. The index applies its + /// composition-specific merge internally: + /// - `Append`: flatten (already done — contributions carry WriteOps). + /// - `Monoidal`: parallel reduce using the declared monoid. + /// - `Fold`: sequential application in chain order. + /// + /// The engine doesn't need to know which strategy is used — it just + /// gets `Vec` back. + fn merge_batch( + &self, + contributions: Vec, + ) -> Result, PipelineError>; +} From 3943f3a10506a40864138425bbfc33b3acee6508 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 13:54:02 -0300 Subject: [PATCH 003/146] Add primitives module with IndexId and BlockHeight newtypes Replaces raw &'static str and u64 across the crate with domain-specific newtypes for compile-time type safety. --- packages/zaino-sync/src/backend.rs | 4 +-- packages/zaino-sync/src/dag.rs | 25 +++++++++--------- packages/zaino-sync/src/descriptor.rs | 8 +++--- packages/zaino-sync/src/lib.rs | 1 + packages/zaino-sync/src/primitives.rs | 7 +++++ .../zaino-sync/src/primitives/block_height.rs | 23 ++++++++++++++++ .../zaino-sync/src/primitives/index_id.rs | 26 +++++++++++++++++++ packages/zaino-sync/src/progress.rs | 4 ++- packages/zaino-sync/src/traits.rs | 9 ++++--- 9 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 packages/zaino-sync/src/primitives.rs create mode 100644 packages/zaino-sync/src/primitives/block_height.rs create mode 100644 packages/zaino-sync/src/primitives/index_id.rs diff --git a/packages/zaino-sync/src/backend.rs b/packages/zaino-sync/src/backend.rs index d1b056070..e4b6e97ae 100644 --- a/packages/zaino-sync/src/backend.rs +++ b/packages/zaino-sync/src/backend.rs @@ -58,6 +58,6 @@ pub trait BackendWriter: Send { /// Read handle. Used by [`DepsReader`](crate::traits::DepsReader) and /// the engine's progress tracking. pub trait BackendReader: Send { - /// Read a single key from a named index. - fn get(&self, index_name: &str, key: &[u8]) -> Result>, BackendError>; + /// Read a single key from the given index. + fn get(&self, index: crate::primitives::IndexId, key: &[u8]) -> Result>, BackendError>; } diff --git a/packages/zaino-sync/src/dag.rs b/packages/zaino-sync/src/dag.rs index 44d3e3bf5..bcf4819a7 100644 --- a/packages/zaino-sync/src/dag.rs +++ b/packages/zaino-sync/src/dag.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use crate::descriptor::{CompositionType, Descriptor, InputScope}; +use crate::primitives::IndexId; /// A node in the dependency DAG, holding the descriptor and computed /// scheduling metadata. @@ -21,17 +22,17 @@ pub struct DagNode { /// scheduling. #[derive(Debug)] pub struct DependencyDag { - nodes: HashMap<&'static str, DagNode>, + nodes: HashMap, edges: Vec, } /// A directed edge in the DAG: `from` must commit before `to` can extract. #[derive(Debug)] pub struct DagEdge { - /// Name of the dependency (upstream index). - pub from: &'static str, - /// Name of the dependent (downstream index). - pub to: &'static str, + /// The dependency (upstream index). + pub from: IndexId, + /// The dependent (downstream index). + pub to: IndexId, /// Scheduling rule derived from the dependency's composition type /// and the dependent's read pattern. pub firing: FiringRule, @@ -53,19 +54,19 @@ pub enum FiringRule { /// Errors during DAG construction. #[derive(Debug, thiserror::Error)] pub enum DagError { - /// A dependency references an index name that was not registered. + /// A dependency references an index that was not registered. #[error("unknown dependency: {from} -> {to}")] UnknownDependency { /// The index that declared the dependency. - from: &'static str, - /// The dependency name that was not found. - to: &'static str, + from: IndexId, + /// The dependency that was not found. + to: IndexId, }, /// The dependency graph contains a cycle. #[error("cycle detected involving: {participants:?}")] CycleDetected { - /// The index names involved in the cycle. - participants: Vec<&'static str>, + /// The indexes involved in the cycle. + participants: Vec, }, } @@ -83,7 +84,7 @@ impl DependencyDag { } /// Return the firing rule for the edge between two indexes. - pub fn firing_rule(&self, _from: &str, _to: &str) -> Option { + pub fn firing_rule(&self, _from: IndexId, _to: IndexId) -> Option { todo!() } diff --git a/packages/zaino-sync/src/descriptor.rs b/packages/zaino-sync/src/descriptor.rs index ac84486bc..f3dc656d4 100644 --- a/packages/zaino-sync/src/descriptor.rs +++ b/packages/zaino-sync/src/descriptor.rs @@ -2,6 +2,8 @@ use bitflags::bitflags; +use crate::primitives::IndexId; + // --------------------------------------------------------------------------- // Sealed marker traits — one per axis. // Implementors live only in this module; downstream code selects but cannot @@ -151,13 +153,13 @@ bitflags! { #[derive(Debug, Clone)] pub struct Descriptor { /// Unique name, used as the key in the DAG and in WriteOps. - pub name: &'static str, + pub name: IndexId, /// What data the extractor needs beyond the block. pub scope: InputScope, /// How per-block deltas are merged. pub composition: CompositionType, - /// Names of indexes this one depends on (must form a DAG). - pub dependencies: &'static [&'static str], + /// Indexes this one depends on (must form a DAG). + pub dependencies: &'static [IndexId], /// What the provisioner must fetch for this index. pub requirements: SourceRequirements, /// Whether extraction may reach the source for non-local data. diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index 3d6320ef0..dc7c4a171 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -11,6 +11,7 @@ pub mod dag; pub mod descriptor; pub mod engine; pub mod pipeline; +pub mod primitives; pub mod progress; pub mod provisioner; pub mod traits; diff --git a/packages/zaino-sync/src/primitives.rs b/packages/zaino-sync/src/primitives.rs new file mode 100644 index 000000000..e640825f5 --- /dev/null +++ b/packages/zaino-sync/src/primitives.rs @@ -0,0 +1,7 @@ +//! Foundational types used across the sync engine. + +mod block_height; +mod index_id; + +pub use block_height::BlockHeight; +pub use index_id::IndexId; diff --git a/packages/zaino-sync/src/primitives/block_height.rs b/packages/zaino-sync/src/primitives/block_height.rs new file mode 100644 index 000000000..5e2b12573 --- /dev/null +++ b/packages/zaino-sync/src/primitives/block_height.rs @@ -0,0 +1,23 @@ +//! Block height newtype. + +/// A block height on the chain. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct BlockHeight(u64); + +impl BlockHeight { + /// Create a block height. + pub const fn new(height: u64) -> Self { + Self(height) + } + + /// The raw numeric value. + pub const fn value(&self) -> u64 { + self.0 + } +} + +impl core::fmt::Display for BlockHeight { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/packages/zaino-sync/src/primitives/index_id.rs b/packages/zaino-sync/src/primitives/index_id.rs new file mode 100644 index 000000000..6e5852492 --- /dev/null +++ b/packages/zaino-sync/src/primitives/index_id.rs @@ -0,0 +1,26 @@ +//! Unique identifier for a registered index. + +/// Unique identifier for a registered index. +/// +/// Newtype over `&'static str` — prevents accidental use of arbitrary +/// strings where an index name is expected. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct IndexId(&'static str); + +impl IndexId { + /// Create an index identifier from a static string. + pub const fn new(name: &'static str) -> Self { + Self(name) + } + + /// The string value. + pub const fn as_str(&self) -> &'static str { + self.0 + } +} + +impl core::fmt::Display for IndexId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(self.0) + } +} diff --git a/packages/zaino-sync/src/progress.rs b/packages/zaino-sync/src/progress.rs index 86e49210d..2898b3132 100644 --- a/packages/zaino-sync/src/progress.rs +++ b/packages/zaino-sync/src/progress.rs @@ -5,10 +5,12 @@ //! Partially committed batches are discarded (the backend's atomic commit //! guarantees no partial state). +use crate::primitives::BlockHeight; + /// Persistent sync progress. #[derive(Debug, Clone, Copy)] pub struct SyncProgress { /// The height of the last fully-flushed batch (inclusive). /// `None` if no batch has been flushed yet. - pub watermark: Option, + pub watermark: Option, } diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index 618e06165..eef6f5485 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -10,6 +10,7 @@ use crate::descriptor::{ Append, BlockLocal, Composition, CrossIndex, Descriptor, Fold, Monoidal, Scope, SelfCumulative, }; +use crate::primitives::IndexId; // --------------------------------------------------------------------------- // Placeholder types — will be fleshed out in their own modules @@ -26,8 +27,8 @@ pub struct SourceHandle; pub enum WriteOp { /// Insert or overwrite a key-value pair. Put { - /// Index (table) name. - index: &'static str, + /// Target index. + index: IndexId, /// Serialised key. key: Vec, /// Serialised value. @@ -35,8 +36,8 @@ pub enum WriteOp { }, /// Remove a key. Delete { - /// Index (table) name. - index: &'static str, + /// Target index. + index: IndexId, /// Serialised key. key: Vec, }, From 2a453c260723a86f9f99e44c3eb335fd73017957 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 13:58:35 -0300 Subject: [PATCH 004/146] Add PhaseIndex primitive, document BlockHeight duplication PhaseIndex wraps u32 for DAG phase indices. BlockHeight documents the overlap with zaino-state's type and notes the path toward a shared zaino-primitives crate. Provisioner Height associated type collapsed to BlockHeight. --- packages/zaino-sync/src/dag.rs | 4 +-- packages/zaino-sync/src/primitives.rs | 2 ++ .../zaino-sync/src/primitives/block_height.rs | 8 ++++++ .../zaino-sync/src/primitives/phase_index.rs | 26 +++++++++++++++++++ packages/zaino-sync/src/provisioner.rs | 3 --- 5 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 packages/zaino-sync/src/primitives/phase_index.rs diff --git a/packages/zaino-sync/src/dag.rs b/packages/zaino-sync/src/dag.rs index bcf4819a7..7eb9b6f19 100644 --- a/packages/zaino-sync/src/dag.rs +++ b/packages/zaino-sync/src/dag.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use crate::descriptor::{CompositionType, Descriptor, InputScope}; -use crate::primitives::IndexId; +use crate::primitives::{IndexId, PhaseIndex}; /// A node in the dependency DAG, holding the descriptor and computed /// scheduling metadata. @@ -12,7 +12,7 @@ pub struct DagNode { /// The index's declarative descriptor. pub descriptor: Descriptor, /// Topological phase (layer in the DAG). Phase 0 has no dependencies. - pub phase: u32, + pub phase: PhaseIndex, } /// The dependency DAG over registered indexes. diff --git a/packages/zaino-sync/src/primitives.rs b/packages/zaino-sync/src/primitives.rs index e640825f5..75562a397 100644 --- a/packages/zaino-sync/src/primitives.rs +++ b/packages/zaino-sync/src/primitives.rs @@ -2,6 +2,8 @@ mod block_height; mod index_id; +mod phase_index; pub use block_height::BlockHeight; pub use index_id::IndexId; +pub use phase_index::PhaseIndex; diff --git a/packages/zaino-sync/src/primitives/block_height.rs b/packages/zaino-sync/src/primitives/block_height.rs index 5e2b12573..79518d9cc 100644 --- a/packages/zaino-sync/src/primitives/block_height.rs +++ b/packages/zaino-sync/src/primitives/block_height.rs @@ -1,4 +1,12 @@ //! Block height newtype. +//! +//! NOTE: zaino-state defines its own `BlockHeight`. Long-term, shared +//! primitives like this should live in a single zero-dependency crate +//! (e.g. `zaino-primitives`) that both zaino-sync and zaino-state depend +//! on. The sync engine's height needs are likely a subset of the domain +//! type's (Ord + Copy + Display), so unification should be +//! straightforward. Until then, this local definition avoids coupling +//! zaino-sync to zaino-state. /// A block height on the chain. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] diff --git a/packages/zaino-sync/src/primitives/phase_index.rs b/packages/zaino-sync/src/primitives/phase_index.rs new file mode 100644 index 000000000..c34505bb9 --- /dev/null +++ b/packages/zaino-sync/src/primitives/phase_index.rs @@ -0,0 +1,26 @@ +//! Phase index newtype. + +/// A topological phase in the dependency DAG. +/// +/// Phase 0 contains indexes with no dependencies. Each subsequent phase +/// contains indexes whose dependencies are all in earlier phases. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct PhaseIndex(u32); + +impl PhaseIndex { + /// Create a phase index. + pub const fn new(index: u32) -> Self { + Self(index) + } + + /// The raw numeric value. + pub const fn value(&self) -> u32 { + self.0 + } +} + +impl core::fmt::Display for PhaseIndex { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/packages/zaino-sync/src/provisioner.rs b/packages/zaino-sync/src/provisioner.rs index 754cd4e5c..0f4cfc158 100644 --- a/packages/zaino-sync/src/provisioner.rs +++ b/packages/zaino-sync/src/provisioner.rs @@ -33,9 +33,6 @@ pub trait Provisioner: Send + Sync { /// registered indexes. type BlockContext: Send + Sync; - /// Height type. - type Height: Copy + Ord + Send + Sync; - /// Configure which source data to fetch, based on the union of all /// registered indexes' requirements. fn configure(&mut self, requirements: SourceRequirements); From 0a2e2f9ece3413000b681c741b91baafa0080bce Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 16:13:33 -0300 Subject: [PATCH 005/146] Add bridge impls and DAG builder with topo sort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridge module connects typed trait hierarchy to IndexPipeline for runtime dispatch. Three BlockLocal bridges (Append, Monoidal, Fold) validate that the scope × composition trait bounds compose. DAG builder implements Kahn's algorithm for topological sort, cycle detection, phase assignment, and firing rule derivation. Pipeline trait refactored from extract_block+merge_batch to process_batch (MVP decision documented — true north is split extract_one+merge_batch with opaque DeltaToken intermediates). --- packages/zaino-sync/src/bridge.rs | 238 +++++++++++++++++++++ packages/zaino-sync/src/dag.rs | 317 ++++++++++++++++++++++++++-- packages/zaino-sync/src/lib.rs | 1 + packages/zaino-sync/src/pipeline.rs | 116 +++++----- 4 files changed, 598 insertions(+), 74 deletions(-) create mode 100644 packages/zaino-sync/src/bridge.rs diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs new file mode 100644 index 000000000..5d0eaba2b --- /dev/null +++ b/packages/zaino-sync/src/bridge.rs @@ -0,0 +1,238 @@ +//! Bridge implementations connecting typed traits to [`IndexPipeline`]. +//! +//! # What this module does +//! +//! The typed trait hierarchy ([`ExtractLocal`], [`MergeAppend`], etc.) +//! gives compile-time safety at the index definition site. The engine +//! needs runtime dispatch via [`IndexPipeline`]. Bridges are the +//! glue: each bridge struct wraps a concrete index type (via +//! `PhantomData`) and implements `IndexPipeline` by calling the +//! type-level extract and merge traits internally. The `Delta` type +//! never leaves the bridge. +//! +//! One bridge per (scope × composition) cell. Currently implemented: +//! - [`LocalAppendBridge`] — BlockLocal × Append +//! - [`LocalMonoidalBridge`] — BlockLocal × Monoidal +//! - [`LocalFoldBridge`] — BlockLocal × Fold +//! +//! SelfCumulative and CrossIndex bridges are not yet implemented — they +//! need backend reader access that the pipeline interface doesn't yet +//! provide. +//! +//! # MVP limitation: sequential extraction within each bridge +//! +//! Every `process_batch` impl currently loops sequentially over +//! `blocks`. For `BlockLocal` indexes, extraction *could* run in +//! parallel across blocks — the type system already proves there are no +//! inter-block dependencies (that is the whole point of the `BlockLocal` +//! scope marker). But because `process_batch` collapses extract + merge +//! into one call, the engine has no way to schedule individual +//! extractions onto a shared thread pool or interleave work across +//! indexes. +//! +//! This is intentional for the MVP: it validates that the type algebra +//! composes end-to-end without solving the intermediate type problem. +//! +//! # True north: split `extract_one` + `merge_batch` +//! +//! The intended production design splits the pipeline interface so the +//! engine controls extraction parallelism: +//! +//! ```text +//! fn extract_one(&self, ctx: &Ctx, ...) -> Result; +//! fn merge_batch(&self, tokens: Vec) -> Result, ...>; +//! ``` +//! +//! `DeltaToken` would be an opaque handle (e.g., an index into a +//! bridge-internal `Vec`) — no `dyn Any`, no downcasting. The +//! bridge owns the typed delta storage; the engine holds and routes +//! opaque tokens. See [`crate::pipeline`] module docs for details. +//! +//! [`IndexPipeline`]: crate::pipeline::IndexPipeline +//! [`ExtractLocal`]: crate::traits::ExtractLocal +//! [`MergeAppend`]: crate::traits::MergeAppend + +use std::marker::PhantomData; + +use crate::descriptor::Descriptor; +use crate::pipeline::{IndexPipeline, PipelineError}; +use crate::traits::{ + DepsReader, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, WriteOp, +}; + +// =========================================================================== +// BlockLocal × Append +// =========================================================================== + +/// Bridge for indexes that are block-local with disjoint-key (append) merge. +/// +/// **Parallelism profile (true north):** +/// - Extraction: fully parallel across blocks (no inter-block deps). +/// - Merge: trivial collect — no combine step needed. Each delta's +/// write ops can be emitted independently. +/// +/// **MVP:** extraction runs sequentially over blocks. No parallelism +/// because `process_batch` owns the full loop. See module docs. +pub struct LocalAppendBridge { + descriptor: Descriptor, + _index: PhantomData, +} + +impl LocalAppendBridge { + fn new() -> Self { + Self { + descriptor: I::descriptor(), + _index: PhantomData, + } + } +} + +impl IndexPipeline for LocalAppendBridge +where + I: ExtractLocal + MergeAppend + IndexDef, + Ctx: Send + Sync + 'static, +{ + fn descriptor(&self) -> &Descriptor { + &self.descriptor + } + + fn process_batch( + &self, + blocks: &[Ctx], + _deps: Option<&DepsReader>, + ) -> Result, PipelineError> { + let mut all_ops = Vec::new(); + for ctx in blocks { + let delta = I::extract(ctx)?; + all_ops.extend(I::to_write_ops(delta)); + } + Ok(all_ops) + } +} + +/// Register a BlockLocal × Append index into the pipeline. +pub fn local_append() -> Box> +where + I: ExtractLocal + MergeAppend + IndexDef, + Ctx: Send + Sync + 'static, +{ + Box::new(LocalAppendBridge::::new()) +} + +// =========================================================================== +// BlockLocal × Monoidal +// =========================================================================== + +/// Bridge for indexes that are block-local with monoidal (associative + +/// commutative) merge. +/// +/// **Parallelism profile (true north):** +/// - Extraction: fully parallel across blocks (no inter-block deps). +/// - Merge: parallel reduce tree — `combine` is associative + +/// commutative, so deltas can be reduced in any order. +/// +/// **MVP:** both extraction and merge run sequentially. See module docs. +pub struct LocalMonoidalBridge { + descriptor: Descriptor, + _index: PhantomData, +} + +impl LocalMonoidalBridge { + fn new() -> Self { + Self { + descriptor: I::descriptor(), + _index: PhantomData, + } + } +} + +impl IndexPipeline for LocalMonoidalBridge +where + I: ExtractLocal + MergeMonoidal + IndexDef, + Ctx: Send + Sync + 'static, +{ + fn descriptor(&self) -> &Descriptor { + &self.descriptor + } + + fn process_batch( + &self, + blocks: &[Ctx], + _deps: Option<&DepsReader>, + ) -> Result, PipelineError> { + let mut acc = I::identity(); + for ctx in blocks { + let delta = I::extract(ctx)?; + acc = I::combine(acc, I::lift(delta)); + } + Ok(I::to_write_ops(acc)) + } +} + +/// Register a BlockLocal × Monoidal index into the pipeline. +pub fn local_monoidal() -> Box> +where + I: ExtractLocal + MergeMonoidal + IndexDef, + Ctx: Send + Sync + 'static, +{ + Box::new(LocalMonoidalBridge::::new()) +} + +// =========================================================================== +// BlockLocal × Fold +// =========================================================================== + +/// Bridge for indexes that are block-local with order-dependent (fold) merge. +/// +/// **Parallelism profile (true north):** +/// - Extraction: fully parallel across blocks (no inter-block deps). +/// - Merge: strictly sequential — `fold` must apply deltas in chain +/// order. This is inherent to the Fold composition type, not an MVP +/// limitation. +/// +/// **MVP:** extraction also runs sequentially. See module docs. +pub struct LocalFoldBridge { + descriptor: Descriptor, + _index: PhantomData, +} + +impl LocalFoldBridge { + fn new() -> Self { + Self { + descriptor: I::descriptor(), + _index: PhantomData, + } + } +} + +impl IndexPipeline for LocalFoldBridge +where + I: ExtractLocal + MergeFold + IndexDef, + Ctx: Send + Sync + 'static, +{ + fn descriptor(&self) -> &Descriptor { + &self.descriptor + } + + fn process_batch( + &self, + blocks: &[Ctx], + _deps: Option<&DepsReader>, + ) -> Result, PipelineError> { + let mut state = I::initial_state(); + for ctx in blocks { + let delta = I::extract(ctx)?; + I::fold(&mut state, delta); + } + Ok(I::to_write_ops(state)) + } +} + +/// Register a BlockLocal × Fold index into the pipeline. +pub fn local_fold() -> Box> +where + I: ExtractLocal + MergeFold + IndexDef, + Ctx: Send + Sync + 'static, +{ + Box::new(LocalFoldBridge::::new()) +} diff --git a/packages/zaino-sync/src/dag.rs b/packages/zaino-sync/src/dag.rs index 7eb9b6f19..aa059c918 100644 --- a/packages/zaino-sync/src/dag.rs +++ b/packages/zaino-sync/src/dag.rs @@ -1,6 +1,14 @@ //! Dependency DAG construction and phase assignment. +//! +//! Built at startup from the set of [`Descriptor`]s. The DAG determines: +//! - **Phase assignment**: topological layers. Phase 0 has no dependencies; +//! each subsequent phase depends only on earlier phases. +//! - **Edges and firing rules**: when downstream indexes may begin work +//! relative to upstream commits. +//! - **Cell scheduling properties**: static parallelism/sequentiality +//! characteristics derived from (scope × composition). -use std::collections::HashMap; +use std::collections::{HashMap, HashSet, VecDeque}; use crate::descriptor::{CompositionType, Descriptor, InputScope}; use crate::primitives::{IndexId, PhaseIndex}; @@ -24,6 +32,7 @@ pub struct DagNode { pub struct DependencyDag { nodes: HashMap, edges: Vec, + phase_count: u32, } /// A directed edge in the DAG: `from` must commit before `to` can extract. @@ -68,37 +77,98 @@ pub enum DagError { /// The indexes involved in the cycle. participants: Vec, }, + /// Two descriptors share the same index name. + #[error("duplicate index name: {0}")] + DuplicateName(IndexId), } impl DependencyDag { /// Build the DAG from a set of descriptors. /// - /// Validates acyclicity and computes phase assignments. - pub fn build(_descriptors: Vec) -> Result { - todo!() + /// Validates uniqueness, dependency existence, and acyclicity, then + /// computes phase assignments and firing rules. + pub fn build(descriptors: Vec) -> Result { + let names = validate_unique_names(&descriptors)?; + let raw_edges = collect_edges(&descriptors, &names)?; + let phase_map = toposort_phases(&descriptors, &raw_edges)?; + + let phase_count = phase_map + .values() + .map(|p| p.value() + 1) + .max() + .unwrap_or(0); + + let desc_map: HashMap = + descriptors.iter().map(|d| (d.name, d)).collect(); + + let edges = raw_edges + .iter() + .map(|&(from, to)| DagEdge { + from, + to, + firing: derive_firing_rule( + desc_map.get(&from).expect("validated"), + desc_map.get(&to).expect("validated"), + ), + }) + .collect(); + + let nodes = descriptors + .into_iter() + .map(|desc| { + let phase = phase_map[&desc.name]; + (desc.name, DagNode { descriptor: desc, phase }) + }) + .collect(); + + Ok(Self { nodes, edges, phase_count }) } /// Return indexes grouped by phase, in phase order. pub fn phases(&self) -> Vec> { - todo!() + let mut result: Vec> = (0..self.phase_count).map(|_| Vec::new()).collect(); + for node in self.nodes.values() { + let idx = usize::try_from(node.phase.value()) + .expect("phase index fits in usize"); + result[idx].push(node); + } + result + } + + /// Total number of phases. + pub fn phase_count(&self) -> u32 { + self.phase_count + } + + /// Look up a node by index id. + pub fn node(&self, id: IndexId) -> Option<&DagNode> { + self.nodes.get(&id) + } + + /// All edges in the DAG. + pub fn edges(&self) -> &[DagEdge] { + &self.edges } /// Return the firing rule for the edge between two indexes. - pub fn firing_rule(&self, _from: IndexId, _to: IndexId) -> Option { - todo!() + pub fn firing_rule(&self, from: IndexId, to: IndexId) -> Option { + self.edges + .iter() + .find(|e| e.from == from && e.to == to) + .map(|e| e.firing) } /// Compute the scheduling properties of a given cell (scope × composition). pub fn cell_properties( - _scope: InputScope, - _composition: CompositionType, + scope: InputScope, + composition: CompositionType, ) -> CellSchedulingProps { CellSchedulingProps { - parallel_extract: matches!(_scope, InputScope::BlockLocal), - parallel_merge: matches!(_composition, CompositionType::Monoidal), - sequential_merge: matches!(_composition, CompositionType::Fold), - requires_phase_gate: matches!(_scope, InputScope::CrossIndex), - requires_self_feedback: matches!(_scope, InputScope::SelfCumulative), + parallel_extract: matches!(scope, InputScope::BlockLocal), + parallel_merge: matches!(composition, CompositionType::Monoidal), + sequential_merge: matches!(composition, CompositionType::Fold), + requires_phase_gate: matches!(scope, InputScope::CrossIndex), + requires_self_feedback: matches!(scope, InputScope::SelfCumulative), } } } @@ -118,3 +188,222 @@ pub struct CellSchedulingProps { /// Must this index read its own prior committed state per block? pub requires_self_feedback: bool, } + +/// Validate that all descriptor names are unique. +fn validate_unique_names(descriptors: &[Descriptor]) -> Result, DagError> { + let mut names = HashSet::new(); + for desc in descriptors { + if !names.insert(desc.name) { + return Err(DagError::DuplicateName(desc.name)); + } + } + Ok(names) +} + +/// Extract (from, to) edges from descriptors, validating that all +/// declared dependencies reference registered indexes. +fn collect_edges( + descriptors: &[Descriptor], + registered: &HashSet, +) -> Result, DagError> { + let mut edges = Vec::new(); + for desc in descriptors { + for &dep in desc.dependencies { + if !registered.contains(&dep) { + return Err(DagError::UnknownDependency { + from: desc.name, + to: dep, + }); + } + edges.push((dep, desc.name)); + } + } + Ok(edges) +} + +/// Kahn's algorithm: topological sort producing a phase assignment. +/// +/// Returns a map from index id to its phase. Phase 0 has no +/// dependencies, phase 1 depends only on phase 0, etc. +/// +/// Detects cycles: if any nodes remain after all zero-in-degree nodes +/// are exhausted, those nodes form a cycle. +fn toposort_phases( + descriptors: &[Descriptor], + edges: &[(IndexId, IndexId)], +) -> Result, DagError> { + let mut in_degree: HashMap = descriptors.iter().map(|d| (d.name, 0)).collect(); + let mut dependents: HashMap> = HashMap::new(); + + for &(from, to) in edges { + *in_degree.entry(to).or_insert(0) += 1; + dependents.entry(from).or_default().push(to); + } + + let mut queue: VecDeque = in_degree + .iter() + .filter(|(_, °)| deg == 0) + .map(|(&id, _)| id) + .collect(); + + let mut phase_map: HashMap = HashMap::new(); + let mut visited = 0usize; + let mut phase_idx = 0u32; + + while !queue.is_empty() { + let layer: Vec = queue.drain(..).collect(); + let phase = PhaseIndex::new(phase_idx); + for &id in &layer { + visited += 1; + phase_map.insert(id, phase); + if let Some(downstream) = dependents.get(&id) { + for &dep in downstream { + let deg = in_degree + .get_mut(&dep) + .expect("all nodes in in_degree map"); + *deg -= 1; + if *deg == 0 { + queue.push_back(dep); + } + } + } + } + phase_idx += 1; + } + + if visited != descriptors.len() { + let participants: Vec = in_degree + .iter() + .filter(|(_, °)| deg > 0) + .map(|(&id, _)| id) + .collect(); + return Err(DagError::CycleDetected { participants }); + } + + Ok(phase_map) +} + +/// Derive the firing rule for an edge. +/// +/// Conservative default: `Pipelined` (downstream can start as soon as +/// upstream commits a batch). `Barrier` when the upstream uses +/// `NonLocal` source access, signalling incomplete output until the +/// full chain is processed. +/// +/// Simplification: the formal model derives firing rules from the +/// dependency's read pattern (R≤ backward = pipelined, R* global = +/// barrier). We use source_access as a proxy until read-pattern +/// declarations are added to the descriptor. +fn derive_firing_rule(upstream: &Descriptor, _downstream: &Descriptor) -> FiringRule { + if upstream.source_access == crate::descriptor::SourceAccess::NonLocal { + FiringRule::Barrier + } else { + FiringRule::Pipelined + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::descriptor::{SourceAccess, SourceRequirements}; + + const A: IndexId = IndexId::new("a"); + const B: IndexId = IndexId::new("b"); + const C: IndexId = IndexId::new("c"); + const MISSING: IndexId = IndexId::new("missing"); + + const DEPS_NONE: &[IndexId] = &[]; + const DEPS_A: &[IndexId] = &[A]; + const DEPS_B: &[IndexId] = &[B]; + const DEPS_AB: &[IndexId] = &[A, B]; + const DEPS_MISSING: &[IndexId] = &[MISSING]; + + fn desc(name: &'static str, deps: &'static [IndexId]) -> Descriptor { + Descriptor { + name: IndexId::new(name), + scope: InputScope::BlockLocal, + composition: CompositionType::Append, + dependencies: deps, + requirements: SourceRequirements::BLOCK, + source_access: SourceAccess::None, + } + } + + #[test] + fn single_index_phase_zero() -> Result<(), DagError> { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)])?; + assert_eq!(dag.phase_count(), 1); + let phases = dag.phases(); + assert_eq!(phases.len(), 1); + assert_eq!(phases[0].len(), 1); + assert_eq!(phases[0][0].descriptor.name, A); + Ok(()) + } + + #[test] + fn two_independent_same_phase() -> Result<(), DagError> { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE), desc("b", DEPS_NONE)])?; + assert_eq!(dag.phase_count(), 1); + let phases = dag.phases(); + assert_eq!(phases[0].len(), 2); + Ok(()) + } + + #[test] + fn linear_chain_three_phases() -> Result<(), DagError> { + let dag = DependencyDag::build(vec![ + desc("a", DEPS_NONE), + desc("b", DEPS_A), + desc("c", DEPS_B), + ])?; + assert_eq!(dag.phase_count(), 3); + + let a_node = dag.node(A).expect("a exists"); + let b_node = dag.node(B).expect("b exists"); + let c_node = dag.node(C).expect("c exists"); + assert_eq!(a_node.phase, PhaseIndex::new(0)); + assert_eq!(b_node.phase, PhaseIndex::new(1)); + assert_eq!(c_node.phase, PhaseIndex::new(2)); + Ok(()) + } + + #[test] + fn diamond_two_phases() -> Result<(), DagError> { + let dag = DependencyDag::build(vec![ + desc("a", DEPS_NONE), + desc("b", DEPS_NONE), + desc("c", DEPS_AB), + ])?; + assert_eq!(dag.phase_count(), 2); + + let c_node = dag.node(C).expect("c exists"); + assert_eq!(c_node.phase, PhaseIndex::new(1)); + Ok(()) + } + + #[test] + fn cycle_detected() { + let result = DependencyDag::build(vec![desc("a", DEPS_B), desc("b", DEPS_A)]); + assert!(matches!(result, Err(DagError::CycleDetected { .. }))); + } + + #[test] + fn unknown_dependency() { + let result = DependencyDag::build(vec![desc("a", DEPS_MISSING)]); + assert!(matches!(result, Err(DagError::UnknownDependency { .. }))); + } + + #[test] + fn duplicate_name() { + let result = DependencyDag::build(vec![desc("a", DEPS_NONE), desc("a", DEPS_NONE)]); + assert!(matches!(result, Err(DagError::DuplicateName(_)))); + } + + #[test] + fn edges_have_pipelined_firing_by_default() -> Result<(), DagError> { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE), desc("b", DEPS_A)])?; + let rule = dag.firing_rule(A, B).expect("edge exists"); + assert_eq!(rule, FiringRule::Pipelined); + Ok(()) + } +} diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index dc7c4a171..8650aec4c 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -7,6 +7,7 @@ #![warn(missing_docs)] pub mod backend; +pub mod bridge; pub mod dag; pub mod descriptor; pub mod engine; diff --git a/packages/zaino-sync/src/pipeline.rs b/packages/zaino-sync/src/pipeline.rs index 3f331c874..a636cd32a 100644 --- a/packages/zaino-sync/src/pipeline.rs +++ b/packages/zaino-sync/src/pipeline.rs @@ -5,41 +5,50 @@ //! needs to hold heterogeneous indexes in a single collection and //! dispatch uniformly. //! -//! The solution: erase only what must cross the trait-object boundary. -//! `Delta`, `Accumulator`, and `FoldState` stay *inside* the index — -//! they never leave its pipeline methods. The engine sees only: +//! [`IndexPipeline`] is the trait-object-safe interface the engine +//! works with. Each index's `Delta`, `Accumulator`, and `FoldState` stay +//! *inside* its bridge implementation — they never cross the trait +//! boundary. The engine sees only `&[Ctx]` in, `Vec` out. //! -//! - `Ctx` in (the provisioner's block context — shared, concrete type) -//! - `Vec` out +//! Bridge types in [`crate::bridge`] connect the typed traits to this +//! interface. //! -//! No `dyn Any`, no downcasting, no runtime type mismatches. +//! # Current limitation: `process_batch` collapses extract and merge +//! +//! The current interface exposes a single `process_batch` method that +//! takes `&[Ctx]` and returns `Vec`. This means the engine +//! cannot control extraction parallelism — it hands a whole batch to +//! each index and gets final results back. Extraction within the bridge +//! runs sequentially. +//! +//! This is a **conscious MVP decision** to validate that the type algebra +//! composes end-to-end without solving the intermediate type problem yet. +//! +//! # True north: split `extract_one` + `merge_batch` +//! +//! The intended design splits the interface into two methods so the +//! engine can schedule per-block extractions onto a shared thread pool +//! and control parallelism across indexes: +//! +//! ```text +//! fn extract_one(&self, ctx: &Ctx, deps: ...) -> Result; +//! fn merge_batch(&self, tokens: Vec) -> Result, ...>; +//! ``` +//! +//! `DeltaToken` would be an opaque handle (e.g., an index into a +//! bridge-internal `Vec`) that does not expose the concrete +//! `Delta` type across the trait boundary — no `dyn Any`, no +//! downcasting. The bridge owns the typed storage; the engine holds +//! and routes opaque tokens. +//! +//! This split is required to unlock: +//! - Per-block parallel extraction for `BlockLocal` indexes. +//! - Engine-controlled work-stealing across indexes sharing a thread pool. +//! - Streaming extraction (process blocks as the provisioner delivers +//! them, rather than buffering a full batch upfront). use crate::descriptor::Descriptor; -use crate::traits::{DepsReader, ExtractError, WriteOp}; - -/// Per-block opaque contribution from one index's extraction. -/// -/// The engine holds these between extract and merge. The concrete -/// content is only meaningful to the index that produced it — the -/// engine never inspects it. -/// -/// Implemented as an index-specific closure over the delta, avoiding -/// `dyn Any`. The merge step calls back into the index to consume it. -pub struct BlockContribution { - write_ops: Vec, -} - -impl BlockContribution { - /// Create a contribution from pre-computed write ops (Append path). - pub fn from_write_ops(ops: Vec) -> Self { - Self { write_ops: ops } - } - - /// Consume into write operations. - pub fn into_write_ops(self) -> Vec { - self.write_ops - } -} +use crate::traits::{ExtractError, WriteOp}; /// Errors during pipeline operations. #[derive(Debug, thiserror::Error)] @@ -58,41 +67,28 @@ pub enum PipelineError { /// indexes, kept concrete (not erased). The engine is generic over /// `Ctx` once, not per-index. /// -/// Each method on this trait encapsulates a full extract-or-merge step. -/// The `Delta` type never crosses this boundary — it lives and dies -/// inside the index's implementation of these methods. +/// Each call to [`process_batch`](Self::process_batch) encapsulates the +/// full extract → merge → to_write_ops pipeline for one batch of blocks. +/// The `Delta` type never crosses this boundary. +/// +/// **MVP shape.** This will be split into `extract_one` + `merge_batch` +/// once the `DeltaToken` intermediate design is resolved. See module docs. pub trait IndexPipeline: Send + Sync { /// The declarative descriptor. fn descriptor(&self) -> &Descriptor; - /// Extract one block's contribution. - /// - /// For `BlockLocal` indexes: `deps` is `None`. - /// For `SelfCumulative` indexes: the index reads its own prior state - /// from the backend internally (the pipeline impl holds a reader). - /// For `CrossIndex` indexes: `deps` provides committed state from - /// dependency indexes. - /// - /// Returns a [`BlockContribution`] — opaque to the engine, consumed - /// by [`merge_batch`](Self::merge_batch). - fn extract_block( - &self, - ctx: &Ctx, - deps: Option<&DepsReader>, - ) -> Result; - - /// Merge a batch of block contributions into final write operations. + /// Process a batch of blocks through the full pipeline. /// - /// `contributions` is in chain order. The index applies its - /// composition-specific merge internally: - /// - `Append`: flatten (already done — contributions carry WriteOps). - /// - `Monoidal`: parallel reduce using the declared monoid. - /// - `Fold`: sequential application in chain order. + /// Internally performs: + /// 1. Extraction: produce a delta per block (scope-specific inputs). + /// 2. Merge: combine deltas according to composition type. + /// 3. Conversion: turn merged result into write operations. /// - /// The engine doesn't need to know which strategy is used — it just - /// gets `Vec` back. - fn merge_batch( + /// `blocks` is in chain order. The engine does not inspect + /// intermediates — it receives final `WriteOp`s ready for commit. + fn process_batch( &self, - contributions: Vec, + blocks: &[Ctx], + deps: Option<&crate::traits::DepsReader>, ) -> Result, PipelineError>; } From c828d16dfc6eaeab0f08e3f9cc597442ea6136a8 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 16:51:14 -0300 Subject: [PATCH 006/146] Add engine sync loop, in-memory backend, and end-to-end tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine implements sync_range: splits blocks into batches, processes each batch phase-by-phase, commits write ops to backend, flushes. Testing module provides InMemoryBackend (HashMap-based), MockProvisioner (generates TestBlockContext with predictable values), and three toy indexes covering all three composition types (Append, Monoidal, Fold). Two end-to-end tests validate the full pipeline: provisioner → engine → bridge dispatch → extract → merge → commit → backend assertions. --- packages/zaino-sync/src/engine.rs | 68 +++- packages/zaino-sync/src/lib.rs | 3 + packages/zaino-sync/src/provisioner.rs | 34 +- packages/zaino-sync/src/testing.rs | 420 +++++++++++++++++++++++++ 4 files changed, 501 insertions(+), 24 deletions(-) create mode 100644 packages/zaino-sync/src/testing.rs diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 3af5cb80f..3437e4162 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -4,17 +4,29 @@ //! merge → commit → flush pipeline, and enforces phase gates. //! Contains no blockchain knowledge. -use crate::backend::Backend; +use std::collections::HashSet; + +use crate::backend::{Backend, BackendError, BackendWriter}; use crate::dag::DependencyDag; -use crate::pipeline::IndexPipeline; +use crate::pipeline::{IndexPipeline, PipelineError}; +use crate::primitives::IndexId; /// Configuration for the sync engine. #[derive(Debug, Clone)] pub struct EngineConfig { /// Number of blocks per persistence batch. pub batch_size: u32, - /// Maximum number of concurrent extraction tasks. - pub extraction_parallelism: usize, +} + +/// Errors during sync. +#[derive(Debug, thiserror::Error)] +pub enum SyncError { + /// An index's extract or merge step failed. + #[error(transparent)] + Pipeline(#[from] PipelineError), + /// The storage backend failed. + #[error(transparent)] + Backend(#[from] BackendError), } /// The sync engine. @@ -49,4 +61,52 @@ impl SyncEngine { config, } } + + /// Process a range of blocks through the full pipeline. + /// + /// `blocks` must be in chain order. The engine splits them into + /// batches, processes each batch phase-by-phase, commits results + /// to the backend, and flushes after each batch. + /// + /// **MVP shape.** Single-threaded, synchronous. Each phase processes + /// sequentially; indexes within a phase also process sequentially. + /// The true north parallelises both across indexes in a phase and + /// across blocks within an index (for BlockLocal scopes). + pub fn sync_range(&mut self, blocks: &[Ctx]) -> Result<(), SyncError> { + let batch_size = self.config.batch_size as usize; + + for batch in blocks.chunks(batch_size) { + self.process_batch(batch)?; + self.backend.flush()?; + } + + Ok(()) + } + + fn process_batch(&mut self, batch: &[Ctx]) -> Result<(), SyncError> { + let phases = self.dag.phases(); + + for phase_nodes in &phases { + let phase_names: HashSet = phase_nodes + .iter() + .map(|node| node.descriptor.name) + .collect(); + + let mut batch_ops = Vec::new(); + + for pipeline in &self.indexes { + if phase_names.contains(&pipeline.descriptor().name) { + let ops = pipeline.process_batch(batch, None)?; + batch_ops.extend(ops); + } + } + + if !batch_ops.is_empty() { + let mut writer = self.backend.writer()?; + writer.commit(batch_ops)?; + } + } + + Ok(()) + } } diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index 8650aec4c..c5b2f1e5a 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -16,3 +16,6 @@ pub mod primitives; pub mod progress; pub mod provisioner; pub mod traits; + +#[cfg(test)] +mod testing; diff --git a/packages/zaino-sync/src/provisioner.rs b/packages/zaino-sync/src/provisioner.rs index 0f4cfc158..d586d7e53 100644 --- a/packages/zaino-sync/src/provisioner.rs +++ b/packages/zaino-sync/src/provisioner.rs @@ -1,12 +1,12 @@ //! Provisioner trait — source data acquisition. //! //! The provisioner owns all source access. Indexes never call the source -//! directly (except via the [`NonLocalSource`](crate::traits::NonLocalSource) -//! escape hatch). The engine tells the provisioner what data to fetch -//! (via [`SourceRequirements`](crate::descriptor::SourceRequirements)), -//! and the provisioner streams block contexts as they become ready. +//! directly. The engine tells the provisioner what data to fetch (via +//! [`SourceRequirements`]), and the provisioner returns block contexts for +//! a requested height range. use crate::descriptor::SourceRequirements; +use crate::primitives::BlockHeight; /// Errors from provisioner operations. #[derive(Debug, thiserror::Error)] @@ -14,9 +14,6 @@ pub enum ProvisionError { /// The source was unreachable or returned an error. #[error("source error: {0}")] Source(String), - /// The channel to the engine was closed. - #[error("channel closed")] - ChannelClosed, } /// The provisioner: first-class owner of all source access. @@ -25,9 +22,10 @@ pub enum ProvisionError { /// is opaque to the engine; extractors know its concrete type through /// [`IndexDef::Context`](crate::traits::IndexDef::Context). /// -/// The provisioner streams contexts as they become ready. It may -/// internally parallelise source fetches. Backpressure propagates -/// from the engine's bounded channel. +/// **MVP shape.** The `provision_range` method returns a `Vec` of all +/// block contexts synchronously. The true north is a streaming interface +/// where the provisioner pushes contexts through a bounded channel as +/// they become ready, enabling pipelined extraction. pub trait Provisioner: Send + Sync { /// Opaque block context type. Matches `IndexDef::Context` for all /// registered indexes. @@ -37,14 +35,10 @@ pub trait Provisioner: Send + Sync { /// registered indexes' requirements. fn configure(&mut self, requirements: SourceRequirements); - // NOTE: the streaming `provision` method is intentionally omitted - // from this initial sketch. It will depend on the async runtime - // choice (tokio channels, crossbeam, etc.) and the engine's pipeline - // design. The signature will look roughly like: - // - // async fn provision( - // &self, - // range: Range, - // tx: Sender<(Self::Height, Self::BlockContext)>, - // ) -> Result<(), ProvisionError>; + /// Fetch block contexts for a height range (inclusive on both ends). + fn provision_range( + &self, + from: BlockHeight, + to: BlockHeight, + ) -> Result, ProvisionError>; } diff --git a/packages/zaino-sync/src/testing.rs b/packages/zaino-sync/src/testing.rs new file mode 100644 index 000000000..0d14b04f0 --- /dev/null +++ b/packages/zaino-sync/src/testing.rs @@ -0,0 +1,420 @@ +//! Test utilities: in-memory backend, mock provisioner, and toy indexes. +//! +//! Everything in this module is `#[cfg(test)]`-gated. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use crate::backend::{Backend, BackendError, BackendReader, BackendWriter, WriterTopology}; +use crate::primitives::{BlockHeight, IndexId}; +use crate::traits::WriteOp; + +// =========================================================================== +// In-memory backend +// =========================================================================== + +/// In-memory backend for tests. Stores key-value pairs per index. +/// +/// Thread-safe via `Arc>` — readers and writers share the +/// same underlying map. Not performant, but correct for test assertions. +#[derive(Clone)] +pub struct InMemoryBackend { + data: Arc, Vec>>>>, +} + +impl InMemoryBackend { + pub fn new() -> Self { + Self { + data: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Read all entries for a given index. For test assertions. + pub fn entries(&self, index: IndexId) -> HashMap, Vec> { + let guard = self.data.lock().expect("test mutex poisoned"); + guard.get(&index).cloned().unwrap_or_default() + } + + /// Read a single value. For test assertions. + pub fn get_value(&self, index: IndexId, key: &[u8]) -> Option> { + let guard = self.data.lock().expect("test mutex poisoned"); + guard.get(&index).and_then(|m| m.get(key).cloned()) + } +} + +impl Backend for InMemoryBackend { + type Reader = InMemoryReader; + type Writer = InMemoryWriter; + + fn reader(&self) -> Result { + Ok(InMemoryReader { + data: Arc::clone(&self.data), + }) + } + + fn writer(&self) -> Result { + Ok(InMemoryWriter { + data: Arc::clone(&self.data), + }) + } + + fn flush(&self) -> Result<(), BackendError> { + Ok(()) + } + + fn topology(&self) -> WriterTopology { + WriterTopology::SharedWriter + } +} + +/// Read handle for the in-memory backend. +pub struct InMemoryReader { + data: Arc, Vec>>>>, +} + +impl BackendReader for InMemoryReader { + fn get(&self, index: IndexId, key: &[u8]) -> Result>, BackendError> { + let guard = self.data.lock().expect("test mutex poisoned"); + Ok(guard.get(&index).and_then(|m| m.get(key).cloned())) + } +} + +/// Write handle for the in-memory backend. +pub struct InMemoryWriter { + data: Arc, Vec>>>>, +} + +impl BackendWriter for InMemoryWriter { + fn commit(&mut self, ops: Vec) -> Result<(), BackendError> { + let mut guard = self.data.lock().expect("test mutex poisoned"); + for op in ops { + match op { + WriteOp::Put { index, key, value } => { + guard.entry(index).or_default().insert(key, value); + } + WriteOp::Delete { index, key } => { + if let Some(map) = guard.get_mut(&index) { + map.remove(&key); + } + } + } + } + Ok(()) + } +} + +// =========================================================================== +// Mock provisioner +// =========================================================================== + +use crate::descriptor::SourceRequirements; +use crate::provisioner::{ProvisionError, Provisioner}; + +/// Minimal block context for tests. +#[derive(Debug, Clone)] +pub struct TestBlockContext { + /// Block height. + pub height: u64, + /// Arbitrary value carried by this block. + pub value: u32, +} + +/// Mock provisioner that generates `TestBlockContext`s with predictable values. +pub struct MockProvisioner { + /// Function that produces the value for a given height. + value_fn: Box u32 + Send + Sync>, +} + +impl MockProvisioner { + /// Create a provisioner where each block's value equals its height. + pub fn identity() -> Self { + Self { + value_fn: Box::new(|h| h as u32), + } + } +} + +impl Provisioner for MockProvisioner { + type BlockContext = TestBlockContext; + + fn configure(&mut self, _requirements: SourceRequirements) {} + + fn provision_range( + &self, + from: BlockHeight, + to: BlockHeight, + ) -> Result, ProvisionError> { + let blocks = (from.value()..=to.value()) + .map(|h| TestBlockContext { + height: h, + value: (self.value_fn)(h), + }) + .collect(); + Ok(blocks) + } +} + +// =========================================================================== +// Test indexes +// =========================================================================== + +use crate::descriptor::{ + Append, BlockLocal, CompositionType, Descriptor, Fold, InputScope, Monoidal, SourceAccess, +}; +use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal}; + +/// BlockLocal × Append: stores (height → value) for each block. +pub struct ValueIndex; + +const VALUE_INDEX_ID: IndexId = IndexId::new("value"); + +impl IndexDef for ValueIndex { + type Scope = BlockLocal; + type Composition = Append; + type Delta = Vec<(Vec, Vec)>; + type Context = TestBlockContext; + + fn descriptor() -> Descriptor { + Descriptor { + name: VALUE_INDEX_ID, + scope: InputScope::BlockLocal, + composition: CompositionType::Append, + dependencies: &[], + requirements: SourceRequirements::BLOCK, + source_access: SourceAccess::None, + } + } +} + +impl ExtractLocal for ValueIndex { + fn extract(ctx: &TestBlockContext) -> Result { + Ok(vec![( + ctx.height.to_le_bytes().to_vec(), + ctx.value.to_le_bytes().to_vec(), + )]) + } +} + +impl MergeAppend for ValueIndex { + fn to_write_ops(delta: Self::Delta) -> Vec { + delta + .into_iter() + .map(|(key, value)| WriteOp::Put { + index: VALUE_INDEX_ID, + key, + value, + }) + .collect() + } +} + +/// BlockLocal × Monoidal: counts total blocks seen in each batch. +pub struct CountIndex; + +const COUNT_INDEX_ID: IndexId = IndexId::new("count"); + +impl IndexDef for CountIndex { + type Scope = BlockLocal; + type Composition = Monoidal; + type Delta = u64; + type Context = TestBlockContext; + + fn descriptor() -> Descriptor { + Descriptor { + name: COUNT_INDEX_ID, + scope: InputScope::BlockLocal, + composition: CompositionType::Monoidal, + dependencies: &[], + requirements: SourceRequirements::BLOCK, + source_access: SourceAccess::None, + } + } +} + +impl ExtractLocal for CountIndex { + fn extract(_ctx: &TestBlockContext) -> Result { + Ok(1) + } +} + +impl MergeMonoidal for CountIndex { + type Accumulator = u64; + + fn identity() -> Self::Accumulator { + 0 + } + + fn lift(delta: Self::Delta) -> Self::Accumulator { + delta + } + + fn combine(a: Self::Accumulator, b: Self::Accumulator) -> Self::Accumulator { + a + b + } + + fn to_write_ops(merged: Self::Accumulator) -> Vec { + vec![WriteOp::Put { + index: COUNT_INDEX_ID, + key: b"total".to_vec(), + value: merged.to_le_bytes().to_vec(), + }] + } +} + +/// BlockLocal × Fold: running sum of values across blocks in a batch. +pub struct RunningSumIndex; + +const RUNNING_SUM_INDEX_ID: IndexId = IndexId::new("running_sum"); + +impl IndexDef for RunningSumIndex { + type Scope = BlockLocal; + type Composition = Fold; + type Delta = u64; + type Context = TestBlockContext; + + fn descriptor() -> Descriptor { + Descriptor { + name: RUNNING_SUM_INDEX_ID, + scope: InputScope::BlockLocal, + composition: CompositionType::Fold, + dependencies: &[], + requirements: SourceRequirements::BLOCK, + source_access: SourceAccess::None, + } + } +} + +impl ExtractLocal for RunningSumIndex { + fn extract(ctx: &TestBlockContext) -> Result { + Ok(u64::from(ctx.value)) + } +} + +impl MergeFold for RunningSumIndex { + type FoldState = u64; + + fn initial_state() -> Self::FoldState { + 0 + } + + fn fold(state: &mut Self::FoldState, delta: Self::Delta) { + *state += delta; + } + + fn to_write_ops(state: Self::FoldState) -> Vec { + vec![WriteOp::Put { + index: RUNNING_SUM_INDEX_ID, + key: b"sum".to_vec(), + value: state.to_le_bytes().to_vec(), + }] + } +} + +// =========================================================================== +// End-to-end tests +// =========================================================================== + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge; + use crate::dag::DependencyDag; + use crate::engine::{EngineConfig, SyncEngine}; + + #[test] + fn end_to_end_single_batch() { + let provisioner = MockProvisioner::identity(); + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(4)) + .expect("provisioning succeeds"); + + let backend = InMemoryBackend::new(); + let mut engine = { + let descriptors = vec![ + ValueIndex::descriptor(), + CountIndex::descriptor(), + RunningSumIndex::descriptor(), + ]; + let dag = DependencyDag::build(descriptors).expect("valid DAG"); + let indexes: Vec>> = vec![ + bridge::local_append::(), + bridge::local_monoidal::(), + bridge::local_fold::(), + ]; + SyncEngine::new(dag, indexes, backend.clone(), EngineConfig { batch_size: 10 }) + }; + + engine.sync_range(&blocks).expect("sync succeeds"); + + // ValueIndex: 5 entries (heights 0..=4), each height → height as value + let values = backend.entries(VALUE_INDEX_ID); + assert_eq!(values.len(), 5); + for h in 0u64..=4 { + let stored = values.get(&h.to_le_bytes().to_vec()).expect("key exists"); + let val = u32::from_le_bytes(stored.as_slice().try_into().expect("4 bytes")); + assert_eq!(val, h as u32); + } + + // CountIndex: one entry "total" = 5 + let count_bytes = backend + .get_value(COUNT_INDEX_ID, b"total") + .expect("count exists"); + let count = u64::from_le_bytes(count_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(count, 5); + + // RunningSumIndex: one entry "sum" = 0+1+2+3+4 = 10 + let sum_bytes = backend + .get_value(RUNNING_SUM_INDEX_ID, b"sum") + .expect("sum exists"); + let sum = u64::from_le_bytes(sum_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(sum, 10); + } + + #[test] + fn multi_batch_splits_correctly() { + let provisioner = MockProvisioner::identity(); + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(9)) + .expect("provisioning succeeds"); + + let backend = InMemoryBackend::new(); + let mut engine = { + let descriptors = vec![ + ValueIndex::descriptor(), + CountIndex::descriptor(), + RunningSumIndex::descriptor(), + ]; + let dag = DependencyDag::build(descriptors).expect("valid DAG"); + let indexes: Vec>> = vec![ + bridge::local_append::(), + bridge::local_monoidal::(), + bridge::local_fold::(), + ]; + // Batch size 3: blocks [0,1,2], [3,4,5], [6,7,8], [9] + SyncEngine::new(dag, indexes, backend.clone(), EngineConfig { batch_size: 3 }) + }; + + engine.sync_range(&blocks).expect("sync succeeds"); + + // ValueIndex: 10 entries, all correct (append across batches) + let values = backend.entries(VALUE_INDEX_ID); + assert_eq!(values.len(), 10); + + // CountIndex: last batch was [9] (1 block), so count = 1. + // Monoidal merge runs per-batch, and each batch overwrites the + // same "total" key — the final value reflects the last batch. + let count_bytes = backend + .get_value(COUNT_INDEX_ID, b"total") + .expect("count exists"); + let count = u64::from_le_bytes(count_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(count, 1); + + // RunningSumIndex: last batch was [9], fold sum = 9. + // Same overwrite semantics as CountIndex. + let sum_bytes = backend + .get_value(RUNNING_SUM_INDEX_ID, b"sum") + .expect("sum exists"); + let sum = u64::from_le_bytes(sum_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(sum, 9); + } +} From 6ac766f7b33562a2a53421bd4b5afb4177ab8e9b Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 19:40:44 -0300 Subject: [PATCH 007/146] Add IndexSet API with BridgeDispatch blanket impl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sealed BridgeDispatch trait maps (Scope, Composition) marker pairs to bridge constructors, enabling a blanket IntoIndexPipeline impl — index authors never write bridge wiring by hand. IndexSet collects indexes declaratively and SyncEngine::from_index_set builds the DAG internally. --- packages/zaino-sync/src/bridge.rs | 55 +++++++++++++++++++++++- packages/zaino-sync/src/engine.rs | 24 ++++++++++- packages/zaino-sync/src/index_set.rs | 64 ++++++++++++++++++++++++++++ packages/zaino-sync/src/lib.rs | 1 + packages/zaino-sync/src/pipeline.rs | 31 +++++++++++++- packages/zaino-sync/src/testing.rs | 49 ++++++++------------- 6 files changed, 190 insertions(+), 34 deletions(-) create mode 100644 packages/zaino-sync/src/index_set.rs diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index 5d0eaba2b..6fcddf637 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -54,12 +54,65 @@ use std::marker::PhantomData; -use crate::descriptor::Descriptor; +use crate::descriptor::{Append, BlockLocal, Descriptor, Fold, Monoidal}; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::traits::{ DepsReader, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, WriteOp, }; +// =========================================================================== +// BridgeDispatch — sealed trait mapping (Scope, Composition) → bridge fn +// =========================================================================== + +mod sealed { + pub trait Sealed {} +} + +/// Maps a (Scope, Composition) marker pair to the correct bridge constructor. +/// +/// Sealed — only implemented in this module for the marker pairs defined +/// in [`crate::descriptor`]. The blanket [`IntoIndexPipeline`] impl in +/// [`crate::pipeline`] delegates to this trait, so index authors never +/// need to write `IntoIndexPipeline` by hand. +/// +/// [`IntoIndexPipeline`]: crate::pipeline::IntoIndexPipeline +pub trait BridgeDispatch: sealed::Sealed { + /// Produce the boxed pipeline for index `I`. + fn dispatch() -> Box>; +} + +// Seal the marker pairs. +impl sealed::Sealed for (BlockLocal, Append) {} +impl sealed::Sealed for (BlockLocal, Monoidal) {} +impl sealed::Sealed for (BlockLocal, Fold) {} + +impl BridgeDispatch for (BlockLocal, Append) +where + I: ExtractLocal + MergeAppend + IndexDef, +{ + fn dispatch() -> Box> { + local_append::() + } +} + +impl BridgeDispatch for (BlockLocal, Monoidal) +where + I: ExtractLocal + MergeMonoidal + IndexDef, +{ + fn dispatch() -> Box> { + local_monoidal::() + } +} + +impl BridgeDispatch for (BlockLocal, Fold) +where + I: ExtractLocal + MergeFold + IndexDef, +{ + fn dispatch() -> Box> { + local_fold::() + } +} + // =========================================================================== // BlockLocal × Append // =========================================================================== diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 3437e4162..20df00cc0 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -7,7 +7,8 @@ use std::collections::HashSet; use crate::backend::{Backend, BackendError, BackendWriter}; -use crate::dag::DependencyDag; +use crate::dag::{DagError, DependencyDag}; +use crate::index_set::IndexSet; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::primitives::IndexId; @@ -21,6 +22,9 @@ pub struct EngineConfig { /// Errors during sync. #[derive(Debug, thiserror::Error)] pub enum SyncError { + /// The dependency graph is invalid. + #[error(transparent)] + Dag(#[from] DagError), /// An index's extract or merge step failed. #[error(transparent)] Pipeline(#[from] PipelineError), @@ -62,6 +66,24 @@ impl SyncEngine { } } + /// Create an engine from a declarative [`IndexSet`]. + /// + /// Builds the dependency DAG from the collected descriptors, + /// validates it, and wires up all pipelines. + pub fn from_index_set( + set: IndexSet, + backend: B, + config: EngineConfig, + ) -> Result { + let (dag, indexes) = set.build()?; + Ok(Self { + dag, + indexes, + backend, + config, + }) + } + /// Process a range of blocks through the full pipeline. /// /// `blocks` must be in chain order. The engine splits them into diff --git a/packages/zaino-sync/src/index_set.rs b/packages/zaino-sync/src/index_set.rs new file mode 100644 index 000000000..0051a3117 --- /dev/null +++ b/packages/zaino-sync/src/index_set.rs @@ -0,0 +1,64 @@ +//! Index set — declarative collection of indexes passed to the engine. +//! +//! The user defines indexes (descriptor + extract + merge), registers them +//! into an `IndexSet`, and hands the set to the engine. The set handles +//! DAG construction and validation internally. + +use crate::dag::{DagError, DependencyDag}; +use crate::pipeline::{IndexPipeline, IntoIndexPipeline}; + +/// A collection of indexes to be processed by the sync engine. +/// +/// Built via the [`with`](Self::with) method, which accepts any type +/// implementing [`IntoIndexPipeline`]. The index set collects pipelines +/// and, on [`build`](Self::build), constructs the dependency DAG. +/// +/// # Example +/// +/// ```text +/// let set = IndexSet::new() +/// .with::() +/// .with::() +/// .with::(); +/// +/// let engine = SyncEngine::from_index_set(set, backend, config)?; +/// ``` +pub struct IndexSet { + pipelines: Vec>>, +} + +impl IndexSet { + /// Create an empty index set. + pub fn new() -> Self { + Self { + pipelines: Vec::new(), + } + } + + /// Register an index. The index must implement [`IntoIndexPipeline`], + /// which provides the bridge to the engine's runtime dispatch. + pub fn with>(mut self) -> Self { + self.pipelines.push(I::into_pipeline()); + self + } + + /// Build the dependency DAG and return the parts the engine needs. + /// + /// Validates uniqueness, dependency existence, and acyclicity. + pub(crate) fn build(self) -> Result<(DependencyDag, Vec>>), DagError> { + let descriptors: Vec<_> = self + .pipelines + .iter() + .map(|p| p.descriptor().clone()) + .collect(); + + let dag = DependencyDag::build(descriptors)?; + Ok((dag, self.pipelines)) + } +} + +impl Default for IndexSet { + fn default() -> Self { + Self::new() + } +} diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index c5b2f1e5a..0a0eefadc 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -11,6 +11,7 @@ pub mod bridge; pub mod dag; pub mod descriptor; pub mod engine; +pub mod index_set; pub mod pipeline; pub mod primitives; pub mod progress; diff --git a/packages/zaino-sync/src/pipeline.rs b/packages/zaino-sync/src/pipeline.rs index a636cd32a..83b03c91f 100644 --- a/packages/zaino-sync/src/pipeline.rs +++ b/packages/zaino-sync/src/pipeline.rs @@ -47,8 +47,9 @@ //! - Streaming extraction (process blocks as the provisioner delivers //! them, rather than buffering a full batch upfront). +use crate::bridge::BridgeDispatch; use crate::descriptor::Descriptor; -use crate::traits::{ExtractError, WriteOp}; +use crate::traits::{ExtractError, IndexDef, WriteOp}; /// Errors during pipeline operations. #[derive(Debug, thiserror::Error)] @@ -92,3 +93,31 @@ pub trait IndexPipeline: Send + Sync { deps: Option<&crate::traits::DepsReader>, ) -> Result, PipelineError>; } + +/// Capstone trait: a fully-defined index that can produce its own pipeline. +/// +/// An index implements [`IndexDef`] + one extract trait + one merge trait +/// to define its behaviour. `IntoIndexPipeline` is then derived +/// automatically via a blanket impl — the (Scope, Composition) marker +/// pair selects the right bridge through [`BridgeDispatch`]. +/// +/// This is the trait that [`IndexSet::with`](crate::index_set::IndexSet::with) +/// requires. Index authors never implement it by hand. +/// +/// [`BridgeDispatch`]: crate::bridge::BridgeDispatch +pub trait IntoIndexPipeline: IndexDef { + /// Produce a boxed pipeline for this index. + fn into_pipeline() -> Box>; +} + +/// Blanket impl: any index whose (Scope, Composition) pair has a +/// [`BridgeDispatch`] impl gets `IntoIndexPipeline` for free. +impl IntoIndexPipeline for I +where + I: IndexDef, + (I::Scope, I::Composition): BridgeDispatch, +{ + fn into_pipeline() -> Box> { + <(I::Scope, I::Composition) as BridgeDispatch>::dispatch() + } +} diff --git a/packages/zaino-sync/src/testing.rs b/packages/zaino-sync/src/testing.rs index 0d14b04f0..e019ce42e 100644 --- a/packages/zaino-sync/src/testing.rs +++ b/packages/zaino-sync/src/testing.rs @@ -317,9 +317,22 @@ impl MergeFold for RunningSumIndex { #[cfg(test)] mod tests { use super::*; - use crate::bridge; - use crate::dag::DependencyDag; use crate::engine::{EngineConfig, SyncEngine}; + use crate::index_set::IndexSet; + + /// Helper: build an engine from the three toy indexes. + fn build_engine( + backend: InMemoryBackend, + batch_size: u32, + ) -> SyncEngine { + let set = IndexSet::new() + .with::() + .with::() + .with::(); + + SyncEngine::from_index_set(set, backend, EngineConfig { batch_size }) + .expect("valid index set") + } #[test] fn end_to_end_single_batch() { @@ -329,20 +342,7 @@ mod tests { .expect("provisioning succeeds"); let backend = InMemoryBackend::new(); - let mut engine = { - let descriptors = vec![ - ValueIndex::descriptor(), - CountIndex::descriptor(), - RunningSumIndex::descriptor(), - ]; - let dag = DependencyDag::build(descriptors).expect("valid DAG"); - let indexes: Vec>> = vec![ - bridge::local_append::(), - bridge::local_monoidal::(), - bridge::local_fold::(), - ]; - SyncEngine::new(dag, indexes, backend.clone(), EngineConfig { batch_size: 10 }) - }; + let mut engine = build_engine(backend.clone(), 10); engine.sync_range(&blocks).expect("sync succeeds"); @@ -378,21 +378,8 @@ mod tests { .expect("provisioning succeeds"); let backend = InMemoryBackend::new(); - let mut engine = { - let descriptors = vec![ - ValueIndex::descriptor(), - CountIndex::descriptor(), - RunningSumIndex::descriptor(), - ]; - let dag = DependencyDag::build(descriptors).expect("valid DAG"); - let indexes: Vec>> = vec![ - bridge::local_append::(), - bridge::local_monoidal::(), - bridge::local_fold::(), - ]; - // Batch size 3: blocks [0,1,2], [3,4,5], [6,7,8], [9] - SyncEngine::new(dag, indexes, backend.clone(), EngineConfig { batch_size: 3 }) - }; + // Batch size 3: blocks [0,1,2], [3,4,5], [6,7,8], [9] + let mut engine = build_engine(backend.clone(), 3); engine.sync_range(&blocks).expect("sync succeeds"); From 8e22a675d01fe70ff80e742b9d5396245af3331d Mon Sep 17 00:00:00 2001 From: nachog00 Date: Wed, 1 Jul 2026 20:29:50 -0300 Subject: [PATCH 008/146] Decouple index BlockContext from set-wide Ctx via ProvideContext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IndexDef::Context renamed to IndexDef::BlockContext — the subset of block data each index needs. ProvideContext trait with identity blanket lets the set-wide context project down to each index's view. Bridges insert .context() before extraction, keeping indexes reusable across different index sets with different provisioner contexts. --- packages/zaino-sync/src/bridge.rs | 64 +++++++++++++++----------- packages/zaino-sync/src/pipeline.rs | 27 +++++++---- packages/zaino-sync/src/provisioner.rs | 4 +- packages/zaino-sync/src/testing.rs | 6 +-- packages/zaino-sync/src/traits.rs | 48 +++++++++++++++++-- 5 files changed, 103 insertions(+), 46 deletions(-) diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index 6fcddf637..7355031db 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -57,7 +57,8 @@ use std::marker::PhantomData; use crate::descriptor::{Append, BlockLocal, Descriptor, Fold, Monoidal}; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::traits::{ - DepsReader, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, WriteOp, + DepsReader, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, ProvideContext, + WriteOp, }; // =========================================================================== @@ -75,10 +76,14 @@ mod sealed { /// [`crate::pipeline`] delegates to this trait, so index authors never /// need to write `IntoIndexPipeline` by hand. /// +/// `I` is the index type, `Ctx` is the set-wide block context. The bridge +/// requires `Ctx: ProvideContext` to project the set-wide +/// context down to what the index needs. +/// /// [`IntoIndexPipeline`]: crate::pipeline::IntoIndexPipeline -pub trait BridgeDispatch: sealed::Sealed { - /// Produce the boxed pipeline for index `I`. - fn dispatch() -> Box>; +pub trait BridgeDispatch: sealed::Sealed { + /// Produce the boxed pipeline for index `I` over set-wide context `Ctx`. + fn dispatch() -> Box>; } // Seal the marker pairs. @@ -86,30 +91,33 @@ impl sealed::Sealed for (BlockLocal, Append) {} impl sealed::Sealed for (BlockLocal, Monoidal) {} impl sealed::Sealed for (BlockLocal, Fold) {} -impl BridgeDispatch for (BlockLocal, Append) +impl BridgeDispatch for (BlockLocal, Append) where I: ExtractLocal + MergeAppend + IndexDef, + Ctx: ProvideContext + Send + Sync + 'static, { - fn dispatch() -> Box> { - local_append::() + fn dispatch() -> Box> { + local_append::() } } -impl BridgeDispatch for (BlockLocal, Monoidal) +impl BridgeDispatch for (BlockLocal, Monoidal) where I: ExtractLocal + MergeMonoidal + IndexDef, + Ctx: ProvideContext + Send + Sync + 'static, { - fn dispatch() -> Box> { - local_monoidal::() + fn dispatch() -> Box> { + local_monoidal::() } } -impl BridgeDispatch for (BlockLocal, Fold) +impl BridgeDispatch for (BlockLocal, Fold) where I: ExtractLocal + MergeFold + IndexDef, + Ctx: ProvideContext + Send + Sync + 'static, { - fn dispatch() -> Box> { - local_fold::() + fn dispatch() -> Box> { + local_fold::() } } @@ -142,8 +150,8 @@ impl LocalAppendBridge { impl IndexPipeline for LocalAppendBridge where - I: ExtractLocal + MergeAppend + IndexDef, - Ctx: Send + Sync + 'static, + I: ExtractLocal + MergeAppend, + Ctx: ProvideContext + Send + Sync + 'static, { fn descriptor(&self) -> &Descriptor { &self.descriptor @@ -156,7 +164,7 @@ where ) -> Result, PipelineError> { let mut all_ops = Vec::new(); for ctx in blocks { - let delta = I::extract(ctx)?; + let delta = I::extract(ctx.context())?; all_ops.extend(I::to_write_ops(delta)); } Ok(all_ops) @@ -166,8 +174,8 @@ where /// Register a BlockLocal × Append index into the pipeline. pub fn local_append() -> Box> where - I: ExtractLocal + MergeAppend + IndexDef, - Ctx: Send + Sync + 'static, + I: ExtractLocal + MergeAppend, + Ctx: ProvideContext + Send + Sync + 'static, { Box::new(LocalAppendBridge::::new()) } @@ -201,8 +209,8 @@ impl LocalMonoidalBridge { impl IndexPipeline for LocalMonoidalBridge where - I: ExtractLocal + MergeMonoidal + IndexDef, - Ctx: Send + Sync + 'static, + I: ExtractLocal + MergeMonoidal, + Ctx: ProvideContext + Send + Sync + 'static, { fn descriptor(&self) -> &Descriptor { &self.descriptor @@ -215,7 +223,7 @@ where ) -> Result, PipelineError> { let mut acc = I::identity(); for ctx in blocks { - let delta = I::extract(ctx)?; + let delta = I::extract(ctx.context())?; acc = I::combine(acc, I::lift(delta)); } Ok(I::to_write_ops(acc)) @@ -225,8 +233,8 @@ where /// Register a BlockLocal × Monoidal index into the pipeline. pub fn local_monoidal() -> Box> where - I: ExtractLocal + MergeMonoidal + IndexDef, - Ctx: Send + Sync + 'static, + I: ExtractLocal + MergeMonoidal, + Ctx: ProvideContext + Send + Sync + 'static, { Box::new(LocalMonoidalBridge::::new()) } @@ -260,8 +268,8 @@ impl LocalFoldBridge { impl IndexPipeline for LocalFoldBridge where - I: ExtractLocal + MergeFold + IndexDef, - Ctx: Send + Sync + 'static, + I: ExtractLocal + MergeFold, + Ctx: ProvideContext + Send + Sync + 'static, { fn descriptor(&self) -> &Descriptor { &self.descriptor @@ -274,7 +282,7 @@ where ) -> Result, PipelineError> { let mut state = I::initial_state(); for ctx in blocks { - let delta = I::extract(ctx)?; + let delta = I::extract(ctx.context())?; I::fold(&mut state, delta); } Ok(I::to_write_ops(state)) @@ -284,8 +292,8 @@ where /// Register a BlockLocal × Fold index into the pipeline. pub fn local_fold() -> Box> where - I: ExtractLocal + MergeFold + IndexDef, - Ctx: Send + Sync + 'static, + I: ExtractLocal + MergeFold, + Ctx: ProvideContext + Send + Sync + 'static, { Box::new(LocalFoldBridge::::new()) } diff --git a/packages/zaino-sync/src/pipeline.rs b/packages/zaino-sync/src/pipeline.rs index 83b03c91f..a91610090 100644 --- a/packages/zaino-sync/src/pipeline.rs +++ b/packages/zaino-sync/src/pipeline.rs @@ -49,7 +49,7 @@ use crate::bridge::BridgeDispatch; use crate::descriptor::Descriptor; -use crate::traits::{ExtractError, IndexDef, WriteOp}; +use crate::traits::{ExtractError, IndexDef, ProvideContext, WriteOp}; /// Errors during pipeline operations. #[derive(Debug, thiserror::Error)] @@ -105,19 +105,30 @@ pub trait IndexPipeline: Send + Sync { /// requires. Index authors never implement it by hand. /// /// [`BridgeDispatch`]: crate::bridge::BridgeDispatch -pub trait IntoIndexPipeline: IndexDef { - /// Produce a boxed pipeline for this index. +/// Capstone trait: a fully-defined index that can produce its own pipeline. +/// +/// `Ctx` is the set-wide block context. The index's [`BlockContext`] may +/// differ — the bridge inserts a [`ProvideContext`] projection. Index +/// authors never implement this trait by hand; the blanket impl below +/// derives it from the (Scope, Composition) marker pair. +/// +/// [`BlockContext`]: IndexDef::BlockContext +/// [`ProvideContext`]: crate::traits::ProvideContext +pub trait IntoIndexPipeline: IndexDef { + /// Produce a boxed pipeline for this index over set-wide context `Ctx`. fn into_pipeline() -> Box>; } /// Blanket impl: any index whose (Scope, Composition) pair has a -/// [`BridgeDispatch`] impl gets `IntoIndexPipeline` for free. -impl IntoIndexPipeline for I +/// [`BridgeDispatch`] impl gets `IntoIndexPipeline` for free, for any +/// `Ctx` that can [`ProvideContext`] the index's [`BlockContext`]. +impl IntoIndexPipeline for I where I: IndexDef, - (I::Scope, I::Composition): BridgeDispatch, + Ctx: ProvideContext + Send + Sync + 'static, + (I::Scope, I::Composition): BridgeDispatch, { - fn into_pipeline() -> Box> { - <(I::Scope, I::Composition) as BridgeDispatch>::dispatch() + fn into_pipeline() -> Box> { + <(I::Scope, I::Composition) as BridgeDispatch>::dispatch() } } diff --git a/packages/zaino-sync/src/provisioner.rs b/packages/zaino-sync/src/provisioner.rs index d586d7e53..0f828578a 100644 --- a/packages/zaino-sync/src/provisioner.rs +++ b/packages/zaino-sync/src/provisioner.rs @@ -27,8 +27,8 @@ pub enum ProvisionError { /// where the provisioner pushes contexts through a bounded channel as /// they become ready, enabling pipelined extraction. pub trait Provisioner: Send + Sync { - /// Opaque block context type. Matches `IndexDef::Context` for all - /// registered indexes. + /// Opaque block context type. Each index's `IndexDef::BlockContext` + /// is projected from this type via `ProvideContext`. type BlockContext: Send + Sync; /// Configure which source data to fetch, based on the union of all diff --git a/packages/zaino-sync/src/testing.rs b/packages/zaino-sync/src/testing.rs index e019ce42e..c5c7d91da 100644 --- a/packages/zaino-sync/src/testing.rs +++ b/packages/zaino-sync/src/testing.rs @@ -172,7 +172,7 @@ impl IndexDef for ValueIndex { type Scope = BlockLocal; type Composition = Append; type Delta = Vec<(Vec, Vec)>; - type Context = TestBlockContext; + type BlockContext = TestBlockContext; fn descriptor() -> Descriptor { Descriptor { @@ -217,7 +217,7 @@ impl IndexDef for CountIndex { type Scope = BlockLocal; type Composition = Monoidal; type Delta = u64; - type Context = TestBlockContext; + type BlockContext = TestBlockContext; fn descriptor() -> Descriptor { Descriptor { @@ -270,7 +270,7 @@ impl IndexDef for RunningSumIndex { type Scope = BlockLocal; type Composition = Fold; type Delta = u64; - type Context = TestBlockContext; + type BlockContext = TestBlockContext; fn descriptor() -> Descriptor { Descriptor { diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index eef6f5485..66185400d 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -12,6 +12,40 @@ use crate::descriptor::{ }; use crate::primitives::IndexId; +// --------------------------------------------------------------------------- +// ProvideContext — projection from set-wide context to index block context +// --------------------------------------------------------------------------- + +/// Projection from a set-wide block context to an index's block context. +/// +/// The provisioner produces one context (`Ctx`) per block for the whole +/// index set. Each index declares its own [`BlockContext`] — the subset +/// of block data it needs. `ProvideContext` lets the set-wide context +/// narrow to a `&T` before the bridge passes it to extraction. +/// +/// The identity blanket impl covers the common case where the index's +/// block context *is* the set-wide context. For richer set-wide contexts, +/// implement this trait for each projection: +/// +/// ```text +/// impl ProvideContext for FullBlockContext { +/// fn context(&self) -> &BlockData { &self.block } +/// } +/// ``` +/// +/// [`BlockContext`]: IndexDef::BlockContext +pub trait ProvideContext { + /// Borrow the block context of type `T`. + fn context(&self) -> &T; +} + +/// Identity projection: any type provides itself. +impl ProvideContext for T { + fn context(&self) -> &T { + self + } +} + // --------------------------------------------------------------------------- // Placeholder types — will be fleshed out in their own modules // --------------------------------------------------------------------------- @@ -61,8 +95,12 @@ pub trait IndexDef: Send + Sync + 'static { /// The per-block contribution produced by extraction. type Delta: Send + Sync; - /// The block context type (matches the provisioner's output). - type Context: Send + Sync; + /// The block context this index needs for extraction. + /// + /// This is the index's view of a block — just the data it cares about. + /// The set-wide context (what the provisioner produces) narrows to this + /// type via [`ProvideContext`]. + type BlockContext: Send + Sync; /// The full declarative descriptor. fn descriptor() -> Descriptor; @@ -81,7 +119,7 @@ pub trait IndexDef: Send + Sync + 'static { /// full parallelism across blocks. pub trait ExtractLocal: IndexDef { /// Produce this block's delta from the block context alone. - fn extract(ctx: &Self::Context) -> Result; + fn extract(ctx: &Self::BlockContext) -> Result; } /// Extraction for self-cumulative indexes. @@ -98,7 +136,7 @@ pub trait ExtractCumulative: IndexDef { /// Produce this block's delta given the block context and this index's /// own committed state up to (but not including) this block. fn extract( - ctx: &Self::Context, + ctx: &Self::BlockContext, prior: &Self::PriorState, ) -> Result; } @@ -111,7 +149,7 @@ pub trait ExtractCumulative: IndexDef { pub trait ExtractCross: IndexDef { /// Produce this block's delta given the block context and committed /// state from dependency indexes. - fn extract(ctx: &Self::Context, deps: &DepsReader) -> Result; + fn extract(ctx: &Self::BlockContext, deps: &DepsReader) -> Result; } // =========================================================================== From a5d3d11f8ab442eac758ec3497f19fab202f8476 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 00:15:56 -0300 Subject: [PATCH 009/146] Narrow per-index contexts with ProvideContext owned projection ProvideContext returns owned T (identity blanket clones). Each toy index gets its own module under testing/toy_indexes/ with a local Context type. ContextRequirements trait added to descriptor.rs for deriving provisioner needs from context types (not yet wired in). --- packages/zaino-sync/src/bridge.rs | 6 +- packages/zaino-sync/src/descriptor.rs | 25 ++ packages/zaino-sync/src/testing.rs | 268 +----------------- .../zaino-sync/src/testing/toy_indexes.rs | 143 ++++++++++ .../src/testing/toy_indexes/count_index.rs | 68 +++++ .../testing/toy_indexes/running_sum_index.rs | 63 ++++ .../src/testing/toy_indexes/value_index.rs | 61 ++++ packages/zaino-sync/src/traits.rs | 22 +- 8 files changed, 387 insertions(+), 269 deletions(-) create mode 100644 packages/zaino-sync/src/testing/toy_indexes.rs create mode 100644 packages/zaino-sync/src/testing/toy_indexes/count_index.rs create mode 100644 packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs create mode 100644 packages/zaino-sync/src/testing/toy_indexes/value_index.rs diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index 7355031db..fe38e7db2 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -164,7 +164,7 @@ where ) -> Result, PipelineError> { let mut all_ops = Vec::new(); for ctx in blocks { - let delta = I::extract(ctx.context())?; + let delta = I::extract(&ctx.context())?; all_ops.extend(I::to_write_ops(delta)); } Ok(all_ops) @@ -223,7 +223,7 @@ where ) -> Result, PipelineError> { let mut acc = I::identity(); for ctx in blocks { - let delta = I::extract(ctx.context())?; + let delta = I::extract(&ctx.context())?; acc = I::combine(acc, I::lift(delta)); } Ok(I::to_write_ops(acc)) @@ -282,7 +282,7 @@ where ) -> Result, PipelineError> { let mut state = I::initial_state(); for ctx in blocks { - let delta = I::extract(ctx.context())?; + let delta = I::extract(&ctx.context())?; I::fold(&mut state, delta); } Ok(I::to_write_ops(state)) diff --git a/packages/zaino-sync/src/descriptor.rs b/packages/zaino-sync/src/descriptor.rs index f3dc656d4..8d1e27db4 100644 --- a/packages/zaino-sync/src/descriptor.rs +++ b/packages/zaino-sync/src/descriptor.rs @@ -139,6 +139,28 @@ bitflags! { } } +// --------------------------------------------------------------------------- +// ContextRequirements — type-level source of truth for provisioner needs +// --------------------------------------------------------------------------- + +/// Declares what data a block context type requires from the provisioner. +/// +/// Each index's [`BlockContext`](super::traits::IndexDef::BlockContext) +/// implements this trait. The engine unions the requirements across all +/// registered indexes and configures the provisioner accordingly. +/// +/// This is the single source of truth — indexes don't declare +/// requirements separately. The context type *is* the requirement. +pub trait ContextRequirements: Send + Sync + 'static { + /// The provisioner requirements needed to populate this context. + const REQUIREMENTS: SourceRequirements; +} + +/// Unit context: no data needed (e.g. a pure counting index). +impl ContextRequirements for () { + const REQUIREMENTS: SourceRequirements = SourceRequirements::empty(); +} + // --------------------------------------------------------------------------- // Descriptor — the full declarative spec of an index // --------------------------------------------------------------------------- @@ -161,6 +183,9 @@ pub struct Descriptor { /// Indexes this one depends on (must form a DAG). pub dependencies: &'static [IndexId], /// What the provisioner must fetch for this index. + /// + /// Derived from the index's `BlockContext` type via + /// [`ContextRequirements`]. Not declared manually. pub requirements: SourceRequirements, /// Whether extraction may reach the source for non-local data. pub source_access: SourceAccess, diff --git a/packages/zaino-sync/src/testing.rs b/packages/zaino-sync/src/testing.rs index c5c7d91da..ac65c5ab3 100644 --- a/packages/zaino-sync/src/testing.rs +++ b/packages/zaino-sync/src/testing.rs @@ -1,6 +1,10 @@ -//! Test utilities: in-memory backend, mock provisioner, and toy indexes. +//! Test utilities: in-memory backend, mock provisioner, and demo index sets. //! -//! Everything in this module is `#[cfg(test)]`-gated. +//! General-purpose utilities (`InMemoryBackend`, `MockProvisioner`, +//! `TestBlockContext`) are defined here. Specific index sets live in +//! sub-modules (e.g. [`toy_indexes`]). + +mod toy_indexes; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -104,13 +108,17 @@ impl BackendWriter for InMemoryWriter { } // =========================================================================== -// Mock provisioner +// Set-wide block context and mock provisioner // =========================================================================== use crate::descriptor::SourceRequirements; use crate::provisioner::{ProvisionError, Provisioner}; -/// Minimal block context for tests. +/// Set-wide block context for tests. +/// +/// The provisioner produces one of these per block. Individual indexes +/// declare narrower [`BlockContext`](crate::traits::IndexDef::BlockContext) +/// types and receive projections via [`ProvideContext`](crate::traits::ProvideContext). #[derive(Debug, Clone)] pub struct TestBlockContext { /// Block height. @@ -153,255 +161,3 @@ impl Provisioner for MockProvisioner { Ok(blocks) } } - -// =========================================================================== -// Test indexes -// =========================================================================== - -use crate::descriptor::{ - Append, BlockLocal, CompositionType, Descriptor, Fold, InputScope, Monoidal, SourceAccess, -}; -use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal}; - -/// BlockLocal × Append: stores (height → value) for each block. -pub struct ValueIndex; - -const VALUE_INDEX_ID: IndexId = IndexId::new("value"); - -impl IndexDef for ValueIndex { - type Scope = BlockLocal; - type Composition = Append; - type Delta = Vec<(Vec, Vec)>; - type BlockContext = TestBlockContext; - - fn descriptor() -> Descriptor { - Descriptor { - name: VALUE_INDEX_ID, - scope: InputScope::BlockLocal, - composition: CompositionType::Append, - dependencies: &[], - requirements: SourceRequirements::BLOCK, - source_access: SourceAccess::None, - } - } -} - -impl ExtractLocal for ValueIndex { - fn extract(ctx: &TestBlockContext) -> Result { - Ok(vec![( - ctx.height.to_le_bytes().to_vec(), - ctx.value.to_le_bytes().to_vec(), - )]) - } -} - -impl MergeAppend for ValueIndex { - fn to_write_ops(delta: Self::Delta) -> Vec { - delta - .into_iter() - .map(|(key, value)| WriteOp::Put { - index: VALUE_INDEX_ID, - key, - value, - }) - .collect() - } -} - -/// BlockLocal × Monoidal: counts total blocks seen in each batch. -pub struct CountIndex; - -const COUNT_INDEX_ID: IndexId = IndexId::new("count"); - -impl IndexDef for CountIndex { - type Scope = BlockLocal; - type Composition = Monoidal; - type Delta = u64; - type BlockContext = TestBlockContext; - - fn descriptor() -> Descriptor { - Descriptor { - name: COUNT_INDEX_ID, - scope: InputScope::BlockLocal, - composition: CompositionType::Monoidal, - dependencies: &[], - requirements: SourceRequirements::BLOCK, - source_access: SourceAccess::None, - } - } -} - -impl ExtractLocal for CountIndex { - fn extract(_ctx: &TestBlockContext) -> Result { - Ok(1) - } -} - -impl MergeMonoidal for CountIndex { - type Accumulator = u64; - - fn identity() -> Self::Accumulator { - 0 - } - - fn lift(delta: Self::Delta) -> Self::Accumulator { - delta - } - - fn combine(a: Self::Accumulator, b: Self::Accumulator) -> Self::Accumulator { - a + b - } - - fn to_write_ops(merged: Self::Accumulator) -> Vec { - vec![WriteOp::Put { - index: COUNT_INDEX_ID, - key: b"total".to_vec(), - value: merged.to_le_bytes().to_vec(), - }] - } -} - -/// BlockLocal × Fold: running sum of values across blocks in a batch. -pub struct RunningSumIndex; - -const RUNNING_SUM_INDEX_ID: IndexId = IndexId::new("running_sum"); - -impl IndexDef for RunningSumIndex { - type Scope = BlockLocal; - type Composition = Fold; - type Delta = u64; - type BlockContext = TestBlockContext; - - fn descriptor() -> Descriptor { - Descriptor { - name: RUNNING_SUM_INDEX_ID, - scope: InputScope::BlockLocal, - composition: CompositionType::Fold, - dependencies: &[], - requirements: SourceRequirements::BLOCK, - source_access: SourceAccess::None, - } - } -} - -impl ExtractLocal for RunningSumIndex { - fn extract(ctx: &TestBlockContext) -> Result { - Ok(u64::from(ctx.value)) - } -} - -impl MergeFold for RunningSumIndex { - type FoldState = u64; - - fn initial_state() -> Self::FoldState { - 0 - } - - fn fold(state: &mut Self::FoldState, delta: Self::Delta) { - *state += delta; - } - - fn to_write_ops(state: Self::FoldState) -> Vec { - vec![WriteOp::Put { - index: RUNNING_SUM_INDEX_ID, - key: b"sum".to_vec(), - value: state.to_le_bytes().to_vec(), - }] - } -} - -// =========================================================================== -// End-to-end tests -// =========================================================================== - -#[cfg(test)] -mod tests { - use super::*; - use crate::engine::{EngineConfig, SyncEngine}; - use crate::index_set::IndexSet; - - /// Helper: build an engine from the three toy indexes. - fn build_engine( - backend: InMemoryBackend, - batch_size: u32, - ) -> SyncEngine { - let set = IndexSet::new() - .with::() - .with::() - .with::(); - - SyncEngine::from_index_set(set, backend, EngineConfig { batch_size }) - .expect("valid index set") - } - - #[test] - fn end_to_end_single_batch() { - let provisioner = MockProvisioner::identity(); - let blocks = provisioner - .provision_range(BlockHeight::new(0), BlockHeight::new(4)) - .expect("provisioning succeeds"); - - let backend = InMemoryBackend::new(); - let mut engine = build_engine(backend.clone(), 10); - - engine.sync_range(&blocks).expect("sync succeeds"); - - // ValueIndex: 5 entries (heights 0..=4), each height → height as value - let values = backend.entries(VALUE_INDEX_ID); - assert_eq!(values.len(), 5); - for h in 0u64..=4 { - let stored = values.get(&h.to_le_bytes().to_vec()).expect("key exists"); - let val = u32::from_le_bytes(stored.as_slice().try_into().expect("4 bytes")); - assert_eq!(val, h as u32); - } - - // CountIndex: one entry "total" = 5 - let count_bytes = backend - .get_value(COUNT_INDEX_ID, b"total") - .expect("count exists"); - let count = u64::from_le_bytes(count_bytes.as_slice().try_into().expect("8 bytes")); - assert_eq!(count, 5); - - // RunningSumIndex: one entry "sum" = 0+1+2+3+4 = 10 - let sum_bytes = backend - .get_value(RUNNING_SUM_INDEX_ID, b"sum") - .expect("sum exists"); - let sum = u64::from_le_bytes(sum_bytes.as_slice().try_into().expect("8 bytes")); - assert_eq!(sum, 10); - } - - #[test] - fn multi_batch_splits_correctly() { - let provisioner = MockProvisioner::identity(); - let blocks = provisioner - .provision_range(BlockHeight::new(0), BlockHeight::new(9)) - .expect("provisioning succeeds"); - - let backend = InMemoryBackend::new(); - // Batch size 3: blocks [0,1,2], [3,4,5], [6,7,8], [9] - let mut engine = build_engine(backend.clone(), 3); - - engine.sync_range(&blocks).expect("sync succeeds"); - - // ValueIndex: 10 entries, all correct (append across batches) - let values = backend.entries(VALUE_INDEX_ID); - assert_eq!(values.len(), 10); - - // CountIndex: last batch was [9] (1 block), so count = 1. - // Monoidal merge runs per-batch, and each batch overwrites the - // same "total" key — the final value reflects the last batch. - let count_bytes = backend - .get_value(COUNT_INDEX_ID, b"total") - .expect("count exists"); - let count = u64::from_le_bytes(count_bytes.as_slice().try_into().expect("8 bytes")); - assert_eq!(count, 1); - - // RunningSumIndex: last batch was [9], fold sum = 9. - // Same overwrite semantics as CountIndex. - let sum_bytes = backend - .get_value(RUNNING_SUM_INDEX_ID, b"sum") - .expect("sum exists"); - let sum = u64::from_le_bytes(sum_bytes.as_slice().try_into().expect("8 bytes")); - assert_eq!(sum, 9); - } -} diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs new file mode 100644 index 000000000..3bc71272c --- /dev/null +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -0,0 +1,143 @@ +//! Toy index set: three indexes demonstrating the three composition types. +//! +//! Each index lives in its own sub-module and declares a narrow +//! [`BlockContext`](crate::traits::IndexDef::BlockContext). The set-wide +//! `TestBlockContext` projects into each via [`ProvideContext`]. +//! +//! [`ProvideContext`]: crate::traits::ProvideContext + +pub mod count_index; +pub mod running_sum_index; +pub mod value_index; + +use crate::traits::ProvideContext; + +use super::TestBlockContext; + +// --------------------------------------------------------------------------- +// ProvideContext projections: set-wide → per-index +// --------------------------------------------------------------------------- + +impl ProvideContext for TestBlockContext { + fn context(&self) -> value_index::Context { + value_index::Context { + height: self.height, + value: self.value, + } + } +} + +impl ProvideContext<()> for TestBlockContext { + fn context(&self) {} +} + +impl ProvideContext for TestBlockContext { + fn context(&self) -> running_sum_index::Context { + running_sum_index::Context { + value: self.value, + } + } +} + +// --------------------------------------------------------------------------- +// End-to-end tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::{EngineConfig, SyncEngine}; + use crate::index_set::IndexSet; + use crate::primitives::BlockHeight; + use crate::provisioner::Provisioner; + use crate::testing::{InMemoryBackend, MockProvisioner}; + + use count_index::CountIndex; + use running_sum_index::RunningSumIndex; + use value_index::ValueIndex; + + /// Helper: build an engine from the three toy indexes. + fn build_engine( + backend: InMemoryBackend, + batch_size: u32, + ) -> SyncEngine { + let set = IndexSet::new() + .with::() + .with::() + .with::(); + + SyncEngine::from_index_set(set, backend, EngineConfig { batch_size }) + .expect("valid index set") + } + + #[test] + fn end_to_end_single_batch() { + let provisioner = MockProvisioner::identity(); + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(4)) + .expect("provisioning succeeds"); + + let backend = InMemoryBackend::new(); + let mut engine = build_engine(backend.clone(), 10); + + engine.sync_range(&blocks).expect("sync succeeds"); + + // ValueIndex: 5 entries (heights 0..=4), each height → height as value + let values = backend.entries(value_index::ID); + assert_eq!(values.len(), 5); + for h in 0u64..=4 { + let stored = values.get(&h.to_le_bytes().to_vec()).expect("key exists"); + let val = u32::from_le_bytes(stored.as_slice().try_into().expect("4 bytes")); + assert_eq!(val, h as u32); + } + + // CountIndex: one entry "total" = 5 + let count_bytes = backend + .get_value(count_index::ID, b"total") + .expect("count exists"); + let count = u64::from_le_bytes(count_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(count, 5); + + // RunningSumIndex: one entry "sum" = 0+1+2+3+4 = 10 + let sum_bytes = backend + .get_value(running_sum_index::ID, b"sum") + .expect("sum exists"); + let sum = u64::from_le_bytes(sum_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(sum, 10); + } + + #[test] + fn multi_batch_splits_correctly() { + let provisioner = MockProvisioner::identity(); + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(9)) + .expect("provisioning succeeds"); + + let backend = InMemoryBackend::new(); + // Batch size 3: blocks [0,1,2], [3,4,5], [6,7,8], [9] + let mut engine = build_engine(backend.clone(), 3); + + engine.sync_range(&blocks).expect("sync succeeds"); + + // ValueIndex: 10 entries, all correct (append across batches) + let values = backend.entries(value_index::ID); + assert_eq!(values.len(), 10); + + // CountIndex: last batch was [9] (1 block), so count = 1. + // Monoidal merge runs per-batch, and each batch overwrites the + // same "total" key — the final value reflects the last batch. + let count_bytes = backend + .get_value(count_index::ID, b"total") + .expect("count exists"); + let count = u64::from_le_bytes(count_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(count, 1); + + // RunningSumIndex: last batch was [9], fold sum = 9. + // Same overwrite semantics as CountIndex. + let sum_bytes = backend + .get_value(running_sum_index::ID, b"sum") + .expect("sum exists"); + let sum = u64::from_le_bytes(sum_bytes.as_slice().try_into().expect("8 bytes")); + assert_eq!(sum, 9); + } +} diff --git a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs new file mode 100644 index 000000000..685471d85 --- /dev/null +++ b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs @@ -0,0 +1,68 @@ +//! BlockLocal × Monoidal: counts total blocks seen in each batch. + +use crate::descriptor::{ + BlockLocal, CompositionType, Descriptor, InputScope, Monoidal, SourceAccess, SourceRequirements, +}; +use crate::primitives::IndexId; +use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, WriteOp}; + +/// Block context for this index: nothing needed. +/// +/// CountIndex only counts blocks — it reads no data from the block. +/// Using `()` means any set-wide context satisfies it via a trivial +/// `ProvideContext<()>` impl. +pub type Context = (); + +/// Counts total blocks seen in each batch. +pub struct CountIndex; + +/// Index identity. +pub const ID: IndexId = IndexId::new("count"); + +impl IndexDef for CountIndex { + type Scope = BlockLocal; + type Composition = Monoidal; + type Delta = u64; + type BlockContext = Context; + + fn descriptor() -> Descriptor { + Descriptor { + name: ID, + scope: InputScope::BlockLocal, + composition: CompositionType::Monoidal, + dependencies: &[], + requirements: SourceRequirements::BLOCK, + source_access: SourceAccess::None, + } + } +} + +impl ExtractLocal for CountIndex { + fn extract(_ctx: &Context) -> Result { + Ok(1) + } +} + +impl MergeMonoidal for CountIndex { + type Accumulator = u64; + + fn identity() -> Self::Accumulator { + 0 + } + + fn lift(delta: Self::Delta) -> Self::Accumulator { + delta + } + + fn combine(a: Self::Accumulator, b: Self::Accumulator) -> Self::Accumulator { + a + b + } + + fn to_write_ops(merged: Self::Accumulator) -> Vec { + vec![WriteOp::Put { + index: ID, + key: b"total".to_vec(), + value: merged.to_le_bytes().to_vec(), + }] + } +} diff --git a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs new file mode 100644 index 000000000..b4132db6e --- /dev/null +++ b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs @@ -0,0 +1,63 @@ +//! BlockLocal × Fold: running sum of values across blocks in a batch. + +use crate::descriptor::{ + BlockLocal, CompositionType, Descriptor, Fold, InputScope, SourceAccess, SourceRequirements, +}; +use crate::primitives::IndexId; +use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeFold, WriteOp}; + +/// Block context for this index: just the block's value. +pub struct Context { + /// Arbitrary value carried by this block. + pub value: u32, +} + +/// Running sum of values across blocks in a batch. +pub struct RunningSumIndex; + +/// Index identity. +pub const ID: IndexId = IndexId::new("running_sum"); + +impl IndexDef for RunningSumIndex { + type Scope = BlockLocal; + type Composition = Fold; + type Delta = u64; + type BlockContext = Context; + + fn descriptor() -> Descriptor { + Descriptor { + name: ID, + scope: InputScope::BlockLocal, + composition: CompositionType::Fold, + dependencies: &[], + requirements: SourceRequirements::BLOCK, + source_access: SourceAccess::None, + } + } +} + +impl ExtractLocal for RunningSumIndex { + fn extract(ctx: &Context) -> Result { + Ok(u64::from(ctx.value)) + } +} + +impl MergeFold for RunningSumIndex { + type FoldState = u64; + + fn initial_state() -> Self::FoldState { + 0 + } + + fn fold(state: &mut Self::FoldState, delta: Self::Delta) { + *state += delta; + } + + fn to_write_ops(state: Self::FoldState) -> Vec { + vec![WriteOp::Put { + index: ID, + key: b"sum".to_vec(), + value: state.to_le_bytes().to_vec(), + }] + } +} diff --git a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs new file mode 100644 index 000000000..620b642c1 --- /dev/null +++ b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs @@ -0,0 +1,61 @@ +//! BlockLocal × Append: stores (height → value) for each block. + +use crate::descriptor::{ + Append, BlockLocal, CompositionType, Descriptor, InputScope, SourceAccess, SourceRequirements, +}; +use crate::primitives::IndexId; +use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, WriteOp}; + +/// Block context for this index: height and value. +pub struct Context { + /// Block height. + pub height: u64, + /// Arbitrary value carried by this block. + pub value: u32, +} + +/// Stores (height → value) for each block. +pub struct ValueIndex; + +/// Index identity. +pub const ID: IndexId = IndexId::new("value"); + +impl IndexDef for ValueIndex { + type Scope = BlockLocal; + type Composition = Append; + type Delta = Vec<(Vec, Vec)>; + type BlockContext = Context; + + fn descriptor() -> Descriptor { + Descriptor { + name: ID, + scope: InputScope::BlockLocal, + composition: CompositionType::Append, + dependencies: &[], + requirements: SourceRequirements::BLOCK, + source_access: SourceAccess::None, + } + } +} + +impl ExtractLocal for ValueIndex { + fn extract(ctx: &Context) -> Result { + Ok(vec![( + ctx.height.to_le_bytes().to_vec(), + ctx.value.to_le_bytes().to_vec(), + )]) + } +} + +impl MergeAppend for ValueIndex { + fn to_write_ops(delta: Self::Delta) -> Vec { + delta + .into_iter() + .map(|(key, value)| WriteOp::Put { + index: ID, + key, + value, + }) + .collect() + } +} diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index 66185400d..30ffc8ab9 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -20,8 +20,8 @@ use crate::primitives::IndexId; /// /// The provisioner produces one context (`Ctx`) per block for the whole /// index set. Each index declares its own [`BlockContext`] — the subset -/// of block data it needs. `ProvideContext` lets the set-wide context -/// narrow to a `&T` before the bridge passes it to extraction. +/// of block data it needs. `ProvideContext` produces a `T` that the +/// bridge passes to extraction. /// /// The identity blanket impl covers the common case where the index's /// block context *is* the set-wide context. For richer set-wide contexts, @@ -29,20 +29,22 @@ use crate::primitives::IndexId; /// /// ```text /// impl ProvideContext for FullBlockContext { -/// fn context(&self) -> &BlockData { &self.block } +/// fn context(&self) -> BlockData { +/// BlockData { height: self.height, hash: self.hash } +/// } /// } /// ``` /// /// [`BlockContext`]: IndexDef::BlockContext -pub trait ProvideContext { - /// Borrow the block context of type `T`. - fn context(&self) -> &T; +pub trait ProvideContext { + /// Produce the narrowed block context of type `T`. + fn context(&self) -> T; } -/// Identity projection: any type provides itself. -impl ProvideContext for T { - fn context(&self) -> &T { - self +/// Identity projection: any type provides itself via clone. +impl ProvideContext for T { + fn context(&self) -> T { + self.clone() } } From f4e583ae5e55d0b19940f68b3e55a38aacfd840a Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 00:49:36 -0300 Subject: [PATCH 010/146] Remove SourceRequirements and ContextRequirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProvideContext is the single source of truth for provisioner needs. The compiler enforces that the set-wide context can project into each index's BlockContext at IndexSet::with time. No runtime flags needed — unused context fields are dead code, signalling unnecessary RPC calls. Drops bitflags dependency. --- Cargo.lock | 1 - packages/zaino-sync/Cargo.toml | 1 - packages/zaino-sync/src/dag.rs | 3 +- packages/zaino-sync/src/descriptor.rs | 55 ++----------------- packages/zaino-sync/src/provisioner.rs | 23 ++++---- packages/zaino-sync/src/testing.rs | 3 - .../src/testing/toy_indexes/count_index.rs | 3 +- .../testing/toy_indexes/running_sum_index.rs | 3 +- .../src/testing/toy_indexes/value_index.rs | 3 +- packages/zaino-sync/src/traits.rs | 3 +- 10 files changed, 24 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index adcc2354c..3755b8fd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6096,7 +6096,6 @@ dependencies = [ name = "zaino-sync" version = "0.1.0" dependencies = [ - "bitflags 2.13.0", "thiserror 2.0.18", ] diff --git a/packages/zaino-sync/Cargo.toml b/packages/zaino-sync/Cargo.toml index 0c1b4af70..5cce6e13f 100644 --- a/packages/zaino-sync/Cargo.toml +++ b/packages/zaino-sync/Cargo.toml @@ -10,7 +10,6 @@ description = "DAG-driven parallel sync engine for blockchain index building." [dependencies] thiserror.workspace = true -bitflags.workspace = true [lints.rust] unsafe_code = "forbid" diff --git a/packages/zaino-sync/src/dag.rs b/packages/zaino-sync/src/dag.rs index aa059c918..fda1bbce3 100644 --- a/packages/zaino-sync/src/dag.rs +++ b/packages/zaino-sync/src/dag.rs @@ -305,7 +305,7 @@ fn derive_firing_rule(upstream: &Descriptor, _downstream: &Descriptor) -> Firing #[cfg(test)] mod tests { use super::*; - use crate::descriptor::{SourceAccess, SourceRequirements}; + use crate::descriptor::SourceAccess; const A: IndexId = IndexId::new("a"); const B: IndexId = IndexId::new("b"); @@ -324,7 +324,6 @@ mod tests { scope: InputScope::BlockLocal, composition: CompositionType::Append, dependencies: deps, - requirements: SourceRequirements::BLOCK, source_access: SourceAccess::None, } } diff --git a/packages/zaino-sync/src/descriptor.rs b/packages/zaino-sync/src/descriptor.rs index 8d1e27db4..aaccca4ae 100644 --- a/packages/zaino-sync/src/descriptor.rs +++ b/packages/zaino-sync/src/descriptor.rs @@ -1,7 +1,5 @@ //! Declarative index descriptors and type-level axis markers. -use bitflags::bitflags; - use crate::primitives::IndexId; // --------------------------------------------------------------------------- @@ -119,48 +117,6 @@ pub enum SourceAccess { NonLocal, } -// --------------------------------------------------------------------------- -// Source requirements — what the provisioner must fetch per block -// --------------------------------------------------------------------------- - -bitflags! { - /// Declared per-index. The provisioner computes the union across all - /// registered indexes and fetches only what is needed. - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub struct SourceRequirements: u32 { - /// Raw block data (always implied). - const BLOCK = 0b0001; - /// Sapling/Orchard commitment tree roots. - const TREE_ROOTS = 0b0010; - /// Cumulative tree sizes. - const TREE_SIZES = 0b0100; - /// Chainwork of parent block. - const PARENT_WORK = 0b1000; - } -} - -// --------------------------------------------------------------------------- -// ContextRequirements — type-level source of truth for provisioner needs -// --------------------------------------------------------------------------- - -/// Declares what data a block context type requires from the provisioner. -/// -/// Each index's [`BlockContext`](super::traits::IndexDef::BlockContext) -/// implements this trait. The engine unions the requirements across all -/// registered indexes and configures the provisioner accordingly. -/// -/// This is the single source of truth — indexes don't declare -/// requirements separately. The context type *is* the requirement. -pub trait ContextRequirements: Send + Sync + 'static { - /// The provisioner requirements needed to populate this context. - const REQUIREMENTS: SourceRequirements; -} - -/// Unit context: no data needed (e.g. a pure counting index). -impl ContextRequirements for () { - const REQUIREMENTS: SourceRequirements = SourceRequirements::empty(); -} - // --------------------------------------------------------------------------- // Descriptor — the full declarative spec of an index // --------------------------------------------------------------------------- @@ -171,6 +127,12 @@ impl ContextRequirements for () { /// and is associated with type-level markers (via [`IndexDef`]) for /// compile-time enforcement of valid operations. /// +/// Provisioner requirements are not declared here — they are implicit +/// in each index's [`BlockContext`](super::traits::IndexDef::BlockContext) +/// type. The set-wide context must implement +/// [`ProvideContext`](super::traits::ProvideContext) for each index's +/// block context, and the compiler enforces this at registration time. +/// /// [`IndexDef`]: super::traits::IndexDef #[derive(Debug, Clone)] pub struct Descriptor { @@ -182,11 +144,6 @@ pub struct Descriptor { pub composition: CompositionType, /// Indexes this one depends on (must form a DAG). pub dependencies: &'static [IndexId], - /// What the provisioner must fetch for this index. - /// - /// Derived from the index's `BlockContext` type via - /// [`ContextRequirements`]. Not declared manually. - pub requirements: SourceRequirements, /// Whether extraction may reach the source for non-local data. pub source_access: SourceAccess, } diff --git a/packages/zaino-sync/src/provisioner.rs b/packages/zaino-sync/src/provisioner.rs index 0f828578a..c0bfaa315 100644 --- a/packages/zaino-sync/src/provisioner.rs +++ b/packages/zaino-sync/src/provisioner.rs @@ -1,11 +1,11 @@ //! Provisioner trait — source data acquisition. //! //! The provisioner owns all source access. Indexes never call the source -//! directly. The engine tells the provisioner what data to fetch (via -//! [`SourceRequirements`]), and the provisioner returns block contexts for -//! a requested height range. +//! directly. The provisioner's output type determines what data is +//! available — indexes access it through [`ProvideContext`] projections. +//! +//! [`ProvideContext`]: crate::traits::ProvideContext -use crate::descriptor::SourceRequirements; use crate::primitives::BlockHeight; /// Errors from provisioner operations. @@ -19,22 +19,23 @@ pub enum ProvisionError { /// The provisioner: first-class owner of all source access. /// /// Generic — no blockchain knowledge. The `BlockContext` associated type -/// is opaque to the engine; extractors know its concrete type through -/// [`IndexDef::Context`](crate::traits::IndexDef::Context). +/// is the set-wide context that each index's +/// [`BlockContext`](crate::traits::IndexDef::BlockContext) is projected +/// from via [`ProvideContext`](crate::traits::ProvideContext). +/// +/// The provisioner is purpose-built for each index set. Its output type +/// contains exactly the data the set's indexes need — fields that no +/// index projects into are dead code, signalling unnecessary RPC calls. /// /// **MVP shape.** The `provision_range` method returns a `Vec` of all /// block contexts synchronously. The true north is a streaming interface /// where the provisioner pushes contexts through a bounded channel as /// they become ready, enabling pipelined extraction. pub trait Provisioner: Send + Sync { - /// Opaque block context type. Each index's `IndexDef::BlockContext` + /// Set-wide block context type. Each index's `IndexDef::BlockContext` /// is projected from this type via `ProvideContext`. type BlockContext: Send + Sync; - /// Configure which source data to fetch, based on the union of all - /// registered indexes' requirements. - fn configure(&mut self, requirements: SourceRequirements); - /// Fetch block contexts for a height range (inclusive on both ends). fn provision_range( &self, diff --git a/packages/zaino-sync/src/testing.rs b/packages/zaino-sync/src/testing.rs index ac65c5ab3..86f491b98 100644 --- a/packages/zaino-sync/src/testing.rs +++ b/packages/zaino-sync/src/testing.rs @@ -111,7 +111,6 @@ impl BackendWriter for InMemoryWriter { // Set-wide block context and mock provisioner // =========================================================================== -use crate::descriptor::SourceRequirements; use crate::provisioner::{ProvisionError, Provisioner}; /// Set-wide block context for tests. @@ -145,8 +144,6 @@ impl MockProvisioner { impl Provisioner for MockProvisioner { type BlockContext = TestBlockContext; - fn configure(&mut self, _requirements: SourceRequirements) {} - fn provision_range( &self, from: BlockHeight, diff --git a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs index 685471d85..bedbd3e08 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs @@ -1,7 +1,7 @@ //! BlockLocal × Monoidal: counts total blocks seen in each batch. use crate::descriptor::{ - BlockLocal, CompositionType, Descriptor, InputScope, Monoidal, SourceAccess, SourceRequirements, + BlockLocal, CompositionType, Descriptor, InputScope, Monoidal, SourceAccess, }; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, WriteOp}; @@ -31,7 +31,6 @@ impl IndexDef for CountIndex { scope: InputScope::BlockLocal, composition: CompositionType::Monoidal, dependencies: &[], - requirements: SourceRequirements::BLOCK, source_access: SourceAccess::None, } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs index b4132db6e..db1e6ccf0 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs @@ -1,7 +1,7 @@ //! BlockLocal × Fold: running sum of values across blocks in a batch. use crate::descriptor::{ - BlockLocal, CompositionType, Descriptor, Fold, InputScope, SourceAccess, SourceRequirements, + BlockLocal, CompositionType, Descriptor, Fold, InputScope, SourceAccess, }; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeFold, WriteOp}; @@ -30,7 +30,6 @@ impl IndexDef for RunningSumIndex { scope: InputScope::BlockLocal, composition: CompositionType::Fold, dependencies: &[], - requirements: SourceRequirements::BLOCK, source_access: SourceAccess::None, } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs index 620b642c1..1a8f51349 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs @@ -1,7 +1,7 @@ //! BlockLocal × Append: stores (height → value) for each block. use crate::descriptor::{ - Append, BlockLocal, CompositionType, Descriptor, InputScope, SourceAccess, SourceRequirements, + Append, BlockLocal, CompositionType, Descriptor, InputScope, SourceAccess, }; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, WriteOp}; @@ -32,7 +32,6 @@ impl IndexDef for ValueIndex { scope: InputScope::BlockLocal, composition: CompositionType::Append, dependencies: &[], - requirements: SourceRequirements::BLOCK, source_access: SourceAccess::None, } } diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index 30ffc8ab9..7f7116ad9 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -102,7 +102,8 @@ pub trait IndexDef: Send + Sync + 'static { /// This is the index's view of a block — just the data it cares about. /// The set-wide context (what the provisioner produces) narrows to this /// type via [`ProvideContext`]. - type BlockContext: Send + Sync; + /// + type BlockContext: Send + Sync + 'static; /// The full declarative descriptor. fn descriptor() -> Descriptor; From c3a7f0b1745be12d3ceff6b048030fa19b457975 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 19:05:10 -0300 Subject: [PATCH 011/146] Add Scheduler with per-edge firing rules and batch tracking Scheduler combines the static DependencyDag with runtime state to answer what work is ready. Tracks per-index extraction progress, pending merges, and committed batches. Downstream indexes unblock only when all dependency firing rules are satisfied. Six tests cover phase-zero readiness, dependency blocking/unblocking, batch advancement, and independent index parallelism. --- packages/zaino-sync/src/lib.rs | 1 + packages/zaino-sync/src/primitives.rs | 2 + .../zaino-sync/src/primitives/batch_index.rs | 26 ++ packages/zaino-sync/src/scheduler.rs | 380 ++++++++++++++++++ 4 files changed, 409 insertions(+) create mode 100644 packages/zaino-sync/src/primitives/batch_index.rs create mode 100644 packages/zaino-sync/src/scheduler.rs diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index 0a0eefadc..ec1ec29f4 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -15,6 +15,7 @@ pub mod index_set; pub mod pipeline; pub mod primitives; pub mod progress; +pub mod scheduler; pub mod provisioner; pub mod traits; diff --git a/packages/zaino-sync/src/primitives.rs b/packages/zaino-sync/src/primitives.rs index 75562a397..d5c558c1c 100644 --- a/packages/zaino-sync/src/primitives.rs +++ b/packages/zaino-sync/src/primitives.rs @@ -1,9 +1,11 @@ //! Foundational types used across the sync engine. +mod batch_index; mod block_height; mod index_id; mod phase_index; +pub use batch_index::BatchIndex; pub use block_height::BlockHeight; pub use index_id::IndexId; pub use phase_index::PhaseIndex; diff --git a/packages/zaino-sync/src/primitives/batch_index.rs b/packages/zaino-sync/src/primitives/batch_index.rs new file mode 100644 index 000000000..3648d10fb --- /dev/null +++ b/packages/zaino-sync/src/primitives/batch_index.rs @@ -0,0 +1,26 @@ +//! Batch index newtype. + +/// Identifies a batch within a sync run. +/// +/// Batch 0 covers blocks `[start, start + batch_size)`, batch 1 covers +/// `[start + batch_size, start + 2*batch_size)`, etc. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct BatchIndex(u32); + +impl BatchIndex { + /// Create a batch index. + pub const fn new(index: u32) -> Self { + Self(index) + } + + /// The raw numeric value. + pub const fn value(&self) -> u32 { + self.0 + } +} + +impl core::fmt::Display for BatchIndex { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "batch:{}", self.0) + } +} diff --git a/packages/zaino-sync/src/scheduler.rs b/packages/zaino-sync/src/scheduler.rs new file mode 100644 index 000000000..3df7537d6 --- /dev/null +++ b/packages/zaino-sync/src/scheduler.rs @@ -0,0 +1,380 @@ +//! Scheduler — the brain of the sync engine. +//! +//! The scheduler combines the static [`DependencyDag`] with runtime state +//! to answer: "what work is ready right now?" It tracks which extractions +//! have completed, which batches are pending merge, and which batches +//! have been committed — using this to evaluate per-edge firing rules +//! and emit ready work. +//! +//! The scheduler is passive. It does not execute work — it only decides +//! what CAN run. The engine drives it in a loop: +//! +//! 1. Ask for ready extraction jobs. +//! 2. Hand jobs to an executor. +//! 3. Report completed extractions back. +//! 4. Ask which indexes are ready for merge. +//! 5. Report completed merges. +//! 6. Report committed batches (unlocks downstream firing rules). + +use std::collections::{HashMap, HashSet}; + +use crate::dag::{DependencyDag, FiringRule}; +use crate::primitives::{BatchIndex, IndexId}; + +/// A single extraction job the engine can schedule. +#[derive(Debug, Clone)] +pub struct ExtractJob { + /// Which index to extract for. + pub index: IndexId, + /// Which batch this block belongs to. + pub batch: BatchIndex, + /// Offset of this block within the batch (0-based). + pub block_offset: u32, +} + +/// The scheduler: static DAG + runtime progress tracking. +pub struct Scheduler { + dag: DependencyDag, + batch_size: u32, + + /// All index IDs, cached for iteration. + all_indexes: Vec, + + /// Dependencies per index (cached from DAG edges). + deps: HashMap>, + + /// How many blocks have been extracted for each index in its current batch. + extracted_in_batch: HashMap, + + /// The current batch each index is working on. + current_batch: HashMap, + + /// The last batch each index has committed. + /// `None` means nothing committed yet. + committed_through: HashMap>, + + /// Indexes that have completed extraction for their current batch + /// and are waiting for merge. + pending_merge: HashSet, + + /// Indexes that have completed merge and are waiting for persist + commit. + pending_commit: HashSet, +} + +impl Scheduler { + /// Create a scheduler from a DAG and batch size. + pub fn new(dag: DependencyDag, batch_size: u32) -> Self { + let all_indexes: Vec = dag + .phases() + .iter() + .flat_map(|phase| phase.iter().map(|node| node.descriptor.name)) + .collect(); + + let mut deps: HashMap> = HashMap::new(); + for id in &all_indexes { + deps.insert(*id, Vec::new()); + } + for edge in dag.edges() { + deps.entry(edge.to) + .or_default() + .push((edge.from, edge.firing)); + } + + let mut extracted_in_batch = HashMap::new(); + let mut current_batch = HashMap::new(); + let mut committed_through = HashMap::new(); + + for &id in &all_indexes { + extracted_in_batch.insert(id, 0u32); + current_batch.insert(id, BatchIndex::new(0)); + committed_through.insert(id, None); + } + + Self { + dag, + batch_size, + all_indexes, + deps, + extracted_in_batch, + current_batch, + committed_through, + pending_merge: HashSet::new(), + pending_commit: HashSet::new(), + } + } + + /// Which indexes can accept extraction jobs right now? + /// + /// An index is ready for extraction when: + /// - It is not pending merge or commit. + /// - Its current batch has not yet been fully extracted. + /// - All dependency firing rules are satisfied for its current batch. + /// + /// Returns one `ExtractJob` per ready (index, block_offset) pair. + /// The engine decides how many to dispatch (all at once for + /// BlockLocal, one at a time for SelfCumulative). + pub fn ready_extractions(&self) -> Vec { + let mut jobs = Vec::new(); + + for &id in &self.all_indexes { + if self.pending_merge.contains(&id) || self.pending_commit.contains(&id) { + continue; + } + + let extracted = self.extracted_in_batch[&id]; + if extracted >= self.batch_size { + continue; + } + + let batch = self.current_batch[&id]; + if !self.firing_rules_satisfied(id, batch) { + continue; + } + + // Emit one job for the next block to extract. + // The engine may call this repeatedly, or take multiple + // jobs at once for parallel dispatch. + jobs.push(ExtractJob { + index: id, + batch, + block_offset: extracted, + }); + } + + jobs + } + + /// Record that one extraction completed for an index. + /// + /// When the batch is fully extracted, the index transitions to + /// pending-merge. + pub fn extraction_done(&mut self, index: IndexId) { + let count = self.extracted_in_batch.get_mut(&index) + .expect("index exists in scheduler"); + *count += 1; + + if *count >= self.batch_size { + self.pending_merge.insert(index); + } + } + + /// Which indexes have a full batch of deltas ready for merge? + pub fn ready_for_merge(&self) -> Vec { + self.pending_merge.iter().copied().collect() + } + + /// Record that merge completed for an index. + /// + /// The index transitions to pending-commit. + pub fn merge_done(&mut self, index: IndexId) { + self.pending_merge.remove(&index); + self.pending_commit.insert(index); + } + + /// Record that a batch was committed for an index. + /// + /// Updates committed-through tracking, resets extraction counter, + /// and advances the index to the next batch. + pub fn batch_committed(&mut self, index: IndexId) { + let batch = self.current_batch[&index]; + + self.pending_commit.remove(&index); + self.committed_through.insert(index, Some(batch)); + + // Advance to next batch. + let next = BatchIndex::new(batch.value() + 1); + self.current_batch.insert(index, next); + self.extracted_in_batch.insert(index, 0); + } + + /// Check whether all firing rules are satisfied for an index at a + /// given batch. + fn firing_rules_satisfied(&self, index: IndexId, batch: BatchIndex) -> bool { + let deps = &self.deps[&index]; + + for &(dep_id, rule) in deps { + match rule { + FiringRule::Pipelined => { + // Dependency must have committed at least this batch. + match self.committed_through[&dep_id] { + Some(committed) if committed >= batch => {} + _ => return false, + } + } + FiringRule::Barrier => { + // Dependency must have completed the entire chain. + // For now, we can't check this — barrier means "wait + // until the dep finishes everything." The engine must + // signal when a dep's full range is done. + // Conservative: always block. + // TODO: add `completed` tracking for barrier deps. + return false; + } + } + } + + true + } + + /// Access the underlying DAG. + pub fn dag(&self) -> &DependencyDag { + &self.dag + } + + /// Whether all indexes have finished the given batch + /// (committed through it). + pub fn all_committed_through(&self, batch: BatchIndex) -> bool { + self.all_indexes.iter().all(|id| { + matches!(self.committed_through[id], Some(b) if b >= batch) + }) + } + + /// Whether any index has work remaining (not all committed through + /// the target batch). + pub fn has_pending_work(&self, total_batches: u32) -> bool { + if total_batches == 0 { + return false; + } + let last = BatchIndex::new(total_batches - 1); + !self.all_committed_through(last) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::descriptor::{ + CompositionType, Descriptor, InputScope, SourceAccess, + }; + + fn desc(name: &'static str, deps: &'static [IndexId]) -> Descriptor { + Descriptor { + name: IndexId::new(name), + scope: InputScope::BlockLocal, + composition: CompositionType::Append, + dependencies: deps, + source_access: SourceAccess::None, + } + } + + const A: IndexId = IndexId::new("a"); + const B: IndexId = IndexId::new("b"); + + const DEPS_NONE: &[IndexId] = &[]; + const DEPS_A: &[IndexId] = &[A]; + + #[test] + fn phase_zero_indexes_ready_immediately() { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) + .expect("valid dag"); + let sched = Scheduler::new(dag, 3); + + let jobs = sched.ready_extractions(); + assert_eq!(jobs.len(), 1); + assert_eq!(jobs[0].index, A); + assert_eq!(jobs[0].batch, BatchIndex::new(0)); + assert_eq!(jobs[0].block_offset, 0); + } + + #[test] + fn downstream_blocked_until_upstream_commits() { + let dag = DependencyDag::build(vec![ + desc("a", DEPS_NONE), + desc("b", DEPS_A), + ]) + .expect("valid dag"); + let sched = Scheduler::new(dag, 2); + + let jobs = sched.ready_extractions(); + // Only A is ready, B is blocked. + let ready_ids: HashSet = jobs.iter().map(|j| j.index).collect(); + assert!(ready_ids.contains(&A)); + assert!(!ready_ids.contains(&B)); + } + + #[test] + fn downstream_unblocks_after_upstream_commit() { + let dag = DependencyDag::build(vec![ + desc("a", DEPS_NONE), + desc("b", DEPS_A), + ]) + .expect("valid dag"); + let mut sched = Scheduler::new(dag, 2); + + // Extract A's full batch. + sched.extraction_done(A); + sched.extraction_done(A); + + // A is pending merge, not extracting more. + assert!(sched.ready_for_merge().contains(&A)); + + // B still blocked — A hasn't committed yet. + let jobs = sched.ready_extractions(); + assert!(jobs.iter().all(|j| j.index != B)); + + // Merge and commit A. + sched.merge_done(A); + sched.batch_committed(A); + + // Now B is ready. + let jobs = sched.ready_extractions(); + assert!(jobs.iter().any(|j| j.index == B)); + } + + #[test] + fn extraction_advances_through_batch() { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) + .expect("valid dag"); + let mut sched = Scheduler::new(dag, 3); + + // First extraction. + let jobs = sched.ready_extractions(); + assert_eq!(jobs[0].block_offset, 0); + + sched.extraction_done(A); + + let jobs = sched.ready_extractions(); + assert_eq!(jobs[0].block_offset, 1); + + sched.extraction_done(A); + sched.extraction_done(A); + + // Batch fully extracted — A is pending merge, no more extract jobs. + assert!(sched.ready_for_merge().contains(&A)); + let jobs = sched.ready_extractions(); + assert!(jobs.iter().all(|j| j.index != A)); + } + + #[test] + fn commit_advances_to_next_batch() { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) + .expect("valid dag"); + let mut sched = Scheduler::new(dag, 2); + + // Complete batch 0. + sched.extraction_done(A); + sched.extraction_done(A); + sched.merge_done(A); + sched.batch_committed(A); + + // Now on batch 1. + let jobs = sched.ready_extractions(); + assert_eq!(jobs[0].batch, BatchIndex::new(1)); + assert_eq!(jobs[0].block_offset, 0); + } + + #[test] + fn independent_indexes_both_ready() { + let dag = DependencyDag::build(vec![ + desc("a", DEPS_NONE), + desc("b", DEPS_NONE), + ]) + .expect("valid dag"); + let sched = Scheduler::new(dag, 3); + + let jobs = sched.ready_extractions(); + let ready_ids: HashSet = jobs.iter().map(|j| j.index).collect(); + assert!(ready_ids.contains(&A)); + assert!(ready_ids.contains(&B)); + } +} From 51d5ba7938419550307e3ce3561247011bf1054c Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 19:11:47 -0300 Subject: [PATCH 012/146] Add phantom type state machine to scheduler batch lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BatchHandle with FullyExtracted and Merged markers enforces valid transition ordering at compile time. extraction_done returns Option>, merge_done consumes it and returns BatchHandle, batch_committed consumes that. The engine cannot skip or reorder steps — invalid sequences are type errors. --- packages/zaino-sync/src/scheduler.rs | 117 ++++++++++++++++++++------- 1 file changed, 89 insertions(+), 28 deletions(-) diff --git a/packages/zaino-sync/src/scheduler.rs b/packages/zaino-sync/src/scheduler.rs index 3df7537d6..ae19c9d10 100644 --- a/packages/zaino-sync/src/scheduler.rs +++ b/packages/zaino-sync/src/scheduler.rs @@ -17,10 +17,50 @@ //! 6. Report committed batches (unlocks downstream firing rules). use std::collections::{HashMap, HashSet}; +use std::marker::PhantomData; use crate::dag::{DependencyDag, FiringRule}; use crate::primitives::{BatchIndex, IndexId}; +// =========================================================================== +// State markers — phantom types for index batch lifecycle +// =========================================================================== + +/// The batch has been fully extracted — all block deltas are available. +pub struct FullyExtracted(()); + +/// The batch has been merged — deltas combined into index state. +pub struct Merged(()); + +/// A typed handle to an index at a specific batch lifecycle stage. +/// +/// The engine receives these from scheduler transition methods and must +/// pass them to the next step. Invalid orderings (e.g. committing +/// before merging) are compile errors — the engine can only obtain a +/// `BatchHandle` by calling `merge_done` with a +/// `BatchHandle`. +pub struct BatchHandle { + /// Which index this handle is for. + pub index: IndexId, + /// Which batch. + pub batch: BatchIndex, + _state: PhantomData, +} + +impl BatchHandle { + fn new(index: IndexId, batch: BatchIndex) -> Self { + Self { + index, + batch, + _state: PhantomData, + } + } + + fn transition(self) -> BatchHandle { + BatchHandle::new(self.index, self.batch) + } +} + /// A single extraction job the engine can schedule. #[derive(Debug, Clone)] pub struct ExtractJob { @@ -146,37 +186,54 @@ impl Scheduler { /// Record that one extraction completed for an index. /// - /// When the batch is fully extracted, the index transitions to - /// pending-merge. - pub fn extraction_done(&mut self, index: IndexId) { + /// Returns `Some(BatchHandle)` when the batch is + /// fully extracted. The engine must pass this handle to + /// [`merge_done`](Self::merge_done) — it cannot be skipped or + /// reordered. + pub fn extraction_done(&mut self, index: IndexId) -> Option> { let count = self.extracted_in_batch.get_mut(&index) .expect("index exists in scheduler"); *count += 1; if *count >= self.batch_size { self.pending_merge.insert(index); + let batch = self.current_batch[&index]; + Some(BatchHandle::new(index, batch)) + } else { + None } } /// Which indexes have a full batch of deltas ready for merge? - pub fn ready_for_merge(&self) -> Vec { - self.pending_merge.iter().copied().collect() + /// + /// Returns typed handles that can only be consumed by + /// [`merge_done`](Self::merge_done). + pub fn ready_for_merge(&self) -> Vec> { + self.pending_merge + .iter() + .map(|&id| BatchHandle::new(id, self.current_batch[&id])) + .collect() } /// Record that merge completed for an index. /// - /// The index transitions to pending-commit. - pub fn merge_done(&mut self, index: IndexId) { - self.pending_merge.remove(&index); - self.pending_commit.insert(index); + /// Consumes a `FullyExtracted` handle and returns a `Merged` handle. + /// The engine must pass the `Merged` handle to + /// [`batch_committed`](Self::batch_committed). + pub fn merge_done(&mut self, handle: BatchHandle) -> BatchHandle { + self.pending_merge.remove(&handle.index); + self.pending_commit.insert(handle.index); + handle.transition() } /// Record that a batch was committed for an index. /// - /// Updates committed-through tracking, resets extraction counter, - /// and advances the index to the next batch. - pub fn batch_committed(&mut self, index: IndexId) { - let batch = self.current_batch[&index]; + /// Consumes a `Merged` handle. Updates committed-through tracking, + /// resets extraction counter, and advances the index to the next + /// batch. + pub fn batch_committed(&mut self, handle: BatchHandle) { + let index = handle.index; + let batch = handle.batch; self.pending_commit.remove(&index); self.committed_through.insert(index, Some(batch)); @@ -302,19 +359,19 @@ mod tests { let mut sched = Scheduler::new(dag, 2); // Extract A's full batch. - sched.extraction_done(A); - sched.extraction_done(A); + assert!(sched.extraction_done(A).is_none()); + let handle = sched.extraction_done(A).expect("batch complete"); // A is pending merge, not extracting more. - assert!(sched.ready_for_merge().contains(&A)); + assert!(!sched.ready_for_merge().is_empty()); // B still blocked — A hasn't committed yet. let jobs = sched.ready_extractions(); assert!(jobs.iter().all(|j| j.index != B)); - // Merge and commit A. - sched.merge_done(A); - sched.batch_committed(A); + // Merge and commit A — typed handle chain. + let merged = sched.merge_done(handle); + sched.batch_committed(merged); // Now B is ready. let jobs = sched.ready_extractions(); @@ -331,18 +388,22 @@ mod tests { let jobs = sched.ready_extractions(); assert_eq!(jobs[0].block_offset, 0); - sched.extraction_done(A); + assert!(sched.extraction_done(A).is_none()); let jobs = sched.ready_extractions(); assert_eq!(jobs[0].block_offset, 1); - sched.extraction_done(A); - sched.extraction_done(A); + assert!(sched.extraction_done(A).is_none()); + let handle = sched.extraction_done(A).expect("batch complete"); // Batch fully extracted — A is pending merge, no more extract jobs. - assert!(sched.ready_for_merge().contains(&A)); + assert!(!sched.ready_for_merge().is_empty()); let jobs = sched.ready_extractions(); assert!(jobs.iter().all(|j| j.index != A)); + + // Consume handle to prove it exists. + let merged = sched.merge_done(handle); + sched.batch_committed(merged); } #[test] @@ -351,11 +412,11 @@ mod tests { .expect("valid dag"); let mut sched = Scheduler::new(dag, 2); - // Complete batch 0. - sched.extraction_done(A); - sched.extraction_done(A); - sched.merge_done(A); - sched.batch_committed(A); + // Complete batch 0 — full handle chain. + assert!(sched.extraction_done(A).is_none()); + let extracted = sched.extraction_done(A).expect("batch complete"); + let merged = sched.merge_done(extracted); + sched.batch_committed(merged); // Now on batch 1. let jobs = sched.ready_extractions(); From 0ee6471a1b1f0d4f63f33016655bfc95eb529bcb Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 19:27:34 -0300 Subject: [PATCH 013/146] Split IndexPipeline into extract_one / merge / persist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridges become stateful: Mutex> buffer for extraction, Mutex> for merged output. process_batch kept as default method for backward compat with batch-loop engine. Engine and tests unchanged — they call process_batch which delegates to the three phases internally. --- packages/zaino-sync/src/bridge.rs | 208 ++++++++++++++++++---------- packages/zaino-sync/src/pipeline.rs | 115 +++++++-------- 2 files changed, 192 insertions(+), 131 deletions(-) diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index fe38e7db2..31fe3396a 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -5,8 +5,8 @@ //! The typed trait hierarchy ([`ExtractLocal`], [`MergeAppend`], etc.) //! gives compile-time safety at the index definition site. The engine //! needs runtime dispatch via [`IndexPipeline`]. Bridges are the -//! glue: each bridge struct wraps a concrete index type (via -//! `PhantomData`) and implements `IndexPipeline` by calling the +//! glue: each bridge struct holds internal state (delta buffer, merge +//! accumulator) and implements `IndexPipeline` by calling the //! type-level extract and merge traits internally. The `Delta` type //! never leaves the bridge. //! @@ -19,46 +19,31 @@ //! need backend reader access that the pipeline interface doesn't yet //! provide. //! -//! # MVP limitation: sequential extraction within each bridge +//! # Three-phase pipeline //! -//! Every `process_batch` impl currently loops sequentially over -//! `blocks`. For `BlockLocal` indexes, extraction *could* run in -//! parallel across blocks — the type system already proves there are no -//! inter-block dependencies (that is the whole point of the `BlockLocal` -//! scope marker). But because `process_batch` collapses extract + merge -//! into one call, the engine has no way to schedule individual -//! extractions onto a shared thread pool or interleave work across -//! indexes. +//! Each bridge implements `extract_one` / `merge` / `persist`: //! -//! This is intentional for the MVP: it validates that the type algebra -//! composes end-to-end without solving the intermediate type problem. +//! - **`extract_one`**: computes a delta from one block context, pushes +//! it into an internal `Mutex>` buffer. +//! - **`merge`**: drains the delta buffer and combines deltas per the +//! composition type. Stores the result in the merge state slot. +//! - **`persist`**: converts the merged domain state into `WriteOp`s +//! and clears the state for the next batch. //! -//! # True north: split `extract_one` + `merge_batch` -//! -//! The intended production design splits the pipeline interface so the -//! engine controls extraction parallelism: -//! -//! ```text -//! fn extract_one(&self, ctx: &Ctx, ...) -> Result; -//! fn merge_batch(&self, tokens: Vec) -> Result, ...>; -//! ``` -//! -//! `DeltaToken` would be an opaque handle (e.g., an index into a -//! bridge-internal `Vec`) — no `dyn Any`, no downcasting. The -//! bridge owns the typed delta storage; the engine holds and routes -//! opaque tokens. See [`crate::pipeline`] module docs for details. +//! Interior mutability (`Mutex`) is used throughout because +//! `IndexPipeline` methods take `&self` for trait-object safety. //! //! [`IndexPipeline`]: crate::pipeline::IndexPipeline //! [`ExtractLocal`]: crate::traits::ExtractLocal //! [`MergeAppend`]: crate::traits::MergeAppend use std::marker::PhantomData; +use std::sync::Mutex; use crate::descriptor::{Append, BlockLocal, Descriptor, Fold, Monoidal}; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::traits::{ - DepsReader, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, ProvideContext, - WriteOp, + ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, ProvideContext, WriteOp, }; // =========================================================================== @@ -125,17 +110,19 @@ where // BlockLocal × Append // =========================================================================== -/// Bridge for indexes that are block-local with disjoint-key (append) merge. +/// Stateful bridge for BlockLocal × Append indexes. /// -/// **Parallelism profile (true north):** +/// **Parallelism profile:** /// - Extraction: fully parallel across blocks (no inter-block deps). -/// - Merge: trivial collect — no combine step needed. Each delta's -/// write ops can be emitted independently. +/// - Merge: trivial collect — deltas are independent, no combine step. /// -/// **MVP:** extraction runs sequentially over blocks. No parallelism -/// because `process_batch` owns the full loop. See module docs. +/// Internal state: +/// - `deltas`: buffer of per-block deltas, pushed by `extract_one`. +/// - `merged_ops`: WriteOps produced by `merge`, drained by `persist`. pub struct LocalAppendBridge { descriptor: Descriptor, + deltas: Mutex>, + merged_ops: Mutex>, _index: PhantomData, } @@ -143,6 +130,8 @@ impl LocalAppendBridge { fn new() -> Self { Self { descriptor: I::descriptor(), + deltas: Mutex::new(Vec::new()), + merged_ops: Mutex::new(Vec::new()), _index: PhantomData, } } @@ -157,17 +146,37 @@ where &self.descriptor } - fn process_batch( - &self, - blocks: &[Ctx], - _deps: Option<&DepsReader>, - ) -> Result, PipelineError> { - let mut all_ops = Vec::new(); - for ctx in blocks { - let delta = I::extract(&ctx.context())?; - all_ops.extend(I::to_write_ops(delta)); + fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError> { + let delta = I::extract(&ctx.context())?; + self.deltas.lock().expect("delta mutex poisoned").push(delta); + Ok(()) + } + + fn merge(&self) -> Result<(), PipelineError> { + let deltas: Vec = self + .deltas + .lock() + .expect("delta mutex poisoned") + .drain(..) + .collect(); + + let mut ops = Vec::new(); + for delta in deltas { + ops.extend(I::to_write_ops(delta)); } - Ok(all_ops) + + *self.merged_ops.lock().expect("ops mutex poisoned") = ops; + Ok(()) + } + + fn persist(&self) -> Result, PipelineError> { + let ops = self + .merged_ops + .lock() + .expect("ops mutex poisoned") + .drain(..) + .collect(); + Ok(ops) } } @@ -184,24 +193,29 @@ where // BlockLocal × Monoidal // =========================================================================== -/// Bridge for indexes that are block-local with monoidal (associative + -/// commutative) merge. +/// Stateful bridge for BlockLocal × Monoidal indexes. /// -/// **Parallelism profile (true north):** +/// **Parallelism profile:** /// - Extraction: fully parallel across blocks (no inter-block deps). /// - Merge: parallel reduce tree — `combine` is associative + /// commutative, so deltas can be reduced in any order. /// -/// **MVP:** both extraction and merge run sequentially. See module docs. -pub struct LocalMonoidalBridge { +/// Internal state: +/// - `deltas`: buffer of per-block deltas, pushed by `extract_one`. +/// - `merged_ops`: WriteOps produced by `merge`, drained by `persist`. +pub struct LocalMonoidalBridge { descriptor: Descriptor, + deltas: Mutex>, + merged_ops: Mutex>, _index: PhantomData, } -impl LocalMonoidalBridge { +impl LocalMonoidalBridge { fn new() -> Self { Self { descriptor: I::descriptor(), + deltas: Mutex::new(Vec::new()), + merged_ops: Mutex::new(Vec::new()), _index: PhantomData, } } @@ -216,17 +230,37 @@ where &self.descriptor } - fn process_batch( - &self, - blocks: &[Ctx], - _deps: Option<&DepsReader>, - ) -> Result, PipelineError> { + fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError> { + let delta = I::extract(&ctx.context())?; + self.deltas.lock().expect("delta mutex poisoned").push(delta); + Ok(()) + } + + fn merge(&self) -> Result<(), PipelineError> { + let deltas: Vec = self + .deltas + .lock() + .expect("delta mutex poisoned") + .drain(..) + .collect(); + let mut acc = I::identity(); - for ctx in blocks { - let delta = I::extract(&ctx.context())?; + for delta in deltas { acc = I::combine(acc, I::lift(delta)); } - Ok(I::to_write_ops(acc)) + + *self.merged_ops.lock().expect("ops mutex poisoned") = I::to_write_ops(acc); + Ok(()) + } + + fn persist(&self) -> Result, PipelineError> { + let ops = self + .merged_ops + .lock() + .expect("ops mutex poisoned") + .drain(..) + .collect(); + Ok(ops) } } @@ -243,24 +277,30 @@ where // BlockLocal × Fold // =========================================================================== -/// Bridge for indexes that are block-local with order-dependent (fold) merge. +/// Stateful bridge for BlockLocal × Fold indexes. /// -/// **Parallelism profile (true north):** +/// **Parallelism profile:** /// - Extraction: fully parallel across blocks (no inter-block deps). /// - Merge: strictly sequential — `fold` must apply deltas in chain -/// order. This is inherent to the Fold composition type, not an MVP -/// limitation. +/// order. This is inherent to the Fold composition type. /// -/// **MVP:** extraction also runs sequentially. See module docs. -pub struct LocalFoldBridge { +/// Internal state: +/// - `deltas`: buffer of per-block deltas, pushed by `extract_one`. +/// **Must be consumed in insertion order** during merge. +/// - `merged_ops`: WriteOps produced by `merge`, drained by `persist`. +pub struct LocalFoldBridge { descriptor: Descriptor, + deltas: Mutex>, + merged_ops: Mutex>, _index: PhantomData, } -impl LocalFoldBridge { +impl LocalFoldBridge { fn new() -> Self { Self { descriptor: I::descriptor(), + deltas: Mutex::new(Vec::new()), + merged_ops: Mutex::new(Vec::new()), _index: PhantomData, } } @@ -275,17 +315,37 @@ where &self.descriptor } - fn process_batch( - &self, - blocks: &[Ctx], - _deps: Option<&DepsReader>, - ) -> Result, PipelineError> { + fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError> { + let delta = I::extract(&ctx.context())?; + self.deltas.lock().expect("delta mutex poisoned").push(delta); + Ok(()) + } + + fn merge(&self) -> Result<(), PipelineError> { + let deltas: Vec = self + .deltas + .lock() + .expect("delta mutex poisoned") + .drain(..) + .collect(); + let mut state = I::initial_state(); - for ctx in blocks { - let delta = I::extract(&ctx.context())?; + for delta in deltas { I::fold(&mut state, delta); } - Ok(I::to_write_ops(state)) + + *self.merged_ops.lock().expect("ops mutex poisoned") = I::to_write_ops(state); + Ok(()) + } + + fn persist(&self) -> Result, PipelineError> { + let ops = self + .merged_ops + .lock() + .expect("ops mutex poisoned") + .drain(..) + .collect(); + Ok(ops) } } diff --git a/packages/zaino-sync/src/pipeline.rs b/packages/zaino-sync/src/pipeline.rs index a91610090..0f9556eac 100644 --- a/packages/zaino-sync/src/pipeline.rs +++ b/packages/zaino-sync/src/pipeline.rs @@ -8,44 +8,29 @@ //! [`IndexPipeline`] is the trait-object-safe interface the engine //! works with. Each index's `Delta`, `Accumulator`, and `FoldState` stay //! *inside* its bridge implementation — they never cross the trait -//! boundary. The engine sees only `&[Ctx]` in, `Vec` out. +//! boundary. //! //! Bridge types in [`crate::bridge`] connect the typed traits to this //! interface. //! -//! # Current limitation: `process_batch` collapses extract and merge +//! # Three-phase pipeline //! -//! The current interface exposes a single `process_batch` method that -//! takes `&[Ctx]` and returns `Vec`. This means the engine -//! cannot control extraction parallelism — it hands a whole batch to -//! each index and gets final results back. Extraction within the bridge -//! runs sequentially. +//! The interface exposes three methods that the engine calls in sequence, +//! driven by the [`Scheduler`](crate::scheduler::Scheduler): //! -//! This is a **conscious MVP decision** to validate that the type algebra -//! composes end-to-end without solving the intermediate type problem yet. +//! 1. **`extract_one`** — called per block. Computes a delta from the +//! block context and stores it in the bridge's internal buffer. +//! The engine may call this in parallel for `BlockLocal` indexes. //! -//! # True north: split `extract_one` + `merge_batch` +//! 2. **`merge`** — called once per batch after all extractions complete. +//! Combines stored deltas according to the composition type (collect +//! for Append, reduce for Monoidal, sequential fold for Fold). //! -//! The intended design splits the interface into two methods so the -//! engine can schedule per-block extractions onto a shared thread pool -//! and control parallelism across indexes: -//! -//! ```text -//! fn extract_one(&self, ctx: &Ctx, deps: ...) -> Result; -//! fn merge_batch(&self, tokens: Vec) -> Result, ...>; -//! ``` -//! -//! `DeltaToken` would be an opaque handle (e.g., an index into a -//! bridge-internal `Vec`) that does not expose the concrete -//! `Delta` type across the trait boundary — no `dyn Any`, no -//! downcasting. The bridge owns the typed storage; the engine holds -//! and routes opaque tokens. -//! -//! This split is required to unlock: -//! - Per-block parallel extraction for `BlockLocal` indexes. -//! - Engine-controlled work-stealing across indexes sharing a thread pool. -//! - Streaming extraction (process blocks as the provisioner delivers -//! them, rather than buffering a full batch upfront). +//! 3. **`persist`** — called once per batch after merge. Drains the +//! merged state into `WriteOp`s for the backend. This is the +//! serialization boundary — domain types cross into persistence +//! types here. (Currently the merge traits own this step; it will +//! move to a dedicated persistence layer.) use crate::bridge::BridgeDispatch; use crate::descriptor::Descriptor; @@ -57,9 +42,12 @@ pub enum PipelineError { /// Extraction failed. #[error(transparent)] Extract(#[from] ExtractError), - /// Merge/write-ops conversion failed. + /// Merge failed. #[error("merge failed: {0}")] Merge(String), + /// Persist failed. + #[error("persist failed: {0}")] + Persist(String), } /// The trait-object-safe interface the engine dispatches through. @@ -68,43 +56,56 @@ pub enum PipelineError { /// indexes, kept concrete (not erased). The engine is generic over /// `Ctx` once, not per-index. /// -/// Each call to [`process_batch`](Self::process_batch) encapsulates the -/// full extract → merge → to_write_ops pipeline for one batch of blocks. -/// The `Delta` type never crosses this boundary. -/// -/// **MVP shape.** This will be split into `extract_one` + `merge_batch` -/// once the `DeltaToken` intermediate design is resolved. See module docs. +/// The bridge implementations hold internal state (delta buffer, merge +/// accumulator) behind interior mutability (`Mutex`), so all methods +/// take `&self` for trait-object safety. pub trait IndexPipeline: Send + Sync { /// The declarative descriptor. fn descriptor(&self) -> &Descriptor; - /// Process a batch of blocks through the full pipeline. + /// Extract a delta from one block's context. /// - /// Internally performs: - /// 1. Extraction: produce a delta per block (scope-specific inputs). - /// 2. Merge: combine deltas according to composition type. - /// 3. Conversion: turn merged result into write operations. + /// Stores the delta in the bridge's internal buffer. The engine + /// calls this once per block, potentially in parallel for + /// `BlockLocal` indexes. The scheduler tracks completion counts + /// and transitions to merge when the batch is full. + fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError>; + + /// Merge all accumulated deltas for the current batch. + /// + /// Consumes the delta buffer and combines deltas according to the + /// composition type: + /// - **Append**: collect (no-op — deltas are already independent). + /// - **Monoidal**: parallel-reducible fold via `combine`. + /// - **Fold**: strictly sequential application in chain order. + /// + /// The merged state is held internally until [`persist`](Self::persist). + fn merge(&self) -> Result<(), PipelineError>; + + /// Drain the merged state into write operations. /// - /// `blocks` is in chain order. The engine does not inspect - /// intermediates — it receives final `WriteOp`s ready for commit. + /// Converts domain-typed merge results into `WriteOp`s for the + /// backend. This is the serialization boundary. Clears the + /// internal state, readying the bridge for the next batch. + fn persist(&self) -> Result, PipelineError>; + + /// Convenience: run all three phases sequentially on a batch. + /// + /// Exists for backward compatibility with the batch-loop engine. + /// The streaming scheduler calls the three methods individually. fn process_batch( &self, blocks: &[Ctx], - deps: Option<&crate::traits::DepsReader>, - ) -> Result, PipelineError>; + _deps: Option<&crate::traits::DepsReader>, + ) -> Result, PipelineError> { + for ctx in blocks { + self.extract_one(ctx)?; + } + self.merge()?; + self.persist() + } } -/// Capstone trait: a fully-defined index that can produce its own pipeline. -/// -/// An index implements [`IndexDef`] + one extract trait + one merge trait -/// to define its behaviour. `IntoIndexPipeline` is then derived -/// automatically via a blanket impl — the (Scope, Composition) marker -/// pair selects the right bridge through [`BridgeDispatch`]. -/// -/// This is the trait that [`IndexSet::with`](crate::index_set::IndexSet::with) -/// requires. Index authors never implement it by hand. -/// -/// [`BridgeDispatch`]: crate::bridge::BridgeDispatch /// Capstone trait: a fully-defined index that can produce its own pipeline. /// /// `Ctx` is the set-wide block context. The index's [`BlockContext`] may From 888f5daef69925f1533a5f7f5e1fe437e6e4a457 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 19:39:16 -0300 Subject: [PATCH 014/146] DRY bridges into single LocalBridge with MergeStrategy trait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three near-identical bridge structs replaced by one generic struct parameterized by a MergeStrategy marker (AppendStrategy, MonoidalStrategy, FoldStrategy). Merge now stores domain-typed state (Vec, Accumulator, FoldState) — persist is the serialization boundary where domain state becomes WriteOps. --- packages/zaino-sync/src/bridge.rs | 295 ++++++++++-------------------- 1 file changed, 99 insertions(+), 196 deletions(-) diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index 31fe3396a..cba8e7ac7 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -6,14 +6,17 @@ //! gives compile-time safety at the index definition site. The engine //! needs runtime dispatch via [`IndexPipeline`]. Bridges are the //! glue: each bridge struct holds internal state (delta buffer, merge -//! accumulator) and implements `IndexPipeline` by calling the -//! type-level extract and merge traits internally. The `Delta` type -//! never leaves the bridge. +//! result) and implements `IndexPipeline` by calling the type-level +//! extract and merge traits internally. The `Delta` type never leaves +//! the bridge. //! -//! One bridge per (scope × composition) cell. Currently implemented: -//! - [`LocalAppendBridge`] — BlockLocal × Append -//! - [`LocalMonoidalBridge`] — BlockLocal × Monoidal -//! - [`LocalFoldBridge`] — BlockLocal × Fold +//! # Single bridge struct +//! +//! [`LocalBridge`] handles all three BlockLocal composition types. +//! The composition-specific logic (how deltas are combined, how the +//! merged result becomes WriteOps) is dispatched through the +//! [`MergeStrategy`] trait, which each merge trait (`MergeAppend`, +//! `MergeMonoidal`, `MergeFold`) satisfies via blanket impls. //! //! SelfCumulative and CrossIndex bridges are not yet implemented — they //! need backend reader access that the pipeline interface doesn't yet @@ -21,17 +24,11 @@ //! //! # Three-phase pipeline //! -//! Each bridge implements `extract_one` / `merge` / `persist`: -//! -//! - **`extract_one`**: computes a delta from one block context, pushes -//! it into an internal `Mutex>` buffer. -//! - **`merge`**: drains the delta buffer and combines deltas per the -//! composition type. Stores the result in the merge state slot. -//! - **`persist`**: converts the merged domain state into `WriteOp`s -//! and clears the state for the next batch. -//! -//! Interior mutability (`Mutex`) is used throughout because -//! `IndexPipeline` methods take `&self` for trait-object safety. +//! - **`extract_one`**: computes a delta, pushes into internal buffer. +//! - **`merge`**: drains deltas, combines per composition type, stores +//! domain-typed result. No serialization. +//! - **`persist`**: converts domain result to `WriteOp`s. This is the +//! serialization boundary. //! //! [`IndexPipeline`]: crate::pipeline::IndexPipeline //! [`ExtractLocal`]: crate::traits::ExtractLocal @@ -61,17 +58,12 @@ mod sealed { /// [`crate::pipeline`] delegates to this trait, so index authors never /// need to write `IntoIndexPipeline` by hand. /// -/// `I` is the index type, `Ctx` is the set-wide block context. The bridge -/// requires `Ctx: ProvideContext` to project the set-wide -/// context down to what the index needs. -/// /// [`IntoIndexPipeline`]: crate::pipeline::IntoIndexPipeline pub trait BridgeDispatch: sealed::Sealed { /// Produce the boxed pipeline for index `I` over set-wide context `Ctx`. fn dispatch() -> Box>; } -// Seal the marker pairs. impl sealed::Sealed for (BlockLocal, Append) {} impl sealed::Sealed for (BlockLocal, Monoidal) {} impl sealed::Sealed for (BlockLocal, Fold) {} @@ -82,7 +74,7 @@ where Ctx: ProvideContext + Send + Sync + 'static, { fn dispatch() -> Box> { - local_append::() + Box::new(LocalBridge::::new()) } } @@ -92,7 +84,7 @@ where Ctx: ProvideContext + Send + Sync + 'static, { fn dispatch() -> Box> { - local_monoidal::() + Box::new(LocalBridge::::new()) } } @@ -102,213 +94,136 @@ where Ctx: ProvideContext + Send + Sync + 'static, { fn dispatch() -> Box> { - local_fold::() + Box::new(LocalBridge::::new()) } } // =========================================================================== -// BlockLocal × Append +// MergeStrategy — composition-specific logic // =========================================================================== -/// Stateful bridge for BlockLocal × Append indexes. +/// Composition-specific merge and persist logic. /// -/// **Parallelism profile:** -/// - Extraction: fully parallel across blocks (no inter-block deps). -/// - Merge: trivial collect — deltas are independent, no combine step. -/// -/// Internal state: -/// - `deltas`: buffer of per-block deltas, pushed by `extract_one`. -/// - `merged_ops`: WriteOps produced by `merge`, drained by `persist`. -pub struct LocalAppendBridge { - descriptor: Descriptor, - deltas: Mutex>, - merged_ops: Mutex>, - _index: PhantomData, +/// Abstracts the difference between Append, Monoidal, and Fold so that +/// [`LocalBridge`] can be a single generic struct. Each strategy defines +/// how to combine a `Vec` into a merged result, and how to +/// convert that result into `WriteOp`s. +pub(crate) trait MergeStrategy: Send + Sync + 'static { + /// The domain-typed result of merging a batch of deltas. + type MergedState: Send + Sync; + + /// Combine a batch of deltas into a merged domain result. + fn merge_deltas(deltas: Vec) -> Self::MergedState; + + /// Convert the merged domain result into write operations. + /// This is the serialization boundary. + fn to_write_ops(state: Self::MergedState) -> Vec; } -impl LocalAppendBridge { - fn new() -> Self { - Self { - descriptor: I::descriptor(), - deltas: Mutex::new(Vec::new()), - merged_ops: Mutex::new(Vec::new()), - _index: PhantomData, - } - } -} +/// Strategy marker for Append composition. +struct AppendStrategy; -impl IndexPipeline for LocalAppendBridge +impl MergeStrategy for AppendStrategy where - I: ExtractLocal + MergeAppend, - Ctx: ProvideContext + Send + Sync + 'static, + I: MergeAppend, { - fn descriptor(&self) -> &Descriptor { - &self.descriptor - } + // For append, the merged state is the collected deltas themselves — + // each delta independently produces WriteOps. + type MergedState = Vec; - fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError> { - let delta = I::extract(&ctx.context())?; - self.deltas.lock().expect("delta mutex poisoned").push(delta); - Ok(()) + fn merge_deltas(deltas: Vec) -> Self::MergedState { + deltas } - fn merge(&self) -> Result<(), PipelineError> { - let deltas: Vec = self - .deltas - .lock() - .expect("delta mutex poisoned") - .drain(..) - .collect(); - + fn to_write_ops(state: Self::MergedState) -> Vec { let mut ops = Vec::new(); - for delta in deltas { + for delta in state { ops.extend(I::to_write_ops(delta)); } - - *self.merged_ops.lock().expect("ops mutex poisoned") = ops; - Ok(()) - } - - fn persist(&self) -> Result, PipelineError> { - let ops = self - .merged_ops - .lock() - .expect("ops mutex poisoned") - .drain(..) - .collect(); - Ok(ops) + ops } } -/// Register a BlockLocal × Append index into the pipeline. -pub fn local_append() -> Box> -where - I: ExtractLocal + MergeAppend, - Ctx: ProvideContext + Send + Sync + 'static, -{ - Box::new(LocalAppendBridge::::new()) -} +/// Strategy marker for Monoidal composition. +struct MonoidalStrategy; -// =========================================================================== -// BlockLocal × Monoidal -// =========================================================================== - -/// Stateful bridge for BlockLocal × Monoidal indexes. -/// -/// **Parallelism profile:** -/// - Extraction: fully parallel across blocks (no inter-block deps). -/// - Merge: parallel reduce tree — `combine` is associative + -/// commutative, so deltas can be reduced in any order. -/// -/// Internal state: -/// - `deltas`: buffer of per-block deltas, pushed by `extract_one`. -/// - `merged_ops`: WriteOps produced by `merge`, drained by `persist`. -pub struct LocalMonoidalBridge { - descriptor: Descriptor, - deltas: Mutex>, - merged_ops: Mutex>, - _index: PhantomData, -} - -impl LocalMonoidalBridge { - fn new() -> Self { - Self { - descriptor: I::descriptor(), - deltas: Mutex::new(Vec::new()), - merged_ops: Mutex::new(Vec::new()), - _index: PhantomData, - } - } -} - -impl IndexPipeline for LocalMonoidalBridge +impl MergeStrategy for MonoidalStrategy where - I: ExtractLocal + MergeMonoidal, - Ctx: ProvideContext + Send + Sync + 'static, + I: MergeMonoidal, { - fn descriptor(&self) -> &Descriptor { - &self.descriptor - } - - fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError> { - let delta = I::extract(&ctx.context())?; - self.deltas.lock().expect("delta mutex poisoned").push(delta); - Ok(()) - } - - fn merge(&self) -> Result<(), PipelineError> { - let deltas: Vec = self - .deltas - .lock() - .expect("delta mutex poisoned") - .drain(..) - .collect(); + type MergedState = I::Accumulator; + fn merge_deltas(deltas: Vec) -> Self::MergedState { let mut acc = I::identity(); for delta in deltas { acc = I::combine(acc, I::lift(delta)); } - - *self.merged_ops.lock().expect("ops mutex poisoned") = I::to_write_ops(acc); - Ok(()) + acc } - fn persist(&self) -> Result, PipelineError> { - let ops = self - .merged_ops - .lock() - .expect("ops mutex poisoned") - .drain(..) - .collect(); - Ok(ops) + fn to_write_ops(state: Self::MergedState) -> Vec { + I::to_write_ops(state) } } -/// Register a BlockLocal × Monoidal index into the pipeline. -pub fn local_monoidal() -> Box> +/// Strategy marker for Fold composition. +struct FoldStrategy; + +impl MergeStrategy for FoldStrategy where - I: ExtractLocal + MergeMonoidal, - Ctx: ProvideContext + Send + Sync + 'static, + I: MergeFold, { - Box::new(LocalMonoidalBridge::::new()) + type MergedState = I::FoldState; + + fn merge_deltas(deltas: Vec) -> Self::MergedState { + let mut state = I::initial_state(); + for delta in deltas { + I::fold(&mut state, delta); + } + state + } + + fn to_write_ops(state: Self::MergedState) -> Vec { + I::to_write_ops(state) + } } // =========================================================================== -// BlockLocal × Fold +// LocalBridge — single struct for all BlockLocal compositions // =========================================================================== -/// Stateful bridge for BlockLocal × Fold indexes. +/// Stateful bridge for all BlockLocal indexes. /// -/// **Parallelism profile:** -/// - Extraction: fully parallel across blocks (no inter-block deps). -/// - Merge: strictly sequential — `fold` must apply deltas in chain -/// order. This is inherent to the Fold composition type. +/// `I` is the index type, `S` is the [`MergeStrategy`] marker. The +/// bridge stores deltas in a buffer and merged state in an `Option`. /// -/// Internal state: -/// - `deltas`: buffer of per-block deltas, pushed by `extract_one`. -/// **Must be consumed in insertion order** during merge. -/// - `merged_ops`: WriteOps produced by `merge`, drained by `persist`. -pub struct LocalFoldBridge { +/// **Parallelism profile:** +/// - Extraction: fully parallel across blocks (BlockLocal proves no +/// inter-block deps). +/// - Merge: depends on strategy (trivial for Append, parallel-reducible +/// for Monoidal, sequential for Fold). +pub(crate) struct LocalBridge> { descriptor: Descriptor, deltas: Mutex>, - merged_ops: Mutex>, - _index: PhantomData, + merged: Mutex>, + _phantom: PhantomData<(I, S)>, } -impl LocalFoldBridge { +impl> LocalBridge { fn new() -> Self { Self { descriptor: I::descriptor(), deltas: Mutex::new(Vec::new()), - merged_ops: Mutex::new(Vec::new()), - _index: PhantomData, + merged: Mutex::new(None), + _phantom: PhantomData, } } } -impl IndexPipeline for LocalFoldBridge +impl IndexPipeline for LocalBridge where - I: ExtractLocal + MergeFold, + I: ExtractLocal, + S: MergeStrategy, Ctx: ProvideContext + Send + Sync + 'static, { fn descriptor(&self) -> &Descriptor { @@ -329,31 +244,19 @@ where .drain(..) .collect(); - let mut state = I::initial_state(); - for delta in deltas { - I::fold(&mut state, delta); - } - - *self.merged_ops.lock().expect("ops mutex poisoned") = I::to_write_ops(state); + let state = S::merge_deltas(deltas); + *self.merged.lock().expect("merged mutex poisoned") = Some(state); Ok(()) } fn persist(&self) -> Result, PipelineError> { - let ops = self - .merged_ops + let state = self + .merged .lock() - .expect("ops mutex poisoned") - .drain(..) - .collect(); - Ok(ops) + .expect("merged mutex poisoned") + .take() + .ok_or_else(|| PipelineError::Persist("no merged state to persist".into()))?; + Ok(S::to_write_ops(state)) } } -/// Register a BlockLocal × Fold index into the pipeline. -pub fn local_fold() -> Box> -where - I: ExtractLocal + MergeFold, - Ctx: ProvideContext + Send + Sync + 'static, -{ - Box::new(LocalFoldBridge::::new()) -} From 0250e2e5aa56c5d82f0c0dab104f836c8c39a323 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 20:00:49 -0300 Subject: [PATCH 015/146] Reshape engine to scheduler-driven three-phase dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine is now a thin driver loop over the Scheduler. For each block, it asks the scheduler for ready extraction jobs, calls extract_one on pipelines, and reports back. At batch boundaries the scheduler emits BatchHandle which the engine chains through merge → persist → commit via phantom-typed handles. Partial final batches supported via effective_batch_size. Old monolithic process_batch loop removed. --- packages/zaino-sync/src/engine.rs | 191 ++++++++++++++++++--------- packages/zaino-sync/src/scheduler.rs | 45 ++++++- 2 files changed, 171 insertions(+), 65 deletions(-) diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 20df00cc0..e5ee1e56f 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -1,16 +1,31 @@ //! Sync engine — the orchestrator. //! -//! Owns the DAG, dispatches extraction to workers, drives the -//! merge → commit → flush pipeline, and enforces phase gates. +//! The engine is a thin driver loop. The [`Scheduler`] decides what work +//! is ready; the engine executes it. The flow per block: +//! +//! 1. Scheduler emits ready extraction jobs. +//! 2. Engine calls `extract_one` on the corresponding pipelines. +//! 3. Engine reports each extraction to the scheduler. +//! 4. When a batch is fully extracted (scheduler returns a +//! [`BatchHandle`]), engine calls `merge` + `persist` +//! on that pipeline, commits to backend, and reports back. +//! +//! Cross-phase dependencies are enforced by the scheduler's firing +//! rules — downstream indexes only become ready after their +//! dependencies commit. +//! //! Contains no blockchain knowledge. +//! +//! [`Scheduler`]: crate::scheduler::Scheduler -use std::collections::HashSet; +use std::collections::HashMap; use crate::backend::{Backend, BackendError, BackendWriter}; -use crate::dag::{DagError, DependencyDag}; +use crate::dag::DagError; use crate::index_set::IndexSet; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::primitives::IndexId; +use crate::scheduler::Scheduler; /// Configuration for the sync engine. #[derive(Debug, Clone)] @@ -40,95 +55,149 @@ pub enum SyncError { /// all indexes — no type erasure). /// - `B`: the storage backend. /// -/// Holds the DAG and a heterogeneous collection of indexes via -/// `dyn IndexPipeline`. Delta types stay inside each index's -/// pipeline — the engine only sees `Ctx` in and `Vec` out. +/// The engine owns a [`Scheduler`] that tracks progress and enforces +/// ordering via phantom-typed [`BatchHandle`](crate::scheduler::BatchHandle)s. +/// Pipelines are looked up by `IndexId` for O(1) dispatch. pub struct SyncEngine { - dag: DependencyDag, - indexes: Vec>>, + scheduler: Scheduler, + pipelines: HashMap>>, backend: B, - config: EngineConfig, } impl SyncEngine { - /// Create a new engine from a built DAG, registered indexes, and backend. - pub fn new( - dag: DependencyDag, - indexes: Vec>>, - backend: B, - config: EngineConfig, - ) -> Self { - Self { - dag, - indexes, - backend, - config, - } - } - /// Create an engine from a declarative [`IndexSet`]. /// - /// Builds the dependency DAG from the collected descriptors, - /// validates it, and wires up all pipelines. + /// Builds the dependency DAG, constructs the scheduler, and indexes + /// pipelines by name for O(1) lookup during dispatch. pub fn from_index_set( set: IndexSet, backend: B, config: EngineConfig, ) -> Result { - let (dag, indexes) = set.build()?; + let (dag, index_vec) = set.build()?; + + let pipelines: HashMap>> = index_vec + .into_iter() + .map(|p| (p.descriptor().name, p)) + .collect(); + + let scheduler = Scheduler::new(dag, config.batch_size); + Ok(Self { - dag, - indexes, + scheduler, + pipelines, backend, - config, }) } - /// Process a range of blocks through the full pipeline. + /// Sync a range of blocks through the full pipeline. /// - /// `blocks` must be in chain order. The engine splits them into - /// batches, processes each batch phase-by-phase, commits results - /// to the backend, and flushes after each batch. + /// `blocks` must be in chain order. The engine feeds blocks to the + /// scheduler one at a time, executing extraction jobs as they become + /// ready. Merge, persist, and commit happen at batch boundaries. /// - /// **MVP shape.** Single-threaded, synchronous. Each phase processes - /// sequentially; indexes within a phase also process sequentially. - /// The true north parallelises both across indexes in a phase and - /// across blocks within an index (for BlockLocal scopes). + /// **Current shape:** single-threaded, synchronous. Extraction jobs + /// are executed sequentially. The scheduler infrastructure supports + /// parallel dispatch — swapping in a parallel executor is a future + /// step that does not change this method's signature. pub fn sync_range(&mut self, blocks: &[Ctx]) -> Result<(), SyncError> { - let batch_size = self.config.batch_size as usize; + let total_blocks = u32::try_from(blocks.len()) + .expect("block count fits in u32"); + self.scheduler.set_total_blocks(total_blocks); + let total_batches = total_blocks.div_ceil(self.scheduler.batch_size()); + + // Feed blocks to ready indexes, one at a time. + let mut block_cursor = 0u32; + + loop { + // Get all ready extraction jobs. + let jobs = self.scheduler.ready_extractions(); + + if jobs.is_empty() { + // No extractions ready — check if we're done. + if !self.scheduler.has_pending_work(total_batches) { + break; + } + + // Try to flush any pending merges/commits that might + // unblock downstream indexes. + let merge_handles = self.scheduler.ready_for_merge(); + if merge_handles.is_empty() { + // Nothing to do — shouldn't happen in a correct DAG. + break; + } + + for handle in merge_handles { + self.merge_persist_commit(handle)?; + } + continue; + } + + for job in &jobs { + // Resolve the block index within the full range. + let global_offset = job.batch.value() * self.scheduler.batch_size() + + job.block_offset; + + if global_offset >= total_blocks { + // Past the end of the range — this index has fewer + // blocks than a full batch. Force the batch complete + // by reporting extraction done for the remaining slots. + // + // TODO: handle partial final batches more cleanly. + // For now, skip and let the batch complete naturally. + continue; + } + + let ctx = &blocks[global_offset as usize]; + let pipeline = self.pipelines.get(&job.index) + .expect("scheduler only emits registered indexes"); + + pipeline.extract_one(ctx)?; - for batch in blocks.chunks(batch_size) { - self.process_batch(batch)?; - self.backend.flush()?; + if let Some(handle) = self.scheduler.extraction_done(job.index) { + self.merge_persist_commit(handle)?; + } + } + + // Advance the block cursor for bookkeeping. + block_cursor = block_cursor.max( + jobs.iter() + .map(|j| j.batch.value() * self.scheduler.batch_size() + j.block_offset + 1) + .max() + .unwrap_or(block_cursor), + ); } + self.backend.flush()?; Ok(()) } - fn process_batch(&mut self, batch: &[Ctx]) -> Result<(), SyncError> { - let phases = self.dag.phases(); + /// Merge, persist, and commit a fully-extracted batch for one index. + fn merge_persist_commit( + &mut self, + handle: crate::scheduler::BatchHandle, + ) -> Result<(), SyncError> { + let index_id = handle.index; - for phase_nodes in &phases { - let phase_names: HashSet = phase_nodes - .iter() - .map(|node| node.descriptor.name) - .collect(); + let pipeline = self.pipelines.get(&index_id) + .expect("scheduler only emits registered indexes"); - let mut batch_ops = Vec::new(); + // Merge: combine deltas (domain types). + pipeline.merge()?; - for pipeline in &self.indexes { - if phase_names.contains(&pipeline.descriptor().name) { - let ops = pipeline.process_batch(batch, None)?; - batch_ops.extend(ops); - } - } + // Persist: domain → WriteOps. + let ops = pipeline.persist()?; - if !batch_ops.is_empty() { - let mut writer = self.backend.writer()?; - writer.commit(batch_ops)?; - } + // Commit to backend. + if !ops.is_empty() { + let mut writer = self.backend.writer()?; + writer.commit(ops)?; } + // Tell the scheduler this batch is done. + let merged = self.scheduler.merge_done(handle); + self.scheduler.batch_committed(merged); + Ok(()) } } diff --git a/packages/zaino-sync/src/scheduler.rs b/packages/zaino-sync/src/scheduler.rs index ae19c9d10..c00ec0133 100644 --- a/packages/zaino-sync/src/scheduler.rs +++ b/packages/zaino-sync/src/scheduler.rs @@ -77,6 +77,10 @@ pub struct Scheduler { dag: DependencyDag, batch_size: u32, + /// Total number of blocks in the sync range. Used to compute the + /// effective size of the final (possibly partial) batch. + total_blocks: Option, + /// All index IDs, cached for iteration. all_indexes: Vec, @@ -133,6 +137,7 @@ impl Scheduler { Self { dag, batch_size, + total_blocks: None, all_indexes, deps, extracted_in_batch, @@ -161,12 +166,13 @@ impl Scheduler { continue; } + let batch = self.current_batch[&id]; + let effective = self.effective_batch_size(batch); let extracted = self.extracted_in_batch[&id]; - if extracted >= self.batch_size { + if extracted >= effective { continue; } - let batch = self.current_batch[&id]; if !self.firing_rules_satisfied(id, batch) { continue; } @@ -191,13 +197,15 @@ impl Scheduler { /// [`merge_done`](Self::merge_done) — it cannot be skipped or /// reordered. pub fn extraction_done(&mut self, index: IndexId) -> Option> { + let batch = self.current_batch[&index]; + let effective = self.effective_batch_size(batch); + let count = self.extracted_in_batch.get_mut(&index) .expect("index exists in scheduler"); *count += 1; - if *count >= self.batch_size { + if *count >= effective { self.pending_merge.insert(index); - let batch = self.current_batch[&index]; Some(BatchHandle::new(index, batch)) } else { None @@ -278,6 +286,35 @@ impl Scheduler { &self.dag } + /// The batch size this scheduler was configured with. + pub fn batch_size(&self) -> u32 { + self.batch_size + } + + /// Set the total number of blocks in the sync range. + /// + /// Must be called before `sync_range` begins so the scheduler + /// knows the effective size of the final (partial) batch. + pub fn set_total_blocks(&mut self, total: u32) { + self.total_blocks = Some(total); + } + + /// Effective batch size for a given batch index. + /// + /// All batches are `batch_size` except possibly the last one, + /// which may be smaller if `total_blocks` is not a multiple of + /// `batch_size`. + fn effective_batch_size(&self, batch: BatchIndex) -> u32 { + match self.total_blocks { + Some(total) => { + let start = batch.value() * self.batch_size; + let remaining = total.saturating_sub(start); + remaining.min(self.batch_size) + } + None => self.batch_size, + } + } + /// Whether all indexes have finished the given batch /// (committed through it). pub fn all_committed_through(&self, batch: BatchIndex) -> bool { From 5f4916daf7c020dfca3030372b5a639188899582 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 20:28:07 -0300 Subject: [PATCH 016/146] Derive descriptor from type-level markers via provided method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IndexDef gains NAME, DEPENDENCIES, SOURCE_ACCESS as associated constants with defaults. descriptor() becomes a provided method that derives scope and composition from the marker types automatically. Index implementors no longer construct Descriptor manually — they declare axes and name, the trait does the rest. --- .../src/testing/toy_indexes/count_index.rs | 14 ++------- .../testing/toy_indexes/running_sum_index.rs | 14 ++------- .../src/testing/toy_indexes/value_index.rs | 14 ++------- packages/zaino-sync/src/traits.rs | 31 ++++++++++++++++--- 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs index bedbd3e08..b682ab06a 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs @@ -1,8 +1,6 @@ //! BlockLocal × Monoidal: counts total blocks seen in each batch. -use crate::descriptor::{ - BlockLocal, CompositionType, Descriptor, InputScope, Monoidal, SourceAccess, -}; +use crate::descriptor::{BlockLocal, Monoidal}; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, WriteOp}; @@ -25,15 +23,7 @@ impl IndexDef for CountIndex { type Delta = u64; type BlockContext = Context; - fn descriptor() -> Descriptor { - Descriptor { - name: ID, - scope: InputScope::BlockLocal, - composition: CompositionType::Monoidal, - dependencies: &[], - source_access: SourceAccess::None, - } - } + const NAME: IndexId = ID; } impl ExtractLocal for CountIndex { diff --git a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs index db1e6ccf0..d1c384dbe 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs @@ -1,8 +1,6 @@ //! BlockLocal × Fold: running sum of values across blocks in a batch. -use crate::descriptor::{ - BlockLocal, CompositionType, Descriptor, Fold, InputScope, SourceAccess, -}; +use crate::descriptor::{BlockLocal, Fold}; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeFold, WriteOp}; @@ -24,15 +22,7 @@ impl IndexDef for RunningSumIndex { type Delta = u64; type BlockContext = Context; - fn descriptor() -> Descriptor { - Descriptor { - name: ID, - scope: InputScope::BlockLocal, - composition: CompositionType::Fold, - dependencies: &[], - source_access: SourceAccess::None, - } - } + const NAME: IndexId = ID; } impl ExtractLocal for RunningSumIndex { diff --git a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs index 1a8f51349..eb10baa3f 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs @@ -1,8 +1,6 @@ //! BlockLocal × Append: stores (height → value) for each block. -use crate::descriptor::{ - Append, BlockLocal, CompositionType, Descriptor, InputScope, SourceAccess, -}; +use crate::descriptor::{Append, BlockLocal}; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, WriteOp}; @@ -26,15 +24,7 @@ impl IndexDef for ValueIndex { type Delta = Vec<(Vec, Vec)>; type BlockContext = Context; - fn descriptor() -> Descriptor { - Descriptor { - name: ID, - scope: InputScope::BlockLocal, - composition: CompositionType::Append, - dependencies: &[], - source_access: SourceAccess::None, - } - } + const NAME: IndexId = ID; } impl ExtractLocal for ValueIndex { diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index 7f7116ad9..97acc3326 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -8,7 +8,7 @@ use crate::descriptor::{ Append, BlockLocal, Composition, CrossIndex, Descriptor, Fold, Monoidal, Scope, - SelfCumulative, + SelfCumulative, SourceAccess, }; use crate::primitives::IndexId; @@ -86,7 +86,9 @@ pub enum WriteOp { /// Root declaration for any index. Pins the scope and composition axes as /// associated types, which downstream extract/merge traits use as bounds. /// -/// `S` and `C` are marker types from [`crate::descriptor`]. +/// The implementor declares axes, name, and dependencies. The +/// [`descriptor`](Self::descriptor) method is provided — it derives +/// runtime-inspectable values from the type-level markers automatically. pub trait IndexDef: Send + Sync + 'static { /// Type-level scope marker (BlockLocal | SelfCumulative | CrossIndex). type Scope: Scope; @@ -102,11 +104,30 @@ pub trait IndexDef: Send + Sync + 'static { /// This is the index's view of a block — just the data it cares about. /// The set-wide context (what the provisioner produces) narrows to this /// type via [`ProvideContext`]. - /// type BlockContext: Send + Sync + 'static; - /// The full declarative descriptor. - fn descriptor() -> Descriptor; + /// Unique name for this index. Used as the key in the DAG and in + /// write operations. + const NAME: IndexId; + + /// Indexes this one depends on (must form a DAG). + const DEPENDENCIES: &'static [IndexId] = &[]; + + /// Whether extraction may reach the source for non-local data. + const SOURCE_ACCESS: SourceAccess = SourceAccess::None; + + /// The full declarative descriptor, derived from type-level markers. + /// + /// Provided — implementors do not override this. + fn descriptor() -> Descriptor { + Descriptor { + name: Self::NAME, + scope: Self::Scope::VALUE, + composition: Self::Composition::VALUE, + dependencies: Self::DEPENDENCIES, + source_access: Self::SOURCE_ACCESS, + } + } } // =========================================================================== From 54efa845b73e824f020dcd5227e9c873c8fcca26 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Thu, 2 Jul 2026 20:34:09 -0300 Subject: [PATCH 017/146] Use domain types for ValueIndex delta, not raw bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delta is now Entry { height: BlockHeight, value: u32 } instead of Vec<(Vec, Vec)>. Extract produces pure domain data — no serialization. to_write_ops is the single serialization point, isolated from extraction logic. --- .../zaino-sync/src/testing/toy_indexes.rs | 3 +- .../src/testing/toy_indexes/value_index.rs | 35 +++++++++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index 3bc71272c..a963125ae 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -10,6 +10,7 @@ pub mod count_index; pub mod running_sum_index; pub mod value_index; +use crate::primitives::BlockHeight; use crate::traits::ProvideContext; use super::TestBlockContext; @@ -21,7 +22,7 @@ use super::TestBlockContext; impl ProvideContext for TestBlockContext { fn context(&self) -> value_index::Context { value_index::Context { - height: self.height, + height: BlockHeight::new(self.height), value: self.value, } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs index eb10baa3f..92958b7d8 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs @@ -1,17 +1,25 @@ //! BlockLocal × Append: stores (height → value) for each block. use crate::descriptor::{Append, BlockLocal}; -use crate::primitives::IndexId; +use crate::primitives::{BlockHeight, IndexId}; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, WriteOp}; /// Block context for this index: height and value. pub struct Context { /// Block height. - pub height: u64, + pub height: BlockHeight, /// Arbitrary value carried by this block. pub value: u32, } +/// A single height → value entry. Domain type — no serialization. +pub struct Entry { + /// Block height. + pub height: BlockHeight, + /// Block value. + pub value: u32, +} + /// Stores (height → value) for each block. pub struct ValueIndex; @@ -21,7 +29,7 @@ pub const ID: IndexId = IndexId::new("value"); impl IndexDef for ValueIndex { type Scope = BlockLocal; type Composition = Append; - type Delta = Vec<(Vec, Vec)>; + type Delta = Entry; type BlockContext = Context; const NAME: IndexId = ID; @@ -29,22 +37,19 @@ impl IndexDef for ValueIndex { impl ExtractLocal for ValueIndex { fn extract(ctx: &Context) -> Result { - Ok(vec![( - ctx.height.to_le_bytes().to_vec(), - ctx.value.to_le_bytes().to_vec(), - )]) + Ok(Entry { + height: ctx.height, + value: ctx.value, + }) } } impl MergeAppend for ValueIndex { fn to_write_ops(delta: Self::Delta) -> Vec { - delta - .into_iter() - .map(|(key, value)| WriteOp::Put { - index: ID, - key, - value, - }) - .collect() + vec![WriteOp::Put { + index: ID, + key: delta.height.value().to_le_bytes().to_vec(), + value: delta.value.to_le_bytes().to_vec(), + }] } } From 5f7c62a69f25481537cf013a70fc767f0fa897e7 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 01:30:52 -0300 Subject: [PATCH 018/146] Separate persistence from merge via Schema + Encode traits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schema declares key/value types and maps merge results to entries. Encode handles byte serialization on the types themselves. The bridge does the mechanical plumbing — no index code touches bytes. Using a generic type parameter M (not an associated type) avoids the cycle error that arises when the merged type references the index's own associated types (e.g. Vec). --- packages/zaino-sync/src/bridge.rs | 64 +++++++++---------- packages/zaino-sync/src/encode.rs | 64 +++++++++++++++++++ packages/zaino-sync/src/lib.rs | 1 + .../src/testing/toy_indexes/count_index.rs | 15 +++-- .../testing/toy_indexes/running_sum_index.rs | 15 +++-- .../src/testing/toy_indexes/value_index.rs | 20 +++--- packages/zaino-sync/src/traits.rs | 55 ++++++++++++---- 7 files changed, 169 insertions(+), 65 deletions(-) create mode 100644 packages/zaino-sync/src/encode.rs diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index cba8e7ac7..522700f28 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -38,9 +38,10 @@ use std::marker::PhantomData; use std::sync::Mutex; use crate::descriptor::{Append, BlockLocal, Descriptor, Fold, Monoidal}; +use crate::encode::Encode; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::traits::{ - ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, ProvideContext, WriteOp, + ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, ProvideContext, Schema, WriteOp, }; // =========================================================================== @@ -70,7 +71,10 @@ impl sealed::Sealed for (BlockLocal, Fold) {} impl BridgeDispatch for (BlockLocal, Append) where - I: ExtractLocal + MergeAppend + IndexDef, + I: ExtractLocal + + MergeAppend + + Schema<>::MergedState> + + IndexDef, Ctx: ProvideContext + Send + Sync + 'static, { fn dispatch() -> Box> { @@ -80,7 +84,10 @@ where impl BridgeDispatch for (BlockLocal, Monoidal) where - I: ExtractLocal + MergeMonoidal + IndexDef, + I: ExtractLocal + + MergeMonoidal + + Schema<>::MergedState> + + IndexDef, Ctx: ProvideContext + Send + Sync + 'static, { fn dispatch() -> Box> { @@ -90,7 +97,10 @@ where impl BridgeDispatch for (BlockLocal, Fold) where - I: ExtractLocal + MergeFold + IndexDef, + I: ExtractLocal + + MergeFold + + Schema<>::MergedState> + + IndexDef, Ctx: ProvideContext + Send + Sync + 'static, { fn dispatch() -> Box> { @@ -102,22 +112,20 @@ where // MergeStrategy — composition-specific logic // =========================================================================== -/// Composition-specific merge and persist logic. +/// Composition-specific merge logic. Pure domain — no schema, no encoding. /// /// Abstracts the difference between Append, Monoidal, and Fold so that -/// [`LocalBridge`] can be a single generic struct. Each strategy defines -/// how to combine a `Vec` into a merged result, and how to -/// convert that result into `WriteOp`s. +/// [`LocalBridge`] can be a single generic struct. Each strategy only +/// defines how to combine `Vec` into a merged result. +/// +/// Schema and encoding are handled separately in the bridge's `persist` +/// method via [`Schema`] + [`Encode`]. pub(crate) trait MergeStrategy: Send + Sync + 'static { /// The domain-typed result of merging a batch of deltas. type MergedState: Send + Sync; /// Combine a batch of deltas into a merged domain result. fn merge_deltas(deltas: Vec) -> Self::MergedState; - - /// Convert the merged domain result into write operations. - /// This is the serialization boundary. - fn to_write_ops(state: Self::MergedState) -> Vec; } /// Strategy marker for Append composition. @@ -127,21 +135,11 @@ impl MergeStrategy for AppendStrategy where I: MergeAppend, { - // For append, the merged state is the collected deltas themselves — - // each delta independently produces WriteOps. type MergedState = Vec; fn merge_deltas(deltas: Vec) -> Self::MergedState { deltas } - - fn to_write_ops(state: Self::MergedState) -> Vec { - let mut ops = Vec::new(); - for delta in state { - ops.extend(I::to_write_ops(delta)); - } - ops - } } /// Strategy marker for Monoidal composition. @@ -160,10 +158,6 @@ where } acc } - - fn to_write_ops(state: Self::MergedState) -> Vec { - I::to_write_ops(state) - } } /// Strategy marker for Fold composition. @@ -182,10 +176,6 @@ where } state } - - fn to_write_ops(state: Self::MergedState) -> Vec { - I::to_write_ops(state) - } } // =========================================================================== @@ -222,7 +212,7 @@ impl> LocalBridge { impl IndexPipeline for LocalBridge where - I: ExtractLocal, + I: ExtractLocal + Schema, S: MergeStrategy, Ctx: ProvideContext + Send + Sync + 'static, { @@ -256,7 +246,17 @@ where .expect("merged mutex poisoned") .take() .ok_or_else(|| PipelineError::Persist("no merged state to persist".into()))?; - Ok(S::to_write_ops(state)) + + let ops = I::into_entries(state) + .into_iter() + .map(|(key, value)| WriteOp::Put { + index: I::NAME, + key: key.encode(), + value: value.encode(), + }) + .collect(); + + Ok(ops) } } diff --git a/packages/zaino-sync/src/encode.rs b/packages/zaino-sync/src/encode.rs new file mode 100644 index 000000000..3ccdc2632 --- /dev/null +++ b/packages/zaino-sync/src/encode.rs @@ -0,0 +1,64 @@ +//! Encoding trait for types that cross the persistence boundary. +//! +//! Types that appear as keys or values in index entries implement +//! [`Encode`] to define their byte representation. Index authors never +//! call `encode` directly — the bridge does it mechanically when +//! converting typed entries into [`WriteOp`](crate::traits::WriteOp)s. +//! +//! This is the serialization single source of truth. If the encoding +//! for `BlockHeight` changes, it changes in one place — not scattered +//! across every index's persist impl. + +use crate::primitives::BlockHeight; + +/// Serialize a value to its on-disk byte representation. +/// +/// Implementations must be deterministic — the same value must always +/// produce the same bytes. This is required for key lookups and for +/// cross-index consistency. +pub trait Encode { + /// Encode this value into bytes. + fn encode(&self) -> Vec; +} + +impl Encode for u8 { + fn encode(&self) -> Vec { + vec![*self] + } +} + +impl Encode for u16 { + fn encode(&self) -> Vec { + self.to_le_bytes().to_vec() + } +} + +impl Encode for u32 { + fn encode(&self) -> Vec { + self.to_le_bytes().to_vec() + } +} + +impl Encode for u64 { + fn encode(&self) -> Vec { + self.to_le_bytes().to_vec() + } +} + +impl Encode for BlockHeight { + fn encode(&self) -> Vec { + self.value().to_le_bytes().to_vec() + } +} + +impl Encode for &'static [u8] { + fn encode(&self) -> Vec { + self.to_vec() + } +} + +impl Encode for &'static str { + fn encode(&self) -> Vec { + self.as_bytes().to_vec() + } +} diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index ec1ec29f4..f54072b6b 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -16,6 +16,7 @@ pub mod pipeline; pub mod primitives; pub mod progress; pub mod scheduler; +pub mod encode; pub mod provisioner; pub mod traits; diff --git a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs index b682ab06a..e75b6a02e 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs @@ -2,7 +2,7 @@ use crate::descriptor::{BlockLocal, Monoidal}; use crate::primitives::IndexId; -use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, WriteOp}; +use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, Schema}; /// Block context for this index: nothing needed. /// @@ -46,12 +46,13 @@ impl MergeMonoidal for CountIndex { fn combine(a: Self::Accumulator, b: Self::Accumulator) -> Self::Accumulator { a + b } +} + +impl Schema for CountIndex { + type Key = &'static [u8]; + type Value = u64; - fn to_write_ops(merged: Self::Accumulator) -> Vec { - vec![WriteOp::Put { - index: ID, - key: b"total".to_vec(), - value: merged.to_le_bytes().to_vec(), - }] + fn into_entries(count: u64) -> Vec<(Self::Key, Self::Value)> { + vec![(b"total".as_slice(), count)] } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs index d1c384dbe..5518eee15 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs @@ -2,7 +2,7 @@ use crate::descriptor::{BlockLocal, Fold}; use crate::primitives::IndexId; -use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeFold, WriteOp}; +use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeFold, Schema}; /// Block context for this index: just the block's value. pub struct Context { @@ -41,12 +41,13 @@ impl MergeFold for RunningSumIndex { fn fold(state: &mut Self::FoldState, delta: Self::Delta) { *state += delta; } +} + +impl Schema for RunningSumIndex { + type Key = &'static [u8]; + type Value = u64; - fn to_write_ops(state: Self::FoldState) -> Vec { - vec![WriteOp::Put { - index: ID, - key: b"sum".to_vec(), - value: state.to_le_bytes().to_vec(), - }] + fn into_entries(sum: u64) -> Vec<(Self::Key, Self::Value)> { + vec![(b"sum".as_slice(), sum)] } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs index 92958b7d8..897cca1c2 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs @@ -2,7 +2,7 @@ use crate::descriptor::{Append, BlockLocal}; use crate::primitives::{BlockHeight, IndexId}; -use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, WriteOp}; +use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, Schema}; /// Block context for this index: height and value. pub struct Context { @@ -44,12 +44,16 @@ impl ExtractLocal for ValueIndex { } } -impl MergeAppend for ValueIndex { - fn to_write_ops(delta: Self::Delta) -> Vec { - vec![WriteOp::Put { - index: ID, - key: delta.height.value().to_le_bytes().to_vec(), - value: delta.value.to_le_bytes().to_vec(), - }] +impl MergeAppend for ValueIndex {} + +impl Schema> for ValueIndex { + type Key = BlockHeight; + type Value = u32; + + fn into_entries(entries: Vec) -> Vec<(Self::Key, Self::Value)> { + entries + .into_iter() + .map(|entry| (entry.height, entry.value)) + .collect() } } diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index 97acc3326..a70d2540e 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -184,12 +184,10 @@ pub trait ExtractCross: IndexDef { /// Merge for append-type indexes (disjoint keys). /// -/// No actual merge logic — each delta's write ops are collected and applied. -/// The engine can batch writes from multiple blocks without any combine step. -pub trait MergeAppend: IndexDef { - /// Convert a single block's delta directly to write operations. - fn to_write_ops(delta: Self::Delta) -> Vec; -} +/// Marker trait — Append composition has no merge logic. Each delta +/// is independent. The bridge collects deltas and passes them to +/// [`Persist`] at batch boundary. +pub trait MergeAppend: IndexDef {} /// Merge for monoidal-type indexes (associative + commutative combine). /// @@ -197,6 +195,9 @@ pub trait MergeAppend: IndexDef { /// parallel reduce tree. The implementor must ensure `combine` is /// associative and commutative — the type system cannot enforce these /// algebraic properties, but the engine relies on them for correctness. +/// +/// Pure domain algebra — no persistence concern. Serialization of the +/// merged accumulator is handled by [`Persist`]. pub trait MergeMonoidal: IndexDef { /// The intermediate type used during the reduce. type Accumulator: Send + Sync; @@ -209,15 +210,15 @@ pub trait MergeMonoidal: IndexDef { /// Associative, commutative combine of two accumulators. fn combine(a: Self::Accumulator, b: Self::Accumulator) -> Self::Accumulator; - - /// Convert the fully-merged accumulator to write operations. - fn to_write_ops(merged: Self::Accumulator) -> Vec; } /// Merge for fold-type indexes (order-dependent sequential application). /// /// Deltas must be folded in strict chain order. The engine cannot /// parallelise the merge step for this composition type. +/// +/// Pure domain logic — no persistence concern. Serialization of the +/// final fold state is handled by [`Persist`]. pub trait MergeFold: IndexDef { /// The running state threaded through the fold. type FoldState: Send + Sync; @@ -227,9 +228,41 @@ pub trait MergeFold: IndexDef { /// Apply one block's delta to the running state. Called in chain order. fn fold(state: &mut Self::FoldState, delta: Self::Delta); +} + +// =========================================================================== +// Schema — index entry declaration, separate from merge logic. +// +// The index declares its key/value types and how its merged result +// maps to entries. The types handle serialization via `Encode`. +// The bridge does the mechanical conversion to `WriteOp`s. +// =========================================================================== + +use crate::encode::Encode; - /// Convert the final fold state to write operations. - fn to_write_ops(state: Self::FoldState) -> Vec; +/// Declares an index's key-value schema and how to map results to entries. +/// +/// Generic over `M` — the merge result type. Each composition produces +/// a different merge result: +/// - Append: `Vec` +/// - Monoidal: `Self::Accumulator` +/// - Fold: `Self::FoldState` +/// +/// The index implements `Schema` for its composition's output type. +/// The bridge calls `into_entries` and uses `Encode` on the key/value +/// types to produce `WriteOp`s. No index code touches bytes. +/// +/// Using a type parameter instead of an associated type avoids cycle +/// errors that arise when the merged type references the index's own +/// associated types (e.g. `Vec`). +pub trait Schema: IndexDef { + /// The key type for this index's entries. + type Key: Encode + Send + Sync; + /// The value type for this index's entries. + type Value: Encode + Send + Sync; + + /// Map a merge result to typed key-value entries. + fn into_entries(merged: M) -> Vec<(Self::Key, Self::Value)>; } // =========================================================================== From eda1b792f8312acc703b5d2e16a743ea840fc3c6 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 01:33:59 -0300 Subject: [PATCH 019/146] Use domain newtypes for toy index Schema key/value types BlockValue, BlockCount, RunningSum replace raw u32/u64 in Schema impls. Each newtype carries its own Encode impl. --- .../zaino-sync/src/testing/toy_indexes.rs | 2 +- .../src/testing/toy_indexes/count_index.rs | 39 +++++++++++++++---- .../testing/toy_indexes/running_sum_index.rs | 35 ++++++++++++++--- .../src/testing/toy_indexes/value_index.rs | 29 ++++++++++++-- 4 files changed, 87 insertions(+), 18 deletions(-) diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index a963125ae..1f1311082 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -23,7 +23,7 @@ impl ProvideContext for TestBlockContext { fn context(&self) -> value_index::Context { value_index::Context { height: BlockHeight::new(self.height), - value: self.value, + value: value_index::BlockValue::new(self.value), } } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs index e75b6a02e..ab66a1d84 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs @@ -1,6 +1,7 @@ //! BlockLocal × Monoidal: counts total blocks seen in each batch. use crate::descriptor::{BlockLocal, Monoidal}; +use crate::encode::Encode; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, Schema}; @@ -11,6 +12,28 @@ use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, Schema} /// `ProvideContext<()>` impl. pub type Context = (); +/// A count of blocks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlockCount(u64); + +impl BlockCount { + /// Create a block count. + pub const fn new(count: u64) -> Self { + Self(count) + } + + /// The raw numeric value. + pub const fn value(&self) -> u64 { + self.0 + } +} + +impl Encode for BlockCount { + fn encode(&self) -> Vec { + self.0.to_le_bytes().to_vec() + } +} + /// Counts total blocks seen in each batch. pub struct CountIndex; @@ -20,7 +43,7 @@ pub const ID: IndexId = IndexId::new("count"); impl IndexDef for CountIndex { type Scope = BlockLocal; type Composition = Monoidal; - type Delta = u64; + type Delta = BlockCount; type BlockContext = Context; const NAME: IndexId = ID; @@ -28,15 +51,15 @@ impl IndexDef for CountIndex { impl ExtractLocal for CountIndex { fn extract(_ctx: &Context) -> Result { - Ok(1) + Ok(BlockCount::new(1)) } } impl MergeMonoidal for CountIndex { - type Accumulator = u64; + type Accumulator = BlockCount; fn identity() -> Self::Accumulator { - 0 + BlockCount::new(0) } fn lift(delta: Self::Delta) -> Self::Accumulator { @@ -44,15 +67,15 @@ impl MergeMonoidal for CountIndex { } fn combine(a: Self::Accumulator, b: Self::Accumulator) -> Self::Accumulator { - a + b + BlockCount::new(a.0 + b.0) } } -impl Schema for CountIndex { +impl Schema for CountIndex { type Key = &'static [u8]; - type Value = u64; + type Value = BlockCount; - fn into_entries(count: u64) -> Vec<(Self::Key, Self::Value)> { + fn into_entries(count: BlockCount) -> Vec<(Self::Key, Self::Value)> { vec![(b"total".as_slice(), count)] } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs index 5518eee15..15129d677 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs @@ -1,6 +1,7 @@ //! BlockLocal × Fold: running sum of values across blocks in a batch. use crate::descriptor::{BlockLocal, Fold}; +use crate::encode::Encode; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeFold, Schema}; @@ -10,6 +11,28 @@ pub struct Context { pub value: u32, } +/// A running sum of block values. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RunningSum(u64); + +impl RunningSum { + /// Create a running sum. + pub const fn new(sum: u64) -> Self { + Self(sum) + } + + /// The raw numeric value. + pub const fn value(&self) -> u64 { + self.0 + } +} + +impl Encode for RunningSum { + fn encode(&self) -> Vec { + self.0.to_le_bytes().to_vec() + } +} + /// Running sum of values across blocks in a batch. pub struct RunningSumIndex; @@ -32,22 +55,22 @@ impl ExtractLocal for RunningSumIndex { } impl MergeFold for RunningSumIndex { - type FoldState = u64; + type FoldState = RunningSum; fn initial_state() -> Self::FoldState { - 0 + RunningSum::new(0) } fn fold(state: &mut Self::FoldState, delta: Self::Delta) { - *state += delta; + state.0 += delta; } } -impl Schema for RunningSumIndex { +impl Schema for RunningSumIndex { type Key = &'static [u8]; - type Value = u64; + type Value = RunningSum; - fn into_entries(sum: u64) -> Vec<(Self::Key, Self::Value)> { + fn into_entries(sum: RunningSum) -> Vec<(Self::Key, Self::Value)> { vec![(b"sum".as_slice(), sum)] } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs index 897cca1c2..eee4280b7 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs @@ -1,6 +1,7 @@ //! BlockLocal × Append: stores (height → value) for each block. use crate::descriptor::{Append, BlockLocal}; +use crate::encode::Encode; use crate::primitives::{BlockHeight, IndexId}; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, Schema}; @@ -9,7 +10,29 @@ pub struct Context { /// Block height. pub height: BlockHeight, /// Arbitrary value carried by this block. - pub value: u32, + pub value: BlockValue, +} + +/// An arbitrary value carried by a block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlockValue(u32); + +impl BlockValue { + /// Create a block value. + pub const fn new(val: u32) -> Self { + Self(val) + } + + /// The raw numeric value. + pub const fn value(&self) -> u32 { + self.0 + } +} + +impl Encode for BlockValue { + fn encode(&self) -> Vec { + self.0.to_le_bytes().to_vec() + } } /// A single height → value entry. Domain type — no serialization. @@ -17,7 +40,7 @@ pub struct Entry { /// Block height. pub height: BlockHeight, /// Block value. - pub value: u32, + pub value: BlockValue, } /// Stores (height → value) for each block. @@ -48,7 +71,7 @@ impl MergeAppend for ValueIndex {} impl Schema> for ValueIndex { type Key = BlockHeight; - type Value = u32; + type Value = BlockValue; fn into_entries(entries: Vec) -> Vec<(Self::Key, Self::Value)> { entries From bacd9850a4465fb4ec9b30c8b4ce95b87b63f6ea Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 02:18:47 -0300 Subject: [PATCH 020/146] Add block availability gating and unified Task enum to scheduler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scheduler now tracks blocks_available — a watermark updated by the engine as the provisioner supplies blocks. Extractions are only emitted for blocks below this watermark. New Task enum unifies ExtractJob and CompleteBatch into a single ready_work() output. The engine can dispatch all returned tasks concurrently — the scheduler guarantees safety. Also adds provisioner_done() for signaling end-of-stream, separating "how many blocks exist" from "how many are available right now." --- packages/zaino-sync/src/scheduler.rs | 154 +++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 11 deletions(-) diff --git a/packages/zaino-sync/src/scheduler.rs b/packages/zaino-sync/src/scheduler.rs index c00ec0133..b5f70a9cd 100644 --- a/packages/zaino-sync/src/scheduler.rs +++ b/packages/zaino-sync/src/scheduler.rs @@ -72,13 +72,36 @@ pub struct ExtractJob { pub block_offset: u32, } +/// A unit of work the scheduler declares safe to execute. +/// +/// Workers consume these without knowledge of ordering or dependencies — +/// the scheduler guarantees anything it emits can run right now. +#[derive(Debug, Clone)] +pub enum Task { + /// Extract a delta for one index at one block. + Extract(ExtractJob), + /// Merge + persist + commit a fully-extracted batch. + CompleteBatch { + /// Which index. + index: IndexId, + /// Which batch. + batch: BatchIndex, + }, +} + /// The scheduler: static DAG + runtime progress tracking. pub struct Scheduler { dag: DependencyDag, batch_size: u32, - /// Total number of blocks in the sync range. Used to compute the - /// effective size of the final (possibly partial) batch. + /// How many blocks are currently available for extraction. Updated + /// by the engine as the provisioner supplies blocks. Extractions + /// are only emitted for blocks below this watermark. + blocks_available: u32, + + /// Total number of blocks in the sync range. Set when the + /// provisioner signals completion. Determines the effective size + /// of the final (possibly partial) batch. total_blocks: Option, /// All index IDs, cached for iteration. @@ -137,6 +160,7 @@ impl Scheduler { Self { dag, batch_size, + blocks_available: 0, total_blocks: None, all_indexes, deps, @@ -153,6 +177,7 @@ impl Scheduler { /// An index is ready for extraction when: /// - It is not pending merge or commit. /// - Its current batch has not yet been fully extracted. + /// - The block is available (provisioner has supplied it). /// - All dependency firing rules are satisfied for its current batch. /// /// Returns one `ExtractJob` per ready (index, block_offset) pair. @@ -173,13 +198,17 @@ impl Scheduler { continue; } + // Check block availability — the provisioner may not have + // supplied this block yet. + let global_offset = batch.value() * self.batch_size + extracted; + if global_offset >= self.blocks_available { + continue; + } + if !self.firing_rules_satisfied(id, batch) { continue; } - // Emit one job for the next block to extract. - // The engine may call this repeatedly, or take multiple - // jobs at once for parallel dispatch. jobs.push(ExtractJob { index: id, batch, @@ -252,6 +281,47 @@ impl Scheduler { self.extracted_in_batch.insert(index, 0); } + /// All currently safe-to-execute work. + /// + /// Returns a mix of extraction and batch-completion tasks. The + /// engine spawns all of them — the scheduler guarantees they can + /// run concurrently. + pub fn ready_work(&self) -> Vec { + let mut tasks: Vec = self + .ready_extractions() + .into_iter() + .map(Task::Extract) + .collect(); + + for handle in self.ready_for_merge() { + tasks.push(Task::CompleteBatch { + index: handle.index, + batch: handle.batch, + }); + } + + tasks + } + + /// Update the block availability watermark. + /// + /// Called by the engine as the provisioner supplies blocks. The + /// count is cumulative — "5" means blocks 0..5 are available. + pub fn set_blocks_available(&mut self, count: u32) { + self.blocks_available = count; + } + + /// Signal that the provisioner has finished and no more blocks + /// will arrive. + /// + /// This sets `total_blocks` so the scheduler can compute the + /// effective size of the final (possibly partial) batch. Before + /// this is called, the scheduler assumes every batch is full. + pub fn provisioner_done(&mut self, total_blocks: u32) { + self.total_blocks = Some(total_blocks); + self.blocks_available = total_blocks; + } + /// Check whether all firing rules are satisfied for an index at a /// given batch. fn firing_rules_satisfied(&self, index: IndexId, batch: BatchIndex) -> bool { @@ -293,10 +363,11 @@ impl Scheduler { /// Set the total number of blocks in the sync range. /// - /// Must be called before `sync_range` begins so the scheduler - /// knows the effective size of the final (partial) batch. + /// Convenience for the synchronous engine path — sets both + /// `total_blocks` and `blocks_available` at once (all blocks + /// are available immediately when pre-loaded). pub fn set_total_blocks(&mut self, total: u32) { - self.total_blocks = Some(total); + self.provisioner_done(total); } /// Effective batch size for a given batch index. @@ -361,7 +432,8 @@ mod tests { fn phase_zero_indexes_ready_immediately() { let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) .expect("valid dag"); - let sched = Scheduler::new(dag, 3); + let mut sched = Scheduler::new(dag, 3); + sched.set_blocks_available(10); let jobs = sched.ready_extractions(); assert_eq!(jobs.len(), 1); @@ -370,6 +442,39 @@ mod tests { assert_eq!(jobs[0].block_offset, 0); } + #[test] + fn no_extractions_when_no_blocks_available() { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) + .expect("valid dag"); + let sched = Scheduler::new(dag, 3); + + // No blocks supplied yet — nothing is ready. + assert!(sched.ready_extractions().is_empty()); + assert!(sched.ready_work().is_empty()); + } + + #[test] + fn extractions_gate_on_availability() { + let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) + .expect("valid dag"); + let mut sched = Scheduler::new(dag, 3); + + // Only 1 block available out of a batch of 3. + sched.set_blocks_available(1); + let jobs = sched.ready_extractions(); + assert_eq!(jobs.len(), 1); + + // Extract it — now we need block 1 but only have 1 available. + sched.extraction_done(A); + assert!(sched.ready_extractions().is_empty()); + + // Provisioner supplies more. + sched.set_blocks_available(3); + let jobs = sched.ready_extractions(); + assert_eq!(jobs.len(), 1); + assert_eq!(jobs[0].block_offset, 1); + } + #[test] fn downstream_blocked_until_upstream_commits() { let dag = DependencyDag::build(vec![ @@ -377,7 +482,8 @@ mod tests { desc("b", DEPS_A), ]) .expect("valid dag"); - let sched = Scheduler::new(dag, 2); + let mut sched = Scheduler::new(dag, 2); + sched.set_blocks_available(10); let jobs = sched.ready_extractions(); // Only A is ready, B is blocked. @@ -394,6 +500,7 @@ mod tests { ]) .expect("valid dag"); let mut sched = Scheduler::new(dag, 2); + sched.set_blocks_available(10); // Extract A's full batch. assert!(sched.extraction_done(A).is_none()); @@ -420,6 +527,7 @@ mod tests { let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) .expect("valid dag"); let mut sched = Scheduler::new(dag, 3); + sched.set_blocks_available(10); // First extraction. let jobs = sched.ready_extractions(); @@ -448,6 +556,7 @@ mod tests { let dag = DependencyDag::build(vec![desc("a", DEPS_NONE)]) .expect("valid dag"); let mut sched = Scheduler::new(dag, 2); + sched.set_blocks_available(10); // Complete batch 0 — full handle chain. assert!(sched.extraction_done(A).is_none()); @@ -468,11 +577,34 @@ mod tests { desc("b", DEPS_NONE), ]) .expect("valid dag"); - let sched = Scheduler::new(dag, 3); + let mut sched = Scheduler::new(dag, 3); + sched.set_blocks_available(10); let jobs = sched.ready_extractions(); let ready_ids: HashSet = jobs.iter().map(|j| j.index).collect(); assert!(ready_ids.contains(&A)); assert!(ready_ids.contains(&B)); } + + #[test] + fn ready_work_includes_both_extract_and_batch_tasks() { + let dag = DependencyDag::build(vec![ + desc("a", DEPS_NONE), + desc("b", DEPS_NONE), + ]) + .expect("valid dag"); + let mut sched = Scheduler::new(dag, 2); + sched.set_blocks_available(10); + + // Extract A's full batch. + sched.extraction_done(A); + sched.extraction_done(A); + + // B still extracting. A is ready to merge. + let tasks = sched.ready_work(); + let has_extract = tasks.iter().any(|t| matches!(t, Task::Extract(_))); + let has_batch = tasks.iter().any(|t| matches!(t, Task::CompleteBatch { .. })); + assert!(has_extract, "should have extract tasks for B"); + assert!(has_batch, "should have batch task for A"); + } } From d35449aca2fe04db646456f261b6c18275ad36bd Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 02:29:08 -0300 Subject: [PATCH 021/146] Add BlockBuffer and BlockOffset, type scheduler/engine offsets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BlockBuffer is a sliding-window store for provisioned block contexts. Contexts are Arc-wrapped for async worker access. Eviction is per-batch — blocks drop when the slowest index commits past them. BlockOffset newtype distinguishes global sync-range positions from raw u32. ExtractJob now carries global_offset so the engine can look up blocks in the buffer directly. --- packages/zaino-sync/src/block_buffer.rs | 155 ++++++++++++++++++ packages/zaino-sync/src/engine.rs | 19 +-- packages/zaino-sync/src/lib.rs | 1 + packages/zaino-sync/src/primitives.rs | 2 + .../zaino-sync/src/primitives/block_offset.rs | 27 +++ packages/zaino-sync/src/scheduler.rs | 10 +- 6 files changed, 197 insertions(+), 17 deletions(-) create mode 100644 packages/zaino-sync/src/block_buffer.rs create mode 100644 packages/zaino-sync/src/primitives/block_offset.rs diff --git a/packages/zaino-sync/src/block_buffer.rs b/packages/zaino-sync/src/block_buffer.rs new file mode 100644 index 000000000..636ae321a --- /dev/null +++ b/packages/zaino-sync/src/block_buffer.rs @@ -0,0 +1,155 @@ +//! Sliding-window buffer for provisioned block contexts. +//! +//! Sits between the provisioner (supply) and the scheduler/workers +//! (demand). The provisioner pushes block contexts as they arrive; +//! workers read them by global offset; the engine evicts completed +//! batches once all indexes have committed past them. +//! +//! Eviction is per-batch — when the slowest index commits past batch +//! N, all blocks in that batch are dropped together. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use crate::primitives::{BatchIndex, BlockOffset}; + +/// A buffer of block contexts indexed by global offset. +/// +/// Generic over `Ctx` — the provisioner's block context type. Contexts +/// are stored behind `Arc` so workers can hold references without +/// borrowing the buffer. +/// +/// The buffer tracks which range of blocks it holds and supports +/// eviction of completed batches. +pub struct BlockBuffer { + blocks: BTreeMap>, + batch_size: u32, + /// The lowest global offset still in the buffer. + floor: u32, +} + +impl BlockBuffer { + /// Create a buffer with the given batch size. + pub fn new(batch_size: u32) -> Self { + Self { + blocks: BTreeMap::new(), + batch_size, + floor: 0, + } + } + + /// Push a block context at the given global offset. + /// + /// The offset must be monotonically increasing — blocks arrive in + /// chain order from the provisioner. + pub fn push(&mut self, offset: BlockOffset, ctx: Ctx) { + self.blocks.insert(offset.value(), Arc::new(ctx)); + } + + /// Get a reference-counted handle to the block at `offset`. + /// + /// Returns `None` if the block has been evicted or not yet supplied. + pub fn get(&self, offset: BlockOffset) -> Option> { + self.blocks.get(&offset.value()).cloned() + } + + /// How many blocks are currently in the buffer. + pub fn len(&self) -> usize { + self.blocks.len() + } + + /// Whether the buffer is empty. + pub fn is_empty(&self) -> bool { + self.blocks.is_empty() + } + + /// The total number of blocks that have been pushed (including + /// evicted ones). This is the watermark the engine passes to + /// `scheduler.set_blocks_available()`. + pub fn total_pushed(&self) -> u32 { + self.floor + u32::try_from(self.blocks.len()) + .expect("buffer size fits in u32") + } + + /// Evict all blocks in batches up to and including `through_batch`. + /// + /// Called by the engine when the slowest index has committed past + /// this batch — the blocks are no longer needed by any index. + pub fn evict_through_batch(&mut self, through_batch: BatchIndex) { + let cutoff = (through_batch.value() + 1) * self.batch_size; + self.blocks = self.blocks.split_off(&cutoff); + self.floor = cutoff; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn offset(n: u32) -> BlockOffset { + BlockOffset::new(n) + } + + fn batch(n: u32) -> BatchIndex { + BatchIndex::new(n) + } + + #[test] + fn push_and_get() { + let mut buf: BlockBuffer = BlockBuffer::new(3); + buf.push(offset(0), 100); + buf.push(offset(1), 101); + buf.push(offset(2), 102); + + assert_eq!(*buf.get(offset(0)).expect("exists"), 100); + assert_eq!(*buf.get(offset(1)).expect("exists"), 101); + assert_eq!(*buf.get(offset(2)).expect("exists"), 102); + assert!(buf.get(offset(3)).is_none()); + } + + #[test] + fn total_pushed_tracks_watermark() { + let mut buf: BlockBuffer = BlockBuffer::new(3); + assert_eq!(buf.total_pushed(), 0); + + buf.push(offset(0), 10); + buf.push(offset(1), 11); + assert_eq!(buf.total_pushed(), 2); + } + + #[test] + fn evict_drops_completed_batches() { + let mut buf: BlockBuffer = BlockBuffer::new(3); + for i in 0..9 { + buf.push(offset(i), i * 10); + } + assert_eq!(buf.len(), 9); + + // Evict batch 0 (offsets 0, 1, 2). + buf.evict_through_batch(batch(0)); + assert_eq!(buf.len(), 6); + assert!(buf.get(offset(0)).is_none()); + assert!(buf.get(offset(2)).is_none()); + assert_eq!(*buf.get(offset(3)).expect("exists"), 30); + + // Evict through batch 1 (offsets 0..6). + buf.evict_through_batch(batch(1)); + assert_eq!(buf.len(), 3); + assert!(buf.get(offset(5)).is_none()); + assert_eq!(*buf.get(offset(6)).expect("exists"), 60); + } + + #[test] + fn total_pushed_accounts_for_eviction() { + let mut buf: BlockBuffer = BlockBuffer::new(3); + for i in 0..6 { + buf.push(offset(i), i); + } + assert_eq!(buf.total_pushed(), 6); + + buf.evict_through_batch(batch(0)); + // 3 evicted + 3 remaining = 6 total pushed. + assert_eq!(buf.total_pushed(), 6); + assert_eq!(buf.len(), 3); + } +} diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index e5ee1e56f..faa8c6ffe 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -134,21 +134,13 @@ impl SyncEngine { } for job in &jobs { - // Resolve the block index within the full range. - let global_offset = job.batch.value() * self.scheduler.batch_size() - + job.block_offset; - - if global_offset >= total_blocks { - // Past the end of the range — this index has fewer - // blocks than a full batch. Force the batch complete - // by reporting extraction done for the remaining slots. - // - // TODO: handle partial final batches more cleanly. - // For now, skip and let the batch complete naturally. + let offset = job.global_offset.value(); + + if offset >= total_blocks { continue; } - let ctx = &blocks[global_offset as usize]; + let ctx = &blocks[offset as usize]; let pipeline = self.pipelines.get(&job.index) .expect("scheduler only emits registered indexes"); @@ -159,10 +151,9 @@ impl SyncEngine { } } - // Advance the block cursor for bookkeeping. block_cursor = block_cursor.max( jobs.iter() - .map(|j| j.batch.value() * self.scheduler.batch_size() + j.block_offset + 1) + .map(|j| j.global_offset.value() + 1) .max() .unwrap_or(block_cursor), ); diff --git a/packages/zaino-sync/src/lib.rs b/packages/zaino-sync/src/lib.rs index f54072b6b..842df3877 100644 --- a/packages/zaino-sync/src/lib.rs +++ b/packages/zaino-sync/src/lib.rs @@ -7,6 +7,7 @@ #![warn(missing_docs)] pub mod backend; +pub mod block_buffer; pub mod bridge; pub mod dag; pub mod descriptor; diff --git a/packages/zaino-sync/src/primitives.rs b/packages/zaino-sync/src/primitives.rs index d5c558c1c..7d993bc71 100644 --- a/packages/zaino-sync/src/primitives.rs +++ b/packages/zaino-sync/src/primitives.rs @@ -2,10 +2,12 @@ mod batch_index; mod block_height; +mod block_offset; mod index_id; mod phase_index; pub use batch_index::BatchIndex; pub use block_height::BlockHeight; +pub use block_offset::BlockOffset; pub use index_id::IndexId; pub use phase_index::PhaseIndex; diff --git a/packages/zaino-sync/src/primitives/block_offset.rs b/packages/zaino-sync/src/primitives/block_offset.rs new file mode 100644 index 000000000..99eee5460 --- /dev/null +++ b/packages/zaino-sync/src/primitives/block_offset.rs @@ -0,0 +1,27 @@ +//! Global block offset newtype. + +/// A zero-based offset into the full sync range. +/// +/// Block offset 0 is the first block provisioned, offset 1 is the +/// second, etc. This is NOT a chain height — it's a position within +/// the range being synced. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct BlockOffset(u32); + +impl BlockOffset { + /// Create a block offset. + pub const fn new(offset: u32) -> Self { + Self(offset) + } + + /// The raw numeric value. + pub const fn value(&self) -> u32 { + self.0 + } +} + +impl core::fmt::Display for BlockOffset { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "offset:{}", self.0) + } +} diff --git a/packages/zaino-sync/src/scheduler.rs b/packages/zaino-sync/src/scheduler.rs index b5f70a9cd..8a0d5aef2 100644 --- a/packages/zaino-sync/src/scheduler.rs +++ b/packages/zaino-sync/src/scheduler.rs @@ -20,7 +20,7 @@ use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use crate::dag::{DependencyDag, FiringRule}; -use crate::primitives::{BatchIndex, IndexId}; +use crate::primitives::{BatchIndex, BlockOffset, IndexId}; // =========================================================================== // State markers — phantom types for index batch lifecycle @@ -70,6 +70,9 @@ pub struct ExtractJob { pub batch: BatchIndex, /// Offset of this block within the batch (0-based). pub block_offset: u32, + /// Global offset into the sync range — the key for looking up the + /// block context in the buffer. + pub global_offset: BlockOffset, } /// A unit of work the scheduler declares safe to execute. @@ -200,8 +203,8 @@ impl Scheduler { // Check block availability — the provisioner may not have // supplied this block yet. - let global_offset = batch.value() * self.batch_size + extracted; - if global_offset >= self.blocks_available { + let global = batch.value() * self.batch_size + extracted; + if global >= self.blocks_available { continue; } @@ -213,6 +216,7 @@ impl Scheduler { index: id, batch, block_offset: extracted, + global_offset: BlockOffset::new(global), }); } From 339fa68dff2d27043f541bb55c6a184e01dffd0f Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 02:44:07 -0300 Subject: [PATCH 022/146] Reshape engine to buffer-driven supply/demand loop The engine now owns a BlockBuffer and uses scheduler.ready_work() for task dispatch instead of manually iterating over a block slice. Blocks are loaded into the buffer, the scheduler gates on availability, and completed batches are evicted once all indexes commit past them. --- packages/zaino-sync/src/engine.rs | 161 +++++++++++------- .../zaino-sync/src/testing/toy_indexes.rs | 4 +- 2 files changed, 99 insertions(+), 66 deletions(-) diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index faa8c6ffe..7ddf83807 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -1,14 +1,14 @@ //! Sync engine — the orchestrator. //! -//! The engine is a thin driver loop. The [`Scheduler`] decides what work -//! is ready; the engine executes it. The flow per block: +//! The engine sits between supply (blocks in a [`BlockBuffer`]) and +//! demand (the [`Scheduler`]'s ready-work queue). Its loop is: //! -//! 1. Scheduler emits ready extraction jobs. -//! 2. Engine calls `extract_one` on the corresponding pipelines. -//! 3. Engine reports each extraction to the scheduler. -//! 4. When a batch is fully extracted (scheduler returns a -//! [`BatchHandle`]), engine calls `merge` + `persist` -//! on that pipeline, commits to backend, and reports back. +//! 1. Ask the scheduler for ready [`Task`]s. +//! 2. For extraction tasks: look up the block in the buffer, call +//! `extract_one`, report completion to the scheduler. +//! 3. When a batch is fully extracted, merge + persist + commit and +//! report back. +//! 4. Evict buffer entries once all indexes commit past a batch. //! //! Cross-phase dependencies are enforced by the scheduler's firing //! rules — downstream indexes only become ready after their @@ -16,16 +16,19 @@ //! //! Contains no blockchain knowledge. //! +//! [`BlockBuffer`]: crate::block_buffer::BlockBuffer //! [`Scheduler`]: crate::scheduler::Scheduler +//! [`Task`]: crate::scheduler::Task use std::collections::HashMap; use crate::backend::{Backend, BackendError, BackendWriter}; +use crate::block_buffer::BlockBuffer; use crate::dag::DagError; use crate::index_set::IndexSet; use crate::pipeline::{IndexPipeline, PipelineError}; -use crate::primitives::IndexId; -use crate::scheduler::Scheduler; +use crate::primitives::{BatchIndex, BlockOffset, IndexId}; +use crate::scheduler::{Scheduler, Task}; /// Configuration for the sync engine. #[derive(Debug, Clone)] @@ -62,6 +65,8 @@ pub struct SyncEngine { scheduler: Scheduler, pipelines: HashMap>>, backend: B, + buffer: BlockBuffer, + evicted_through: Option, } impl SyncEngine { @@ -81,82 +86,90 @@ impl SyncEngine { .map(|p| (p.descriptor().name, p)) .collect(); - let scheduler = Scheduler::new(dag, config.batch_size); + let batch_size = config.batch_size; + let scheduler = Scheduler::new(dag, batch_size); Ok(Self { scheduler, pipelines, backend, + buffer: BlockBuffer::new(batch_size), + evicted_through: None, }) } /// Sync a range of blocks through the full pipeline. /// - /// `blocks` must be in chain order. The engine feeds blocks to the - /// scheduler one at a time, executing extraction jobs as they become - /// ready. Merge, persist, and commit happen at batch boundaries. + /// Blocks must be in chain order. The engine loads them into an + /// internal [`BlockBuffer`], signals the scheduler that all blocks + /// are available, and then runs a demand-driven loop: /// - /// **Current shape:** single-threaded, synchronous. Extraction jobs - /// are executed sequentially. The scheduler infrastructure supports - /// parallel dispatch — swapping in a parallel executor is a future - /// step that does not change this method's signature. - pub fn sync_range(&mut self, blocks: &[Ctx]) -> Result<(), SyncError> { + /// 1. Ask the scheduler for ready [`Task`]s. + /// 2. Execute each task (extract via buffer lookup, or merge/persist/commit). + /// 3. Report completions back to the scheduler. + /// 4. Evict buffer entries once all indexes commit past a batch. + /// + /// **Current shape:** single-threaded, synchronous. The scheduler + /// infrastructure supports parallel dispatch — swapping in an async + /// executor is a future step that does not change this method's + /// contract. + pub fn sync_range(&mut self, blocks: Vec) -> Result<(), SyncError> { let total_blocks = u32::try_from(blocks.len()) .expect("block count fits in u32"); - self.scheduler.set_total_blocks(total_blocks); - let total_batches = total_blocks.div_ceil(self.scheduler.batch_size()); - // Feed blocks to ready indexes, one at a time. - let mut block_cursor = 0u32; - - loop { - // Get all ready extraction jobs. - let jobs = self.scheduler.ready_extractions(); + if total_blocks == 0 { + return Ok(()); + } - if jobs.is_empty() { - // No extractions ready — check if we're done. - if !self.scheduler.has_pending_work(total_batches) { - break; - } + // Supply: load all blocks into the buffer. + for (i, ctx) in blocks.into_iter().enumerate() { + self.buffer.push( + BlockOffset::new(u32::try_from(i).expect("block index fits in u32")), + ctx, + ); + } + self.scheduler.provisioner_done(total_blocks); - // Try to flush any pending merges/commits that might - // unblock downstream indexes. - let merge_handles = self.scheduler.ready_for_merge(); - if merge_handles.is_empty() { - // Nothing to do — shouldn't happen in a correct DAG. - break; - } + let total_batches = total_blocks.div_ceil(self.scheduler.batch_size()); - for handle in merge_handles { - self.merge_persist_commit(handle)?; - } - continue; + // Demand loop: pull tasks, execute, report. + loop { + if !self.scheduler.has_pending_work(total_batches) { + break; } - for job in &jobs { - let offset = job.global_offset.value(); - - if offset >= total_blocks { - continue; - } - - let ctx = &blocks[offset as usize]; - let pipeline = self.pipelines.get(&job.index) - .expect("scheduler only emits registered indexes"); - - pipeline.extract_one(ctx)?; + let tasks = self.scheduler.ready_work(); + if tasks.is_empty() { + break; + } - if let Some(handle) = self.scheduler.extraction_done(job.index) { - self.merge_persist_commit(handle)?; + for task in tasks { + match task { + Task::Extract(job) => { + let ctx = self.buffer.get(job.global_offset) + .expect("block available — scheduler verified watermark"); + let pipeline = self.pipelines.get(&job.index) + .expect("scheduler only emits registered indexes"); + + pipeline.extract_one(&ctx)?; + + if let Some(handle) = self.scheduler.extraction_done(job.index) { + self.merge_persist_commit(handle)?; + self.try_evict(); + } + } + Task::CompleteBatch { index, .. } => { + let handle = self.scheduler.ready_for_merge() + .into_iter() + .find(|h| h.index == index); + + if let Some(handle) = handle { + self.merge_persist_commit(handle)?; + self.try_evict(); + } + } } } - - block_cursor = block_cursor.max( - jobs.iter() - .map(|j| j.global_offset.value() + 1) - .max() - .unwrap_or(block_cursor), - ); } self.backend.flush()?; @@ -191,4 +204,24 @@ impl SyncEngine { Ok(()) } + + /// Advance the eviction frontier. + /// + /// After each batch commit, checks whether all indexes have moved + /// past the next unevicted batch. If so, drops those blocks from + /// the buffer — they are no longer needed by any index. + fn try_evict(&mut self) { + loop { + let candidate = match self.evicted_through { + None => BatchIndex::new(0), + Some(b) => BatchIndex::new(b.value() + 1), + }; + if self.scheduler.all_committed_through(candidate) { + self.buffer.evict_through_batch(candidate); + self.evicted_through = Some(candidate); + } else { + break; + } + } + } } diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index 1f1311082..9a3b50cdf 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -81,7 +81,7 @@ mod tests { let backend = InMemoryBackend::new(); let mut engine = build_engine(backend.clone(), 10); - engine.sync_range(&blocks).expect("sync succeeds"); + engine.sync_range(blocks).expect("sync succeeds"); // ValueIndex: 5 entries (heights 0..=4), each height → height as value let values = backend.entries(value_index::ID); @@ -118,7 +118,7 @@ mod tests { // Batch size 3: blocks [0,1,2], [3,4,5], [6,7,8], [9] let mut engine = build_engine(backend.clone(), 3); - engine.sync_range(&blocks).expect("sync succeeds"); + engine.sync_range(blocks).expect("sync succeeds"); // ValueIndex: 10 entries, all correct (append across batches) let values = backend.entries(value_index::ID); From 165a316ce55808fb9e9d4f9bbc4c6b80ee7359f9 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 02:47:28 -0300 Subject: [PATCH 023/146] Add eviction test and test-only engine accessors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verifies the buffer is fully evicted after multi-batch sync — proves the supply/demand lifecycle from load through eviction works end to end. --- packages/zaino-sync/src/engine.rs | 11 ++++++++++ .../zaino-sync/src/testing/toy_indexes.rs | 21 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 7ddf83807..62495525f 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -225,3 +225,14 @@ impl SyncEngine { } } } + +#[cfg(test)] +impl SyncEngine { + pub(crate) fn buffer_len(&self) -> usize { + self.buffer.len() + } + + pub(crate) fn evicted_through(&self) -> Option { + self.evicted_through + } +} diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index 9a3b50cdf..15221fe20 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -53,6 +53,8 @@ mod tests { use crate::provisioner::Provisioner; use crate::testing::{InMemoryBackend, MockProvisioner}; + use crate::primitives::BatchIndex; + use count_index::CountIndex; use running_sum_index::RunningSumIndex; use value_index::ValueIndex; @@ -141,4 +143,23 @@ mod tests { let sum = u64::from_le_bytes(sum_bytes.as_slice().try_into().expect("8 bytes")); assert_eq!(sum, 9); } + + #[test] + fn buffer_evicted_during_multi_batch_sync() { + let provisioner = MockProvisioner::identity(); + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(9)) + .expect("provisioning succeeds"); + + let backend = InMemoryBackend::new(); + // Batch size 3: batches [0,1,2], [3,4,5], [6,7,8], [9]. + let mut engine = build_engine(backend, 3); + + engine.sync_range(blocks).expect("sync succeeds"); + + // All blocks should be evicted — buffer empty. + assert_eq!(engine.buffer_len(), 0); + // Eviction frontier covers all 4 batches (0..=3). + assert_eq!(engine.evicted_through(), Some(BatchIndex::new(3))); + } } From 1e6e4bceb9b0498e26df4a29407320ededbae3f8 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 02:55:38 -0300 Subject: [PATCH 024/146] Extract sync_streaming as core loop, sync_range delegates to it sync_streaming pulls blocks incrementally from an IntoIterator, interleaving supply (batch_size blocks per pull) with demand (task dispatch). sync_range is now a thin wrapper. Streaming test verifies incremental arrival produces equivalent results with full eviction. --- packages/zaino-sync/src/engine.rs | 82 +++++++++++-------- .../zaino-sync/src/testing/toy_indexes.rs | 21 +++++ 2 files changed, 71 insertions(+), 32 deletions(-) diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 62495525f..3fdf1de01 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -98,49 +98,67 @@ impl SyncEngine { }) } - /// Sync a range of blocks through the full pipeline. + /// Sync a pre-loaded range of blocks. /// - /// Blocks must be in chain order. The engine loads them into an - /// internal [`BlockBuffer`], signals the scheduler that all blocks - /// are available, and then runs a demand-driven loop: + /// Convenience wrapper around [`sync_streaming`](Self::sync_streaming) + /// for the common case where all blocks are available upfront. + pub fn sync_range(&mut self, blocks: Vec) -> Result<(), SyncError> { + self.sync_streaming(blocks) + } + + /// Sync blocks from an incremental source. + /// + /// Pulls blocks from `source` in batches, pushes them into the + /// internal [`BlockBuffer`], and interleaves with task processing. + /// Each iteration: + /// + /// 1. **Supply**: pull up to `batch_size` blocks from the source. + /// 2. **Demand**: execute all ready [`Task`]s from the scheduler. + /// 3. **Evict**: drop buffer entries once all indexes commit past a batch. /// - /// 1. Ask the scheduler for ready [`Task`]s. - /// 2. Execute each task (extract via buffer lookup, or merge/persist/commit). - /// 3. Report completions back to the scheduler. - /// 4. Evict buffer entries once all indexes commit past a batch. + /// The loop terminates when the source is exhausted and no work + /// remains. This is the core sync loop — [`sync_range`](Self::sync_range) + /// delegates to it. /// /// **Current shape:** single-threaded, synchronous. The scheduler /// infrastructure supports parallel dispatch — swapping in an async - /// executor is a future step that does not change this method's - /// contract. - pub fn sync_range(&mut self, blocks: Vec) -> Result<(), SyncError> { - let total_blocks = u32::try_from(blocks.len()) - .expect("block count fits in u32"); - - if total_blocks == 0 { - return Ok(()); - } - - // Supply: load all blocks into the buffer. - for (i, ctx) in blocks.into_iter().enumerate() { - self.buffer.push( - BlockOffset::new(u32::try_from(i).expect("block index fits in u32")), - ctx, - ); - } - self.scheduler.provisioner_done(total_blocks); + /// executor is a future step. + pub(crate) fn sync_streaming(&mut self, source: I) -> Result<(), SyncError> + where + I: IntoIterator, + { + let mut source = source.into_iter(); + let mut provisioner_done = false; - let total_batches = total_blocks.div_ceil(self.scheduler.batch_size()); - - // Demand loop: pull tasks, execute, report. loop { - if !self.scheduler.has_pending_work(total_batches) { - break; + // Supply: pull up to one batch of blocks from the source. + if !provisioner_done { + for _ in 0..self.scheduler.batch_size() { + match source.next() { + Some(ctx) => { + let offset = BlockOffset::new(self.buffer.total_pushed()); + self.buffer.push(offset, ctx); + } + None => { + self.scheduler.provisioner_done(self.buffer.total_pushed()); + provisioner_done = true; + break; + } + } + } + if !provisioner_done { + self.scheduler.set_blocks_available(self.buffer.total_pushed()); + } } + // Demand: execute all ready tasks. let tasks = self.scheduler.ready_work(); + if tasks.is_empty() { - break; + if provisioner_done { + break; + } + continue; } for task in tasks { diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index 15221fe20..7182d8f33 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -144,6 +144,27 @@ mod tests { assert_eq!(sum, 9); } + #[test] + fn streaming_iterator_produces_same_results() { + let backend = InMemoryBackend::new(); + let mut engine = build_engine(backend.clone(), 3); + + let blocks = (0u64..=9).map(|h| TestBlockContext { + height: h, + value: h as u32, + }); + + engine.sync_streaming(blocks).expect("sync succeeds"); + + // Incremental arrival produces the same entry count as pre-loaded. + assert_eq!(backend.entries(value_index::ID).len(), 10); + assert!(backend.get_value(count_index::ID, b"total").is_some()); + assert!(backend.get_value(running_sum_index::ID, b"sum").is_some()); + + assert_eq!(engine.buffer_len(), 0); + assert_eq!(engine.evicted_through(), Some(BatchIndex::new(3))); + } + #[test] fn buffer_evicted_during_multi_batch_sync() { let provisioner = MockProvisioner::identity(); From dc45add02036ba4ad9607f9e47bc4209fe224d57 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 02:59:08 -0300 Subject: [PATCH 025/146] Extract dispatch_tasks helper from sync_streaming loop DRYs the task dispatch logic into a shared method so the upcoming async sync_channel can reuse it without duplicating the match arms. --- packages/zaino-sync/src/engine.rs | 50 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 3fdf1de01..2adc2bdcc 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -161,36 +161,42 @@ impl SyncEngine { continue; } - for task in tasks { - match task { - Task::Extract(job) => { - let ctx = self.buffer.get(job.global_offset) - .expect("block available — scheduler verified watermark"); - let pipeline = self.pipelines.get(&job.index) - .expect("scheduler only emits registered indexes"); + self.dispatch_tasks(tasks)?; + } + + self.backend.flush()?; + Ok(()) + } - pipeline.extract_one(&ctx)?; + /// Execute a batch of tasks from the scheduler. + fn dispatch_tasks(&mut self, tasks: Vec) -> Result<(), SyncError> { + for task in tasks { + match task { + Task::Extract(job) => { + let ctx = self.buffer.get(job.global_offset) + .expect("block available — scheduler verified watermark"); + let pipeline = self.pipelines.get(&job.index) + .expect("scheduler only emits registered indexes"); - if let Some(handle) = self.scheduler.extraction_done(job.index) { - self.merge_persist_commit(handle)?; - self.try_evict(); - } + pipeline.extract_one(&ctx)?; + + if let Some(handle) = self.scheduler.extraction_done(job.index) { + self.merge_persist_commit(handle)?; + self.try_evict(); } - Task::CompleteBatch { index, .. } => { - let handle = self.scheduler.ready_for_merge() - .into_iter() - .find(|h| h.index == index); + } + Task::CompleteBatch { index, .. } => { + let handle = self.scheduler.ready_for_merge() + .into_iter() + .find(|h| h.index == index); - if let Some(handle) = handle { - self.merge_persist_commit(handle)?; - self.try_evict(); - } + if let Some(handle) = handle { + self.merge_persist_commit(handle)?; + self.try_evict(); } } } } - - self.backend.flush()?; Ok(()) } From 5116cfccafcefcd653c99520a46f27c9c8a6e3ab Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 03:02:27 -0300 Subject: [PATCH 026/146] Add async sync_channel with channel-driven supply/demand loop The provisioner runs as a separate task and sends blocks through an mpsc channel. The engine drains available blocks (drain_channel), processes ready tasks (dispatch_tasks), and awaits more blocks when idle (await_block). Channel close signals provisioner completion. Adds tokio dependency (sync feature for library, macros+rt for tests). --- packages/zaino-sync/Cargo.toml | 4 + packages/zaino-sync/src/engine.rs | 86 +++++++++++++++++++ .../zaino-sync/src/testing/toy_indexes.rs | 27 ++++++ 3 files changed, 117 insertions(+) diff --git a/packages/zaino-sync/Cargo.toml b/packages/zaino-sync/Cargo.toml index 5cce6e13f..923cba1e7 100644 --- a/packages/zaino-sync/Cargo.toml +++ b/packages/zaino-sync/Cargo.toml @@ -10,6 +10,10 @@ description = "DAG-driven parallel sync engine for blockchain index building." [dependencies] thiserror.workspace = true +tokio = { workspace = true, features = ["sync"] } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } [lints.rust] unsafe_code = "forbid" diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 2adc2bdcc..41f2b5b2f 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -168,6 +168,92 @@ impl SyncEngine { Ok(()) } + /// Sync blocks arriving through an async channel. + /// + /// The provisioner runs independently (typically a spawned task) and + /// sends blocks through the channel. The engine drains available + /// blocks, processes ready tasks, and awaits more blocks when idle. + /// The channel closing signals provisioner completion. + /// + /// This is the production entry point — the provisioner and engine + /// run concurrently, with the [`BlockBuffer`] absorbing the rate + /// difference between supply and demand. + pub async fn sync_channel( + &mut self, + mut rx: tokio::sync::mpsc::Receiver, + ) -> Result<(), SyncError> { + let mut provisioner_done = false; + + loop { + if !provisioner_done { + provisioner_done = self.drain_channel(&mut rx); + } + + let tasks = self.scheduler.ready_work(); + + if tasks.is_empty() { + if provisioner_done { + break; + } + provisioner_done = self.await_block(&mut rx).await; + continue; + } + + self.dispatch_tasks(tasks)?; + } + + self.backend.flush()?; + Ok(()) + } + + /// Non-blocking drain: pull all available blocks from the channel. + /// + /// Returns `true` if the channel disconnected (provisioner done). + fn drain_channel( + &mut self, + rx: &mut tokio::sync::mpsc::Receiver, + ) -> bool { + loop { + match rx.try_recv() { + Ok(ctx) => self.push_block(ctx), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) => return false, + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => { + self.scheduler + .provisioner_done(self.buffer.total_pushed()); + return true; + } + } + } + } + + /// Blocking wait for the next block from the channel. + /// + /// Returns `true` if the channel closed (provisioner done). + async fn await_block( + &mut self, + rx: &mut tokio::sync::mpsc::Receiver, + ) -> bool { + match rx.recv().await { + Some(ctx) => { + self.push_block(ctx); + false + } + None => { + self.scheduler + .provisioner_done(self.buffer.total_pushed()); + true + } + } + } + + /// Push a single block into the buffer and update availability. + fn push_block(&mut self, ctx: Ctx) { + let offset = BlockOffset::new(self.buffer.total_pushed()); + self.buffer.push(offset, ctx); + self.scheduler + .set_blocks_available(self.buffer.total_pushed()); + } + /// Execute a batch of tasks from the scheduler. fn dispatch_tasks(&mut self, tasks: Vec) -> Result<(), SyncError> { for task in tasks { diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index 7182d8f33..e608378c0 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -165,6 +165,33 @@ mod tests { assert_eq!(engine.evicted_through(), Some(BatchIndex::new(3))); } + #[tokio::test] + async fn async_channel_produces_same_results() { + let backend = InMemoryBackend::new(); + let mut engine = build_engine(backend.clone(), 3); + + let (tx, rx) = tokio::sync::mpsc::channel(16); + + tokio::spawn(async move { + for h in 0u64..=9 { + tx.send(TestBlockContext { + height: h, + value: h as u32, + }) + .await + .expect("channel open"); + } + }); + + engine.sync_channel(rx).await.expect("sync succeeds"); + + assert_eq!(backend.entries(value_index::ID).len(), 10); + assert!(backend.get_value(count_index::ID, b"total").is_some()); + assert!(backend.get_value(running_sum_index::ID, b"sum").is_some()); + assert_eq!(engine.buffer_len(), 0); + assert_eq!(engine.evicted_through(), Some(BatchIndex::new(3))); + } + #[test] fn buffer_evicted_during_multi_batch_sync() { let provisioner = MockProvisioner::identity(); From 91dd26044fa894fcff0378ba8401abcc33f4f136 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 03:07:36 -0300 Subject: [PATCH 027/146] Change pipeline storage from Box to Arc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enables sharing pipeline references with spawned extraction tasks. No behavior change — Arc derefs identically to Box. --- packages/zaino-sync/src/engine.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 41f2b5b2f..db8ca40f6 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -21,6 +21,7 @@ //! [`Task`]: crate::scheduler::Task use std::collections::HashMap; +use std::sync::Arc; use crate::backend::{Backend, BackendError, BackendWriter}; use crate::block_buffer::BlockBuffer; @@ -63,7 +64,7 @@ pub enum SyncError { /// Pipelines are looked up by `IndexId` for O(1) dispatch. pub struct SyncEngine { scheduler: Scheduler, - pipelines: HashMap>>, + pipelines: HashMap>>, backend: B, buffer: BlockBuffer, evicted_through: Option, @@ -81,9 +82,12 @@ impl SyncEngine { ) -> Result { let (dag, index_vec) = set.build()?; - let pipelines: HashMap>> = index_vec + let pipelines: HashMap>> = index_vec .into_iter() - .map(|p| (p.descriptor().name, p)) + .map(|p| { + let name = p.descriptor().name; + (name, Arc::from(p)) + }) .collect(); let batch_size = config.batch_size; From 1274dccc29571b1dcb09d9fe062142ae19c5fdf7 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 03:09:52 -0300 Subject: [PATCH 028/146] Add parallel extraction dispatch via spawn_blocking sync_channel now uses dispatch_tasks_async which spawns each extraction on the blocking thread pool, waits for all to complete, then reports completions sequentially. The scheduler guarantees at most one extraction per index per ready_work() call, so spawned tasks are fully independent. --- packages/zaino-sync/Cargo.toml | 2 +- packages/zaino-sync/src/engine.rs | 59 ++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/packages/zaino-sync/Cargo.toml b/packages/zaino-sync/Cargo.toml index 923cba1e7..d1d63df43 100644 --- a/packages/zaino-sync/Cargo.toml +++ b/packages/zaino-sync/Cargo.toml @@ -10,7 +10,7 @@ description = "DAG-driven parallel sync engine for blockchain index building." [dependencies] thiserror.workspace = true -tokio = { workspace = true, features = ["sync"] } +tokio = { workspace = true, features = ["sync", "rt"] } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index db8ca40f6..73d0170a2 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -203,7 +203,7 @@ impl SyncEngine { continue; } - self.dispatch_tasks(tasks)?; + self.dispatch_tasks_async(tasks).await?; } self.backend.flush()?; @@ -290,6 +290,63 @@ impl SyncEngine { Ok(()) } + /// Execute tasks with parallel extraction dispatch. + /// + /// Extractions for different indexes are independent — the scheduler + /// guarantees at most one extraction per index per `ready_work()` + /// call. This method spawns them on the blocking thread pool, waits + /// for all to complete, then reports completions sequentially. + /// + /// Batch completions are handled first (they may unblock downstream + /// extractions on the next iteration). + async fn dispatch_tasks_async(&mut self, tasks: Vec) -> Result<(), SyncError> { + let mut extract_jobs = Vec::new(); + + for task in tasks { + match task { + Task::Extract(job) => extract_jobs.push(job), + Task::CompleteBatch { index, .. } => { + let handle = self.scheduler.ready_for_merge() + .into_iter() + .find(|h| h.index == index); + + if let Some(handle) = handle { + self.merge_persist_commit(handle)?; + self.try_evict(); + } + } + } + } + + // Spawn extractions in parallel on the blocking thread pool. + let mut handles = Vec::with_capacity(extract_jobs.len()); + for job in &extract_jobs { + let ctx = self.buffer.get(job.global_offset) + .expect("block available — scheduler verified watermark"); + let pipeline = Arc::clone( + self.pipelines.get(&job.index) + .expect("scheduler only emits registered indexes"), + ); + handles.push(tokio::task::spawn_blocking(move || { + pipeline.extract_one(&ctx) + })); + } + + for handle in handles { + handle.await.expect("extraction task panicked")?; + } + + // Report completions sequentially — may trigger merges. + for job in extract_jobs { + if let Some(handle) = self.scheduler.extraction_done(job.index) { + self.merge_persist_commit(handle)?; + self.try_evict(); + } + } + + Ok(()) + } + /// Merge, persist, and commit a fully-extracted batch for one index. fn merge_persist_commit( &mut self, From 27d3892407978422fe00424ed7e87ca41da27cb2 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 03:12:46 -0300 Subject: [PATCH 029/146] Decompose dispatch into flush/extract/report helpers Both sync and async dispatch now share flush_batch_completions and report_extractions. The async path adds run_extractions_parallel in between. Each method does one thing. --- packages/zaino-sync/src/engine.rs | 85 ++++++++++++++----------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 73d0170a2..0990f796e 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -29,7 +29,7 @@ use crate::dag::DagError; use crate::index_set::IndexSet; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::primitives::{BatchIndex, BlockOffset, IndexId}; -use crate::scheduler::{Scheduler, Task}; +use crate::scheduler::{ExtractJob, Scheduler, Task}; /// Configuration for the sync engine. #[derive(Debug, Clone)] @@ -258,50 +258,35 @@ impl SyncEngine { .set_blocks_available(self.buffer.total_pushed()); } - /// Execute a batch of tasks from the scheduler. + /// Execute tasks sequentially (sync path). fn dispatch_tasks(&mut self, tasks: Vec) -> Result<(), SyncError> { - for task in tasks { - match task { - Task::Extract(job) => { - let ctx = self.buffer.get(job.global_offset) - .expect("block available — scheduler verified watermark"); - let pipeline = self.pipelines.get(&job.index) - .expect("scheduler only emits registered indexes"); - - pipeline.extract_one(&ctx)?; - - if let Some(handle) = self.scheduler.extraction_done(job.index) { - self.merge_persist_commit(handle)?; - self.try_evict(); - } - } - Task::CompleteBatch { index, .. } => { - let handle = self.scheduler.ready_for_merge() - .into_iter() - .find(|h| h.index == index); - - if let Some(handle) = handle { - self.merge_persist_commit(handle)?; - self.try_evict(); - } - } - } + let jobs = self.flush_batch_completions(tasks)?; + for job in &jobs { + let ctx = self.buffer.get(job.global_offset) + .expect("block available — scheduler verified watermark"); + let pipeline = self.pipelines.get(&job.index) + .expect("scheduler only emits registered indexes"); + pipeline.extract_one(&ctx)?; } - Ok(()) + self.report_extractions(jobs) } - /// Execute tasks with parallel extraction dispatch. - /// - /// Extractions for different indexes are independent — the scheduler - /// guarantees at most one extraction per index per `ready_work()` - /// call. This method spawns them on the blocking thread pool, waits - /// for all to complete, then reports completions sequentially. + /// Execute tasks with parallel extraction (async path). /// - /// Batch completions are handled first (they may unblock downstream - /// extractions on the next iteration). + /// The scheduler guarantees at most one extraction per index per + /// `ready_work()` call, so spawned tasks are fully independent. async fn dispatch_tasks_async(&mut self, tasks: Vec) -> Result<(), SyncError> { - let mut extract_jobs = Vec::new(); + let jobs = self.flush_batch_completions(tasks)?; + self.run_extractions_parallel(&jobs).await?; + self.report_extractions(jobs) + } + /// Handle all batch-completion tasks, return remaining extract jobs. + fn flush_batch_completions( + &mut self, + tasks: Vec, + ) -> Result, SyncError> { + let mut extract_jobs = Vec::new(); for task in tasks { match task { Task::Extract(job) => extract_jobs.push(job), @@ -309,7 +294,6 @@ impl SyncEngine { let handle = self.scheduler.ready_for_merge() .into_iter() .find(|h| h.index == index); - if let Some(handle) = handle { self.merge_persist_commit(handle)?; self.try_evict(); @@ -317,10 +301,16 @@ impl SyncEngine { } } } + Ok(extract_jobs) + } - // Spawn extractions in parallel on the blocking thread pool. - let mut handles = Vec::with_capacity(extract_jobs.len()); - for job in &extract_jobs { + /// Spawn extractions on the blocking thread pool and await all. + async fn run_extractions_parallel( + &self, + jobs: &[ExtractJob], + ) -> Result<(), SyncError> { + let mut handles = Vec::with_capacity(jobs.len()); + for job in jobs { let ctx = self.buffer.get(job.global_offset) .expect("block available — scheduler verified watermark"); let pipeline = Arc::clone( @@ -331,19 +321,22 @@ impl SyncEngine { pipeline.extract_one(&ctx) })); } - for handle in handles { handle.await.expect("extraction task panicked")?; } + Ok(()) + } - // Report completions sequentially — may trigger merges. - for job in extract_jobs { + /// Report completed extractions to the scheduler. + /// + /// May trigger merges when a batch becomes fully extracted. + fn report_extractions(&mut self, jobs: Vec) -> Result<(), SyncError> { + for job in jobs { if let Some(handle) = self.scheduler.extraction_done(job.index) { self.merge_persist_commit(handle)?; self.try_evict(); } } - Ok(()) } From 7fe67330504555248367fd645941b6923849ae32 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 03:26:26 -0300 Subject: [PATCH 030/146] Update Cargo.lock for tokio dependency --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 3755b8fd1..4406eff16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6097,6 +6097,7 @@ name = "zaino-sync" version = "0.1.0" dependencies = [ "thiserror 2.0.18", + "tokio", ] [[package]] From 77a665f1cf8637b014ff1cbabc926b20602f9467 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 03:33:05 -0300 Subject: [PATCH 031/146] Replace spawn_blocking with rayon for parallel extraction rayon's work-stealing pool is the right tool for CPU-bound extraction. This eliminates the Arc requirement on pipelines (rayon borrows via par_iter), removes dispatch_tasks_async (both sync and async paths share one dispatch_tasks), and drops the tokio rt feature from library deps. --- Cargo.lock | 1 + Cargo.toml | 3 ++ packages/zaino-sync/Cargo.toml | 3 +- packages/zaino-sync/src/engine.rs | 66 ++++++++++++------------------- 4 files changed, 31 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4406eff16..fe2e2e7e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6096,6 +6096,7 @@ dependencies = [ name = "zaino-sync" version = "0.1.0" dependencies = [ + "rayon", "thiserror 2.0.18", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 9c0e50422..96d2a9ef7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,6 +105,9 @@ dashmap = "6.1" lmdb = "0.8" lmdb-sys = "0.8" +# Parallelism +rayon = "1.10" + # Async async-stream = "0.3" futures = "0.3.30" diff --git a/packages/zaino-sync/Cargo.toml b/packages/zaino-sync/Cargo.toml index d1d63df43..2dde78dc5 100644 --- a/packages/zaino-sync/Cargo.toml +++ b/packages/zaino-sync/Cargo.toml @@ -9,8 +9,9 @@ homepage.workspace = true description = "DAG-driven parallel sync engine for blockchain index building." [dependencies] +rayon.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = ["sync", "rt"] } +tokio = { workspace = true, features = ["sync"] } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 0990f796e..802b901ed 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -21,7 +21,8 @@ //! [`Task`]: crate::scheduler::Task use std::collections::HashMap; -use std::sync::Arc; + +use rayon::prelude::*; use crate::backend::{Backend, BackendError, BackendWriter}; use crate::block_buffer::BlockBuffer; @@ -64,7 +65,7 @@ pub enum SyncError { /// Pipelines are looked up by `IndexId` for O(1) dispatch. pub struct SyncEngine { scheduler: Scheduler, - pipelines: HashMap>>, + pipelines: HashMap>>, backend: B, buffer: BlockBuffer, evicted_through: Option, @@ -82,12 +83,9 @@ impl SyncEngine { ) -> Result { let (dag, index_vec) = set.build()?; - let pipelines: HashMap>> = index_vec + let pipelines: HashMap>> = index_vec .into_iter() - .map(|p| { - let name = p.descriptor().name; - (name, Arc::from(p)) - }) + .map(|p| (p.descriptor().name, p)) .collect(); let batch_size = config.batch_size; @@ -203,7 +201,7 @@ impl SyncEngine { continue; } - self.dispatch_tasks_async(tasks).await?; + self.dispatch_tasks(tasks)?; } self.backend.flush()?; @@ -258,26 +256,13 @@ impl SyncEngine { .set_blocks_available(self.buffer.total_pushed()); } - /// Execute tasks sequentially (sync path). - fn dispatch_tasks(&mut self, tasks: Vec) -> Result<(), SyncError> { - let jobs = self.flush_batch_completions(tasks)?; - for job in &jobs { - let ctx = self.buffer.get(job.global_offset) - .expect("block available — scheduler verified watermark"); - let pipeline = self.pipelines.get(&job.index) - .expect("scheduler only emits registered indexes"); - pipeline.extract_one(&ctx)?; - } - self.report_extractions(jobs) - } - - /// Execute tasks with parallel extraction (async path). + /// Execute a batch of tasks from the scheduler. /// - /// The scheduler guarantees at most one extraction per index per - /// `ready_work()` call, so spawned tasks are fully independent. - async fn dispatch_tasks_async(&mut self, tasks: Vec) -> Result<(), SyncError> { + /// Handles batch completions first, then runs all extractions in + /// parallel via rayon, then reports completions sequentially. + fn dispatch_tasks(&mut self, tasks: Vec) -> Result<(), SyncError> { let jobs = self.flush_batch_completions(tasks)?; - self.run_extractions_parallel(&jobs).await?; + self.run_extractions_parallel(&jobs)?; self.report_extractions(jobs) } @@ -304,26 +289,25 @@ impl SyncEngine { Ok(extract_jobs) } - /// Spawn extractions on the blocking thread pool and await all. - async fn run_extractions_parallel( + /// Run extractions in parallel via rayon's work-stealing pool. + /// + /// Prepares (pipeline, context) pairs on the calling thread, then + /// fans out via `par_iter`. Borrows from `self` — no Arc cloning. + fn run_extractions_parallel( &self, jobs: &[ExtractJob], ) -> Result<(), SyncError> { - let mut handles = Vec::with_capacity(jobs.len()); - for job in jobs { + let work: Vec<_> = jobs.iter().map(|job| { let ctx = self.buffer.get(job.global_offset) .expect("block available — scheduler verified watermark"); - let pipeline = Arc::clone( - self.pipelines.get(&job.index) - .expect("scheduler only emits registered indexes"), - ); - handles.push(tokio::task::spawn_blocking(move || { - pipeline.extract_one(&ctx) - })); - } - for handle in handles { - handle.await.expect("extraction task panicked")?; - } + let pipeline = self.pipelines.get(&job.index) + .expect("scheduler only emits registered indexes"); + (pipeline, ctx) + }).collect(); + + work.par_iter() + .try_for_each(|(pipeline, ctx)| pipeline.extract_one(ctx))?; + Ok(()) } From c7c16bb38fe311fc3db301223d6f4e427ad4ea4f Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 04:10:44 -0300 Subject: [PATCH 032/146] Add CumulativeBridge for SelfCumulative index scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend MergeStrategy with initial_state/accumulate_one primitives, making merge_deltas a provided method. CumulativeBridge threads running state through sequential extractions via accumulate_one — no separate advance_state needed, the composition algebra handles it. CumulativeSumIndex toy (S,M) proves batch-size-independent results: extraction depends on prior accumulated state, doubling contributions past a threshold. --- packages/zaino-sync/src/bridge.rs | 194 ++++++++++++++++-- .../zaino-sync/src/testing/toy_indexes.rs | 124 ++++++++++- .../toy_indexes/cumulative_sum_index.rs | 98 +++++++++ packages/zaino-sync/src/traits.rs | 24 ++- 4 files changed, 409 insertions(+), 31 deletions(-) create mode 100644 packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index 522700f28..f93361363 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -18,9 +18,13 @@ //! [`MergeStrategy`] trait, which each merge trait (`MergeAppend`, //! `MergeMonoidal`, `MergeFold`) satisfies via blanket impls. //! -//! SelfCumulative and CrossIndex bridges are not yet implemented — they -//! need backend reader access that the pipeline interface doesn't yet -//! provide. +//! [`CumulativeBridge`] handles all SelfCumulative composition types. +//! It threads a running [`PriorState`](crate::traits::ExtractCumulative::PriorState) +//! through sequential extractions and snapshots the accumulated state +//! at batch end for persistence. +//! +//! CrossIndex bridges are not yet implemented — they need backend reader +//! access that the pipeline interface doesn't yet provide. //! //! # Three-phase pipeline //! @@ -37,11 +41,12 @@ use std::marker::PhantomData; use std::sync::Mutex; -use crate::descriptor::{Append, BlockLocal, Descriptor, Fold, Monoidal}; +use crate::descriptor::{Append, BlockLocal, Descriptor, Fold, Monoidal, SelfCumulative}; use crate::encode::Encode; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::traits::{ - ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, ProvideContext, Schema, WriteOp, + ExtractCumulative, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, + ProvideContext, Schema, WriteOp, }; // =========================================================================== @@ -69,6 +74,9 @@ impl sealed::Sealed for (BlockLocal, Append) {} impl sealed::Sealed for (BlockLocal, Monoidal) {} impl sealed::Sealed for (BlockLocal, Fold) {} +impl sealed::Sealed for (SelfCumulative, Monoidal) {} +impl sealed::Sealed for (SelfCumulative, Fold) {} + impl BridgeDispatch for (BlockLocal, Append) where I: ExtractLocal @@ -108,6 +116,34 @@ where } } +impl BridgeDispatch for (SelfCumulative, Monoidal) +where + I: ExtractCumulative>::MergedState> + + MergeMonoidal + + Schema<>::MergedState> + + IndexDef, + >::MergedState: Clone, + Ctx: ProvideContext + Send + Sync + 'static, +{ + fn dispatch() -> Box> { + Box::new(CumulativeBridge::::new()) + } +} + +impl BridgeDispatch for (SelfCumulative, Fold) +where + I: ExtractCumulative>::MergedState> + + MergeFold + + Schema<>::MergedState> + + IndexDef, + >::MergedState: Clone, + Ctx: ProvideContext + Send + Sync + 'static, +{ + fn dispatch() -> Box> { + Box::new(CumulativeBridge::::new()) + } +} + // =========================================================================== // MergeStrategy — composition-specific logic // =========================================================================== @@ -115,8 +151,10 @@ where /// Composition-specific merge logic. Pure domain — no schema, no encoding. /// /// Abstracts the difference between Append, Monoidal, and Fold so that -/// [`LocalBridge`] can be a single generic struct. Each strategy only -/// defines how to combine `Vec` into a merged result. +/// [`LocalBridge`] and [`CumulativeBridge`] can each be a single generic +/// struct. Two primitive methods — [`initial_state`](Self::initial_state) +/// and [`accumulate_one`](Self::accumulate_one) — define the algebra. +/// [`merge_deltas`](Self::merge_deltas) is provided from them. /// /// Schema and encoding are handled separately in the bridge's `persist` /// method via [`Schema`] + [`Encode`]. @@ -124,8 +162,20 @@ pub(crate) trait MergeStrategy: Send + Sync + 'static { /// The domain-typed result of merging a batch of deltas. type MergedState: Send + Sync; + /// The identity/initial state before any deltas. + fn initial_state() -> Self::MergedState; + + /// Fold one delta into the running state. + fn accumulate_one(state: &mut Self::MergedState, delta: I::Delta); + /// Combine a batch of deltas into a merged domain result. - fn merge_deltas(deltas: Vec) -> Self::MergedState; + fn merge_deltas(deltas: Vec) -> Self::MergedState { + let mut state = Self::initial_state(); + for delta in deltas { + Self::accumulate_one(&mut state, delta); + } + state + } } /// Strategy marker for Append composition. @@ -137,8 +187,12 @@ where { type MergedState = Vec; - fn merge_deltas(deltas: Vec) -> Self::MergedState { - deltas + fn initial_state() -> Self::MergedState { + Vec::new() + } + + fn accumulate_one(state: &mut Self::MergedState, delta: I::Delta) { + state.push(delta); } } @@ -151,12 +205,13 @@ where { type MergedState = I::Accumulator; - fn merge_deltas(deltas: Vec) -> Self::MergedState { - let mut acc = I::identity(); - for delta in deltas { - acc = I::combine(acc, I::lift(delta)); - } - acc + fn initial_state() -> Self::MergedState { + I::identity() + } + + fn accumulate_one(state: &mut Self::MergedState, delta: I::Delta) { + let prev = std::mem::replace(state, I::identity()); + *state = I::combine(prev, I::lift(delta)); } } @@ -169,12 +224,12 @@ where { type MergedState = I::FoldState; - fn merge_deltas(deltas: Vec) -> Self::MergedState { - let mut state = I::initial_state(); - for delta in deltas { - I::fold(&mut state, delta); - } - state + fn initial_state() -> Self::MergedState { + I::initial_state() + } + + fn accumulate_one(state: &mut Self::MergedState, delta: I::Delta) { + I::fold(state, delta); } } @@ -260,3 +315,98 @@ where } } +// =========================================================================== +// CumulativeBridge — single struct for all SelfCumulative compositions +// =========================================================================== + +/// Stateful bridge for all SelfCumulative indexes. +/// +/// Unlike [`LocalBridge`], extraction is sequential within each index: +/// the bridge maintains a `running_state` that threads through blocks. +/// Different SelfCumulative indexes still extract in parallel with each +/// other — the scheduler guarantees at most one pending extraction per +/// index. +/// +/// **State threading:** +/// - Starts at the merge strategy's +/// [`initial_state`](MergeStrategy::initial_state). +/// - After each extraction, the delta is folded into the running state +/// via [`accumulate_one`](MergeStrategy::accumulate_one). No separate +/// delta buffer — the running state IS the accumulated merge result. +/// - At batch end, the running state is snapshotted for persistence. +/// - The running state carries across batch boundaries — no reset. +/// +/// **Persistence:** +/// The merge result (running state snapshot) is mapped to entries via +/// [`Schema`]. For (S, M) indexes where `PriorState = Accumulator`, +/// this persists the cumulative accumulator. +pub(crate) struct CumulativeBridge> { + descriptor: Descriptor, + running_state: Mutex, + merged: Mutex>, + _phantom: PhantomData<(I, S)>, +} + +impl> CumulativeBridge { + fn new() -> Self + where + S::MergedState: Clone, + { + Self { + descriptor: I::descriptor(), + running_state: Mutex::new(S::initial_state()), + merged: Mutex::new(None), + _phantom: PhantomData, + } + } +} + +impl IndexPipeline for CumulativeBridge +where + I: ExtractCumulative + Schema, + S: MergeStrategy, + S::MergedState: Clone, + Ctx: ProvideContext + Send + Sync + 'static, +{ + fn descriptor(&self) -> &Descriptor { + &self.descriptor + } + + fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError> { + let mut running = self.running_state.lock().expect("running state mutex poisoned"); + let delta = I::extract(&ctx.context(), &running)?; + S::accumulate_one(&mut running, delta); + Ok(()) + } + + fn merge(&self) -> Result<(), PipelineError> { + let snapshot = self + .running_state + .lock() + .expect("running state mutex poisoned") + .clone(); + *self.merged.lock().expect("merged mutex poisoned") = Some(snapshot); + Ok(()) + } + + fn persist(&self) -> Result, PipelineError> { + let state = self + .merged + .lock() + .expect("merged mutex poisoned") + .take() + .ok_or_else(|| PipelineError::Persist("no merged state to persist".into()))?; + + let ops = I::into_entries(state) + .into_iter() + .map(|(key, value)| WriteOp::Put { + index: I::NAME, + key: key.encode(), + value: value.encode(), + }) + .collect(); + + Ok(ops) + } +} + diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index e608378c0..d01e562d4 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -1,12 +1,18 @@ -//! Toy index set: three indexes demonstrating the three composition types. +//! Toy index set: four indexes demonstrating composition and scope types. //! //! Each index lives in its own sub-module and declares a narrow //! [`BlockContext`](crate::traits::IndexDef::BlockContext). The set-wide //! `TestBlockContext` projects into each via [`ProvideContext`]. //! +//! BlockLocal indexes: [`ValueIndex`](value_index), [`CountIndex`](count_index), +//! [`RunningSumIndex`](running_sum_index). +//! +//! SelfCumulative indexes: [`CumulativeSumIndex`](cumulative_sum_index). +//! //! [`ProvideContext`]: crate::traits::ProvideContext pub mod count_index; +pub mod cumulative_sum_index; pub mod running_sum_index; pub mod value_index; @@ -40,6 +46,14 @@ impl ProvideContext for TestBlockContext { } } +impl ProvideContext for TestBlockContext { + fn context(&self) -> cumulative_sum_index::Context { + cumulative_sum_index::Context { + value: self.value, + } + } +} + // --------------------------------------------------------------------------- // End-to-end tests // --------------------------------------------------------------------------- @@ -56,10 +70,11 @@ mod tests { use crate::primitives::BatchIndex; use count_index::CountIndex; + use cumulative_sum_index::CumulativeSumIndex; use running_sum_index::RunningSumIndex; use value_index::ValueIndex; - /// Helper: build an engine from the three toy indexes. + /// Helper: build an engine from the three BlockLocal toy indexes. fn build_engine( backend: InMemoryBackend, batch_size: u32, @@ -73,6 +88,29 @@ mod tests { .expect("valid index set") } + /// Helper: build an engine that includes the CumulativeSumIndex. + fn build_engine_with_cumulative( + backend: InMemoryBackend, + batch_size: u32, + ) -> SyncEngine { + let set = IndexSet::new() + .with::() + .with::() + .with::() + .with::(); + + SyncEngine::from_index_set(set, backend, EngineConfig { batch_size }) + .expect("valid index set") + } + + /// Read the cumulative sum from the backend. + fn read_cumulative_sum(backend: &InMemoryBackend) -> u64 { + let bytes = backend + .get_value(cumulative_sum_index::ID, b"sum") + .expect("cumulative sum exists"); + u64::from_le_bytes(bytes.as_slice().try_into().expect("8 bytes")) + } + #[test] fn end_to_end_single_batch() { let provisioner = MockProvisioner::identity(); @@ -210,4 +248,86 @@ mod tests { // Eviction frontier covers all 4 batches (0..=3). assert_eq!(engine.evicted_through(), Some(BatchIndex::new(3))); } + + // ----------------------------------------------------------------------- + // SelfCumulative tests + // ----------------------------------------------------------------------- + + #[test] + fn cumulative_sum_single_batch() { + // Blocks 0..=6, values = heights, all in one batch. + // Threshold = 10. Prior sums: 0,0,1,3,6,10,15 + // block 0: prior=0, delta=0 → sum=0 + // block 1: prior=0, delta=1 → sum=1 + // block 2: prior=1, delta=2 → sum=3 + // block 3: prior=3, delta=3 → sum=6 + // block 4: prior=6, delta=4 → sum=10 + // block 5: prior=10, delta=5 → sum=15 + // block 6: prior=15, delta=6*2=12 → sum=27 + // + // Only block 6 exceeds the threshold (prior=15 > 10). + let provisioner = MockProvisioner::identity(); + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(6)) + .expect("provisioning succeeds"); + + let backend = InMemoryBackend::new(); + let mut engine = build_engine_with_cumulative(backend.clone(), 20); + + engine.sync_range(blocks).expect("sync succeeds"); + + assert_eq!(read_cumulative_sum(&backend), 27); + } + + #[test] + fn cumulative_sum_deterministic_across_batch_sizes() { + // The cumulative result must be identical regardless of batch + // boundaries. This is the key property of SelfCumulative: the + // running state threads correctly across batches. + let provisioner = MockProvisioner::identity(); + + let expected = { + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(6)) + .expect("provisioning succeeds"); + let backend = InMemoryBackend::new(); + let mut engine = build_engine_with_cumulative(backend.clone(), 20); + engine.sync_range(blocks).expect("sync succeeds"); + read_cumulative_sum(&backend) + }; + + for batch_size in [1, 2, 3, 4, 5, 7] { + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(6)) + .expect("provisioning succeeds"); + let backend = InMemoryBackend::new(); + let mut engine = build_engine_with_cumulative(backend.clone(), batch_size); + engine.sync_range(blocks).expect("sync succeeds"); + + assert_eq!( + read_cumulative_sum(&backend), + expected, + "batch_size={batch_size} produced different result" + ); + } + } + + #[test] + fn cumulative_sum_state_threads_across_batches() { + // Batch size 3: batches [0,1,2], [3,4,5], [6]. + // After batch 0: sum = 0+1+2 = 3 (no doubling, all priors ≤ 10) + // After batch 1: sum = 3+3+4+5 = 15 (no doubling, priors 3,6,10 ≤ 10) + // After batch 2: sum = 15 + 6*2 = 27 (block 6: prior=15 > 10, doubled) + let provisioner = MockProvisioner::identity(); + let blocks = provisioner + .provision_range(BlockHeight::new(0), BlockHeight::new(6)) + .expect("provisioning succeeds"); + + let backend = InMemoryBackend::new(); + let mut engine = build_engine_with_cumulative(backend.clone(), 3); + + engine.sync_range(blocks).expect("sync succeeds"); + + assert_eq!(read_cumulative_sum(&backend), 27); + } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs new file mode 100644 index 000000000..5a0bce891 --- /dev/null +++ b/packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs @@ -0,0 +1,98 @@ +//! SelfCumulative x Monoidal: running sum where extraction depends on +//! the accumulated state. +//! +//! Blocks whose prior running total exceeds a threshold contribute +//! double their value. This makes extraction genuinely dependent on +//! prior state — a BlockLocal index could not reproduce the same +//! result. + +use crate::descriptor::{Monoidal, SelfCumulative}; +use crate::encode::Encode; +use crate::primitives::IndexId; +use crate::traits::{ + ExtractCumulative, ExtractError, IndexDef, MergeMonoidal, Schema, +}; + +/// Block context for this index: just the block's value. +pub struct Context { + /// Arbitrary value carried by this block. + pub value: u32, +} + +/// The accumulated sum — serves as both PriorState and Accumulator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CumulativeSum(u64); + +impl CumulativeSum { + /// Create a cumulative sum. + pub const fn new(sum: u64) -> Self { + Self(sum) + } + + /// The raw numeric value. + pub const fn value(&self) -> u64 { + self.0 + } +} + +impl Encode for CumulativeSum { + fn encode(&self) -> Vec { + self.0.to_le_bytes().to_vec() + } +} + +/// Cumulative sum where blocks past a threshold contribute double. +pub struct CumulativeSumIndex; + +/// Index identity. +pub const ID: IndexId = IndexId::new("cumulative_sum"); + +/// Prior sums above this value cause blocks to contribute double. +const DOUBLING_THRESHOLD: u64 = 10; + +impl IndexDef for CumulativeSumIndex { + type Scope = SelfCumulative; + type Composition = Monoidal; + type Delta = u64; + type BlockContext = Context; + + const NAME: IndexId = ID; +} + +impl ExtractCumulative for CumulativeSumIndex { + type PriorState = CumulativeSum; + + fn extract(ctx: &Context, prior: &CumulativeSum) -> Result { + let base = u64::from(ctx.value); + if prior.value() > DOUBLING_THRESHOLD { + Ok(base * 2) + } else { + Ok(base) + } + } +} + +impl MergeMonoidal for CumulativeSumIndex { + type Accumulator = CumulativeSum; + + fn identity() -> CumulativeSum { + CumulativeSum::new(0) + } + + fn lift(delta: u64) -> CumulativeSum { + CumulativeSum::new(delta) + } + + fn combine(a: CumulativeSum, b: CumulativeSum) -> CumulativeSum { + CumulativeSum::new(a.0 + b.0) + } +} + +impl Schema for CumulativeSumIndex { + type Key = &'static [u8]; + type Value = CumulativeSum; + + fn into_entries(sum: CumulativeSum) -> Vec<(Self::Key, Self::Value)> { + vec![(b"sum".as_slice(), sum)] + } +} diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index a70d2540e..6c353f2fa 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -148,17 +148,27 @@ pub trait ExtractLocal: IndexDef { /// Extraction for self-cumulative indexes. /// -/// The extract signature includes a read handle to this index's own -/// committed prior state. Extraction for this index is sequential (each -/// block depends on the previous block's committed output), but the index -/// runs in parallel with other indexes that have no dependency on it. +/// The extract signature includes this index's own accumulated state +/// through the prior block. Extraction for this index is sequential +/// within a batch (each block depends on the previous block's state), +/// but the index runs in parallel with other indexes that have no +/// dependency on it. +/// +/// The bridge threads state automatically using the index's merge +/// trait: [`MergeMonoidal`] provides `identity` + `combine` + `lift`; +/// [`MergeFold`] provides `initial_state` + `fold`. No separate +/// `advance_state` method is needed — the composition axis already +/// declares the algebra. pub trait ExtractCumulative: IndexDef { - /// Opaque type representing this index's accumulated state, as read - /// from the backend after the prior batch's commit. + /// The accumulated state threaded through extractions. + /// + /// For (S, M) indexes this is the same type as the monoidal + /// `Accumulator`. For (S, F) indexes it matches `FoldState`. + /// The bridge enforces this via a type equality bound. type PriorState: Send + Sync; /// Produce this block's delta given the block context and this index's - /// own committed state up to (but not including) this block. + /// own accumulated state up to (but not including) this block. fn extract( ctx: &Self::BlockContext, prior: &Self::PriorState, From 1edd0ad87723e5d2ee8fa9b2b513b78d60088e95 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 05:47:35 -0300 Subject: [PATCH 033/146] Add backend reader, atomic batch commit, and watermark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the backend reader into engine construction so SelfCumulative pipelines hydrate their running accumulators from previously committed data. Restructure the commit path: merge_persist stashes WriteOps per-batch, and try_commit flushes all indexes' ops plus the height watermark in a single atomic backend write when every index has persisted. The watermark enables resume — callers read it via SyncEngine::committed_height before constructing the next engine. Key changes: - BackendReader gains scan() for bulk state loading - CumulativeBridge implements load_state (decode from backend) - Decode trait + impls for all Schema key/value types - Engine splits merge_persist (stash) from try_commit (atomic write) - EngineConfig gains start_height for absolute watermark computation - InMemoryBackend implements the new reader/writer surface --- packages/zaino-sync/src/backend.rs | 10 +- packages/zaino-sync/src/bridge.rs | 28 +++- packages/zaino-sync/src/encode.rs | 84 ++++++++++- packages/zaino-sync/src/engine.rs | 133 ++++++++++++++---- packages/zaino-sync/src/pipeline.rs | 11 ++ packages/zaino-sync/src/testing.rs | 8 ++ .../zaino-sync/src/testing/toy_indexes.rs | 106 +++++++++++++- .../src/testing/toy_indexes/count_index.rs | 40 +++++- .../toy_indexes/cumulative_sum_index.rs | 40 +++++- .../testing/toy_indexes/running_sum_index.rs | 40 +++++- .../src/testing/toy_indexes/value_index.rs | 15 +- packages/zaino-sync/src/traits.rs | 17 ++- 12 files changed, 481 insertions(+), 51 deletions(-) diff --git a/packages/zaino-sync/src/backend.rs b/packages/zaino-sync/src/backend.rs index e4b6e97ae..741360aca 100644 --- a/packages/zaino-sync/src/backend.rs +++ b/packages/zaino-sync/src/backend.rs @@ -55,9 +55,15 @@ pub trait BackendWriter: Send { fn commit(&mut self, ops: Vec) -> Result<(), BackendError>; } -/// Read handle. Used by [`DepsReader`](crate::traits::DepsReader) and -/// the engine's progress tracking. +/// Read handle. Used by [`DepsReader`](crate::traits::DepsReader), +/// the engine's progress tracking, and S-scope state loading. pub trait BackendReader: Send { /// Read a single key from the given index. fn get(&self, index: crate::primitives::IndexId, key: &[u8]) -> Result>, BackendError>; + + /// Return all entries for an index as raw key-value byte pairs. + /// + /// Used by the bridge to load accumulated state on resume. The + /// bridge handles decoding; the backend returns raw bytes. + fn scan(&self, index: crate::primitives::IndexId) -> Result, Vec)>, BackendError>; } diff --git a/packages/zaino-sync/src/bridge.rs b/packages/zaino-sync/src/bridge.rs index f93361363..ace284c7a 100644 --- a/packages/zaino-sync/src/bridge.rs +++ b/packages/zaino-sync/src/bridge.rs @@ -41,8 +41,9 @@ use std::marker::PhantomData; use std::sync::Mutex; +use crate::backend::BackendReader; use crate::descriptor::{Append, BlockLocal, Descriptor, Fold, Monoidal, SelfCumulative}; -use crate::encode::Encode; +use crate::encode::{Decode, Encode}; use crate::pipeline::{IndexPipeline, PipelineError}; use crate::traits::{ ExtractCumulative, ExtractLocal, IndexDef, MergeAppend, MergeFold, MergeMonoidal, @@ -372,6 +373,31 @@ where &self.descriptor } + fn load_state(&self, reader: &dyn BackendReader) -> Result<(), PipelineError> { + let raw_entries = reader + .scan(I::NAME) + .map_err(|e| PipelineError::Persist(e.to_string()))?; + + if raw_entries.is_empty() { + return Ok(()); + } + + let entries: Vec<_> = raw_entries + .into_iter() + .map(|(k, v)| { + let key = >::Key::decode(&k) + .map_err(|e| PipelineError::Persist(e.to_string()))?; + let value = >::Value::decode(&v) + .map_err(|e| PipelineError::Persist(e.to_string()))?; + Ok((key, value)) + }) + .collect::>()?; + + let state = I::from_entries(entries); + *self.running_state.lock().expect("running state mutex poisoned") = state; + Ok(()) + } + fn extract_one(&self, ctx: &Ctx) -> Result<(), PipelineError> { let mut running = self.running_state.lock().expect("running state mutex poisoned"); let delta = I::extract(&ctx.context(), &running)?; diff --git a/packages/zaino-sync/src/encode.rs b/packages/zaino-sync/src/encode.rs index 3ccdc2632..4d69fc0e0 100644 --- a/packages/zaino-sync/src/encode.rs +++ b/packages/zaino-sync/src/encode.rs @@ -1,9 +1,11 @@ -//! Encoding trait for types that cross the persistence boundary. +//! Encoding and decoding traits for types that cross the persistence boundary. //! //! Types that appear as keys or values in index entries implement -//! [`Encode`] to define their byte representation. Index authors never -//! call `encode` directly — the bridge does it mechanically when -//! converting typed entries into [`WriteOp`](crate::traits::WriteOp)s. +//! [`Encode`] and [`Decode`] to define their byte representation. +//! Index authors never call these directly — the bridge does it +//! mechanically when converting typed entries into +//! [`WriteOp`](crate::traits::WriteOp)s and when loading state back +//! from the backend. //! //! This is the serialization single source of truth. If the encoding //! for `BlockHeight` changes, it changes in one place — not scattered @@ -21,36 +23,110 @@ pub trait Encode { fn encode(&self) -> Vec; } +/// Deserialize a value from its on-disk byte representation. +/// +/// The inverse of [`Encode`]. Implementations must round-trip: +/// `Decode::decode(&value.encode()) == Ok(value)`. +pub trait Decode: Sized { + /// Decode a value from bytes. + fn decode(bytes: &[u8]) -> Result; +} + +/// Errors during decoding. +#[derive(Debug, thiserror::Error)] +pub enum DecodeError { + /// The byte slice has the wrong length. + #[error("invalid length: expected {expected}, got {got}")] + InvalidLength { + /// Expected byte count. + expected: usize, + /// Actual byte count. + got: usize, + }, + /// A generic decode failure. + #[error("decode failed: {0}")] + Failed(String), +} + impl Encode for u8 { fn encode(&self) -> Vec { vec![*self] } } +impl Decode for u8 { + fn decode(bytes: &[u8]) -> Result { + if bytes.len() != 1 { + return Err(DecodeError::InvalidLength { + expected: 1, + got: bytes.len(), + }); + } + Ok(bytes[0]) + } +} + impl Encode for u16 { fn encode(&self) -> Vec { self.to_le_bytes().to_vec() } } +impl Decode for u16 { + fn decode(bytes: &[u8]) -> Result { + let arr: [u8; 2] = bytes.try_into().map_err(|_| DecodeError::InvalidLength { + expected: 2, + got: bytes.len(), + })?; + Ok(Self::from_le_bytes(arr)) + } +} + impl Encode for u32 { fn encode(&self) -> Vec { self.to_le_bytes().to_vec() } } +impl Decode for u32 { + fn decode(bytes: &[u8]) -> Result { + let arr: [u8; 4] = bytes.try_into().map_err(|_| DecodeError::InvalidLength { + expected: 4, + got: bytes.len(), + })?; + Ok(Self::from_le_bytes(arr)) + } +} + impl Encode for u64 { fn encode(&self) -> Vec { self.to_le_bytes().to_vec() } } +impl Decode for u64 { + fn decode(bytes: &[u8]) -> Result { + let arr: [u8; 8] = bytes.try_into().map_err(|_| DecodeError::InvalidLength { + expected: 8, + got: bytes.len(), + })?; + Ok(Self::from_le_bytes(arr)) + } +} + impl Encode for BlockHeight { fn encode(&self) -> Vec { self.value().to_le_bytes().to_vec() } } +impl Decode for BlockHeight { + fn decode(bytes: &[u8]) -> Result { + let raw = u64::decode(bytes)?; + Ok(Self::new(raw)) + } +} + impl Encode for &'static [u8] { fn encode(&self) -> Vec { self.to_vec() diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 802b901ed..8f2ad0424 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -24,19 +24,33 @@ use std::collections::HashMap; use rayon::prelude::*; -use crate::backend::{Backend, BackendError, BackendWriter}; +use crate::backend::{Backend, BackendError, BackendReader, BackendWriter}; use crate::block_buffer::BlockBuffer; use crate::dag::DagError; +use crate::encode::{Decode, Encode}; use crate::index_set::IndexSet; use crate::pipeline::{IndexPipeline, PipelineError}; -use crate::primitives::{BatchIndex, BlockOffset, IndexId}; +use crate::primitives::{BatchIndex, BlockHeight, BlockOffset, IndexId}; use crate::scheduler::{ExtractJob, Scheduler, Task}; +use crate::traits::WriteOp; + +/// Well-known index ID for engine metadata (watermark, etc.). +const METADATA_INDEX: IndexId = IndexId::new("_engine_meta"); + +/// Key for the committed-height watermark entry. +const WATERMARK_KEY: &[u8] = b"committed_height"; /// Configuration for the sync engine. #[derive(Debug, Clone)] pub struct EngineConfig { /// Number of blocks per persistence batch. pub batch_size: u32, + /// The block height of the first block in this sync run. + /// + /// Used to compute absolute committed heights for the watermark. + /// On a fresh sync this is genesis (0); on resume the caller + /// reads the prior watermark and sets this to `watermark + 1`. + pub start_height: BlockHeight, } /// Errors during sync. @@ -68,14 +82,23 @@ pub struct SyncEngine { pipelines: HashMap>>, backend: B, buffer: BlockBuffer, + start_height: BlockHeight, + /// Write ops waiting for an atomic batch commit. Each index's + /// persist step pushes ops here; the actual backend write happens + /// when all indexes have persisted for that batch. + pending_ops: HashMap>, evicted_through: Option, } impl SyncEngine { /// Create an engine from a declarative [`IndexSet`]. /// - /// Builds the dependency DAG, constructs the scheduler, and indexes - /// pipelines by name for O(1) lookup during dispatch. + /// Builds the dependency DAG, constructs the scheduler, indexes + /// pipelines by name for O(1) lookup, and hydrates pipeline state + /// from the backend. SelfCumulative pipelines decode their running + /// accumulators from previously committed data; BlockLocal + /// pipelines have no state to load (no-op). If the backend is + /// empty the load is a no-op for all pipelines. pub fn from_index_set( set: IndexSet, backend: B, @@ -88,6 +111,11 @@ impl SyncEngine { .map(|p| (p.descriptor().name, p)) .collect(); + let reader = backend.reader()?; + for pipeline in pipelines.values() { + pipeline.load_state(&reader)?; + } + let batch_size = config.batch_size; let scheduler = Scheduler::new(dag, batch_size); @@ -96,10 +124,30 @@ impl SyncEngine { pipelines, backend, buffer: BlockBuffer::new(batch_size), + start_height: config.start_height, + pending_ops: HashMap::new(), evicted_through: None, }) } + /// The committed-height watermark from a prior sync run, if any. + /// + /// Read from the backend before engine construction. Returns `None` + /// on a fresh backend. The caller uses this to decide what + /// `start_height` to pass and where to begin provisioning. + pub fn committed_height(backend: &B) -> Result, SyncError> { + let reader = backend.reader()?; + let raw = reader.get(METADATA_INDEX, WATERMARK_KEY)?; + match raw { + Some(bytes) => { + let height = BlockHeight::decode(&bytes) + .map_err(|e| PipelineError::Persist(e.to_string()))?; + Ok(Some(height)) + } + None => Ok(None), + } + } + /// Sync a pre-loaded range of blocks. /// /// Convenience wrapper around [`sync_streaming`](Self::sync_streaming) @@ -280,8 +328,8 @@ impl SyncEngine { .into_iter() .find(|h| h.index == index); if let Some(handle) = handle { - self.merge_persist_commit(handle)?; - self.try_evict(); + self.merge_persist(handle)?; + self.try_commit()?; } } } @@ -317,18 +365,25 @@ impl SyncEngine { fn report_extractions(&mut self, jobs: Vec) -> Result<(), SyncError> { for job in jobs { if let Some(handle) = self.scheduler.extraction_done(job.index) { - self.merge_persist_commit(handle)?; - self.try_evict(); + self.merge_persist(handle)?; + self.try_commit()?; } } Ok(()) } - /// Merge, persist, and commit a fully-extracted batch for one index. - fn merge_persist_commit( + /// Merge and persist a fully-extracted batch for one index. + /// + /// Combines deltas into domain state, serializes to [`WriteOp`]s, + /// and stashes the ops in [`pending_ops`](Self::pending_ops). The + /// actual backend write happens later in [`try_commit_and_evict`] + /// when ALL indexes have persisted for the batch — a single atomic + /// commit covers every index's data plus the watermark. + fn merge_persist( &mut self, handle: crate::scheduler::BatchHandle, ) -> Result<(), SyncError> { + let batch = handle.batch; let index_id = handle.index; let pipeline = self.pipelines.get(&index_id) @@ -337,40 +392,64 @@ impl SyncEngine { // Merge: combine deltas (domain types). pipeline.merge()?; - // Persist: domain → WriteOps. + // Persist: domain → WriteOps, stash for atomic commit. let ops = pipeline.persist()?; + self.pending_ops.entry(batch).or_default().extend(ops); - // Commit to backend. - if !ops.is_empty() { - let mut writer = self.backend.writer()?; - writer.commit(ops)?; - } - - // Tell the scheduler this batch is done. + // Advance the scheduler — downstream indexes can proceed. let merged = self.scheduler.merge_done(handle); self.scheduler.batch_committed(merged); Ok(()) } - /// Advance the eviction frontier. + /// Atomically commit all stashed ops for fully-persisted batches. /// - /// After each batch commit, checks whether all indexes have moved - /// past the next unevicted batch. If so, drops those blocks from - /// the buffer — they are no longer needed by any index. - fn try_evict(&mut self) { + /// When all indexes have persisted for a batch, drains the + /// stashed [`WriteOp`]s, appends the watermark (highest committed + /// block height), and writes everything in a single backend call. + /// The watermark never leads any index's data. + fn try_commit(&mut self) -> Result<(), SyncError> { loop { let candidate = match self.evicted_through { None => BatchIndex::new(0), Some(b) => BatchIndex::new(b.value() + 1), }; - if self.scheduler.all_committed_through(candidate) { - self.buffer.evict_through_batch(candidate); - self.evicted_through = Some(candidate); - } else { + if !self.scheduler.all_committed_through(candidate) { break; } + + let mut ops = self.pending_ops.remove(&candidate).unwrap_or_default(); + + // Watermark: highest committed height for this batch. + let batch_size = u64::from(self.scheduler.batch_size()); + let max_offset = + ((u64::from(candidate.value()) + 1) * batch_size) + .min(u64::from(self.buffer.total_pushed())); + let committed_height = + BlockHeight::new(self.start_height.value() + max_offset - 1); + + ops.push(WriteOp::Put { + index: METADATA_INDEX, + key: WATERMARK_KEY.to_vec(), + value: committed_height.encode(), + }); + + let mut writer = self.backend.writer()?; + writer.commit(ops)?; + + self.try_evict(candidate); } + Ok(()) + } + + /// Evict a committed batch's blocks from the buffer. + /// + /// Called after the atomic commit succeeds. Drops block contexts + /// that are no longer needed by any index. + fn try_evict(&mut self, batch: BatchIndex) { + self.buffer.evict_through_batch(batch); + self.evicted_through = Some(batch); } } diff --git a/packages/zaino-sync/src/pipeline.rs b/packages/zaino-sync/src/pipeline.rs index 0f9556eac..e9fd6a42c 100644 --- a/packages/zaino-sync/src/pipeline.rs +++ b/packages/zaino-sync/src/pipeline.rs @@ -32,6 +32,7 @@ //! types here. (Currently the merge traits own this step; it will //! move to a dedicated persistence layer.) +use crate::backend::BackendReader; use crate::bridge::BridgeDispatch; use crate::descriptor::Descriptor; use crate::traits::{ExtractError, IndexDef, ProvideContext, WriteOp}; @@ -89,6 +90,16 @@ pub trait IndexPipeline: Send + Sync { /// internal state, readying the bridge for the next batch. fn persist(&self) -> Result, PipelineError>; + /// Load persisted state from the backend on startup. + /// + /// Called once per pipeline before the first batch. The default + /// implementation is a no-op — BlockLocal indexes have no state + /// to resume. SelfCumulative bridges override this to reload the + /// running accumulator from the backend. + fn load_state(&self, _reader: &dyn BackendReader) -> Result<(), PipelineError> { + Ok(()) + } + /// Convenience: run all three phases sequentially on a batch. /// /// Exists for backward compatibility with the batch-loop engine. diff --git a/packages/zaino-sync/src/testing.rs b/packages/zaino-sync/src/testing.rs index 86f491b98..a0f4bf95d 100644 --- a/packages/zaino-sync/src/testing.rs +++ b/packages/zaino-sync/src/testing.rs @@ -81,6 +81,14 @@ impl BackendReader for InMemoryReader { let guard = self.data.lock().expect("test mutex poisoned"); Ok(guard.get(&index).and_then(|m| m.get(key).cloned())) } + + fn scan(&self, index: IndexId) -> Result, Vec)>, BackendError> { + let guard = self.data.lock().expect("test mutex poisoned"); + Ok(guard + .get(&index) + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default()) + } } /// Write handle for the in-memory backend. diff --git a/packages/zaino-sync/src/testing/toy_indexes.rs b/packages/zaino-sync/src/testing/toy_indexes.rs index d01e562d4..959dfd027 100644 --- a/packages/zaino-sync/src/testing/toy_indexes.rs +++ b/packages/zaino-sync/src/testing/toy_indexes.rs @@ -78,13 +78,23 @@ mod tests { fn build_engine( backend: InMemoryBackend, batch_size: u32, + ) -> SyncEngine { + build_engine_at(backend, batch_size, BlockHeight::new(0)) + } + + /// Helper: build an engine from the three BlockLocal toy indexes + /// starting at a given height. + fn build_engine_at( + backend: InMemoryBackend, + batch_size: u32, + start_height: BlockHeight, ) -> SyncEngine { let set = IndexSet::new() .with::() .with::() .with::(); - SyncEngine::from_index_set(set, backend, EngineConfig { batch_size }) + SyncEngine::from_index_set(set, backend, EngineConfig { batch_size, start_height }) .expect("valid index set") } @@ -92,6 +102,16 @@ mod tests { fn build_engine_with_cumulative( backend: InMemoryBackend, batch_size: u32, + ) -> SyncEngine { + build_engine_with_cumulative_at(backend, batch_size, BlockHeight::new(0)) + } + + /// Helper: build an engine with cumulative index starting at a + /// given height. + fn build_engine_with_cumulative_at( + backend: InMemoryBackend, + batch_size: u32, + start_height: BlockHeight, ) -> SyncEngine { let set = IndexSet::new() .with::() @@ -99,7 +119,7 @@ mod tests { .with::() .with::(); - SyncEngine::from_index_set(set, backend, EngineConfig { batch_size }) + SyncEngine::from_index_set(set, backend, EngineConfig { batch_size, start_height }) .expect("valid index set") } @@ -330,4 +350,86 @@ mod tests { assert_eq!(read_cumulative_sum(&backend), 27); } + + #[test] + fn cumulative_sum_resumes_from_backend() { + // Sync blocks 0..=4, drop the engine, build a new one on the + // same backend, sync blocks 5..=6. The new engine must load + // the committed accumulator so that extraction sees the correct + // prior state (and triggers doubling at the right threshold). + // + // Phase 1 (blocks 0..=4): + // block 0: prior=0, delta=0 → sum=0 + // block 1: prior=0, delta=1 → sum=1 + // block 2: prior=1, delta=2 → sum=3 + // block 3: prior=3, delta=3 → sum=6 + // block 4: prior=6, delta=4 → sum=10 + // + // Phase 2 (blocks 5..=6, new engine, loaded prior=10): + // block 5: prior=10, delta=5 → sum=15 (10 is NOT > 10) + // block 6: prior=15, delta=12 → sum=27 (15 > 10, doubled) + // + // Without load_state the new engine would start from prior=0, + // and the result would be 0+5+6 = 11 — wrong. + let backend = InMemoryBackend::new(); + + // Phase 1. + { + let blocks: Vec<_> = (0u64..=4) + .map(|h| TestBlockContext { height: h, value: h as u32 }) + .collect(); + let mut engine = build_engine_with_cumulative(backend.clone(), 20); + engine.sync_range(blocks).expect("phase 1 sync succeeds"); + assert_eq!(read_cumulative_sum(&backend), 10); + } + + // Watermark should reflect phase 1. + let watermark = SyncEngine::::committed_height(&backend) + .expect("read succeeds") + .expect("watermark exists"); + assert_eq!(watermark, BlockHeight::new(4)); + + // Phase 2: new engine, same backend, starting from watermark + 1. + { + let start = BlockHeight::new(watermark.value() + 1); + let blocks: Vec<_> = (5u64..=6) + .map(|h| TestBlockContext { height: h, value: h as u32 }) + .collect(); + let mut engine = build_engine_with_cumulative_at(backend.clone(), 20, start); + engine.sync_range(blocks).expect("phase 2 sync succeeds"); + assert_eq!(read_cumulative_sum(&backend), 27); + } + + // Watermark should now reflect phase 2. + let watermark = SyncEngine::::committed_height(&backend) + .expect("read succeeds") + .expect("watermark exists"); + assert_eq!(watermark, BlockHeight::new(6)); + } + + #[test] + fn watermark_advances_per_batch() { + // Batch size 3, blocks 0..=9 → batches [0,1,2], [3,4,5], [6,7,8], [9]. + // After sync, watermark should be 9. + let backend = InMemoryBackend::new(); + let mut engine = build_engine(backend.clone(), 3); + + let blocks: Vec<_> = (0u64..=9) + .map(|h| TestBlockContext { height: h, value: h as u32 }) + .collect(); + engine.sync_range(blocks).expect("sync succeeds"); + + let watermark = SyncEngine::::committed_height(&backend) + .expect("read succeeds") + .expect("watermark exists"); + assert_eq!(watermark, BlockHeight::new(9)); + } + + #[test] + fn watermark_none_on_fresh_backend() { + let backend = InMemoryBackend::new(); + let watermark = SyncEngine::::committed_height(&backend) + .expect("read succeeds"); + assert!(watermark.is_none()); + } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs index ab66a1d84..e447b2899 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/count_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/count_index.rs @@ -1,7 +1,7 @@ //! BlockLocal × Monoidal: counts total blocks seen in each batch. use crate::descriptor::{BlockLocal, Monoidal}; -use crate::encode::Encode; +use crate::encode::{Decode, DecodeError, Encode}; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeMonoidal, Schema}; @@ -34,6 +34,32 @@ impl Encode for BlockCount { } } +impl Decode for BlockCount { + fn decode(bytes: &[u8]) -> Result { + Ok(Self(u64::decode(bytes)?)) + } +} + +/// Unit key type for the single "total" entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TotalKey; + +impl Encode for TotalKey { + fn encode(&self) -> Vec { + b"total".to_vec() + } +} + +impl Decode for TotalKey { + fn decode(bytes: &[u8]) -> Result { + if bytes == b"total" { + Ok(Self) + } else { + Err(DecodeError::Failed("expected 'total' key".into())) + } + } +} + /// Counts total blocks seen in each batch. pub struct CountIndex; @@ -72,10 +98,18 @@ impl MergeMonoidal for CountIndex { } impl Schema for CountIndex { - type Key = &'static [u8]; + type Key = TotalKey; type Value = BlockCount; fn into_entries(count: BlockCount) -> Vec<(Self::Key, Self::Value)> { - vec![(b"total".as_slice(), count)] + vec![(TotalKey, count)] + } + + fn from_entries(entries: Vec<(Self::Key, Self::Value)>) -> BlockCount { + entries + .into_iter() + .next() + .map(|(_, v)| v) + .unwrap_or(BlockCount::new(0)) } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs index 5a0bce891..6b5b282aa 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/cumulative_sum_index.rs @@ -7,7 +7,7 @@ //! result. use crate::descriptor::{Monoidal, SelfCumulative}; -use crate::encode::Encode; +use crate::encode::{Decode, DecodeError, Encode}; use crate::primitives::IndexId; use crate::traits::{ ExtractCumulative, ExtractError, IndexDef, MergeMonoidal, Schema, @@ -41,6 +41,32 @@ impl Encode for CumulativeSum { } } +impl Decode for CumulativeSum { + fn decode(bytes: &[u8]) -> Result { + Ok(Self(u64::decode(bytes)?)) + } +} + +/// Unit key type for the single "sum" entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CumSumKey; + +impl Encode for CumSumKey { + fn encode(&self) -> Vec { + b"sum".to_vec() + } +} + +impl Decode for CumSumKey { + fn decode(bytes: &[u8]) -> Result { + if bytes == b"sum" { + Ok(Self) + } else { + Err(DecodeError::Failed("expected 'sum' key".into())) + } + } +} + /// Cumulative sum where blocks past a threshold contribute double. pub struct CumulativeSumIndex; @@ -89,10 +115,18 @@ impl MergeMonoidal for CumulativeSumIndex { } impl Schema for CumulativeSumIndex { - type Key = &'static [u8]; + type Key = CumSumKey; type Value = CumulativeSum; fn into_entries(sum: CumulativeSum) -> Vec<(Self::Key, Self::Value)> { - vec![(b"sum".as_slice(), sum)] + vec![(CumSumKey, sum)] + } + + fn from_entries(entries: Vec<(Self::Key, Self::Value)>) -> CumulativeSum { + entries + .into_iter() + .next() + .map(|(_, v)| v) + .unwrap_or(CumulativeSum::new(0)) } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs index 15129d677..4475ced00 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/running_sum_index.rs @@ -1,7 +1,7 @@ //! BlockLocal × Fold: running sum of values across blocks in a batch. use crate::descriptor::{BlockLocal, Fold}; -use crate::encode::Encode; +use crate::encode::{Decode, DecodeError, Encode}; use crate::primitives::IndexId; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeFold, Schema}; @@ -33,6 +33,32 @@ impl Encode for RunningSum { } } +impl Decode for RunningSum { + fn decode(bytes: &[u8]) -> Result { + Ok(Self(u64::decode(bytes)?)) + } +} + +/// Unit key type for the single "sum" entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SumKey; + +impl Encode for SumKey { + fn encode(&self) -> Vec { + b"sum".to_vec() + } +} + +impl Decode for SumKey { + fn decode(bytes: &[u8]) -> Result { + if bytes == b"sum" { + Ok(Self) + } else { + Err(DecodeError::Failed("expected 'sum' key".into())) + } + } +} + /// Running sum of values across blocks in a batch. pub struct RunningSumIndex; @@ -67,10 +93,18 @@ impl MergeFold for RunningSumIndex { } impl Schema for RunningSumIndex { - type Key = &'static [u8]; + type Key = SumKey; type Value = RunningSum; fn into_entries(sum: RunningSum) -> Vec<(Self::Key, Self::Value)> { - vec![(b"sum".as_slice(), sum)] + vec![(SumKey, sum)] + } + + fn from_entries(entries: Vec<(Self::Key, Self::Value)>) -> RunningSum { + entries + .into_iter() + .next() + .map(|(_, v)| v) + .unwrap_or(RunningSum::new(0)) } } diff --git a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs index eee4280b7..06e41c01f 100644 --- a/packages/zaino-sync/src/testing/toy_indexes/value_index.rs +++ b/packages/zaino-sync/src/testing/toy_indexes/value_index.rs @@ -1,7 +1,7 @@ //! BlockLocal × Append: stores (height → value) for each block. use crate::descriptor::{Append, BlockLocal}; -use crate::encode::Encode; +use crate::encode::{Decode, DecodeError, Encode}; use crate::primitives::{BlockHeight, IndexId}; use crate::traits::{ExtractError, ExtractLocal, IndexDef, MergeAppend, Schema}; @@ -35,6 +35,12 @@ impl Encode for BlockValue { } } +impl Decode for BlockValue { + fn decode(bytes: &[u8]) -> Result { + Ok(Self(u32::decode(bytes)?)) + } +} + /// A single height → value entry. Domain type — no serialization. pub struct Entry { /// Block height. @@ -79,4 +85,11 @@ impl Schema> for ValueIndex { .map(|entry| (entry.height, entry.value)) .collect() } + + fn from_entries(entries: Vec<(Self::Key, Self::Value)>) -> Vec { + entries + .into_iter() + .map(|(height, value)| Entry { height, value }) + .collect() + } } diff --git a/packages/zaino-sync/src/traits.rs b/packages/zaino-sync/src/traits.rs index 6c353f2fa..5e64090b3 100644 --- a/packages/zaino-sync/src/traits.rs +++ b/packages/zaino-sync/src/traits.rs @@ -248,7 +248,7 @@ pub trait MergeFold: IndexDef { // The bridge does the mechanical conversion to `WriteOp`s. // =========================================================================== -use crate::encode::Encode; +use crate::encode::{Decode, Encode}; /// Declares an index's key-value schema and how to map results to entries. /// @@ -259,20 +259,27 @@ use crate::encode::Encode; /// - Fold: `Self::FoldState` /// /// The index implements `Schema` for its composition's output type. -/// The bridge calls `into_entries` and uses `Encode` on the key/value -/// types to produce `WriteOp`s. No index code touches bytes. +/// The bridge calls `into_entries` / `from_entries` and uses `Encode` / +/// `Decode` on the key/value types to produce `WriteOp`s or reconstruct +/// state. No index code touches bytes. /// /// Using a type parameter instead of an associated type avoids cycle /// errors that arise when the merged type references the index's own /// associated types (e.g. `Vec`). pub trait Schema: IndexDef { /// The key type for this index's entries. - type Key: Encode + Send + Sync; + type Key: Encode + Decode + Send + Sync; /// The value type for this index's entries. - type Value: Encode + Send + Sync; + type Value: Encode + Decode + Send + Sync; /// Map a merge result to typed key-value entries. fn into_entries(merged: M) -> Vec<(Self::Key, Self::Value)>; + + /// Reconstruct a merge result from typed key-value entries. + /// + /// The mechanical inverse of [`into_entries`](Self::into_entries). + /// The bridge calls this after decoding raw bytes from the backend. + fn from_entries(entries: Vec<(Self::Key, Self::Value)>) -> M; } // =========================================================================== From 940d42565ec2f3b492a76120e785f5685b00acae Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 11:01:49 -0300 Subject: [PATCH 034/146] Add tracing instrumentation, bench harness, and SlowBackend Add optional tracing feature flag with #[instrument] on key engine methods (sync_range, sync_streaming, sync_channel, dispatch_tasks, run_extractions_parallel, merge_persist, try_commit). Spans carry structured fields (batch, index, op_count, committed_height) that can feed log subscribers, Prometheus, or a future Bevy visualization. Add SlowBackend wrapper with configurable commit delay for simulating LMDB fsync latency. Add bench module with scenarios: baseline throughput, batch size sensitivity, slow backend, slow provisioner via channel, and combined slow provisioner + backend. --- Cargo.lock | 2 + packages/zaino-sync/Cargo.toml | 6 + packages/zaino-sync/src/engine.rs | 25 +- packages/zaino-sync/src/testing.rs | 66 +++++ packages/zaino-sync/src/testing/bench.rs | 326 +++++++++++++++++++++++ 5 files changed, 421 insertions(+), 4 deletions(-) create mode 100644 packages/zaino-sync/src/testing/bench.rs diff --git a/Cargo.lock b/Cargo.lock index fe2e2e7e4..323af99b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6099,6 +6099,8 @@ dependencies = [ "rayon", "thiserror 2.0.18", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] diff --git a/packages/zaino-sync/Cargo.toml b/packages/zaino-sync/Cargo.toml index 2dde78dc5..3116a3492 100644 --- a/packages/zaino-sync/Cargo.toml +++ b/packages/zaino-sync/Cargo.toml @@ -8,13 +8,19 @@ repository.workspace = true homepage.workspace = true description = "DAG-driven parallel sync engine for blockchain index building." +[features] +tracing = ["dep:tracing"] + [dependencies] rayon.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } [lints.rust] unsafe_code = "forbid" diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 8f2ad0424..81f55725c 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -152,6 +152,7 @@ impl SyncEngine { /// /// Convenience wrapper around [`sync_streaming`](Self::sync_streaming) /// for the common case where all blocks are available upfront. + #[cfg_attr(feature = "tracing", tracing::instrument(skip_all, fields(block_count = blocks.len())))] pub fn sync_range(&mut self, blocks: Vec) -> Result<(), SyncError> { self.sync_streaming(blocks) } @@ -173,6 +174,7 @@ impl SyncEngine { /// **Current shape:** single-threaded, synchronous. The scheduler /// infrastructure supports parallel dispatch — swapping in an async /// executor is a future step. + #[cfg_attr(feature = "tracing", tracing::instrument(skip_all))] pub(crate) fn sync_streaming(&mut self, source: I) -> Result<(), SyncError> where I: IntoIterator, @@ -228,6 +230,7 @@ impl SyncEngine { /// This is the production entry point — the provisioner and engine /// run concurrently, with the [`BlockBuffer`] absorbing the rate /// difference between supply and demand. + #[cfg_attr(feature = "tracing", tracing::instrument(skip_all))] pub async fn sync_channel( &mut self, mut rx: tokio::sync::mpsc::Receiver, @@ -308,6 +311,7 @@ impl SyncEngine { /// /// Handles batch completions first, then runs all extractions in /// parallel via rayon, then reports completions sequentially. + #[cfg_attr(feature = "tracing", tracing::instrument(skip_all, fields(task_count = tasks.len())))] fn dispatch_tasks(&mut self, tasks: Vec) -> Result<(), SyncError> { let jobs = self.flush_batch_completions(tasks)?; self.run_extractions_parallel(&jobs)?; @@ -341,10 +345,8 @@ impl SyncEngine { /// /// Prepares (pipeline, context) pairs on the calling thread, then /// fans out via `par_iter`. Borrows from `self` — no Arc cloning. - fn run_extractions_parallel( - &self, - jobs: &[ExtractJob], - ) -> Result<(), SyncError> { + #[cfg_attr(feature = "tracing", tracing::instrument(skip_all, fields(job_count = jobs.len())))] + fn run_extractions_parallel(&self, jobs: &[ExtractJob]) -> Result<(), SyncError> { let work: Vec<_> = jobs.iter().map(|job| { let ctx = self.buffer.get(job.global_offset) .expect("block available — scheduler verified watermark"); @@ -379,6 +381,10 @@ impl SyncEngine { /// actual backend write happens later in [`try_commit_and_evict`] /// when ALL indexes have persisted for the batch — a single atomic /// commit covers every index's data plus the watermark. + #[cfg_attr(feature = "tracing", tracing::instrument( + skip_all, + fields(index = %handle.index, batch = handle.batch.value()) + ))] fn merge_persist( &mut self, handle: crate::scheduler::BatchHandle, @@ -394,6 +400,8 @@ impl SyncEngine { // Persist: domain → WriteOps, stash for atomic commit. let ops = pipeline.persist()?; + #[cfg(feature = "tracing")] + tracing::debug!(op_count = ops.len(), "persist produced ops"); self.pending_ops.entry(batch).or_default().extend(ops); // Advance the scheduler — downstream indexes can proceed. @@ -409,6 +417,7 @@ impl SyncEngine { /// stashed [`WriteOp`]s, appends the watermark (highest committed /// block height), and writes everything in a single backend call. /// The watermark never leads any index's data. + #[cfg_attr(feature = "tracing", tracing::instrument(skip_all))] fn try_commit(&mut self) -> Result<(), SyncError> { loop { let candidate = match self.evicted_through { @@ -435,6 +444,14 @@ impl SyncEngine { value: committed_height.encode(), }); + #[cfg(feature = "tracing")] + tracing::info!( + batch = candidate.value(), + op_count = ops.len(), + committed_height = committed_height.value(), + "atomic batch commit" + ); + let mut writer = self.backend.writer()?; writer.commit(ops)?; diff --git a/packages/zaino-sync/src/testing.rs b/packages/zaino-sync/src/testing.rs index a0f4bf95d..d2ea7da5b 100644 --- a/packages/zaino-sync/src/testing.rs +++ b/packages/zaino-sync/src/testing.rs @@ -4,6 +4,7 @@ //! `TestBlockContext`) are defined here. Specific index sets live in //! sub-modules (e.g. [`toy_indexes`]). +mod bench; mod toy_indexes; use std::collections::HashMap; @@ -115,6 +116,71 @@ impl BackendWriter for InMemoryWriter { } } +// =========================================================================== +// Slow backend — wraps any backend with configurable IO latency +// =========================================================================== + +use std::time::Duration; + +/// Backend wrapper that adds configurable latency to commits. +/// +/// Simulates the cost of durable writes (e.g. LMDB fsync) by sleeping +/// in the writer's `commit` method. Reads are unaffected — real backends +/// typically have fast reads. +#[derive(Clone)] +pub struct SlowBackend { + inner: B, + commit_delay: Duration, +} + +impl SlowBackend { + /// Wrap `inner` with a fixed delay per `commit` call. + pub fn new(inner: B, commit_delay: Duration) -> Self { + Self { + inner, + commit_delay, + } + } +} + +impl Backend for SlowBackend { + type Reader = B::Reader; + type Writer = SlowWriter; + + fn reader(&self) -> Result { + self.inner.reader() + } + + fn writer(&self) -> Result { + let inner = self.inner.writer()?; + Ok(SlowWriter { + inner, + delay: self.commit_delay, + }) + } + + fn flush(&self) -> Result<(), BackendError> { + self.inner.flush() + } + + fn topology(&self) -> WriterTopology { + self.inner.topology() + } +} + +/// Writer that sleeps before delegating to the inner writer. +pub struct SlowWriter { + inner: W, + delay: Duration, +} + +impl BackendWriter for SlowWriter { + fn commit(&mut self, ops: Vec) -> Result<(), BackendError> { + std::thread::sleep(self.delay); + self.inner.commit(ops) + } +} + // =========================================================================== // Set-wide block context and mock provisioner // =========================================================================== diff --git a/packages/zaino-sync/src/testing/bench.rs b/packages/zaino-sync/src/testing/bench.rs new file mode 100644 index 000000000..b82e15164 --- /dev/null +++ b/packages/zaino-sync/src/testing/bench.rs @@ -0,0 +1,326 @@ +//! Benchmark scenarios for the sync engine. +//! +//! Each test is `#[ignore]` so it doesn't run in the normal test suite. +//! Run with: +//! `cargo test --package zaino-sync --features tracing -- bench --ignored --nocapture` +//! +//! Tracing output is controlled by `RUST_LOG`. Examples: +//! `RUST_LOG=info` — batch commits and totals only +//! `RUST_LOG=debug` — per-dispatch detail +//! `RUST_LOG=off` — pure timing (default if unset) + +#[cfg(test)] +mod tests { + use std::sync::Once; + use std::time::{Duration, Instant}; + + use crate::engine::{EngineConfig, SyncEngine}; + use crate::index_set::IndexSet; + use crate::primitives::BlockHeight; + use crate::testing::toy_indexes::count_index::CountIndex; + use crate::testing::toy_indexes::cumulative_sum_index::CumulativeSumIndex; + use crate::testing::toy_indexes::running_sum_index::RunningSumIndex; + use crate::testing::toy_indexes::value_index::ValueIndex; + use crate::testing::{InMemoryBackend, SlowBackend, TestBlockContext}; + + static INIT_TRACING: Once = Once::new(); + + fn init_tracing() { + INIT_TRACING.call_once(|| { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("off")), + ) + .with_target(false) + .compact() + .init(); + }); + } + + /// Build blocks 0..n with value = height. + fn make_blocks(n: u64) -> Vec { + (0..n) + .map(|h| TestBlockContext { + height: h, + value: h as u32, + }) + .collect() + } + + /// Build the full 4-index set (3 BlockLocal + 1 SelfCumulative). + fn full_index_set() -> IndexSet { + IndexSet::new() + .with::() + .with::() + .with::() + .with::() + } + + /// Report throughput. + fn report(label: &str, block_count: u64, elapsed: Duration) { + let secs = elapsed.as_secs_f64(); + let blocks_per_sec = block_count as f64 / secs; + println!( + " {label:<40} {block_count:>8} blocks in {secs:>8.3}s ({blocks_per_sec:>10.0} blocks/sec)" + ); + } + + // ----------------------------------------------------------------------- + // Baseline: instant IO, measure pure engine overhead + // ----------------------------------------------------------------------- + + #[test] + #[ignore] + fn baseline_sync_range() { + init_tracing(); + println!("\n=== Baseline (in-memory, no IO delay) ==="); + for &n in &[1_000, 10_000, 100_000, 500_000] { + let blocks = make_blocks(n); + let backend = InMemoryBackend::new(); + let config = EngineConfig { + batch_size: 1_000, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let start = Instant::now(); + engine.sync_range(blocks).expect("sync succeeds"); + report("sync_range", n, start.elapsed()); + } + } + + #[test] + #[ignore] + fn baseline_sync_streaming() { + init_tracing(); + println!("\n=== Baseline streaming (in-memory, no IO delay) ==="); + for &n in &[1_000, 10_000, 100_000, 500_000] { + let backend = InMemoryBackend::new(); + let config = EngineConfig { + batch_size: 1_000, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let blocks = (0..n).map(|h| TestBlockContext { + height: h, + value: h as u32, + }); + + let start = Instant::now(); + engine.sync_streaming(blocks).expect("sync succeeds"); + report("sync_streaming", n, start.elapsed()); + } + } + + // ----------------------------------------------------------------------- + // Batch size sensitivity + // ----------------------------------------------------------------------- + + #[test] + #[ignore] + fn batch_size_sensitivity() { + init_tracing(); + println!("\n=== Batch size sensitivity (100k blocks, no IO delay) ==="); + let n = 100_000u64; + for &batch_size in &[10, 50, 100, 500, 1_000, 5_000, 10_000] { + let blocks = make_blocks(n); + let backend = InMemoryBackend::new(); + let config = EngineConfig { + batch_size, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let start = Instant::now(); + engine.sync_range(blocks).expect("sync succeeds"); + report(&format!("batch_size={batch_size}"), n, start.elapsed()); + } + } + + // ----------------------------------------------------------------------- + // Backend-bound: slow commits + // ----------------------------------------------------------------------- + + #[test] + #[ignore] + fn slow_backend_commit() { + init_tracing(); + println!("\n=== Slow backend (1ms commit delay) ==="); + for &n in &[1_000, 10_000] { + for &batch_size in &[100, 500, 1_000, 5_000] { + let blocks = make_blocks(n); + let inner = InMemoryBackend::new(); + let backend = SlowBackend::new(inner, Duration::from_millis(1)); + let config = EngineConfig { + batch_size, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let start = Instant::now(); + engine.sync_range(blocks).expect("sync succeeds"); + report( + &format!("n={n}, batch={batch_size}, commit=1ms"), + n, + start.elapsed(), + ); + } + } + } + + #[test] + #[ignore] + fn slow_backend_heavy_commit() { + init_tracing(); + println!("\n=== Slow backend (5ms commit delay, simulating fsync) ==="); + for &n in &[1_000, 5_000] { + for &batch_size in &[500, 1_000, 2_500] { + let blocks = make_blocks(n); + let inner = InMemoryBackend::new(); + let backend = SlowBackend::new(inner, Duration::from_millis(5)); + let config = EngineConfig { + batch_size, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let start = Instant::now(); + engine.sync_range(blocks).expect("sync succeeds"); + report( + &format!("n={n}, batch={batch_size}, commit=5ms"), + n, + start.elapsed(), + ); + } + } + } + + // ----------------------------------------------------------------------- + // Channel-based provisioner with delay (simulated RPC latency) + // ----------------------------------------------------------------------- + + #[tokio::test] + #[ignore] + async fn slow_provisioner_channel() { + init_tracing(); + println!("\n=== Slow provisioner (50μs/block via channel) ==="); + for &n in &[1_000u64, 10_000] { + let backend = InMemoryBackend::new(); + let config = EngineConfig { + batch_size: 1_000, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let (tx, rx) = tokio::sync::mpsc::channel(256); + let delay = Duration::from_micros(50); + + tokio::spawn(async move { + for h in 0..n { + let ctx = TestBlockContext { + height: h, + value: h as u32, + }; + tx.send(ctx).await.expect("channel open"); + tokio::time::sleep(delay).await; + } + }); + + let start = Instant::now(); + engine.sync_channel(rx).await.expect("sync succeeds"); + report(&format!("n={n}, prov_delay=50μs"), n, start.elapsed()); + } + } + + #[tokio::test] + #[ignore] + async fn fast_provisioner_channel() { + init_tracing(); + println!("\n=== Fast provisioner (no delay, channel backpressure only) ==="); + for &n in &[10_000u64, 100_000, 500_000] { + let backend = InMemoryBackend::new(); + let config = EngineConfig { + batch_size: 1_000, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let (tx, rx) = tokio::sync::mpsc::channel(1_024); + + tokio::spawn(async move { + for h in 0..n { + let ctx = TestBlockContext { + height: h, + value: h as u32, + }; + tx.send(ctx).await.expect("channel open"); + } + }); + + let start = Instant::now(); + engine.sync_channel(rx).await.expect("sync succeeds"); + report(&format!("channel, n={n}"), n, start.elapsed()); + } + } + + // ----------------------------------------------------------------------- + // Combined: slow provisioner + slow backend + // ----------------------------------------------------------------------- + + #[tokio::test] + #[ignore] + async fn combined_slow_provisioner_and_backend() { + init_tracing(); + println!("\n=== Combined: 50μs/block provisioner + 1ms commit ==="); + for &batch_size in &[100, 500, 1_000] { + let n = 5_000u64; + let inner = InMemoryBackend::new(); + let backend = SlowBackend::new(inner, Duration::from_millis(1)); + let config = EngineConfig { + batch_size, + start_height: BlockHeight::new(0), + }; + let mut engine = + SyncEngine::from_index_set(full_index_set(), backend, config) + .expect("valid index set"); + + let (tx, rx) = tokio::sync::mpsc::channel(256); + let delay = Duration::from_micros(50); + + tokio::spawn(async move { + for h in 0..n { + let ctx = TestBlockContext { + height: h, + value: h as u32, + }; + tx.send(ctx).await.expect("channel open"); + tokio::time::sleep(delay).await; + } + }); + + let start = Instant::now(); + engine.sync_channel(rx).await.expect("sync succeeds"); + report( + &format!("batch={batch_size}, prov=50μs, commit=1ms"), + n, + start.elapsed(), + ); + } + } +} From 4eb6ed1dbb21692576f45b3950fb069b42edcf7d Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 11:38:01 -0300 Subject: [PATCH 035/146] Add debug_assert invariant guards across engine, scheduler, and buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guards at key state transitions to catch violations early during development and testing: - BlockBuffer: push offset must be monotonic, eviction must advance - Scheduler: extraction count must not exceed batch size, committed_through must advance monotonically, batch_committed requires pending_commit - Engine: eviction monotonicity, watermark monotonicity in try_commit, post-sync invariants (buffer empty, pending_ops drained, eviction covers all batches) All are debug_assert — zero cost in release builds. --- packages/zaino-sync/src/block_buffer.rs | 13 ++++++ packages/zaino-sync/src/engine.rs | 62 ++++++++++++++++++++++++- packages/zaino-sync/src/scheduler.rs | 22 +++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/packages/zaino-sync/src/block_buffer.rs b/packages/zaino-sync/src/block_buffer.rs index 636ae321a..715239b6e 100644 --- a/packages/zaino-sync/src/block_buffer.rs +++ b/packages/zaino-sync/src/block_buffer.rs @@ -43,6 +43,13 @@ impl BlockBuffer { /// The offset must be monotonically increasing — blocks arrive in /// chain order from the provisioner. pub fn push(&mut self, offset: BlockOffset, ctx: Ctx) { + debug_assert_eq!( + offset.value(), + self.total_pushed(), + "blocks must be pushed in order: expected offset {}, got {}", + self.total_pushed(), + offset.value(), + ); self.blocks.insert(offset.value(), Arc::new(ctx)); } @@ -77,6 +84,12 @@ impl BlockBuffer { /// this batch — the blocks are no longer needed by any index. pub fn evict_through_batch(&mut self, through_batch: BatchIndex) { let cutoff = (through_batch.value() + 1) * self.batch_size; + debug_assert!( + cutoff >= self.floor, + "eviction must advance: cutoff {} < floor {}", + cutoff, + self.floor, + ); self.blocks = self.blocks.split_off(&cutoff); self.floor = cutoff; } diff --git a/packages/zaino-sync/src/engine.rs b/packages/zaino-sync/src/engine.rs index 81f55725c..b410f46dc 100644 --- a/packages/zaino-sync/src/engine.rs +++ b/packages/zaino-sync/src/engine.rs @@ -154,7 +154,11 @@ impl SyncEngine { /// for the common case where all blocks are available upfront. #[cfg_attr(feature = "tracing", tracing::instrument(skip_all, fields(block_count = blocks.len())))] pub fn sync_range(&mut self, blocks: Vec) -> Result<(), SyncError> { - self.sync_streaming(blocks) + let result = self.sync_streaming(blocks); + if result.is_ok() { + self.assert_post_sync_invariants(); + } + result } /// Sync blocks from an incremental source. @@ -256,6 +260,7 @@ impl SyncEngine { } self.backend.flush()?; + self.assert_post_sync_invariants(); Ok(()) } @@ -392,6 +397,11 @@ impl SyncEngine { let batch = handle.batch; let index_id = handle.index; + // The pipeline must exist — the scheduler only emits registered indexes. + debug_assert!( + self.pipelines.contains_key(&index_id), + "merge_persist called for unknown index {index_id}", + ); let pipeline = self.pipelines.get(&index_id) .expect("scheduler only emits registered indexes"); @@ -452,6 +462,14 @@ impl SyncEngine { "atomic batch commit" ); + // Watermark must advance monotonically. + debug_assert!( + self.evicted_through.map_or(true, |prev| candidate.value() > prev.value()), + "try_commit batch {} but already committed through {:?}", + candidate.value(), + self.evicted_through.map(|b| b.value()), + ); + let mut writer = self.backend.writer()?; writer.commit(ops)?; @@ -465,9 +483,51 @@ impl SyncEngine { /// Called after the atomic commit succeeds. Drops block contexts /// that are no longer needed by any index. fn try_evict(&mut self, batch: BatchIndex) { + // Eviction must be monotonic. + debug_assert!( + self.evicted_through.map_or(true, |prev| batch.value() > prev.value()), + "eviction must advance: evicting batch {} but already evicted through {:?}", + batch.value(), + self.evicted_through.map(|b| b.value()), + ); + self.buffer.evict_through_batch(batch); self.evicted_through = Some(batch); } + + // ----------------------------------------------------------------------- + // Invariant assertions + // ----------------------------------------------------------------------- + + /// Invariants that must hold after a successful sync run. + fn assert_post_sync_invariants(&self) { + debug_assert!( + self.buffer.is_empty(), + "buffer must be empty after sync, has {} blocks remaining", + self.buffer.len(), + ); + + debug_assert!( + self.pending_ops.is_empty(), + "pending_ops must be drained after sync, {} batches remain: {:?}", + self.pending_ops.len(), + self.pending_ops.keys().map(|b| b.value()).collect::>(), + ); + + // If any blocks were processed, eviction must have covered all batches. + if self.buffer.total_pushed() > 0 { + let batch_size = self.scheduler.batch_size(); + let total = self.buffer.total_pushed(); + let expected_last_batch = (total - 1) / batch_size; + debug_assert_eq!( + self.evicted_through.map(|b| b.value()), + Some(expected_last_batch), + "eviction must cover all batches: expected through batch {}, got {:?}", + expected_last_batch, + self.evicted_through.map(|b| b.value()), + ); + } + } } #[cfg(test)] diff --git a/packages/zaino-sync/src/scheduler.rs b/packages/zaino-sync/src/scheduler.rs index 8a0d5aef2..713210afc 100644 --- a/packages/zaino-sync/src/scheduler.rs +++ b/packages/zaino-sync/src/scheduler.rs @@ -237,6 +237,12 @@ impl Scheduler { .expect("index exists in scheduler"); *count += 1; + debug_assert!( + *count <= effective, + "extraction count {} exceeds effective batch size {} for index {} batch {}", + *count, effective, index, batch.value(), + ); + if *count >= effective { self.pending_merge.insert(index); Some(BatchHandle::new(index, batch)) @@ -276,6 +282,22 @@ impl Scheduler { let index = handle.index; let batch = handle.batch; + // committed_through must advance monotonically per index. + debug_assert!( + self.committed_through[&index].map_or(true, |prev| batch.value() > prev.value()), + "committed_through must advance for index {}: committing batch {} but already at {:?}", + index, + batch.value(), + self.committed_through[&index].map(|b| b.value()), + ); + + // The index must be pending commit (came through merge_done). + debug_assert!( + self.pending_commit.contains(&index), + "batch_committed called for index {} which is not pending commit", + index, + ); + self.pending_commit.remove(&index); self.committed_through.insert(index, Some(batch)); From aac17b66d20915204204a23e4fd34127be508dc5 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 10 Jul 2026 22:46:23 -0300 Subject: [PATCH 036/146] Add zaino-primitives crate: Height, BlockHash, TransactionHash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero-dependency vocabulary types for the Zaino stack. All inner representations are private with checked construction and arithmetic. - Height: u32 with protocol-limit validation (≤ 2^31-1), checked_add/sub - BlockHash: [u8; 32] with big-endian display, truncated debug - TransactionHash: parallel structure to BlockHash Design: insights on semantic type safety, private inner representation, parse-don't-validate, and virgin crate isolation from zaino-design. --- packages/zaino-primitives/Cargo.toml | 14 ++ packages/zaino-primitives/src/lib.rs | 6 + packages/zaino-primitives/src/types.rs | 9 + .../zaino-primitives/src/types/block_hash.rs | 97 ++++++++++ packages/zaino-primitives/src/types/height.rs | 167 ++++++++++++++++++ .../src/types/transaction_hash.rs | 63 +++++++ 6 files changed, 356 insertions(+) create mode 100644 packages/zaino-primitives/Cargo.toml create mode 100644 packages/zaino-primitives/src/lib.rs create mode 100644 packages/zaino-primitives/src/types.rs create mode 100644 packages/zaino-primitives/src/types/block_hash.rs create mode 100644 packages/zaino-primitives/src/types/height.rs create mode 100644 packages/zaino-primitives/src/types/transaction_hash.rs diff --git a/packages/zaino-primitives/Cargo.toml b/packages/zaino-primitives/Cargo.toml new file mode 100644 index 000000000..6aeed76bc --- /dev/null +++ b/packages/zaino-primitives/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "zaino-primitives" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +license.workspace = true + +[dependencies] + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" diff --git a/packages/zaino-primitives/src/lib.rs b/packages/zaino-primitives/src/lib.rs new file mode 100644 index 000000000..5e79ceb8b --- /dev/null +++ b/packages/zaino-primitives/src/lib.rs @@ -0,0 +1,6 @@ +//! Zaino primitives — vocabulary types for the Zcash chain. +//! +//! Zero-dependency crate. All Zaino crates that need chain-level types +//! (heights, hashes) depend on this crate instead of on each other. + +pub mod types; diff --git a/packages/zaino-primitives/src/types.rs b/packages/zaino-primitives/src/types.rs new file mode 100644 index 000000000..df5f8ccf9 --- /dev/null +++ b/packages/zaino-primitives/src/types.rs @@ -0,0 +1,9 @@ +//! Core vocabulary types shared across the Zaino stack. + +mod block_hash; +mod height; +mod transaction_hash; + +pub use block_hash::BlockHash; +pub use height::{Height, HeightOverflow}; +pub use transaction_hash::TransactionHash; diff --git a/packages/zaino-primitives/src/types/block_hash.rs b/packages/zaino-primitives/src/types/block_hash.rs new file mode 100644 index 000000000..347e6548f --- /dev/null +++ b/packages/zaino-primitives/src/types/block_hash.rs @@ -0,0 +1,97 @@ +//! SHA-256d block hash. + +use core::fmt; + +/// SHA-256d block hash (32 bytes, internal byte order). +/// +/// Internal byte order = little-endian as produced by the double-SHA256 +/// digest. Display and RPC use reversed (big-endian) order. +/// +/// The inner bytes are private. Use `From<[u8; 32]>` to construct and +/// `From for [u8; 32]` at boundaries that need raw bytes. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct BlockHash([u8; 32]); + +impl BlockHash { + /// The zero hash, used as a sentinel (e.g. genesis `prev_hash`). + pub const ZERO: Self = Self([0u8; 32]); +} + +impl From<[u8; 32]> for BlockHash { + fn from(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} + +impl From for [u8; 32] { + fn from(h: BlockHash) -> Self { + h.0 + } +} + +impl fmt::Debug for BlockHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // First 4 bytes in display (big-endian) order for log readability. + write!( + f, + "BlockHash({:02x}{:02x}{:02x}{:02x}…)", + self.0[31], self.0[30], self.0[29], self.0[28] + ) + } +} + +impl fmt::Display for BlockHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Full 64-char hex in display (big-endian) order. + for &byte in self.0.iter().rev() { + write!(f, "{byte:02x}")?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn zero_is_all_zeroes() { + assert_eq!(<[u8; 32]>::from(BlockHash::ZERO), [0u8; 32]); + } + + #[test] + fn roundtrip_bytes() { + let bytes = [0xAB; 32]; + let hash = BlockHash::from(bytes); + assert_eq!(<[u8; 32]>::from(hash), bytes); + } + + #[test] + fn display_is_big_endian_hex() { + let mut bytes = [0u8; 32]; + bytes[31] = 0xAB; + let hash = BlockHash::from(bytes); + let display = format!("{hash}"); + assert!(display.starts_with("ab"), "got: {display}"); + assert_eq!(display.len(), 64); + } + + #[test] + fn debug_shows_truncated_prefix() { + let mut bytes = [0u8; 32]; + bytes[31] = 0xDE; + bytes[30] = 0xAD; + let hash = BlockHash::from(bytes); + let debug = format!("{hash:?}"); + assert!(debug.contains("dead"), "got: {debug}"); + assert!(debug.contains('…'), "got: {debug}"); + } + + #[test] + fn equality_and_ordering() { + let a = BlockHash::from([0u8; 32]); + let b = BlockHash::from([1u8; 32]); + assert_ne!(a, b); + assert!(a < b); + } +} diff --git a/packages/zaino-primitives/src/types/height.rs b/packages/zaino-primitives/src/types/height.rs new file mode 100644 index 000000000..cc3488480 --- /dev/null +++ b/packages/zaino-primitives/src/types/height.rs @@ -0,0 +1,167 @@ +//! Block height on the Zcash chain. + +use core::fmt; + +/// Maximum valid block height (Zcash protocol limit, matches Zebra). +/// +/// `2^31 - 1`. Heights above this are rejected at construction. +const MAX_HEIGHT: u32 = (1 << 31) - 1; + +/// Block height. +/// +/// Invariant: the inner value is `≤ MAX_HEIGHT` (`2^31 - 1`). +/// Enforced at construction; all arithmetic is checked. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Height(u32); + +/// Error returned when a `u32` exceeds the protocol height limit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HeightOverflow { + /// The value that was rejected. + pub got: u32, +} + +impl fmt::Display for HeightOverflow { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "height {} exceeds protocol maximum {}", + self.got, MAX_HEIGHT + ) + } +} + +impl Height { + /// The genesis block. + pub const GENESIS: Self = Self(0); + + /// Add a delta, returning `None` on overflow or protocol-limit violation. + pub fn checked_add(self, delta: u32) -> Option { + let sum = self.0.checked_add(delta)?; + if sum > MAX_HEIGHT { + return None; + } + Some(Self(sum)) + } + + /// Subtract a delta, returning `None` on underflow. + pub fn checked_sub(self, delta: u32) -> Option { + self.0.checked_sub(delta).map(Self) + } + + /// Subtract, saturating at zero. + pub fn saturating_sub(self, delta: u32) -> Self { + Self(self.0.saturating_sub(delta)) + } + + /// Distance between two heights (absolute value). + pub fn abs_diff(self, other: Self) -> u32 { + self.0.abs_diff(other.0) + } +} + +impl TryFrom for Height { + type Error = HeightOverflow; + + fn try_from(h: u32) -> Result { + if h > MAX_HEIGHT { + Err(HeightOverflow { got: h }) + } else { + Ok(Self(h)) + } + } +} + +impl From for u32 { + fn from(h: Height) -> Self { + h.0 + } +} + +impl From for u64 { + fn from(h: Height) -> Self { + u64::from(h.0) + } +} + +impl fmt::Debug for Height { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Height({})", self.0) + } +} + +impl fmt::Display for Height { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn genesis() { + assert_eq!(u32::from(Height::GENESIS), 0); + } + + #[test] + fn valid_construction() { + let h = Height::try_from(100).expect("valid height"); + assert_eq!(u32::from(h), 100); + } + + #[test] + fn max_is_valid() { + assert!(Height::try_from(MAX_HEIGHT).is_ok()); + } + + #[test] + fn above_max_rejected() { + let err = Height::try_from(MAX_HEIGHT + 1).unwrap_err(); + assert_eq!(err.got, MAX_HEIGHT + 1); + } + + #[test] + fn checked_add_within_limit() { + let h = Height::try_from(10).expect("valid"); + assert_eq!(u32::from(h.checked_add(5).expect("ok")), 15); + } + + #[test] + fn checked_add_overflow_returns_none() { + let h = Height::try_from(MAX_HEIGHT).expect("valid"); + assert!(h.checked_add(1).is_none()); + } + + #[test] + fn checked_sub_underflow_returns_none() { + assert!(Height::GENESIS.checked_sub(1).is_none()); + } + + #[test] + fn saturating_sub_floors_at_zero() { + assert_eq!(Height::GENESIS.saturating_sub(100), Height::GENESIS); + } + + #[test] + fn abs_diff_commutative() { + let a = Height::try_from(10).expect("valid"); + let b = Height::try_from(25).expect("valid"); + assert_eq!(a.abs_diff(b), 15); + assert_eq!(b.abs_diff(a), 15); + } + + #[test] + fn ordering() { + let a = Height::try_from(1).expect("valid"); + let b = Height::try_from(2).expect("valid"); + assert!(a < b); + } + + #[test] + fn into_u64() { + let h = Height::try_from(42).expect("valid"); + assert_eq!(u64::from(h), 42u64); + } +} diff --git a/packages/zaino-primitives/src/types/transaction_hash.rs b/packages/zaino-primitives/src/types/transaction_hash.rs new file mode 100644 index 000000000..8b1774b54 --- /dev/null +++ b/packages/zaino-primitives/src/types/transaction_hash.rs @@ -0,0 +1,63 @@ +//! Transaction hash (txid). + +use core::fmt; + +/// Transaction hash / txid (32 bytes, internal byte order). +/// +/// Same byte-order convention as [`super::BlockHash`]: internal +/// little-endian, display big-endian. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct TransactionHash([u8; 32]); + +impl From<[u8; 32]> for TransactionHash { + fn from(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} + +impl From for [u8; 32] { + fn from(h: TransactionHash) -> Self { + h.0 + } +} + +impl fmt::Debug for TransactionHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "TxHash({:02x}{:02x}{:02x}{:02x}…)", + self.0[31], self.0[30], self.0[29], self.0[28] + ) + } +} + +impl fmt::Display for TransactionHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for &byte in self.0.iter().rev() { + write!(f, "{byte:02x}")?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrip_bytes() { + let bytes = [0x42; 32]; + let hash = TransactionHash::from(bytes); + assert_eq!(<[u8; 32]>::from(hash), bytes); + } + + #[test] + fn display_is_big_endian_hex() { + let mut bytes = [0u8; 32]; + bytes[31] = 0xFF; + let hash = TransactionHash::from(bytes); + let display = format!("{hash}"); + assert!(display.starts_with("ff"), "got: {display}"); + assert_eq!(display.len(), 64); + } +} From 894c17b06580c517940ff2ff6cc176852340f47d Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 10 Jul 2026 22:46:37 -0300 Subject: [PATCH 037/146] Add zaino-source crate: driven port traits for validator queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One trait per question: GetBlockBytes, GetChainTip, GetTreestate. Each has its own domain error type; all share TransportError for transport-level failures. Depends only on zaino-primitives. This is the driven-port abstraction layer — adapters (JSON-RPC, ReadState, mock) implement these traits, consumers compose them via trait bounds. --- Cargo.lock | 11 +++ Cargo.toml | 4 + packages/zaino-source/Cargo.toml | 15 ++++ packages/zaino-source/src/error.rs | 30 +++++++ packages/zaino-source/src/get_block_bytes.rs | 67 ++++++++++++++++ packages/zaino-source/src/get_chain_tip.rs | 70 ++++++++++++++++ packages/zaino-source/src/get_treestate.rs | 84 ++++++++++++++++++++ packages/zaino-source/src/lib.rs | 20 +++++ 8 files changed, 301 insertions(+) create mode 100644 packages/zaino-source/Cargo.toml create mode 100644 packages/zaino-source/src/error.rs create mode 100644 packages/zaino-source/src/get_block_bytes.rs create mode 100644 packages/zaino-source/src/get_chain_tip.rs create mode 100644 packages/zaino-source/src/get_treestate.rs create mode 100644 packages/zaino-source/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 323af99b1..6687ec2ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6005,6 +6005,10 @@ dependencies = [ "zebra-rpc 10.0.1", ] +[[package]] +name = "zaino-primitives" +version = "0.1.0" + [[package]] name = "zaino-proto" version = "0.1.3" @@ -6038,6 +6042,13 @@ dependencies = [ "zebra-rpc 10.0.1", ] +[[package]] +name = "zaino-source" +version = "0.1.0" +dependencies = [ + "zaino-primitives", +] + [[package]] name = "zaino-state" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 96d2a9ef7..0c112777b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,8 @@ # Workspace Members are in dependency order, do not change this without prior consideration. [workspace] members = [ + "packages/zaino-primitives", + "packages/zaino-source", "packages/zaino-common", "packages/zainod", "packages/zaino-serve", @@ -18,6 +20,8 @@ members = [ # default-members — so the heavy live-test crates are built and tested only # when selected explicitly (`-p e2e` / `-p clientless`). See docs/adr/0002. default-members = [ + "packages/zaino-primitives", + "packages/zaino-source", "packages/zaino-common", "packages/zainod", "packages/zaino-serve", diff --git a/packages/zaino-source/Cargo.toml b/packages/zaino-source/Cargo.toml new file mode 100644 index 000000000..6fe5b1534 --- /dev/null +++ b/packages/zaino-source/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "zaino-source" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +license.workspace = true + +[dependencies] +zaino-primitives = { path = "../zaino-primitives" } + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" diff --git a/packages/zaino-source/src/error.rs b/packages/zaino-source/src/error.rs new file mode 100644 index 000000000..aef8a5f9f --- /dev/null +++ b/packages/zaino-source/src/error.rs @@ -0,0 +1,30 @@ +//! Transport-level error shared across all query traits. + +use core::fmt; + +/// Transport-level failure (connection refused, timeout, deserialization). +/// +/// Shared across all query traits. Domain-specific errors (block not found, +/// height out of range) are per-trait; transport errors are uniform because +/// they depend on the adapter, not the question. +#[derive(Debug)] +pub struct TransportError { + message: String, +} + +impl TransportError { + /// Wrap an arbitrary transport failure. + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl fmt::Display for TransportError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "transport error: {}", self.message) + } +} + +impl std::error::Error for TransportError {} diff --git a/packages/zaino-source/src/get_block_bytes.rs b/packages/zaino-source/src/get_block_bytes.rs new file mode 100644 index 000000000..40dd81fc4 --- /dev/null +++ b/packages/zaino-source/src/get_block_bytes.rs @@ -0,0 +1,67 @@ +//! Query: fetch raw serialized block bytes at a given height. + +use core::fmt; +use std::future::Future; + +use zaino_primitives::types::Height; + +use super::TransportError; + +/// Domain error for [`GetBlockBytes`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GetBlockBytesError { + /// No block exists at this height. + HeightNotFound(Height), +} + +impl fmt::Display for GetBlockBytesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::HeightNotFound(h) => write!(f, "no block at height {h}"), + } + } +} + +/// Fetch the raw serialized block at a given height. +/// +/// Maps to `getblock(height, 0)` over JSON-RPC, or the equivalent +/// ReadState query. +pub trait GetBlockBytes: Send + Sync { + /// Fetch raw block bytes. + fn get_block_bytes( + &self, + height: Height, + ) -> impl Future, QueryError>> + Send; +} + +/// Combined domain + transport error for this query. +#[derive(Debug)] +pub enum QueryError { + /// The question has a valid answer: "no such block." + Domain(GetBlockBytesError), + /// The question couldn't be delivered or the response couldn't be parsed. + Transport(TransportError), +} + +impl fmt::Display for QueryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Domain(e) => write!(f, "{e}"), + Self::Transport(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for QueryError {} + +impl From for QueryError { + fn from(e: GetBlockBytesError) -> Self { + Self::Domain(e) + } +} + +impl From for QueryError { + fn from(e: TransportError) -> Self { + Self::Transport(e) + } +} diff --git a/packages/zaino-source/src/get_chain_tip.rs b/packages/zaino-source/src/get_chain_tip.rs new file mode 100644 index 000000000..e8ce5c09f --- /dev/null +++ b/packages/zaino-source/src/get_chain_tip.rs @@ -0,0 +1,70 @@ +//! Query: fetch the current best chain tip. + +use core::fmt; +use std::future::Future; + +use zaino_primitives::types::{BlockHash, Height}; + +use super::TransportError; + +/// Domain error for [`GetChainTip`]. +/// +/// There are no domain-level failure modes for "what is the tip?" beyond +/// transport failure — the validator always has a tip. This enum exists +/// for forward compatibility (e.g. a validator that reports "still syncing"). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GetChainTipError { + /// The validator is not ready to report a tip (e.g. still syncing). + NotReady, +} + +impl fmt::Display for GetChainTipError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotReady => write!(f, "validator not ready"), + } + } +} + +/// Fetch the current best chain tip (hash + height). +/// +/// Maps to `getbestblockhash()` + `getblock(hash, 0)` over JSON-RPC, +/// or the equivalent ReadState query. +pub trait GetChainTip: Send + Sync { + /// Fetch current tip. + fn get_chain_tip( + &self, + ) -> impl Future> + Send; +} + +/// Combined domain + transport error for this query. +#[derive(Debug)] +pub enum QueryError { + /// Domain-level failure. + Domain(GetChainTipError), + /// Transport-level failure. + Transport(TransportError), +} + +impl fmt::Display for QueryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Domain(e) => write!(f, "{e}"), + Self::Transport(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for QueryError {} + +impl From for QueryError { + fn from(e: GetChainTipError) -> Self { + Self::Domain(e) + } +} + +impl From for QueryError { + fn from(e: TransportError) -> Self { + Self::Transport(e) + } +} diff --git a/packages/zaino-source/src/get_treestate.rs b/packages/zaino-source/src/get_treestate.rs new file mode 100644 index 000000000..211844dbf --- /dev/null +++ b/packages/zaino-source/src/get_treestate.rs @@ -0,0 +1,84 @@ +//! Query: fetch commitment tree state at a given height. + +use core::fmt; +use std::future::Future; + +use zaino_primitives::types::Height; + +use super::TransportError; + +/// Serialized commitment tree bytes for one pool. +/// +/// Opaque to the primitives crate. Interpretation (Sapling vs Orchard, +/// deserialization into tree structures) happens in consumer crates. +pub type TreeBytes = Vec; + +/// Commitment tree state at a block: Sapling and Orchard trees. +/// +/// Either pool may be absent if the block predates that pool's activation. +#[derive(Debug, Clone)] +pub struct TreestateResponse { + /// Serialized Sapling commitment tree, if active at this height. + pub sapling: Option, + /// Serialized Orchard commitment tree, if active at this height. + pub orchard: Option, +} + +/// Domain error for [`GetTreestate`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GetTreestateError { + /// No block exists at this height (can't compute treestate). + HeightNotFound(Height), +} + +impl fmt::Display for GetTreestateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::HeightNotFound(h) => write!(f, "no treestate at height {h}"), + } + } +} + +/// Fetch the commitment tree state at a given height. +/// +/// Maps to `z_gettreestate(height)` over JSON-RPC, or the equivalent +/// ReadState query. +pub trait GetTreestate: Send + Sync { + /// Fetch treestate. + fn get_treestate( + &self, + height: Height, + ) -> impl Future> + Send; +} + +/// Combined domain + transport error for this query. +#[derive(Debug)] +pub enum QueryError { + /// Domain-level failure. + Domain(GetTreestateError), + /// Transport-level failure. + Transport(TransportError), +} + +impl fmt::Display for QueryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Domain(e) => write!(f, "{e}"), + Self::Transport(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for QueryError {} + +impl From for QueryError { + fn from(e: GetTreestateError) -> Self { + Self::Domain(e) + } +} + +impl From for QueryError { + fn from(e: TransportError) -> Self { + Self::Transport(e) + } +} diff --git a/packages/zaino-source/src/lib.rs b/packages/zaino-source/src/lib.rs new file mode 100644 index 000000000..2996fecdb --- /dev/null +++ b/packages/zaino-source/src/lib.rs @@ -0,0 +1,20 @@ +//! Zaino source — driven port traits for validator access. +//! +//! One trait per question a consumer can ask about the chain. +//! Implementations (adapters) bridge to a specific transport +//! (JSON-RPC, Zebra ReadState, mock). +//! +//! Consumers compose traits via bounds: +//! ```ignore +//! fn sync(validator: &V) { ... } +//! ``` + +mod error; +mod get_block_bytes; +mod get_chain_tip; +mod get_treestate; + +pub use error::TransportError; +pub use get_block_bytes::{GetBlockBytes, GetBlockBytesError}; +pub use get_chain_tip::{GetChainTip, GetChainTipError}; +pub use get_treestate::{GetTreestate, GetTreestateError}; From 29b1cc63f645c9bce3772f7b9bc4de8a1d2e6e80 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 10 Jul 2026 22:50:33 -0300 Subject: [PATCH 038/146] Extract generic QueryError, use thiserror across zaino-source Replace three identical per-trait QueryError enums with a single generic QueryError in error.rs. Switch all error types to thiserror derives, eliminating manual Display + Error impls. Net: -112 lines, zero duplication. --- packages/zaino-source/Cargo.toml | 1 + packages/zaino-source/src/error.rs | 23 +++++---- packages/zaino-source/src/get_block_bytes.rs | 48 ++----------------- packages/zaino-source/src/get_chain_tip.rs | 48 ++----------------- packages/zaino-source/src/get_treestate.rs | 50 ++------------------ packages/zaino-source/src/lib.rs | 4 +- 6 files changed, 31 insertions(+), 143 deletions(-) diff --git a/packages/zaino-source/Cargo.toml b/packages/zaino-source/Cargo.toml index 6fe5b1534..793819b0d 100644 --- a/packages/zaino-source/Cargo.toml +++ b/packages/zaino-source/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true [dependencies] zaino-primitives = { path = "../zaino-primitives" } +thiserror = { workspace = true } [lints.rust] unsafe_code = "forbid" diff --git a/packages/zaino-source/src/error.rs b/packages/zaino-source/src/error.rs index aef8a5f9f..b083d4595 100644 --- a/packages/zaino-source/src/error.rs +++ b/packages/zaino-source/src/error.rs @@ -1,4 +1,4 @@ -//! Transport-level error shared across all query traits. +//! Error types shared across all query traits. use core::fmt; @@ -7,7 +7,8 @@ use core::fmt; /// Shared across all query traits. Domain-specific errors (block not found, /// height out of range) are per-trait; transport errors are uniform because /// they depend on the adapter, not the question. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] +#[error("transport error: {message}")] pub struct TransportError { message: String, } @@ -21,10 +22,16 @@ impl TransportError { } } -impl fmt::Display for TransportError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "transport error: {}", self.message) - } +/// Combined domain + transport error for a query. +/// +/// Generic over the domain error `E`. Each query trait defines its own +/// domain error; this wrapper adds the transport layer uniformly. +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + /// The question has a domain-level answer: "not found", "not ready", etc. + #[error("{0}")] + Domain(E), + /// The question couldn't be delivered or the response couldn't be parsed. + #[error("{0}")] + Transport(#[from] TransportError), } - -impl std::error::Error for TransportError {} diff --git a/packages/zaino-source/src/get_block_bytes.rs b/packages/zaino-source/src/get_block_bytes.rs index 40dd81fc4..526f11f6e 100644 --- a/packages/zaino-source/src/get_block_bytes.rs +++ b/packages/zaino-source/src/get_block_bytes.rs @@ -1,27 +1,19 @@ //! Query: fetch raw serialized block bytes at a given height. -use core::fmt; use std::future::Future; use zaino_primitives::types::Height; -use super::TransportError; +use super::QueryError; /// Domain error for [`GetBlockBytes`]. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] pub enum GetBlockBytesError { /// No block exists at this height. + #[error("no block at height {0}")] HeightNotFound(Height), } -impl fmt::Display for GetBlockBytesError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::HeightNotFound(h) => write!(f, "no block at height {h}"), - } - } -} - /// Fetch the raw serialized block at a given height. /// /// Maps to `getblock(height, 0)` over JSON-RPC, or the equivalent @@ -31,37 +23,5 @@ pub trait GetBlockBytes: Send + Sync { fn get_block_bytes( &self, height: Height, - ) -> impl Future, QueryError>> + Send; -} - -/// Combined domain + transport error for this query. -#[derive(Debug)] -pub enum QueryError { - /// The question has a valid answer: "no such block." - Domain(GetBlockBytesError), - /// The question couldn't be delivered or the response couldn't be parsed. - Transport(TransportError), -} - -impl fmt::Display for QueryError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Domain(e) => write!(f, "{e}"), - Self::Transport(e) => write!(f, "{e}"), - } - } -} - -impl std::error::Error for QueryError {} - -impl From for QueryError { - fn from(e: GetBlockBytesError) -> Self { - Self::Domain(e) - } -} - -impl From for QueryError { - fn from(e: TransportError) -> Self { - Self::Transport(e) - } + ) -> impl Future, QueryError>> + Send; } diff --git a/packages/zaino-source/src/get_chain_tip.rs b/packages/zaino-source/src/get_chain_tip.rs index e8ce5c09f..64c79597c 100644 --- a/packages/zaino-source/src/get_chain_tip.rs +++ b/packages/zaino-source/src/get_chain_tip.rs @@ -1,31 +1,23 @@ //! Query: fetch the current best chain tip. -use core::fmt; use std::future::Future; use zaino_primitives::types::{BlockHash, Height}; -use super::TransportError; +use super::QueryError; /// Domain error for [`GetChainTip`]. /// /// There are no domain-level failure modes for "what is the tip?" beyond /// transport failure — the validator always has a tip. This enum exists /// for forward compatibility (e.g. a validator that reports "still syncing"). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] pub enum GetChainTipError { /// The validator is not ready to report a tip (e.g. still syncing). + #[error("validator not ready")] NotReady, } -impl fmt::Display for GetChainTipError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::NotReady => write!(f, "validator not ready"), - } - } -} - /// Fetch the current best chain tip (hash + height). /// /// Maps to `getbestblockhash()` + `getblock(hash, 0)` over JSON-RPC, @@ -34,37 +26,5 @@ pub trait GetChainTip: Send + Sync { /// Fetch current tip. fn get_chain_tip( &self, - ) -> impl Future> + Send; -} - -/// Combined domain + transport error for this query. -#[derive(Debug)] -pub enum QueryError { - /// Domain-level failure. - Domain(GetChainTipError), - /// Transport-level failure. - Transport(TransportError), -} - -impl fmt::Display for QueryError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Domain(e) => write!(f, "{e}"), - Self::Transport(e) => write!(f, "{e}"), - } - } -} - -impl std::error::Error for QueryError {} - -impl From for QueryError { - fn from(e: GetChainTipError) -> Self { - Self::Domain(e) - } -} - -impl From for QueryError { - fn from(e: TransportError) -> Self { - Self::Transport(e) - } + ) -> impl Future>> + Send; } diff --git a/packages/zaino-source/src/get_treestate.rs b/packages/zaino-source/src/get_treestate.rs index 211844dbf..89caf80e8 100644 --- a/packages/zaino-source/src/get_treestate.rs +++ b/packages/zaino-source/src/get_treestate.rs @@ -1,15 +1,14 @@ //! Query: fetch commitment tree state at a given height. -use core::fmt; use std::future::Future; use zaino_primitives::types::Height; -use super::TransportError; +use super::QueryError; /// Serialized commitment tree bytes for one pool. /// -/// Opaque to the primitives crate. Interpretation (Sapling vs Orchard, +/// Opaque to this crate. Interpretation (Sapling vs Orchard, /// deserialization into tree structures) happens in consumer crates. pub type TreeBytes = Vec; @@ -25,20 +24,13 @@ pub struct TreestateResponse { } /// Domain error for [`GetTreestate`]. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] pub enum GetTreestateError { /// No block exists at this height (can't compute treestate). + #[error("no treestate at height {0}")] HeightNotFound(Height), } -impl fmt::Display for GetTreestateError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::HeightNotFound(h) => write!(f, "no treestate at height {h}"), - } - } -} - /// Fetch the commitment tree state at a given height. /// /// Maps to `z_gettreestate(height)` over JSON-RPC, or the equivalent @@ -48,37 +40,5 @@ pub trait GetTreestate: Send + Sync { fn get_treestate( &self, height: Height, - ) -> impl Future> + Send; -} - -/// Combined domain + transport error for this query. -#[derive(Debug)] -pub enum QueryError { - /// Domain-level failure. - Domain(GetTreestateError), - /// Transport-level failure. - Transport(TransportError), -} - -impl fmt::Display for QueryError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Domain(e) => write!(f, "{e}"), - Self::Transport(e) => write!(f, "{e}"), - } - } -} - -impl std::error::Error for QueryError {} - -impl From for QueryError { - fn from(e: GetTreestateError) -> Self { - Self::Domain(e) - } -} - -impl From for QueryError { - fn from(e: TransportError) -> Self { - Self::Transport(e) - } + ) -> impl Future>> + Send; } diff --git a/packages/zaino-source/src/lib.rs b/packages/zaino-source/src/lib.rs index 2996fecdb..d8d5cf870 100644 --- a/packages/zaino-source/src/lib.rs +++ b/packages/zaino-source/src/lib.rs @@ -14,7 +14,7 @@ mod get_block_bytes; mod get_chain_tip; mod get_treestate; -pub use error::TransportError; +pub use error::{QueryError, TransportError}; pub use get_block_bytes::{GetBlockBytes, GetBlockBytesError}; pub use get_chain_tip::{GetChainTip, GetChainTipError}; -pub use get_treestate::{GetTreestate, GetTreestateError}; +pub use get_treestate::{GetTreestate, GetTreestateError, TreestateResponse}; From 9fb332356def97914bb4100002dac65955db8cd8 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 10 Jul 2026 22:52:19 -0300 Subject: [PATCH 039/146] Add thiserror to zaino-primitives, derive HeightOverflow No reason to hand-roll Display+Error for a workspace-managed dep. --- packages/zaino-primitives/Cargo.toml | 1 + packages/zaino-primitives/src/types/height.rs | 13 ++----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/zaino-primitives/Cargo.toml b/packages/zaino-primitives/Cargo.toml index 6aeed76bc..7dcfd0332 100644 --- a/packages/zaino-primitives/Cargo.toml +++ b/packages/zaino-primitives/Cargo.toml @@ -8,6 +8,7 @@ homepage.workspace = true license.workspace = true [dependencies] +thiserror = { workspace = true } [lints.rust] unsafe_code = "forbid" diff --git a/packages/zaino-primitives/src/types/height.rs b/packages/zaino-primitives/src/types/height.rs index cc3488480..ccc3ea9ca 100644 --- a/packages/zaino-primitives/src/types/height.rs +++ b/packages/zaino-primitives/src/types/height.rs @@ -15,22 +15,13 @@ const MAX_HEIGHT: u32 = (1 << 31) - 1; pub struct Height(u32); /// Error returned when a `u32` exceeds the protocol height limit. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("height {got} exceeds protocol maximum {MAX_HEIGHT}")] pub struct HeightOverflow { /// The value that was rejected. pub got: u32, } -impl fmt::Display for HeightOverflow { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "height {} exceeds protocol maximum {}", - self.got, MAX_HEIGHT - ) - } -} - impl Height { /// The genesis block. pub const GENESIS: Self = Self(0); From 3d106c0c1c481f6ce3d6958f06be30c1150b8979 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 10 Jul 2026 23:23:12 -0300 Subject: [PATCH 040/146] Add domain newtypes and aliases to zaino-primitives Hard-typed newtypes with private inners: - Zatoshis / SignedZatoshis (validated monetary amounts) - TransparentAddress, Script, ChainWork, TreeRoot - ShieldedPool enum Composite types with typed pub fields (no accessors needed): - Utxo, AddressDelta, AddressBalance, SubtreeRoot - TreeRoots / TreeRootInfo, Treestate, BlockVerbose - TransactionLocation enum Aliases for borderline cases (grep targets for future promotion): - OutputIndex, Confirmations, Difficulty, TreeSize, SubtreeIndex --- packages/zaino-primitives/src/types.rs | 30 +++++ .../src/types/address_balance.rs | 12 ++ .../src/types/address_delta.rs | 18 +++ .../zaino-primitives/src/types/aliases.rs | 19 +++ .../src/types/block_verbose.rs | 18 +++ .../zaino-primitives/src/types/chain_work.rs | 24 ++++ packages/zaino-primitives/src/types/script.rs | 24 ++++ .../src/types/shielded_pool.rs | 19 +++ .../src/types/subtree_root.rs | 12 ++ .../src/types/transaction_location.rs | 14 ++ .../src/types/transparent_address.rs | 32 +++++ .../zaino-primitives/src/types/tree_root.rs | 27 ++++ .../zaino-primitives/src/types/tree_roots.rs | 21 +++ .../zaino-primitives/src/types/treestate.rs | 18 +++ packages/zaino-primitives/src/types/utxo.rs | 20 +++ .../zaino-primitives/src/types/zatoshis.rs | 122 ++++++++++++++++++ 16 files changed, 430 insertions(+) create mode 100644 packages/zaino-primitives/src/types/address_balance.rs create mode 100644 packages/zaino-primitives/src/types/address_delta.rs create mode 100644 packages/zaino-primitives/src/types/aliases.rs create mode 100644 packages/zaino-primitives/src/types/block_verbose.rs create mode 100644 packages/zaino-primitives/src/types/chain_work.rs create mode 100644 packages/zaino-primitives/src/types/script.rs create mode 100644 packages/zaino-primitives/src/types/shielded_pool.rs create mode 100644 packages/zaino-primitives/src/types/subtree_root.rs create mode 100644 packages/zaino-primitives/src/types/transaction_location.rs create mode 100644 packages/zaino-primitives/src/types/transparent_address.rs create mode 100644 packages/zaino-primitives/src/types/tree_root.rs create mode 100644 packages/zaino-primitives/src/types/tree_roots.rs create mode 100644 packages/zaino-primitives/src/types/treestate.rs create mode 100644 packages/zaino-primitives/src/types/utxo.rs create mode 100644 packages/zaino-primitives/src/types/zatoshis.rs diff --git a/packages/zaino-primitives/src/types.rs b/packages/zaino-primitives/src/types.rs index df5f8ccf9..6b9f71119 100644 --- a/packages/zaino-primitives/src/types.rs +++ b/packages/zaino-primitives/src/types.rs @@ -1,9 +1,39 @@ //! Core vocabulary types shared across the Zaino stack. +mod address_balance; +mod address_delta; +mod aliases; mod block_hash; +mod block_verbose; +mod chain_work; mod height; +mod script; +mod shielded_pool; +mod subtree_root; mod transaction_hash; +mod transaction_location; +mod transparent_address; +mod tree_root; +mod tree_roots; +mod treestate; +mod utxo; +mod zatoshis; +pub use address_balance::AddressBalance; +pub use address_delta::AddressDelta; +pub use aliases::{Confirmations, Difficulty, OutputIndex, SubtreeIndex, TreeSize}; pub use block_hash::BlockHash; +pub use block_verbose::BlockVerbose; +pub use chain_work::ChainWork; pub use height::{Height, HeightOverflow}; +pub use script::Script; +pub use shielded_pool::ShieldedPool; +pub use subtree_root::SubtreeRoot; pub use transaction_hash::TransactionHash; +pub use transaction_location::TransactionLocation; +pub use transparent_address::TransparentAddress; +pub use tree_root::TreeRoot; +pub use tree_roots::{TreeRootInfo, TreeRoots}; +pub use treestate::{TreeBytes, Treestate}; +pub use utxo::Utxo; +pub use zatoshis::{SignedZatoshis, Zatoshis, ZatoshisOverflow}; diff --git a/packages/zaino-primitives/src/types/address_balance.rs b/packages/zaino-primitives/src/types/address_balance.rs new file mode 100644 index 000000000..23fbcd600 --- /dev/null +++ b/packages/zaino-primitives/src/types/address_balance.rs @@ -0,0 +1,12 @@ +//! Transparent address balance. + +use super::Zatoshis; + +/// Balance information for a set of transparent addresses. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AddressBalance { + /// Total current balance in zatoshis. + pub balance: Zatoshis, + /// Total received in zatoshis (lifetime). + pub received: Zatoshis, +} diff --git a/packages/zaino-primitives/src/types/address_delta.rs b/packages/zaino-primitives/src/types/address_delta.rs new file mode 100644 index 000000000..ec5ecd8c7 --- /dev/null +++ b/packages/zaino-primitives/src/types/address_delta.rs @@ -0,0 +1,18 @@ +//! Transparent address balance delta. + +use super::{Height, OutputIndex, SignedZatoshis, TransactionHash, TransparentAddress}; + +/// A single balance change for a transparent address. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AddressDelta { + /// Change in zatoshis (negative for spends, positive for receives). + pub satoshis: SignedZatoshis, + /// The transaction that caused this delta. + pub txid: TransactionHash, + /// Input or output index within the transaction. + pub index: OutputIndex, + /// Block height where this delta occurred. + pub height: Height, + /// The transparent address affected. + pub address: TransparentAddress, +} diff --git a/packages/zaino-primitives/src/types/aliases.rs b/packages/zaino-primitives/src/types/aliases.rs new file mode 100644 index 000000000..234c79d05 --- /dev/null +++ b/packages/zaino-primitives/src/types/aliases.rs @@ -0,0 +1,19 @@ +//! Type aliases for fields not yet promoted to newtypes. +//! +//! Each alias is a grep target: when misuse surfaces, promote +//! the alias to a newtype with private inner + constructor. + +/// Output index within a transaction. +pub type OutputIndex = u32; + +/// Number of confirmations (depth from tip). Ephemeral, query-time only. +pub type Confirmations = i64; + +/// Difficulty target. Protocol-specific float representation. +pub type Difficulty = f64; + +/// Cumulative number of notes in a commitment tree. +pub type TreeSize = u64; + +/// Subtree index within a shielded pool's commitment tree. +pub type SubtreeIndex = u16; diff --git a/packages/zaino-primitives/src/types/block_verbose.rs b/packages/zaino-primitives/src/types/block_verbose.rs new file mode 100644 index 000000000..75d0e3645 --- /dev/null +++ b/packages/zaino-primitives/src/types/block_verbose.rs @@ -0,0 +1,18 @@ +//! Verbose block metadata (cumulative chain state). + +use super::{ChainWork, Confirmations, Difficulty}; + +/// Verbose block metadata not present in the raw block bytes. +/// +/// Fields that require cumulative chain state (chainwork, +/// confirmations) live here; fields derivable from the raw block +/// (header, transactions) do not. +#[derive(Debug, Clone)] +pub struct BlockVerbose { + /// Cumulative chainwork at this block. + pub chainwork: ChainWork, + /// Current difficulty target. + pub difficulty: Difficulty, + /// Number of confirmations (depth from tip). + pub confirmations: Confirmations, +} diff --git a/packages/zaino-primitives/src/types/chain_work.rs b/packages/zaino-primitives/src/types/chain_work.rs new file mode 100644 index 000000000..03ae79c7a --- /dev/null +++ b/packages/zaino-primitives/src/types/chain_work.rs @@ -0,0 +1,24 @@ +//! Cumulative proof-of-work. + +/// Cumulative chainwork at a block (256-bit big-endian). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChainWork([u8; 32]); + +impl ChainWork { + /// Wrap raw chainwork bytes (256-bit big-endian). + pub fn new(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} + +impl From for [u8; 32] { + fn from(cw: ChainWork) -> Self { + cw.0 + } +} + +impl From<[u8; 32]> for ChainWork { + fn from(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/packages/zaino-primitives/src/types/script.rs b/packages/zaino-primitives/src/types/script.rs new file mode 100644 index 000000000..f9dffda70 --- /dev/null +++ b/packages/zaino-primitives/src/types/script.rs @@ -0,0 +1,24 @@ +//! Transaction output script. + +/// A transparent output script (raw bytes). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Script(Vec); + +impl Script { + /// Wrap raw script bytes. + pub fn new(bytes: Vec) -> Self { + Self(bytes) + } +} + +impl From