From f6bdbbe213e6902d7becbdde9b052e6d89316ba5 Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Mon, 1 Jun 2026 13:20:30 +0530 Subject: [PATCH 1/9] feat(prover-db-indexer): add configurable include set to gate captured tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By default the producer hook captures every Create/Drop/Insert event that reaches `T::EventCapture::capture_events`. This adds a sudo-only extrinsic, `ProverDbIndexer::set_include_rules`, that replaces an on-chain rule list against which every captured event is matched. An empty rule list (the default) preserves the current "index everything" behavior; a non-empty list gates the producer so only events for the listed namespaces or fully-qualified tables reach the offchain DB — saving offchain storage and OCW work for chains that index a strict subset of their tables. Shape: - `sxt-core::prover_db_indexer::IncludeRule` (`Namespace(TableNamespace)` | `Table(TableIdentifier)`), stored as `BoundedVec>`. - Pure helper `table_matches_rules(&TableIdentifier, &[IncludeRule])` with the "empty = match all" semantics. - `pallet_prover_db_indexer::IncludeSet` value-query storage, with a matching `IncludeRulesSet { count }` event. - `Pallet::capture_events` reads the rule set once per call and skips the offchain-DB write entirely when nothing passes the filter, so there's no per-event storage cost in the default case. Tests cover the extrinsic (root succeeds + event deposited; non-root rejected; storage unchanged on reject) and the producer side via `offchain_index::set` → `persist_offchain_overlay()` → `ext.offchain_db()`: empty set passes everything, namespace rules pass only that namespace, table rules pass only the exact identifier, and a non-matching set writes neither the payload nor the high-water-mark. --- pallets/prover_db_indexer/src/lib.rs | 55 +++++++- pallets/prover_db_indexer/src/tests.rs | 182 ++++++++++++++++++++++++- sxt-core/src/prover_db_indexer.rs | 42 +++++- 3 files changed, 274 insertions(+), 5 deletions(-) diff --git a/pallets/prover_db_indexer/src/lib.rs b/pallets/prover_db_indexer/src/lib.rs index ae70b3b9..a60cbe72 100644 --- a/pallets/prover_db_indexer/src/lib.rs +++ b/pallets/prover_db_indexer/src/lib.rs @@ -136,8 +136,10 @@ pub mod pallet { use sxt_core::prover_db_indexer::{ key_for_event, key_for_high_water, + table_matches_rules, BlockEvent, EventCapture, + IncludeRules, }; use crate::ConsumerError; @@ -181,8 +183,22 @@ pub mod pallet { /// Block number that the forwarder failed on. block_number: u64, }, + /// The set of namespace/table rules that gate which captured + /// events get forwarded to the indexer has been replaced. + IncludeRulesSet { + /// New number of rules in the include set. Zero means + /// "index everything". + count: u32, + }, } + /// Include-set: which tables the producer should capture events for. + /// An empty value (the default) means "capture every table". A + /// non-empty value gates `capture_events` so only events whose table + /// matches at least one rule reach the offchain DB. + #[pallet::storage] + pub type IncludeSet = StorageValue<_, IncludeRules, ValueQuery>; + #[pallet::hooks] impl Hooks> for Pallet { // Fires at chain tip only (not during sync). Drains the offchain @@ -198,12 +214,49 @@ pub mod pallet { } } + #[pallet::call] + impl Pallet { + /// Replace the include set used by `capture_events` to gate which + /// tables' events get forwarded to the indexer. An empty list + /// means "index every table" (the default). Root-only. + #[pallet::call_index(0)] + #[pallet::weight(polkadot_sdk::frame_support::weights::Weight::from_parts(10_000, 0))] + pub fn set_include_rules(origin: OriginFor, rules: IncludeRules) -> DispatchResult { + polkadot_sdk::frame_system::ensure_root(origin)?; + let count = rules.len() as u32; + IncludeSet::::put(rules); + Self::deposit_event(Event::IncludeRulesSet { count }); + Ok(()) + } + } + impl EventCapture for Pallet { fn capture_events(events: Vec>) { if events.is_empty() { return; } + // Filter against the configured include set. Empty set means + // "match all", so this is a cheap pass-through in the default + // config. We read storage once per `capture_events` call + // (i.e. once per extrinsic that produces events). + let rules = IncludeSet::::get(); + let filtered: Vec> = events + .into_iter() + .filter(|e| { + let table = match e { + BlockEvent::Create(entry) => entry.ident.as_ref(), + BlockEvent::Drop(ident) => ident.as_ref(), + BlockEvent::Insert(entry) => entry.table.as_ref(), + }; + table_matches_rules(table, &rules) + }) + .collect(); + + if filtered.is_empty() { + return; + } + // `block_number()` always fits in u64 in our runtime (BlockNumber // is u32); `extrinsic_index()` is always `Some` while we're inside // an extrinsic, which is the only path that reaches `capture_events`. @@ -216,7 +269,7 @@ pub mod pallet { // Per-extrinsic event payload. polkadot_sdk::sp_io::offchain_index::set( &key_for_event(block, ext_idx), - &events.encode(), + &filtered.encode(), ); // Per-block high-water-mark. Always overwrites; the OCW only diff --git a/pallets/prover_db_indexer/src/tests.rs b/pallets/prover_db_indexer/src/tests.rs index 05a8fd3f..f1e64f3e 100644 --- a/pallets/prover_db_indexer/src/tests.rs +++ b/pallets/prover_db_indexer/src/tests.rs @@ -21,12 +21,15 @@ use sxt_core::prover_db_indexer::{ key_for_high_water, BlockEvent, CreateEntry, + EventCapture, + IncludeRule, + IncludeRules, InsertEntry, }; -use sxt_core::tables::TableIdentifier; +use sxt_core::tables::{TableIdentifier, TableNamespace}; use crate::mock::*; -use crate::{proto, PROVER_DB_URL_KEY}; +use crate::{proto, IncludeSet, PROVER_DB_URL_KEY}; type StateArc = std::sync::Arc>; @@ -395,3 +398,178 @@ fn ocw_walks_multiple_extrinsics_in_one_block() { assert!(s.persistent_storage.get(&key_for_event(1, 3)).is_none()); assert!(s.persistent_storage.get(&key_for_high_water(1)).is_none()); } + +// ─── Include-set tests ────────────────────────────────────────────────── + +/// Helper: build a Drop event for `(name, namespace)`. +fn drop_event(name: &str, namespace: &str) -> BlockEvent<'static> { + BlockEvent::Drop(Cow::Owned(TableIdentifier::from_str_unchecked( + name, namespace, + ))) +} + +/// Helper: build a Create event for `(name, namespace)` with arbitrary DDL. +fn create_event(name: &str, namespace: &str) -> BlockEvent<'static> { + BlockEvent::Create(CreateEntry { + ident: Cow::Owned(TableIdentifier::from_str_unchecked(name, namespace)), + ddl: Cow::Owned(b"CREATE TABLE ...".to_vec()), + }) +} + +/// Helper: build an Insert event for `(name, namespace)` with arbitrary data. +fn insert_event(name: &str, namespace: &str) -> BlockEvent<'static> { + BlockEvent::Insert(InsertEntry { + table: Cow::Owned(TableIdentifier::from_str_unchecked(name, namespace)), + data: Cow::Owned(b"rows".to_vec()), + }) +} + +/// Set up an externalities for exercising the producer side +/// (`capture_events`). No `TestOffchainExt` needed — `offchain_index::set` +/// goes through the runtime overlay into `ext.offchain_db()`, which the +/// tests inspect after calling `persist_offchain_overlay`. +fn capture_ext() -> polkadot_sdk::sp_io::TestExternalities { + new_test_ext() +} + +#[test] +fn set_include_rules_works_for_root_and_emits_event() { + new_test_ext().execute_with(|| { + System::set_block_number(1); + let rules = vec![ + IncludeRule::Namespace(TableNamespace::try_from(b"PUBLIC".to_vec()).unwrap()), + IncludeRule::Table(TableIdentifier::from_str_unchecked("BAR", "FOO")), + ] + .try_into() + .unwrap(); + + assert!(ProverDbIndexer::set_include_rules(RuntimeOrigin::root(), rules).is_ok()); + + // Storage now holds two rules. + assert_eq!(IncludeSet::::get().len(), 2); + + // And an event was deposited. + let events = System::events(); + assert!(events.iter().any(|er| matches!( + er.event, + RuntimeEvent::ProverDbIndexer(crate::Event::IncludeRulesSet { count: 2 }), + ))); + }); +} + +#[test] +fn set_include_rules_rejects_non_root() { + new_test_ext().execute_with(|| { + System::set_block_number(1); + let signed = RuntimeOrigin::signed(polkadot_sdk::sp_runtime::AccountId32::new([1; 32])); + let rules = vec![IncludeRule::Namespace( + TableNamespace::try_from(b"PUBLIC".to_vec()).unwrap(), + )] + .try_into() + .unwrap(); + assert!(ProverDbIndexer::set_include_rules(signed, rules).is_err()); + // Storage unchanged. + assert!(IncludeSet::::get().is_empty()); + }); +} + +#[test] +fn capture_events_writes_everything_when_include_set_is_empty() { + let mut ext = capture_ext(); + ext.execute_with(|| { + System::set_block_number(1); + polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); + + let events = vec![ + create_event("A", "ALPHA"), + insert_event("B", "BETA"), + drop_event("C", "GAMMA"), + ]; + ::capture_events(events); + }); + ext.persist_offchain_overlay(); + let db = ext.offchain_db(); + let stored = db.get(&key_for_event(1, 0)).unwrap(); + let decoded: Vec> = codec::Decode::decode(&mut &stored[..]).unwrap(); + assert_eq!(decoded.len(), 3); +} + +#[test] +fn capture_events_keeps_only_namespace_matches() { + let mut ext = capture_ext(); + ext.execute_with(|| { + System::set_block_number(1); + polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); + + IncludeSet::::put( + IncludeRules::try_from(vec![IncludeRule::Namespace( + TableNamespace::try_from(b"ALPHA".to_vec()).unwrap(), + )]) + .unwrap(), + ); + + let events = vec![ + create_event("A", "ALPHA"), // pass + insert_event("B", "BETA"), // filter out + drop_event("C", "ALPHA"), // pass + ]; + ::capture_events(events); + }); + ext.persist_offchain_overlay(); + let db = ext.offchain_db(); + let stored = db.get(&key_for_event(1, 0)).unwrap(); + let decoded: Vec> = codec::Decode::decode(&mut &stored[..]).unwrap(); + assert_eq!(decoded.len(), 2); +} + +#[test] +fn capture_events_keeps_only_specific_table_matches() { + let mut ext = capture_ext(); + ext.execute_with(|| { + System::set_block_number(1); + polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); + + IncludeSet::::put( + IncludeRules::try_from(vec![IncludeRule::Table( + TableIdentifier::from_str_unchecked("B", "BETA"), + )]) + .unwrap(), + ); + + let events = vec![ + create_event("A", "ALPHA"), // filter out + insert_event("B", "BETA"), // pass + drop_event("C", "BETA"), // filter out: namespace match but rule is table-scoped + ]; + ::capture_events(events); + }); + ext.persist_offchain_overlay(); + let db = ext.offchain_db(); + let stored = db.get(&key_for_event(1, 0)).unwrap(); + let decoded: Vec> = codec::Decode::decode(&mut &stored[..]).unwrap(); + assert_eq!(decoded.len(), 1); + assert!(matches!(decoded[0], BlockEvent::Insert(_))); +} + +#[test] +fn capture_events_writes_nothing_when_no_event_matches() { + let mut ext = capture_ext(); + ext.execute_with(|| { + System::set_block_number(1); + polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); + + IncludeSet::::put( + IncludeRules::try_from(vec![IncludeRule::Namespace( + TableNamespace::try_from(b"NOT_PRESENT".to_vec()).unwrap(), + )]) + .unwrap(), + ); + + let events = vec![create_event("A", "ALPHA"), insert_event("B", "BETA")]; + ::capture_events(events); + }); + ext.persist_offchain_overlay(); + let db = ext.offchain_db(); + assert!(db.get(&key_for_event(1, 0)).is_none()); + assert!(db.get(&key_for_high_water(1)).is_none()); +} diff --git a/sxt-core/src/prover_db_indexer.rs b/sxt-core/src/prover_db_indexer.rs index 8989732d..3298bb36 100644 --- a/sxt-core/src/prover_db_indexer.rs +++ b/sxt-core/src/prover_db_indexer.rs @@ -5,9 +5,12 @@ use alloc::borrow::Cow; use alloc::vec::Vec; -use codec::{Decode, Encode}; +use codec::{Decode, Encode, MaxEncodedLen}; +use polkadot_sdk::sp_core::ConstU32; +use polkadot_sdk::sp_runtime::BoundedVec; +use scale_info::TypeInfo; -use crate::tables::TableIdentifier; +use crate::tables::{TableIdentifier, TableNamespace}; /// Offchain local-storage key holding the prover-db indexer URL. /// @@ -98,3 +101,38 @@ pub trait EventCapture { impl EventCapture for () { fn capture_events(_events: Vec>) {} } + +/// Maximum number of include-set rules the chain accepts. Bounds the +/// storage footprint and the per-extrinsic match cost in `capture_events`. +pub const MAX_INCLUDE_RULES: u32 = 256; + +/// A single entry in the prover-db indexer's include set. An empty list +/// of these means "index everything"; a non-empty list means "index only +/// events whose table matches at least one rule". +/// +/// Matches are byte-exact against the on-chain identifiers, so callers +/// that need case-insensitive matching should normalize their inputs +/// (e.g. via [`TableIdentifier::from_str_unchecked`] which uppercases) +/// before submitting the rule. +#[derive(Encode, Decode, MaxEncodedLen, TypeInfo, Debug, Clone, Eq, PartialEq)] +pub enum IncludeRule { + /// Match every table within the given namespace. + Namespace(TableNamespace), + /// Match exactly one fully-qualified table identifier. + Table(TableIdentifier), +} + +/// Bounded list of include rules suitable for on-chain storage. +pub type IncludeRules = BoundedVec>; + +/// Returns true if `table` matches at least one rule in `rules`. An +/// empty rule set is treated as "match all". +pub fn table_matches_rules(table: &TableIdentifier, rules: &[IncludeRule]) -> bool { + if rules.is_empty() { + return true; + } + rules.iter().any(|rule| match rule { + IncludeRule::Namespace(ns) => &table.namespace == ns, + IncludeRule::Table(t) => t == table, + }) +} From 8231a1300808924857eeda12dc6c02dbdf4c8a2b Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Tue, 2 Jun 2026 13:11:10 +0530 Subject: [PATCH 2/9] refactor(prover-db-indexer): move include set from on-chain to per-node CLI config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback: the include set is a node-local indexing policy ("which tables does *this* node's prb-service receive?"), not a chain-wide one. Different indexer nodes can subscribe to different subsets. Capture stays unfiltered on every validator so the offchain queue mirrors the full block; the filter applies only when the OCW forwards events. Removed (on-chain): - `IncludeSet` storage value - `set_include_rules` extrinsic - `IncludeRulesSet` event - Producer-side filter in `EventCapture::capture_events` - `IncludeRules` BoundedVec alias, `MAX_INCLUDE_RULES`, `MaxEncodedLen` / `TypeInfo` derives on `IncludeRule` Added (per-node): - `PROVER_DB_INCLUDE_KEY` offchain-storage key in `sxt-core` - `--prover-db-include-file ` flag on the node binary, plus a `PROVER_DB_INCLUDE_FILE` env var. JSON schema: [ { "kind": "namespace", "value": "ETHEREUM_MT" }, { "kind": "table", "namespace": "OTHER_NS", "name": "SPECIAL" } ] Identifiers get uppercased before storage so the byte-match against the on-chain canonical form works without per-operator vigilance. - Service-side `configure_prover_db_include` parses the file and SCALE-encodes a `Vec` into offchain local storage at startup, before the first block. Mirrors the existing `configure_prover_db_url` shape exactly. Rejects the include-file-without-url misconfiguration up front. - Consumer-side filter: `run_consumer` reads the rules once per OCW round and threads them through `forward_block` → `forward_events`, where each `BlockEvent` is matched against `table_matches_rules` and skipped (Drops included, uniformly) when it doesn't pass. Tests dropped the four producer-filter tests and the two extrinsic tests; added `ocw_forwards_only_events_matching_include_set` (mixed events, only matchers POST) and `ocw_with_empty_include_set_forwards_everything` (absent key ⇒ match-all). Kept the existing `capture_events_writes_all_events_unconditionally` regression. 12/12 pass. --- node/src/cli.rs | 18 ++ node/src/service.rs | 92 +++++++++ pallets/prover_db_indexer/src/lib.rs | 109 +++++------ pallets/prover_db_indexer/src/tests.rs | 257 +++++++++++++------------ sxt-core/src/prover_db_indexer.rs | 37 ++-- 5 files changed, 314 insertions(+), 199 deletions(-) diff --git a/node/src/cli.rs b/node/src/cli.rs index c5438cbd..b800f099 100644 --- a/node/src/cli.rs +++ b/node/src/cli.rs @@ -24,6 +24,24 @@ pub struct Cli { #[clap(long, env)] pub prover_db_url: Option, + /// Path to a JSON file listing which tables/namespaces this node + /// should forward to its prover-db indexer. Schema: + /// + /// ```json + /// [ + /// { "kind": "namespace", "value": "ETHEREUM_MT" }, + /// { "kind": "table", "namespace": "OTHER_NS", "name": "SPECIAL" } + /// ] + /// ``` + /// + /// An empty array (or omitting this flag) means "forward every + /// captured event" — the on-chain capture itself is unfiltered, so + /// the choice is purely per-node. Identifiers are uppercased before + /// being written to offchain storage, to match how the chain stores + /// them. + #[clap(long, env)] + pub prover_db_include_file: Option, + #[allow(missing_docs)] #[clap(flatten)] pub storage_monitor: sc_storage_monitor::StorageMonitorParams, diff --git a/node/src/service.rs b/node/src/service.rs index 184e9366..21d0b71c 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -103,6 +103,83 @@ fn configure_prover_db_url(backend: &FullBackend, url: &url::Url) -> Result<(), Ok(()) } +/// JSON rule type as it appears in the user-supplied `--prover-db-include-file`. +/// Decoupled from the runtime [`sxt_core::prover_db_indexer::IncludeRule`] +/// so the on-disk format can use natural string fields while the wire +/// type uses byte-bounded `TableNamespace` / `TableIdentifier`. +#[derive(serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum IncludeRuleJson { + Namespace { value: String }, + Table { namespace: String, name: String }, +} + +/// Parse the include-file JSON and seed the rule set into offchain +/// persistent local storage at startup, before the first block is +/// authored. +/// +/// Stored under `sxt_core::prover_db_indexer::PROVER_DB_INCLUDE_KEY` as +/// a SCALE-encoded `Vec`. Identifiers are uppercased on +/// the way in via `TableIdentifier::from_str_unchecked`, so the matches +/// the OCW does later are byte-exact against the on-chain canonical +/// form. +#[expect( + clippy::result_large_err, + reason = "ServiceError is from substrate and cannot be modified" +)] +fn configure_prover_db_include(backend: &FullBackend, path: &Path) -> Result<(), ServiceError> { + let raw = std::fs::read_to_string(path).map_err(|e| { + ServiceError::Other(format!( + "failed to read --prover-db-include-file {}: {}", + path.display(), + e + )) + })?; + let json_rules: Vec = serde_json::from_str(&raw).map_err(|e| { + ServiceError::Other(format!( + "failed to parse --prover-db-include-file {} as JSON: {}", + path.display(), + e + )) + })?; + + let rules: Vec = json_rules + .into_iter() + .map(|r| match r { + IncludeRuleJson::Namespace { value } => { + let bytes = value.to_uppercase().into_bytes(); + let ns = sxt_core::tables::TableNamespace::try_from(bytes).map_err(|_| { + ServiceError::Other(format!( + "namespace '{}' in {} exceeds the maximum identifier length", + value, + path.display(), + )) + })?; + Ok(sxt_core::prover_db_indexer::IncludeRule::Namespace(ns)) + } + IncludeRuleJson::Table { namespace, name } => { + let ident = + sxt_core::tables::TableIdentifier::from_str_unchecked(&name, &namespace); + Ok(sxt_core::prover_db_indexer::IncludeRule::Table(ident)) + } + }) + .collect::>()?; + + let Some(mut storage) = backend.offchain_storage() else { + return Err(ServiceError::Other( + "backend did not expose an offchain storage handle; \ + cannot apply --prover-db-include-file" + .into(), + )); + }; + storage.set( + STORAGE_PREFIX, + sxt_core::prover_db_indexer::PROVER_DB_INCLUDE_KEY, + &rules.encode(), + ); + Ok(()) +} + #[allow(clippy::type_complexity)] #[expect( clippy::result_large_err, @@ -368,6 +445,21 @@ pub fn new_full_base::Hash>>( configure_prover_db_url(backend.as_ref(), url)?; } + // `--prover-db-include-file` is only meaningful when the OCW will + // actually forward something, i.e. when the URL is set. Reject the + // common misconfiguration up front. + if let Some(path) = cli.prover_db_include_file.as_ref() { + if cli.prover_db_url.is_none() { + return Err(ServiceError::Other( + "--prover-db-include-file was set but --prover-db-url is not; \ + the include set would have no effect because the OCW only \ + runs when an indexer URL is configured." + .into(), + )); + } + configure_prover_db_include(backend.as_ref(), path)?; + } + let metrics = N::register_notification_metrics( config.prometheus_config.as_ref().map(|cfg| &cfg.registry), ); diff --git a/pallets/prover_db_indexer/src/lib.rs b/pallets/prover_db_indexer/src/lib.rs index a60cbe72..992c1cc7 100644 --- a/pallets/prover_db_indexer/src/lib.rs +++ b/pallets/prover_db_indexer/src/lib.rs @@ -59,10 +59,11 @@ mod proto { } pub use pallet::*; -/// Re-export of the canonical offchain local-storage key (defined in `sxt-core`), -/// kept here so existing call sites that reach for `pallet_prover_db_indexer::PROVER_DB_URL_KEY` -/// keep working. -pub use sxt_core::prover_db_indexer::PROVER_DB_URL_KEY; +/// Re-export of the canonical offchain local-storage keys (defined in `sxt-core`), +/// kept here so existing call sites that reach for +/// `pallet_prover_db_indexer::PROVER_DB_URL_KEY` keep working and so the node +/// service has a single import for both. +pub use sxt_core::prover_db_indexer::{PROVER_DB_INCLUDE_KEY, PROVER_DB_URL_KEY}; /// Offchain DB key for the lock that serializes OCW consumer rounds. const OCW_LOCK_KEY: &[u8] = b"prover_db_indexer/ocw_lock"; @@ -139,7 +140,7 @@ pub mod pallet { table_matches_rules, BlockEvent, EventCapture, - IncludeRules, + IncludeRule, }; use crate::ConsumerError; @@ -183,22 +184,8 @@ pub mod pallet { /// Block number that the forwarder failed on. block_number: u64, }, - /// The set of namespace/table rules that gate which captured - /// events get forwarded to the indexer has been replaced. - IncludeRulesSet { - /// New number of rules in the include set. Zero means - /// "index everything". - count: u32, - }, } - /// Include-set: which tables the producer should capture events for. - /// An empty value (the default) means "capture every table". A - /// non-empty value gates `capture_events` so only events whose table - /// matches at least one rule reach the offchain DB. - #[pallet::storage] - pub type IncludeSet = StorageValue<_, IncludeRules, ValueQuery>; - #[pallet::hooks] impl Hooks> for Pallet { // Fires at chain tip only (not during sync). Drains the offchain @@ -214,48 +201,17 @@ pub mod pallet { } } - #[pallet::call] - impl Pallet { - /// Replace the include set used by `capture_events` to gate which - /// tables' events get forwarded to the indexer. An empty list - /// means "index every table" (the default). Root-only. - #[pallet::call_index(0)] - #[pallet::weight(polkadot_sdk::frame_support::weights::Weight::from_parts(10_000, 0))] - pub fn set_include_rules(origin: OriginFor, rules: IncludeRules) -> DispatchResult { - polkadot_sdk::frame_system::ensure_root(origin)?; - let count = rules.len() as u32; - IncludeSet::::put(rules); - Self::deposit_event(Event::IncludeRulesSet { count }); - Ok(()) - } - } - impl EventCapture for Pallet { fn capture_events(events: Vec>) { if events.is_empty() { return; } - // Filter against the configured include set. Empty set means - // "match all", so this is a cheap pass-through in the default - // config. We read storage once per `capture_events` call - // (i.e. once per extrinsic that produces events). - let rules = IncludeSet::::get(); - let filtered: Vec> = events - .into_iter() - .filter(|e| { - let table = match e { - BlockEvent::Create(entry) => entry.ident.as_ref(), - BlockEvent::Drop(ident) => ident.as_ref(), - BlockEvent::Insert(entry) => entry.table.as_ref(), - }; - table_matches_rules(table, &rules) - }) - .collect(); - - if filtered.is_empty() { - return; - } + // Capture every event unconditionally. Per-node filtering is + // applied later by the OCW consumer when forwarding to the + // indexer; the offchain queue mirrors the full block so every + // validator carries the same data, regardless of which subset + // each indexer cares about. // `block_number()` always fits in u64 in our runtime (BlockNumber // is u32); `extrinsic_index()` is always `Some` while we're inside @@ -269,7 +225,7 @@ pub mod pallet { // Per-extrinsic event payload. polkadot_sdk::sp_io::offchain_index::set( &key_for_event(block, ext_idx), - &filtered.encode(), + &events.encode(), ); // Per-block high-water-mark. Always overwrites; the OCW only @@ -321,10 +277,21 @@ pub mod pallet { let url = url::Url::parse(url_str).map_err(|error| ConsumerError::InvalidUrl { error })?; + // 1b. Load this node's include set from offchain local storage. + // Absent or empty ⇒ "forward every event". The rules are + // written by the node service at startup from a CLI config + // file; see `sxt_core::prover_db_indexer::PROVER_DB_INCLUDE_KEY`. + let include_ref = StorageValueRef::persistent(crate::PROVER_DB_INCLUDE_KEY); + let include_rules: Vec = include_ref + .get::>() + .ok() + .flatten() + .unwrap_or_default(); + polkadot_sdk::sp_tracing::debug!( target: "prover_db_indexer", - "consumer round starting; indexer base URL = {}", - url, + "consumer round starting; indexer base URL = {}; {} include rules", + url, include_rules.len(), ); // 2. Current block number — passed in by `offchain_worker`. @@ -366,7 +333,7 @@ pub mod pallet { ); for block_num in start..=end { - Self::forward_block(&url, block_num)?; + Self::forward_block(&url, block_num, &include_rules)?; // Checkpoint on the server (always, even for empty blocks). crate::http_client::checkpoint(&url, block_num) @@ -379,7 +346,11 @@ pub mod pallet { /// Forward a single block's events in extrinsic-index order, then /// clear the offchain entries we consumed. If the block had no /// captures (no high-water-mark key), this is a no-op. - fn forward_block(url: &url::Url, block_num: u64) -> Result<(), ConsumerError> { + fn forward_block( + url: &url::Url, + block_num: u64, + include_rules: &[IncludeRule], + ) -> Result<(), ConsumerError> { let Some(hwm) = crate::offchain_consumer::read_high_water(block_num) else { return Ok(()); }; @@ -395,7 +366,7 @@ pub mod pallet { let Some(events) = crate::offchain_consumer::read_events(block_num, ext_idx) else { continue; }; - Self::forward_events(url, block_num, &events)?; + Self::forward_events(url, block_num, &events, include_rules)?; crate::offchain_consumer::clear_events(block_num, ext_idx); } @@ -403,15 +374,29 @@ pub mod pallet { Ok(()) } - /// POST one extrinsic's captured events to the indexer in deposit order. + /// POST one extrinsic's captured events to the indexer in deposit + /// order, skipping any whose table doesn't match this node's + /// include set. The capture queue is unfiltered (every validator + /// records the full block), so the filter applies only to the + /// HTTP forwarding done by this node's OCW. fn forward_events( url: &url::Url, block_num: u64, events: &[BlockEvent<'_>], + include_rules: &[IncludeRule], ) -> Result<(), ConsumerError> { let seq = block_num; for event in events { + let table = match event { + BlockEvent::Create(entry) => entry.ident.as_ref(), + BlockEvent::Drop(ident) => ident.as_ref(), + BlockEvent::Insert(entry) => entry.table.as_ref(), + }; + if !table_matches_rules(table, include_rules) { + continue; + } + match event { BlockEvent::Drop(ident) => { crate::http_client::drop_table(url, seq, ident) diff --git a/pallets/prover_db_indexer/src/tests.rs b/pallets/prover_db_indexer/src/tests.rs index f1e64f3e..637281a0 100644 --- a/pallets/prover_db_indexer/src/tests.rs +++ b/pallets/prover_db_indexer/src/tests.rs @@ -23,13 +23,12 @@ use sxt_core::prover_db_indexer::{ CreateEntry, EventCapture, IncludeRule, - IncludeRules, InsertEntry, }; use sxt_core::tables::{TableIdentifier, TableNamespace}; use crate::mock::*; -use crate::{proto, IncludeSet, PROVER_DB_URL_KEY}; +use crate::{proto, PROVER_DB_INCLUDE_KEY, PROVER_DB_URL_KEY}; type StateArc = std::sync::Arc>; @@ -399,7 +398,7 @@ fn ocw_walks_multiple_extrinsics_in_one_block() { assert!(s.persistent_storage.get(&key_for_high_water(1)).is_none()); } -// ─── Include-set tests ────────────────────────────────────────────────── +// ─── Include-set tests (consumer-side filter) ────────────────────────── /// Helper: build a Drop event for `(name, namespace)`. fn drop_event(name: &str, namespace: &str) -> BlockEvent<'static> { @@ -408,7 +407,7 @@ fn drop_event(name: &str, namespace: &str) -> BlockEvent<'static> { ))) } -/// Helper: build a Create event for `(name, namespace)` with arbitrary DDL. +/// Helper: build a Create event for `(name, namespace)`. fn create_event(name: &str, namespace: &str) -> BlockEvent<'static> { BlockEvent::Create(CreateEntry { ident: Cow::Owned(TableIdentifier::from_str_unchecked(name, namespace)), @@ -416,7 +415,7 @@ fn create_event(name: &str, namespace: &str) -> BlockEvent<'static> { }) } -/// Helper: build an Insert event for `(name, namespace)` with arbitrary data. +/// Helper: build an Insert event for `(name, namespace)`. fn insert_event(name: &str, namespace: &str) -> BlockEvent<'static> { BlockEvent::Insert(InsertEntry { table: Cow::Owned(TableIdentifier::from_str_unchecked(name, namespace)), @@ -424,58 +423,12 @@ fn insert_event(name: &str, namespace: &str) -> BlockEvent<'static> { }) } -/// Set up an externalities for exercising the producer side -/// (`capture_events`). No `TestOffchainExt` needed — `offchain_index::set` -/// goes through the runtime overlay into `ext.offchain_db()`, which the -/// tests inspect after calling `persist_offchain_overlay`. -fn capture_ext() -> polkadot_sdk::sp_io::TestExternalities { - new_test_ext() -} - -#[test] -fn set_include_rules_works_for_root_and_emits_event() { - new_test_ext().execute_with(|| { - System::set_block_number(1); - let rules = vec![ - IncludeRule::Namespace(TableNamespace::try_from(b"PUBLIC".to_vec()).unwrap()), - IncludeRule::Table(TableIdentifier::from_str_unchecked("BAR", "FOO")), - ] - .try_into() - .unwrap(); - - assert!(ProverDbIndexer::set_include_rules(RuntimeOrigin::root(), rules).is_ok()); - - // Storage now holds two rules. - assert_eq!(IncludeSet::::get().len(), 2); - - // And an event was deposited. - let events = System::events(); - assert!(events.iter().any(|er| matches!( - er.event, - RuntimeEvent::ProverDbIndexer(crate::Event::IncludeRulesSet { count: 2 }), - ))); - }); -} - -#[test] -fn set_include_rules_rejects_non_root() { - new_test_ext().execute_with(|| { - System::set_block_number(1); - let signed = RuntimeOrigin::signed(polkadot_sdk::sp_runtime::AccountId32::new([1; 32])); - let rules = vec![IncludeRule::Namespace( - TableNamespace::try_from(b"PUBLIC".to_vec()).unwrap(), - )] - .try_into() - .unwrap(); - assert!(ProverDbIndexer::set_include_rules(signed, rules).is_err()); - // Storage unchanged. - assert!(IncludeSet::::get().is_empty()); - }); -} - +/// `capture_events` is now unconditional — every event reaches the +/// offchain queue regardless of any per-node configuration. This test +/// is the single producer-side regression check. #[test] -fn capture_events_writes_everything_when_include_set_is_empty() { - let mut ext = capture_ext(); +fn capture_events_writes_all_events_unconditionally() { + let mut ext = new_test_ext(); ext.execute_with(|| { System::set_block_number(1); polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); @@ -494,82 +447,144 @@ fn capture_events_writes_everything_when_include_set_is_empty() { assert_eq!(decoded.len(), 3); } +/// End-to-end consumer test: seed the node's include set into offchain +/// local storage (where the OCW reads it), seed a multi-event block, +/// and confirm only the matching events are POSTed to the indexer. #[test] -fn capture_events_keeps_only_namespace_matches() { - let mut ext = capture_ext(); - ext.execute_with(|| { - System::set_block_number(1); - polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); +fn ocw_forwards_only_events_matching_include_set() { + let (mut ext, state) = setup_with_url(); - IncludeSet::::put( - IncludeRules::try_from(vec![IncludeRule::Namespace( - TableNamespace::try_from(b"ALPHA".to_vec()).unwrap(), - )]) - .unwrap(), - ); + // Seed the node-local include set: namespace = "ALPHA", and a + // specific table B.BETA. Encoded as SCALE Vec, exactly + // as the node service would write it from the CLI config file. + let rules = vec![ + IncludeRule::Namespace(TableNamespace::try_from(b"ALPHA".to_vec()).unwrap()), + IncludeRule::Table(TableIdentifier::from_str_unchecked("BETA_T", "BETA_NS")), + ]; + state + .write() + .persistent_storage + .set(b"", PROVER_DB_INCLUDE_KEY, &rules.encode()); - let events = vec![ - create_event("A", "ALPHA"), // pass - insert_event("B", "BETA"), // filter out - drop_event("C", "ALPHA"), // pass - ]; - ::capture_events(events); - }); - ext.persist_offchain_overlay(); - let db = ext.offchain_db(); - let stored = db.get(&key_for_event(1, 0)).unwrap(); - let decoded: Vec> = codec::Decode::decode(&mut &stored[..]).unwrap(); - assert_eq!(decoded.len(), 2); -} + // Block 1, extrinsic 0: a mix of matching and non-matching events. + seed_block_events( + &state, + 1, + 0, + vec![ + create_event("A", "ALPHA"), // match (namespace) + insert_event("BETA_T", "BETA_NS"), // match (table) + drop_event("OTHER", "GAMMA"), // skip + create_event("X", "GAMMA"), // skip + insert_event("Y", "ALPHA"), // match (namespace) + ], + ); + + // Only the three matching events should reach the indexer. Drops + // for the skipped GAMMA namespace get filtered too — uniform. + { + let mut s = state.write(); + s.expect_request(expected_request( + "/v1/get_last_checkpoint", + vec![], + no_checkpoint_response(), + )); + s.expect_request(expected_request( + "/v1/create_table", + proto::CreateTableRequest { + sequence_number: 1, + table_name: "ALPHA.A".into(), + arrow_schema: b"CREATE TABLE ...".to_vec(), + key: "META_ROW_NUMBER".into(), + } + .encode_to_vec(), + vec![], + )); + s.expect_request(expected_request( + "/v1/put_batches", + proto::PutBatchesRequest { + sequence_number: 1, + batches: vec![proto::TableBatch { + table_name: "BETA_NS.BETA_T".into(), + record_batch: b"rows".to_vec(), + }], + } + .encode_to_vec(), + vec![], + )); + s.expect_request(expected_request( + "/v1/put_batches", + proto::PutBatchesRequest { + sequence_number: 1, + batches: vec![proto::TableBatch { + table_name: "ALPHA.Y".into(), + record_batch: b"rows".to_vec(), + }], + } + .encode_to_vec(), + vec![], + )); + s.expect_request(expected_request( + "/v1/checkpoint", + proto::CheckpointRequest { sequence_number: 1 }.encode_to_vec(), + vec![], + )); + } -#[test] -fn capture_events_keeps_only_specific_table_matches() { - let mut ext = capture_ext(); ext.execute_with(|| { System::set_block_number(1); - polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); - - IncludeSet::::put( - IncludeRules::try_from(vec![IncludeRule::Table( - TableIdentifier::from_str_unchecked("B", "BETA"), - )]) - .unwrap(), - ); - - let events = vec![ - create_event("A", "ALPHA"), // filter out - insert_event("B", "BETA"), // pass - drop_event("C", "BETA"), // filter out: namespace match but rule is table-scoped - ]; - ::capture_events(events); + ProverDbIndexer::offchain_worker(1); }); - ext.persist_offchain_overlay(); - let db = ext.offchain_db(); - let stored = db.get(&key_for_event(1, 0)).unwrap(); - let decoded: Vec> = codec::Decode::decode(&mut &stored[..]).unwrap(); - assert_eq!(decoded.len(), 1); - assert!(matches!(decoded[0], BlockEvent::Insert(_))); + + let s = state.read(); + assert!(s.persistent_storage.get(&key_for_event(1, 0)).is_none()); + assert!(s.persistent_storage.get(&key_for_high_water(1)).is_none()); } +/// Empty include set in offchain storage ⇒ forward every event (default). #[test] -fn capture_events_writes_nothing_when_no_event_matches() { - let mut ext = capture_ext(); - ext.execute_with(|| { - System::set_block_number(1); - polkadot_sdk::frame_system::Pallet::::set_extrinsic_index(0); +fn ocw_with_empty_include_set_forwards_everything() { + let (mut ext, state) = setup_with_url(); - IncludeSet::::put( - IncludeRules::try_from(vec![IncludeRule::Namespace( - TableNamespace::try_from(b"NOT_PRESENT".to_vec()).unwrap(), - )]) - .unwrap(), - ); + // No PROVER_DB_INCLUDE_KEY written — absence ⇒ empty rules ⇒ match all. + let ddl = b"CREATE TABLE ANY.ANY (ID BIGINT NOT NULL)"; + seed_block_events( + &state, + 1, + 0, + vec![BlockEvent::Create(CreateEntry { + ident: Cow::Owned(TableIdentifier::from_str_unchecked("ANY", "ANY")), + ddl: ddl.to_vec().into(), + })], + ); - let events = vec![create_event("A", "ALPHA"), insert_event("B", "BETA")]; - ::capture_events(events); + { + let mut s = state.write(); + s.expect_request(expected_request( + "/v1/get_last_checkpoint", + vec![], + no_checkpoint_response(), + )); + s.expect_request(expected_request( + "/v1/create_table", + proto::CreateTableRequest { + sequence_number: 1, + table_name: "ANY.ANY".into(), + arrow_schema: ddl.to_vec(), + key: "META_ROW_NUMBER".into(), + } + .encode_to_vec(), + vec![], + )); + s.expect_request(expected_request( + "/v1/checkpoint", + proto::CheckpointRequest { sequence_number: 1 }.encode_to_vec(), + vec![], + )); + } + + ext.execute_with(|| { + System::set_block_number(1); + ProverDbIndexer::offchain_worker(1); }); - ext.persist_offchain_overlay(); - let db = ext.offchain_db(); - assert!(db.get(&key_for_event(1, 0)).is_none()); - assert!(db.get(&key_for_high_water(1)).is_none()); } diff --git a/sxt-core/src/prover_db_indexer.rs b/sxt-core/src/prover_db_indexer.rs index 3298bb36..9f00be00 100644 --- a/sxt-core/src/prover_db_indexer.rs +++ b/sxt-core/src/prover_db_indexer.rs @@ -5,10 +5,7 @@ use alloc::borrow::Cow; use alloc::vec::Vec; -use codec::{Decode, Encode, MaxEncodedLen}; -use polkadot_sdk::sp_core::ConstU32; -use polkadot_sdk::sp_runtime::BoundedVec; -use scale_info::TypeInfo; +use codec::{Decode, Encode}; use crate::tables::{TableIdentifier, TableNamespace}; @@ -20,6 +17,19 @@ use crate::tables::{TableIdentifier, TableNamespace}; /// forward events. If the key is unset the OCW stays dormant. pub const PROVER_DB_URL_KEY: &[u8] = b"prover_db_indexer/prover_db_url"; +/// Offchain local-storage key holding this node's per-node include set. +/// +/// Stored as a SCALE-encoded `Vec`. Set by the node service +/// at startup from a CLI-supplied config file. The OCW consumer reads +/// this once per round and only forwards events whose table matches one +/// of the configured rules. An empty list (the default if the key is +/// unset) means "forward every captured event" — the on-chain capture +/// side is unfiltered, so every node still maintains the same offchain +/// queue regardless of what this filter is set to. Two indexer nodes +/// can therefore subscribe to different subsets without affecting each +/// other or the chain. +pub const PROVER_DB_INCLUDE_KEY: &[u8] = b"prover_db_indexer/include_rules"; + /// Offchain DB key prefix for per-extrinsic event payloads. SCALE-encoded /// as part of a `(prefix, block, ext_idx)` tuple, so the tuple structure /// (not any trailing separator) provides the boundary between fields. @@ -102,19 +112,17 @@ impl EventCapture for () { fn capture_events(_events: Vec>) {} } -/// Maximum number of include-set rules the chain accepts. Bounds the -/// storage footprint and the per-extrinsic match cost in `capture_events`. -pub const MAX_INCLUDE_RULES: u32 = 256; - -/// A single entry in the prover-db indexer's include set. An empty list -/// of these means "index everything"; a non-empty list means "index only -/// events whose table matches at least one rule". +/// A single entry in the prover-db indexer's include set. Stored only +/// in the indexer node's offchain local storage (under +/// [`PROVER_DB_INCLUDE_KEY`]); not part of on-chain state. An empty list +/// of these means "forward every event"; a non-empty list means "only +/// forward events whose table matches at least one rule". /// /// Matches are byte-exact against the on-chain identifiers, so callers /// that need case-insensitive matching should normalize their inputs /// (e.g. via [`TableIdentifier::from_str_unchecked`] which uppercases) -/// before submitting the rule. -#[derive(Encode, Decode, MaxEncodedLen, TypeInfo, Debug, Clone, Eq, PartialEq)] +/// before writing the rule to offchain storage. +#[derive(Encode, Decode, Debug, Clone, Eq, PartialEq)] pub enum IncludeRule { /// Match every table within the given namespace. Namespace(TableNamespace), @@ -122,9 +130,6 @@ pub enum IncludeRule { Table(TableIdentifier), } -/// Bounded list of include rules suitable for on-chain storage. -pub type IncludeRules = BoundedVec>; - /// Returns true if `table` matches at least one rule in `rules`. An /// empty rule set is treated as "match all". pub fn table_matches_rules(table: &TableIdentifier, rules: &[IncludeRule]) -> bool { From eac0c270a254df06f720fc00dd76f4d430831c22 Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Wed, 3 Jun 2026 17:21:57 +0530 Subject: [PATCH 3/9] refactor(prover-db-indexer): unify URL + include into one offchain config; drop the file flag for a wildcard CLI flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review points addressed together: 1. Make invalid states unrepresentable. The consumer used to live across two offchain keys — PROVER_DB_URL_KEY (`Vec`) and PROVER_DB_INCLUDE_KEY (`Vec`) — which let an operator accidentally have a stale include set with no URL. Replaced both with a single PROVER_DB_CONFIG_KEY holding a SCALE-encoded `ProverDbConsumerConfig { url: String, include: Vec }` in sxt-core. URL is non-optional in the type: writing the config *is* the act of enabling the consumer. 2. Replace `--prover-db-include-file` with `--prover-db-include`. A file path complicates deployment (volume mounts, ConfigMap + mountPath coordination) for what's now a tiny string list. The new flag is a clap `value_delimiter = ','` Vec with env-var fallback (`PROVER_DB_INCLUDE`), and the grammar moves to a wildcard form: NAMESPACE.NAME — exact table NAMESPACE.* — every table in NAMESPACE *.* — match everything (degenerate; same as omitting) *.NAME — rejected (runtime rule type can't express it) In K8s this is one ConfigMap key referenced by `envFrom` from every pod in the prover stack instead of a mounted file. CLI shape uses clap::Args + #[clap(flatten)]: a new ProverDbConsumerCli group holds both fields. The service-side resolver in `configure_prover_db_consumer` is the single seam where the optional CLI shape is validated into the non-optional `ProverDbConsumerConfig` — fails loud if include patterns are set without a URL, or if any pattern is malformed. Pallet OCW: one StorageValueRef read replaces the previous two; URL encoding error variant dropped (String can't be invalid UTF-8). Tests seed the unified key via a new `setup_with_config(include)` helper; existing `setup_with_url()` becomes a thin alias around an empty include set. 12/12 pallet tests pass. End-to-end: the harness orchestrator passes `--prover-db-include ` instead of rendering a JSON file. All 9 example scenarios (89 actions) green under `--auto-start --all`. --- node/src/cli.rs | 67 +++++--- node/src/service.rs | 214 ++++++++++++------------- pallets/prover_db_indexer/src/lib.rs | 40 ++--- pallets/prover_db_indexer/src/tests.rs | 46 ++++-- sxt-core/src/prover_db_indexer.rs | 50 +++--- 5 files changed, 221 insertions(+), 196 deletions(-) diff --git a/node/src/cli.rs b/node/src/cli.rs index b800f099..fdfc6bf0 100644 --- a/node/src/cli.rs +++ b/node/src/cli.rs @@ -19,28 +19,12 @@ pub struct Cli { #[clap(long)] pub event_forwarder_rpc: Option, - /// URL of the prover-db indexer to forward indexed events to. Also - /// configurable via the `PROVER_DB_URL` environment variable. - #[clap(long, env)] - pub prover_db_url: Option, - - /// Path to a JSON file listing which tables/namespaces this node - /// should forward to its prover-db indexer. Schema: - /// - /// ```json - /// [ - /// { "kind": "namespace", "value": "ETHEREUM_MT" }, - /// { "kind": "table", "namespace": "OTHER_NS", "name": "SPECIAL" } - /// ] - /// ``` - /// - /// An empty array (or omitting this flag) means "forward every - /// captured event" — the on-chain capture itself is unfiltered, so - /// the choice is purely per-node. Identifiers are uppercased before - /// being written to offchain storage, to match how the chain stores - /// them. - #[clap(long, env)] - pub prover_db_include_file: Option, + /// Prover-db consumer settings. Setting `--prover-db-url` is what + /// turns this node into a prover-db indexer; the other fields in + /// the group only take effect when the URL is set. See the field + /// docs on [`ProverDbConsumerCli`] for grammar. + #[clap(flatten)] + pub prover_db: ProverDbConsumerCli, #[allow(missing_docs)] #[clap(flatten)] @@ -51,6 +35,45 @@ pub struct Cli { pub proof_of_sql_public_setup_args: ProofOfSqlPublicSetupArgs, } +/// Flattened consumer-side configuration. Lives on the top-level +/// `Cli` via `#[clap(flatten)]`. Validated at service init into +/// `sxt_core::prover_db_indexer::ProverDbConsumerConfig` before being +/// written to offchain local storage; see +/// `service::configure_prover_db_consumer`. +/// +/// On the CLI side the fields are individually optional / repeatable — +/// clap can't express "if URL is set, the group is active" in the type. +/// The resolver in `service.rs` enforces that constraint and produces a +/// concrete `ProverDbConsumerConfig` (URL non-optional) or refuses to +/// start. +#[derive(Debug, clap::Args)] +pub struct ProverDbConsumerCli { + /// URL of the prover-db indexer to forward indexed events to. + /// Also configurable via the `PROVER_DB_URL` env var. + /// + /// Presence of this flag is what enables the consumer. If omitted, + /// the OCW stays dormant regardless of `--prover-db-include`. + #[clap(long, env)] + pub prover_db_url: Option, + + /// Wildcard patterns naming which tables this node should forward + /// to the indexer. Repeat the flag, or pass a single comma- + /// separated value, or set `PROVER_DB_INCLUDE`. + /// + /// Grammar (always exactly one dot, two segments): + /// - `*.*` — every captured event (default if omitted). + /// - `NAMESPACE.*` — every table in NAMESPACE. + /// - `NAMESPACE.NAME` — exactly that one table. + /// `*.NAME` is rejected: the runtime rule type can't express + /// "any namespace, this name". + /// + /// Identifiers are uppercased before being written to offchain + /// storage so the byte-match against the on-chain canonical form + /// works without per-operator vigilance. + #[clap(long, env, value_delimiter = ',')] + pub prover_db_include: Vec, +} + #[derive(Debug, clap::Subcommand)] #[allow(clippy::large_enum_variant)] pub enum Subcommand { diff --git a/node/src/service.rs b/node/src/service.rs index 21d0b71c..18b71af6 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -77,105 +77,116 @@ pub type TransactionPool = sc_transaction_pool::FullPool; /// imported and generated. const GRANDPA_JUSTIFICATION_PERIOD: u32 = 512; -/// Seed the prover-db indexer URL into offchain persistent local storage -/// at startup, before the first block is authored. +/// Parse one entry from `--prover-db-include`. Grammar (always exactly +/// one dot, two non-empty segments): /// -/// Stored under `sxt_core::prover_db_indexer::PROVER_DB_URL_KEY` as a -/// SCALE-encoded `Vec` of the URL bytes. -#[expect( - clippy::result_large_err, - reason = "ServiceError is from substrate and cannot be modified" -)] -fn configure_prover_db_url(backend: &FullBackend, url: &url::Url) -> Result<(), ServiceError> { - let Some(mut storage) = backend.offchain_storage() else { - return Err(ServiceError::Other( - "backend did not expose an offchain storage handle; \ - cannot apply --prover-db-url" - .into(), - )); +/// - `"*.*"` → no rule emitted (degenerate; equivalent to +/// passing no patterns at all). +/// - `"NAMESPACE.*"` → `IncludeRule::Namespace(NAMESPACE)`. +/// - `"NAMESPACE.NAME"` → `IncludeRule::Table(NAMESPACE.NAME)`. +/// +/// `*.NAME` (left wildcard) is rejected — the runtime rule type can't +/// express "any namespace, this name". Identifiers are uppercased so +/// matches against the on-chain canonical form are byte-exact. +fn parse_include_pattern( + pattern: &str, +) -> Result, ServiceError> { + let Some((ns, name)) = pattern.split_once('.') else { + return Err(ServiceError::Other(format!( + "include pattern '{}' must be of the form \ + 'NAMESPACE.NAME', 'NAMESPACE.*', or '*.*'", + pattern, + ))); }; - let encoded = url.as_str().as_bytes().to_vec().encode(); - storage.set( - STORAGE_PREFIX, - sxt_core::prover_db_indexer::PROVER_DB_URL_KEY, - &encoded, - ); - Ok(()) -} - -/// JSON rule type as it appears in the user-supplied `--prover-db-include-file`. -/// Decoupled from the runtime [`sxt_core::prover_db_indexer::IncludeRule`] -/// so the on-disk format can use natural string fields while the wire -/// type uses byte-bounded `TableNamespace` / `TableIdentifier`. -#[derive(serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum IncludeRuleJson { - Namespace { value: String }, - Table { namespace: String, name: String }, + if ns.is_empty() || name.is_empty() { + return Err(ServiceError::Other(format!( + "include pattern '{}' has an empty segment", + pattern, + ))); + } + match (ns, name) { + ("*", "*") => Ok(None), + ("*", _) => Err(ServiceError::Other(format!( + "include pattern '{}': left wildcard ('*.NAME') is not \ + supported; use 'NAMESPACE.*' or 'NAMESPACE.NAME'", + pattern, + ))), + (ns, "*") => { + let bytes = ns.to_uppercase().into_bytes(); + let ns_bv = sxt_core::tables::TableNamespace::try_from(bytes).map_err(|_| { + ServiceError::Other(format!( + "namespace '{}' exceeds the maximum identifier length", + ns, + )) + })?; + Ok(Some(sxt_core::prover_db_indexer::IncludeRule::Namespace( + ns_bv, + ))) + } + (ns, name) => { + let ident = sxt_core::tables::TableIdentifier::from_str_unchecked(name, ns); + Ok(Some(sxt_core::prover_db_indexer::IncludeRule::Table(ident))) + } + } } -/// Parse the include-file JSON and seed the rule set into offchain -/// persistent local storage at startup, before the first block is -/// authored. +/// Validate the consumer CLI group into a concrete +/// `ProverDbConsumerConfig` (URL non-optional) and seed it under +/// `PROVER_DB_CONFIG_KEY` in offchain persistent local storage. Atomic +/// shape: one offchain key per consumer, so half-states like "include +/// patterns set but URL missing" are caught up front instead of becoming +/// stale data in the OCW DB. /// -/// Stored under `sxt_core::prover_db_indexer::PROVER_DB_INCLUDE_KEY` as -/// a SCALE-encoded `Vec`. Identifiers are uppercased on -/// the way in via `TableIdentifier::from_str_unchecked`, so the matches -/// the OCW does later are byte-exact against the on-chain canonical -/// form. +/// Returns: +/// - `Ok(())` and writes the key when `--prover-db-url` is set. +/// - `Ok(())` and **does not** write when nothing is set (this node +/// isn't an indexer; OCW stays dormant). +/// - `Err(_)` if the operator tried to set patterns without a URL, or +/// if any pattern is malformed. #[expect( clippy::result_large_err, reason = "ServiceError is from substrate and cannot be modified" )] -fn configure_prover_db_include(backend: &FullBackend, path: &Path) -> Result<(), ServiceError> { - let raw = std::fs::read_to_string(path).map_err(|e| { - ServiceError::Other(format!( - "failed to read --prover-db-include-file {}: {}", - path.display(), - e - )) - })?; - let json_rules: Vec = serde_json::from_str(&raw).map_err(|e| { - ServiceError::Other(format!( - "failed to parse --prover-db-include-file {} as JSON: {}", - path.display(), - e - )) - })?; +fn configure_prover_db_consumer( + backend: &FullBackend, + cli: &crate::cli::ProverDbConsumerCli, +) -> Result<(), ServiceError> { + let Some(url) = cli.prover_db_url.as_ref() else { + if !cli.prover_db_include.is_empty() { + return Err(ServiceError::Other( + "--prover-db-include was set but --prover-db-url is not; \ + the include set would have no effect because the OCW \ + only runs when the indexer URL is configured." + .into(), + )); + } + return Ok(()); + }; - let rules: Vec = json_rules - .into_iter() - .map(|r| match r { - IncludeRuleJson::Namespace { value } => { - let bytes = value.to_uppercase().into_bytes(); - let ns = sxt_core::tables::TableNamespace::try_from(bytes).map_err(|_| { - ServiceError::Other(format!( - "namespace '{}' in {} exceeds the maximum identifier length", - value, - path.display(), - )) - })?; - Ok(sxt_core::prover_db_indexer::IncludeRule::Namespace(ns)) - } - IncludeRuleJson::Table { namespace, name } => { - let ident = - sxt_core::tables::TableIdentifier::from_str_unchecked(&name, &namespace); - Ok(sxt_core::prover_db_indexer::IncludeRule::Table(ident)) - } - }) - .collect::>()?; + let mut rules: Vec = + Vec::with_capacity(cli.prover_db_include.len()); + for pat in &cli.prover_db_include { + if let Some(rule) = parse_include_pattern(pat)? { + rules.push(rule); + } + } + + let cfg = sxt_core::prover_db_indexer::ProverDbConsumerConfig { + url: url.as_str().to_string(), + include: rules, + }; let Some(mut storage) = backend.offchain_storage() else { return Err(ServiceError::Other( "backend did not expose an offchain storage handle; \ - cannot apply --prover-db-include-file" + cannot apply --prover-db-url / --prover-db-include" .into(), )); }; storage.set( STORAGE_PREFIX, - sxt_core::prover_db_indexer::PROVER_DB_INCLUDE_KEY, - &rules.encode(), + sxt_core::prover_db_indexer::PROVER_DB_CONFIG_KEY, + &cfg.encode(), ); Ok(()) } @@ -428,37 +439,20 @@ pub fn new_full_base::Hash>>( other: (rpc_builder, import_setup, rpc_setup, mut telemetry, statement_store), } = new_partial(&config)?; - // `--prover-db-url` only takes effect when offchain indexing is - // enabled, since the prover-db-indexer OCW only runs in that case. - // Fail loud if the operator set the URL without enabling indexing, - // rather than silently ignoring it. - if let Some(url) = cli.prover_db_url.as_ref() { - if !config.offchain_worker.indexing_enabled { - return Err(ServiceError::Other( - "--prover-db-url was set but --enable-offchain-indexing is not \ - true; the prover-db-indexer offchain worker would be inactive. \ - Restart with --enable-offchain-indexing=true, or omit \ - --prover-db-url." - .into(), - )); - } - configure_prover_db_url(backend.as_ref(), url)?; - } - - // `--prover-db-include-file` is only meaningful when the OCW will - // actually forward something, i.e. when the URL is set. Reject the - // common misconfiguration up front. - if let Some(path) = cli.prover_db_include_file.as_ref() { - if cli.prover_db_url.is_none() { - return Err(ServiceError::Other( - "--prover-db-include-file was set but --prover-db-url is not; \ - the include set would have no effect because the OCW only \ - runs when an indexer URL is configured." - .into(), - )); - } - configure_prover_db_include(backend.as_ref(), path)?; + // The prover-db consumer (URL + include set) only takes effect when + // offchain indexing is enabled, since the OCW only runs in that + // case. Fail loud if the operator set --prover-db-url without + // enabling indexing, rather than silently ignoring it. + if cli.prover_db.prover_db_url.is_some() && !config.offchain_worker.indexing_enabled { + return Err(ServiceError::Other( + "--prover-db-url was set but --enable-offchain-indexing is not \ + true; the prover-db-indexer offchain worker would be inactive. \ + Restart with --enable-offchain-indexing=true, or omit \ + --prover-db-url." + .into(), + )); } + configure_prover_db_consumer(backend.as_ref(), &cli.prover_db)?; let metrics = N::register_notification_metrics( config.prometheus_config.as_ref().map(|cfg| &cfg.registry), diff --git a/pallets/prover_db_indexer/src/lib.rs b/pallets/prover_db_indexer/src/lib.rs index 992c1cc7..22736f9d 100644 --- a/pallets/prover_db_indexer/src/lib.rs +++ b/pallets/prover_db_indexer/src/lib.rs @@ -59,11 +59,9 @@ mod proto { } pub use pallet::*; -/// Re-export of the canonical offchain local-storage keys (defined in `sxt-core`), -/// kept here so existing call sites that reach for -/// `pallet_prover_db_indexer::PROVER_DB_URL_KEY` keep working and so the node -/// service has a single import for both. -pub use sxt_core::prover_db_indexer::{PROVER_DB_INCLUDE_KEY, PROVER_DB_URL_KEY}; +/// Re-export of the canonical offchain local-storage key (defined in `sxt-core`), +/// so the node service and any external tooling have a single import. +pub use sxt_core::prover_db_indexer::PROVER_DB_CONFIG_KEY; /// Offchain DB key for the lock that serializes OCW consumer rounds. const OCW_LOCK_KEY: &[u8] = b"prover_db_indexer/ocw_lock"; @@ -79,9 +77,6 @@ const OCW_LOCK_KEY: &[u8] = b"prover_db_indexer/ocw_lock"; /// every variant looks the same. #[derive(Debug, snafu::Snafu)] pub enum ConsumerError { - /// The configured indexer URL was not valid UTF-8. - #[snafu(display("indexer URL was not valid UTF-8"))] - InvalidUrlEncoding, /// The configured indexer URL couldn't be parsed. #[snafu(display("indexer URL is not a valid URL: {error}"))] InvalidUrl { @@ -141,6 +136,7 @@ pub mod pallet { BlockEvent, EventCapture, IncludeRule, + ProverDbConsumerConfig, }; use crate::ConsumerError; @@ -264,29 +260,17 @@ pub mod pallet { } }; - // 1. Check if this node is configured as an indexer. - let url_ref = StorageValueRef::persistent(crate::PROVER_DB_URL_KEY); - let url_bytes: Option> = url_ref.get::>().ok().flatten(); - - let Some(url_bytes) = url_bytes else { + // 1. Read the single offchain config. Absence ⇒ this node + // isn't an indexer; OCW is dormant. The node service + // writes the encoded `ProverDbConsumerConfig` at startup + // iff `--prover-db-url` was provided. + let cfg_ref = StorageValueRef::persistent(crate::PROVER_DB_CONFIG_KEY); + let Some(cfg) = cfg_ref.get::().ok().flatten() else { return Ok(()); }; - - let url_str = - core::str::from_utf8(&url_bytes).map_err(|_| ConsumerError::InvalidUrlEncoding)?; let url = - url::Url::parse(url_str).map_err(|error| ConsumerError::InvalidUrl { error })?; - - // 1b. Load this node's include set from offchain local storage. - // Absent or empty ⇒ "forward every event". The rules are - // written by the node service at startup from a CLI config - // file; see `sxt_core::prover_db_indexer::PROVER_DB_INCLUDE_KEY`. - let include_ref = StorageValueRef::persistent(crate::PROVER_DB_INCLUDE_KEY); - let include_rules: Vec = include_ref - .get::>() - .ok() - .flatten() - .unwrap_or_default(); + url::Url::parse(&cfg.url).map_err(|error| ConsumerError::InvalidUrl { error })?; + let include_rules: Vec = cfg.include; polkadot_sdk::sp_tracing::debug!( target: "prover_db_indexer", diff --git a/pallets/prover_db_indexer/src/tests.rs b/pallets/prover_db_indexer/src/tests.rs index 637281a0..bec0bc27 100644 --- a/pallets/prover_db_indexer/src/tests.rs +++ b/pallets/prover_db_indexer/src/tests.rs @@ -24,19 +24,25 @@ use sxt_core::prover_db_indexer::{ EventCapture, IncludeRule, InsertEntry, + ProverDbConsumerConfig, }; use sxt_core::tables::{TableIdentifier, TableNamespace}; use crate::mock::*; -use crate::{proto, PROVER_DB_INCLUDE_KEY, PROVER_DB_URL_KEY}; +use crate::{proto, PROVER_DB_CONFIG_KEY}; type StateArc = std::sync::Arc>; const MOCK_URL: &str = "http://127.0.0.1:9999"; -fn encode_url() -> Vec { - codec::Encode::encode(&MOCK_URL.as_bytes().to_vec()) +/// SCALE-encode the unified consumer config the same way the node +/// service does at startup. +fn encode_config(include: Vec) -> Vec { + codec::Encode::encode(&ProverDbConsumerConfig { + url: MOCK_URL.to_string(), + include, + }) } fn expected_request(path: &str, body: Vec, response: Vec) -> PendingRequest { @@ -67,7 +73,18 @@ fn no_checkpoint_response() -> Vec { .encode_to_vec() } +/// Seed an empty include set — equivalent to the operator running +/// without `--prover-db-include`, which is what every existing test +/// expects. fn setup_with_url() -> (polkadot_sdk::sp_io::TestExternalities, StateArc) { + setup_with_config(Vec::new()) +} + +/// Seed a non-empty include set under the unified config key. Used by +/// the consumer-side filter tests. +fn setup_with_config( + include: Vec, +) -> (polkadot_sdk::sp_io::TestExternalities, StateArc) { let mut ext = new_test_ext(); let (offchain, state) = TestOffchainExt::new(); ext.register_extension(OffchainWorkerExt::new(offchain.clone())); @@ -75,7 +92,7 @@ fn setup_with_url() -> (polkadot_sdk::sp_io::TestExternalities, StateArc) { state .write() .persistent_storage - .set(b"", PROVER_DB_URL_KEY, &encode_url()); + .set(b"", PROVER_DB_CONFIG_KEY, &encode_config(include)); (ext, state) } @@ -452,19 +469,15 @@ fn capture_events_writes_all_events_unconditionally() { /// and confirm only the matching events are POSTed to the indexer. #[test] fn ocw_forwards_only_events_matching_include_set() { - let (mut ext, state) = setup_with_url(); - - // Seed the node-local include set: namespace = "ALPHA", and a - // specific table B.BETA. Encoded as SCALE Vec, exactly - // as the node service would write it from the CLI config file. - let rules = vec![ + // Seed the node-local config with two include rules: any table in + // namespace ALPHA, plus the specific table BETA_NS.BETA_T. The + // service writes both URL + include atomically via a single + // SCALE-encoded `ProverDbConsumerConfig` — same shape this test + // mirrors. + let (mut ext, state) = setup_with_config(vec![ IncludeRule::Namespace(TableNamespace::try_from(b"ALPHA".to_vec()).unwrap()), IncludeRule::Table(TableIdentifier::from_str_unchecked("BETA_T", "BETA_NS")), - ]; - state - .write() - .persistent_storage - .set(b"", PROVER_DB_INCLUDE_KEY, &rules.encode()); + ]); // Block 1, extrinsic 0: a mix of matching and non-matching events. seed_block_events( @@ -546,7 +559,8 @@ fn ocw_forwards_only_events_matching_include_set() { fn ocw_with_empty_include_set_forwards_everything() { let (mut ext, state) = setup_with_url(); - // No PROVER_DB_INCLUDE_KEY written — absence ⇒ empty rules ⇒ match all. + // `setup_with_url` writes a config whose `include` is empty — + // same effect as the operator not passing `--prover-db-include`. let ddl = b"CREATE TABLE ANY.ANY (ID BIGINT NOT NULL)"; seed_block_events( &state, diff --git a/sxt-core/src/prover_db_indexer.rs b/sxt-core/src/prover_db_indexer.rs index 9f00be00..4038d24a 100644 --- a/sxt-core/src/prover_db_indexer.rs +++ b/sxt-core/src/prover_db_indexer.rs @@ -3,32 +3,42 @@ //! `pallet-tables` and `pallet-indexing`. use alloc::borrow::Cow; +use alloc::string::String; use alloc::vec::Vec; use codec::{Decode, Encode}; use crate::tables::{TableIdentifier, TableNamespace}; -/// Offchain local-storage key holding the prover-db indexer URL. +/// Offchain local-storage key holding the prover-db indexer consumer's +/// configuration. The embedding node writes a SCALE-encoded +/// [`ProverDbConsumerConfig`] under this key at startup; the OCW reads +/// it once per round. Absence of the key means "OCW is dormant". /// -/// The embedding node is expected to write the configured URL to this -/// key in OCW persistent storage before block authoring begins; the -/// prover-db-indexer pallet's offchain worker reads it to know where to -/// forward events. If the key is unset the OCW stays dormant. -pub const PROVER_DB_URL_KEY: &[u8] = b"prover_db_indexer/prover_db_url"; - -/// Offchain local-storage key holding this node's per-node include set. +/// One key for the whole consumer rather than one-per-field so the +/// "consumer is enabled" and "consumer is disabled with stale filter +/// settings" states are unrepresentable. +pub const PROVER_DB_CONFIG_KEY: &[u8] = b"prover_db_indexer/consumer_config"; + +/// Per-node configuration that turns this node into a prover-db +/// indexer: the upstream URL to POST forwarded events to, plus the +/// optional include set that gates which events get forwarded. /// -/// Stored as a SCALE-encoded `Vec`. Set by the node service -/// at startup from a CLI-supplied config file. The OCW consumer reads -/// this once per round and only forwards events whose table matches one -/// of the configured rules. An empty list (the default if the key is -/// unset) means "forward every captured event" — the on-chain capture -/// side is unfiltered, so every node still maintains the same offchain -/// queue regardless of what this filter is set to. Two indexer nodes -/// can therefore subscribe to different subsets without affecting each -/// other or the chain. -pub const PROVER_DB_INCLUDE_KEY: &[u8] = b"prover_db_indexer/include_rules"; +/// Lives in offchain local storage under [`PROVER_DB_CONFIG_KEY`]. +/// `url` is non-optional: writing the config _is_ the act of enabling +/// the consumer, and there's no enabled-without-URL state to express. +/// `include` may be empty, which the OCW treats as "match every table" +/// (same default as if the operator hadn't passed any include patterns +/// at all). +#[derive(Encode, Decode, Debug, Clone, Eq, PartialEq)] +pub struct ProverDbConsumerConfig { + /// HTTP base URL of the upstream prover-db indexer. Validated at + /// node start; stored as raw bytes so the pallet doesn't need to + /// pull in the `url` crate's full parsing surface. + pub url: String, + /// Per-node include set. Empty ⇒ forward every captured event. + pub include: Vec, +} /// Offchain DB key prefix for per-extrinsic event payloads. SCALE-encoded /// as part of a `(prefix, block, ext_idx)` tuple, so the tuple structure @@ -113,8 +123,8 @@ impl EventCapture for () { } /// A single entry in the prover-db indexer's include set. Stored only -/// in the indexer node's offchain local storage (under -/// [`PROVER_DB_INCLUDE_KEY`]); not part of on-chain state. An empty list +/// in the indexer node's offchain local storage (as part of +/// [`ProverDbConsumerConfig`]); not part of on-chain state. An empty list /// of these means "forward every event"; a non-empty list means "only /// forward events whose table matches at least one rule". /// From a64adb220da8144d4059ecf86619ac5c56976af3 Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Wed, 3 Jun 2026 18:24:36 +0530 Subject: [PATCH 4/9] fix(prover-db-indexer): clippy doc-formatting + result_large_err on parse_include_pattern - cli.rs: blank-line separator before the '`*.NAME` is rejected' paragraph so it stops being parsed as a list-item continuation (doc_lazy_continuation). - service.rs: dedent the wrap line under the '`"*.*"`' bullet from 23 spaces to 2 (doc_overindented_list_items). - service.rs: `parse_include_pattern` returns `Result<_, ServiceError>`, which clippy now flags at 176 bytes (result_large_err). `ServiceError` is a substrate type we don't own, so apply the same `#[expect(..., reason = ...)]` already used on `configure_prover_db_consumer`. --- node/src/cli.rs | 1 + node/src/service.rs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/node/src/cli.rs b/node/src/cli.rs index fdfc6bf0..bd9fbf65 100644 --- a/node/src/cli.rs +++ b/node/src/cli.rs @@ -64,6 +64,7 @@ pub struct ProverDbConsumerCli { /// - `*.*` — every captured event (default if omitted). /// - `NAMESPACE.*` — every table in NAMESPACE. /// - `NAMESPACE.NAME` — exactly that one table. + /// /// `*.NAME` is rejected: the runtime rule type can't express /// "any namespace, this name". /// diff --git a/node/src/service.rs b/node/src/service.rs index 18b71af6..169afd53 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -81,13 +81,17 @@ const GRANDPA_JUSTIFICATION_PERIOD: u32 = 512; /// one dot, two non-empty segments): /// /// - `"*.*"` → no rule emitted (degenerate; equivalent to -/// passing no patterns at all). +/// passing no patterns at all). /// - `"NAMESPACE.*"` → `IncludeRule::Namespace(NAMESPACE)`. /// - `"NAMESPACE.NAME"` → `IncludeRule::Table(NAMESPACE.NAME)`. /// /// `*.NAME` (left wildcard) is rejected — the runtime rule type can't /// express "any namespace, this name". Identifiers are uppercased so /// matches against the on-chain canonical form are byte-exact. +#[expect( + clippy::result_large_err, + reason = "ServiceError is from substrate and cannot be modified" +)] fn parse_include_pattern( pattern: &str, ) -> Result, ServiceError> { From daca8c86dd3edca4ff13c8fd04e1a2aaaa55373e Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Mon, 8 Jun 2026 09:10:52 +0530 Subject: [PATCH 5/9] refactor(prover-db-indexer): include set is Vec parsed via FromStr Per review: split each side of the dot into IdentFilter {Wildcard, Ident(String)} and combine into TableIdentifierFilter, both with matches() + FromStr. Drops IncludeRule, table_matches_rules, and service-side parse_include_pattern; clap parses --prover-db-include values directly. *.NAME now allowed. --- node/src/cli.rs | 12 +- node/src/service.rs | 75 ++--------- pallets/prover_db_indexer/src/lib.rs | 20 +-- pallets/prover_db_indexer/src/tests.rs | 12 +- sxt-core/src/prover_db_indexer.rs | 165 ++++++++++++++++++++----- 5 files changed, 167 insertions(+), 117 deletions(-) diff --git a/node/src/cli.rs b/node/src/cli.rs index bd9fbf65..94d30e26 100644 --- a/node/src/cli.rs +++ b/node/src/cli.rs @@ -60,19 +60,21 @@ pub struct ProverDbConsumerCli { /// to the indexer. Repeat the flag, or pass a single comma- /// separated value, or set `PROVER_DB_INCLUDE`. /// - /// Grammar (always exactly one dot, two segments): + /// Each entry parses as a [`TableIdentifierFilter`]: two + /// dot-separated sides, each either a literal identifier or `*`. + /// Both sides match independently, so: /// - `*.*` — every captured event (default if omitted). /// - `NAMESPACE.*` — every table in NAMESPACE. + /// - `*.NAME` — every table called NAME, regardless of namespace. /// - `NAMESPACE.NAME` — exactly that one table. /// - /// `*.NAME` is rejected: the runtime rule type can't express - /// "any namespace, this name". - /// /// Identifiers are uppercased before being written to offchain /// storage so the byte-match against the on-chain canonical form /// works without per-operator vigilance. + /// + /// [`TableIdentifierFilter`]: sxt_core::prover_db_indexer::TableIdentifierFilter #[clap(long, env, value_delimiter = ',')] - pub prover_db_include: Vec, + pub prover_db_include: Vec, } #[derive(Debug, clap::Subcommand)] diff --git a/node/src/service.rs b/node/src/service.rs index 169afd53..3569574b 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -77,63 +77,6 @@ pub type TransactionPool = sc_transaction_pool::FullPool; /// imported and generated. const GRANDPA_JUSTIFICATION_PERIOD: u32 = 512; -/// Parse one entry from `--prover-db-include`. Grammar (always exactly -/// one dot, two non-empty segments): -/// -/// - `"*.*"` → no rule emitted (degenerate; equivalent to -/// passing no patterns at all). -/// - `"NAMESPACE.*"` → `IncludeRule::Namespace(NAMESPACE)`. -/// - `"NAMESPACE.NAME"` → `IncludeRule::Table(NAMESPACE.NAME)`. -/// -/// `*.NAME` (left wildcard) is rejected — the runtime rule type can't -/// express "any namespace, this name". Identifiers are uppercased so -/// matches against the on-chain canonical form are byte-exact. -#[expect( - clippy::result_large_err, - reason = "ServiceError is from substrate and cannot be modified" -)] -fn parse_include_pattern( - pattern: &str, -) -> Result, ServiceError> { - let Some((ns, name)) = pattern.split_once('.') else { - return Err(ServiceError::Other(format!( - "include pattern '{}' must be of the form \ - 'NAMESPACE.NAME', 'NAMESPACE.*', or '*.*'", - pattern, - ))); - }; - if ns.is_empty() || name.is_empty() { - return Err(ServiceError::Other(format!( - "include pattern '{}' has an empty segment", - pattern, - ))); - } - match (ns, name) { - ("*", "*") => Ok(None), - ("*", _) => Err(ServiceError::Other(format!( - "include pattern '{}': left wildcard ('*.NAME') is not \ - supported; use 'NAMESPACE.*' or 'NAMESPACE.NAME'", - pattern, - ))), - (ns, "*") => { - let bytes = ns.to_uppercase().into_bytes(); - let ns_bv = sxt_core::tables::TableNamespace::try_from(bytes).map_err(|_| { - ServiceError::Other(format!( - "namespace '{}' exceeds the maximum identifier length", - ns, - )) - })?; - Ok(Some(sxt_core::prover_db_indexer::IncludeRule::Namespace( - ns_bv, - ))) - } - (ns, name) => { - let ident = sxt_core::tables::TableIdentifier::from_str_unchecked(name, ns); - Ok(Some(sxt_core::prover_db_indexer::IncludeRule::Table(ident))) - } - } -} - /// Validate the consumer CLI group into a concrete /// `ProverDbConsumerConfig` (URL non-optional) and seed it under /// `PROVER_DB_CONFIG_KEY` in offchain persistent local storage. Atomic @@ -141,12 +84,16 @@ fn parse_include_pattern( /// patterns set but URL missing" are caught up front instead of becoming /// stale data in the OCW DB. /// +/// Each `--prover-db-include` value has already been parsed into a +/// `TableIdentifierFilter` by clap (via the type's `FromStr` impl), so +/// this function only enforces the URL/include consistency rule and +/// writes the encoded config. +/// /// Returns: /// - `Ok(())` and writes the key when `--prover-db-url` is set. /// - `Ok(())` and **does not** write when nothing is set (this node /// isn't an indexer; OCW stays dormant). -/// - `Err(_)` if the operator tried to set patterns without a URL, or -/// if any pattern is malformed. +/// - `Err(_)` if the operator tried to set patterns without a URL. #[expect( clippy::result_large_err, reason = "ServiceError is from substrate and cannot be modified" @@ -167,17 +114,9 @@ fn configure_prover_db_consumer( return Ok(()); }; - let mut rules: Vec = - Vec::with_capacity(cli.prover_db_include.len()); - for pat in &cli.prover_db_include { - if let Some(rule) = parse_include_pattern(pat)? { - rules.push(rule); - } - } - let cfg = sxt_core::prover_db_indexer::ProverDbConsumerConfig { url: url.as_str().to_string(), - include: rules, + include: cli.prover_db_include.clone(), }; let Some(mut storage) = backend.offchain_storage() else { diff --git a/pallets/prover_db_indexer/src/lib.rs b/pallets/prover_db_indexer/src/lib.rs index 22736f9d..0f5d089a 100644 --- a/pallets/prover_db_indexer/src/lib.rs +++ b/pallets/prover_db_indexer/src/lib.rs @@ -132,11 +132,11 @@ pub mod pallet { use sxt_core::prover_db_indexer::{ key_for_event, key_for_high_water, - table_matches_rules, + table_matches_filters, BlockEvent, EventCapture, - IncludeRule, ProverDbConsumerConfig, + TableIdentifierFilter, }; use crate::ConsumerError; @@ -270,12 +270,12 @@ pub mod pallet { }; let url = url::Url::parse(&cfg.url).map_err(|error| ConsumerError::InvalidUrl { error })?; - let include_rules: Vec = cfg.include; + let include_filters: Vec = cfg.include; polkadot_sdk::sp_tracing::debug!( target: "prover_db_indexer", - "consumer round starting; indexer base URL = {}; {} include rules", - url, include_rules.len(), + "consumer round starting; indexer base URL = {}; {} include filters", + url, include_filters.len(), ); // 2. Current block number — passed in by `offchain_worker`. @@ -317,7 +317,7 @@ pub mod pallet { ); for block_num in start..=end { - Self::forward_block(&url, block_num, &include_rules)?; + Self::forward_block(&url, block_num, &include_filters)?; // Checkpoint on the server (always, even for empty blocks). crate::http_client::checkpoint(&url, block_num) @@ -333,7 +333,7 @@ pub mod pallet { fn forward_block( url: &url::Url, block_num: u64, - include_rules: &[IncludeRule], + include_filters: &[TableIdentifierFilter], ) -> Result<(), ConsumerError> { let Some(hwm) = crate::offchain_consumer::read_high_water(block_num) else { return Ok(()); @@ -350,7 +350,7 @@ pub mod pallet { let Some(events) = crate::offchain_consumer::read_events(block_num, ext_idx) else { continue; }; - Self::forward_events(url, block_num, &events, include_rules)?; + Self::forward_events(url, block_num, &events, include_filters)?; crate::offchain_consumer::clear_events(block_num, ext_idx); } @@ -367,7 +367,7 @@ pub mod pallet { url: &url::Url, block_num: u64, events: &[BlockEvent<'_>], - include_rules: &[IncludeRule], + include_filters: &[TableIdentifierFilter], ) -> Result<(), ConsumerError> { let seq = block_num; @@ -377,7 +377,7 @@ pub mod pallet { BlockEvent::Drop(ident) => ident.as_ref(), BlockEvent::Insert(entry) => entry.table.as_ref(), }; - if !table_matches_rules(table, include_rules) { + if !table_matches_filters(table, include_filters) { continue; } diff --git a/pallets/prover_db_indexer/src/tests.rs b/pallets/prover_db_indexer/src/tests.rs index bec0bc27..00036a1a 100644 --- a/pallets/prover_db_indexer/src/tests.rs +++ b/pallets/prover_db_indexer/src/tests.rs @@ -22,11 +22,11 @@ use sxt_core::prover_db_indexer::{ BlockEvent, CreateEntry, EventCapture, - IncludeRule, InsertEntry, ProverDbConsumerConfig, + TableIdentifierFilter, }; -use sxt_core::tables::{TableIdentifier, TableNamespace}; +use sxt_core::tables::TableIdentifier; use crate::mock::*; use crate::{proto, PROVER_DB_CONFIG_KEY}; @@ -38,7 +38,7 @@ const MOCK_URL: &str = "http://127.0.0.1:9999"; /// SCALE-encode the unified consumer config the same way the node /// service does at startup. -fn encode_config(include: Vec) -> Vec { +fn encode_config(include: Vec) -> Vec { codec::Encode::encode(&ProverDbConsumerConfig { url: MOCK_URL.to_string(), include, @@ -83,7 +83,7 @@ fn setup_with_url() -> (polkadot_sdk::sp_io::TestExternalities, StateArc) { /// Seed a non-empty include set under the unified config key. Used by /// the consumer-side filter tests. fn setup_with_config( - include: Vec, + include: Vec, ) -> (polkadot_sdk::sp_io::TestExternalities, StateArc) { let mut ext = new_test_ext(); let (offchain, state) = TestOffchainExt::new(); @@ -475,8 +475,8 @@ fn ocw_forwards_only_events_matching_include_set() { // SCALE-encoded `ProverDbConsumerConfig` — same shape this test // mirrors. let (mut ext, state) = setup_with_config(vec![ - IncludeRule::Namespace(TableNamespace::try_from(b"ALPHA".to_vec()).unwrap()), - IncludeRule::Table(TableIdentifier::from_str_unchecked("BETA_T", "BETA_NS")), + "ALPHA.*".parse().unwrap(), + "BETA_NS.BETA_T".parse().unwrap(), ]); // Block 1, extrinsic 0: a mix of matching and non-matching events. diff --git a/sxt-core/src/prover_db_indexer.rs b/sxt-core/src/prover_db_indexer.rs index 4038d24a..0c61952b 100644 --- a/sxt-core/src/prover_db_indexer.rs +++ b/sxt-core/src/prover_db_indexer.rs @@ -3,12 +3,15 @@ //! `pallet-tables` and `pallet-indexing`. use alloc::borrow::Cow; -use alloc::string::String; +use alloc::string::{String, ToString}; use alloc::vec::Vec; +use core::str::FromStr; use codec::{Decode, Encode}; +use snafu::Snafu; -use crate::tables::{TableIdentifier, TableNamespace}; +use crate::tables::TableIdentifier; +use crate::IDENT_LENGTH; /// Offchain local-storage key holding the prover-db indexer consumer's /// configuration. The embedding node writes a SCALE-encoded @@ -37,7 +40,7 @@ pub struct ProverDbConsumerConfig { /// pull in the `url` crate's full parsing surface. pub url: String, /// Per-node include set. Empty ⇒ forward every captured event. - pub include: Vec, + pub include: Vec, } /// Offchain DB key prefix for per-extrinsic event payloads. SCALE-encoded @@ -122,32 +125,138 @@ impl EventCapture for () { fn capture_events(_events: Vec>) {} } -/// A single entry in the prover-db indexer's include set. Stored only -/// in the indexer node's offchain local storage (as part of -/// [`ProverDbConsumerConfig`]); not part of on-chain state. An empty list -/// of these means "forward every event"; a non-empty list means "only -/// forward events whose table matches at least one rule". +/// Filter for one side (namespace or name) of a `TableIdentifier`. +/// Either matches every identifier (`Wildcard`) or matches exactly one +/// byte sequence (`Ident`). `Ident` values are stored uppercased so +/// matches against the on-chain canonical form are byte-exact without +/// per-callsite vigilance. +#[derive(Encode, Decode, Debug, Clone, Eq, PartialEq)] +pub enum IdentFilter { + /// Matches any identifier on this side of the dot. + Wildcard, + /// Matches an identifier whose bytes equal these. + Ident(String), +} + +impl IdentFilter { + /// Returns true if `bytes` (the on-chain canonical, uppercased form) + /// matches this filter. + pub fn matches(&self, bytes: &[u8]) -> bool { + match self { + Self::Wildcard => true, + Self::Ident(s) => s.as_bytes() == bytes, + } + } +} + +/// `FromStr` parse error for [`IdentFilter`]. +#[derive(Debug, Snafu)] +pub enum IdentFilterParseError { + /// The input was empty (neither `*` nor a non-empty identifier). + #[snafu(display("identifier filter is empty"))] + Empty, + /// The identifier exceeded the on-chain identifier length cap and so + /// could never match a captured table identifier. + #[snafu(display("identifier filter exceeds the {max}-byte on-chain length cap"))] + TooLong { + /// The cap, in bytes (`IDENT_LENGTH`). + max: u32, + }, +} + +impl FromStr for IdentFilter { + type Err = IdentFilterParseError; + + fn from_str(s: &str) -> Result { + if s == "*" { + return Ok(Self::Wildcard); + } + if s.is_empty() { + return Err(IdentFilterParseError::Empty); + } + let upper = s.to_uppercase(); + if upper.len() as u32 > IDENT_LENGTH { + return Err(IdentFilterParseError::TooLong { max: IDENT_LENGTH }); + } + Ok(Self::Ident(upper)) + } +} + +/// Filter against a fully-qualified [`TableIdentifier`]. Matches when +/// both sides match — so `*.*` is "everything", `NS.*` is "every table +/// in NS", `NS.NAME` is exact, and `*.NAME` is "any namespace, this +/// name". /// -/// Matches are byte-exact against the on-chain identifiers, so callers -/// that need case-insensitive matching should normalize their inputs -/// (e.g. via [`TableIdentifier::from_str_unchecked`] which uppercases) -/// before writing the rule to offchain storage. +/// Stored only in the indexer node's offchain local storage (as the +/// `include` field of [`ProverDbConsumerConfig`]); not part of on-chain +/// state. An empty list is treated as "forward every event". #[derive(Encode, Decode, Debug, Clone, Eq, PartialEq)] -pub enum IncludeRule { - /// Match every table within the given namespace. - Namespace(TableNamespace), - /// Match exactly one fully-qualified table identifier. - Table(TableIdentifier), -} - -/// Returns true if `table` matches at least one rule in `rules`. An -/// empty rule set is treated as "match all". -pub fn table_matches_rules(table: &TableIdentifier, rules: &[IncludeRule]) -> bool { - if rules.is_empty() { - return true; +pub struct TableIdentifierFilter { + /// Filter applied to `ident.namespace`. + pub namespace_filter: IdentFilter, + /// Filter applied to `ident.name`. + pub name_filter: IdentFilter, +} + +impl TableIdentifierFilter { + /// Returns true if `ident` passes both sides of this filter. + pub fn matches(&self, ident: &TableIdentifier) -> bool { + self.namespace_filter.matches(&ident.namespace) && self.name_filter.matches(&ident.name) } - rules.iter().any(|rule| match rule { - IncludeRule::Namespace(ns) => &table.namespace == ns, - IncludeRule::Table(t) => t == table, - }) +} + +/// `FromStr` parse error for [`TableIdentifierFilter`]. +#[derive(Debug, Snafu)] +pub enum TableIdentifierFilterParseError { + /// The input didn't contain exactly one `.` separator. + #[snafu(display("expected 'NAMESPACE.NAME' form with exactly one dot, got '{input}'"))] + MalformedShape { + /// The original input string. + input: String, + }, + /// The namespace side failed to parse as an [`IdentFilter`]. + #[snafu(display("invalid namespace filter: {source}"))] + Namespace { + /// Underlying [`IdentFilter`] parse error. + source: IdentFilterParseError, + }, + /// The name side failed to parse as an [`IdentFilter`]. + #[snafu(display("invalid name filter: {source}"))] + Name { + /// Underlying [`IdentFilter`] parse error. + source: IdentFilterParseError, + }, +} + +impl FromStr for TableIdentifierFilter { + type Err = TableIdentifierFilterParseError; + + fn from_str(s: &str) -> Result { + let Some((ns, name)) = s.split_once('.') else { + return Err(TableIdentifierFilterParseError::MalformedShape { + input: s.to_string(), + }); + }; + if s.matches('.').count() != 1 { + return Err(TableIdentifierFilterParseError::MalformedShape { + input: s.to_string(), + }); + } + let namespace_filter = ns + .parse() + .map_err(|source| TableIdentifierFilterParseError::Namespace { source })?; + let name_filter = name + .parse() + .map_err(|source| TableIdentifierFilterParseError::Name { source })?; + Ok(Self { + namespace_filter, + name_filter, + }) + } +} + +/// Returns true if `table` matches at least one filter in `filters`. An +/// empty filter set is treated as "match all". +pub fn table_matches_filters(table: &TableIdentifier, filters: &[TableIdentifierFilter]) -> bool { + filters.is_empty() || filters.iter().any(|f| f.matches(table)) } From 03cf6108fadbd496551b2cdbc674f8e90e312e9d Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Mon, 8 Jun 2026 12:54:30 +0530 Subject: [PATCH 6/9] feat(prover-db-indexer): default --prover-db-include to *.* Per review: surface the match-all default in --help and let the include vec stay non-empty. Cross-flag guard on URL-absent dropped since the default now makes that branch unreachable; URL absence remains the single dormancy signal. --- node/src/cli.rs | 2 +- node/src/service.rs | 28 ++++++---------------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/node/src/cli.rs b/node/src/cli.rs index 94d30e26..8d1a49f3 100644 --- a/node/src/cli.rs +++ b/node/src/cli.rs @@ -73,7 +73,7 @@ pub struct ProverDbConsumerCli { /// works without per-operator vigilance. /// /// [`TableIdentifierFilter`]: sxt_core::prover_db_indexer::TableIdentifierFilter - #[clap(long, env, value_delimiter = ',')] + #[clap(long, env, value_delimiter = ',', default_value = "*.*")] pub prover_db_include: Vec, } diff --git a/node/src/service.rs b/node/src/service.rs index 3569574b..58829eb0 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -79,21 +79,13 @@ const GRANDPA_JUSTIFICATION_PERIOD: u32 = 512; /// Validate the consumer CLI group into a concrete /// `ProverDbConsumerConfig` (URL non-optional) and seed it under -/// `PROVER_DB_CONFIG_KEY` in offchain persistent local storage. Atomic -/// shape: one offchain key per consumer, so half-states like "include -/// patterns set but URL missing" are caught up front instead of becoming -/// stale data in the OCW DB. +/// `PROVER_DB_CONFIG_KEY` in offchain persistent local storage. /// -/// Each `--prover-db-include` value has already been parsed into a -/// `TableIdentifierFilter` by clap (via the type's `FromStr` impl), so -/// this function only enforces the URL/include consistency rule and -/// writes the encoded config. -/// -/// Returns: -/// - `Ok(())` and writes the key when `--prover-db-url` is set. -/// - `Ok(())` and **does not** write when nothing is set (this node -/// isn't an indexer; OCW stays dormant). -/// - `Err(_)` if the operator tried to set patterns without a URL. +/// `--prover-db-url` is the single switch that enables the consumer: +/// present ⇒ write the config; absent ⇒ the OCW stays dormant and we +/// don't write anything. `--prover-db-include` is allowed (with its +/// `*.*` default) regardless of whether the URL is set; without a URL +/// it just has nothing to filter. #[expect( clippy::result_large_err, reason = "ServiceError is from substrate and cannot be modified" @@ -103,14 +95,6 @@ fn configure_prover_db_consumer( cli: &crate::cli::ProverDbConsumerCli, ) -> Result<(), ServiceError> { let Some(url) = cli.prover_db_url.as_ref() else { - if !cli.prover_db_include.is_empty() { - return Err(ServiceError::Other( - "--prover-db-include was set but --prover-db-url is not; \ - the include set would have no effect because the OCW \ - only runs when the indexer URL is configured." - .into(), - )); - } return Ok(()); }; From dd409748f40125a8881806779642cbbd5cd6c638 Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Tue, 9 Jun 2026 10:29:45 +0530 Subject: [PATCH 7/9] test(prover-db-indexer): unit-test IdentFilter, TableIdentifierFilter, and table_matches_filters 13 tests covering wildcard/literal matching, FromStr happy + error paths (empty side, over-length, missing dot, multi-dot), and the match-all-on-empty behaviour of table_matches_filters. --- sxt-core/src/prover_db_indexer.rs | 167 ++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/sxt-core/src/prover_db_indexer.rs b/sxt-core/src/prover_db_indexer.rs index 0c61952b..be21ff57 100644 --- a/sxt-core/src/prover_db_indexer.rs +++ b/sxt-core/src/prover_db_indexer.rs @@ -260,3 +260,170 @@ impl FromStr for TableIdentifierFilter { pub fn table_matches_filters(table: &TableIdentifier, filters: &[TableIdentifierFilter]) -> bool { filters.is_empty() || filters.iter().any(|f| f.matches(table)) } + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + + fn ident(name: &str, namespace: &str) -> TableIdentifier { + TableIdentifier::from_str_unchecked(name, namespace) + } + + // ── IdentFilter::matches ───────────────────────────────────────── + + #[test] + fn ident_filter_wildcard_matches_anything() { + assert!(IdentFilter::Wildcard.matches(b"")); + assert!(IdentFilter::Wildcard.matches(b"ANYTHING")); + } + + #[test] + fn ident_filter_ident_matches_exact_bytes_only() { + let f = IdentFilter::Ident("FOO".into()); + assert!(f.matches(b"FOO")); + assert!(!f.matches(b"foo")); // case-sensitive at match time + assert!(!f.matches(b"FOOBAR")); + assert!(!f.matches(b"FO")); + assert!(!f.matches(b"")); + } + + // ── IdentFilter::FromStr ───────────────────────────────────────── + + #[test] + fn ident_filter_parses_star_as_wildcard() { + assert_eq!("*".parse::().unwrap(), IdentFilter::Wildcard); + } + + #[test] + fn ident_filter_parses_ident_uppercased() { + assert_eq!( + "foo".parse::().unwrap(), + IdentFilter::Ident("FOO".into()), + ); + assert_eq!( + "MIXEDcase".parse::().unwrap(), + IdentFilter::Ident("MIXEDCASE".into()), + ); + } + + #[test] + fn ident_filter_rejects_empty_string() { + assert!(matches!( + "".parse::(), + Err(IdentFilterParseError::Empty), + )); + } + + #[test] + fn ident_filter_rejects_over_length_input() { + // IDENT_LENGTH + 1 ASCII bytes — uppercase is a no-op so the + // post-uppercase length is also IDENT_LENGTH + 1. + let too_long: alloc::string::String = + core::iter::repeat_n('A', IDENT_LENGTH as usize + 1).collect(); + assert!(matches!( + too_long.parse::(), + Err(IdentFilterParseError::TooLong { max }) if max == IDENT_LENGTH, + )); + } + + // ── TableIdentifierFilter::matches ─────────────────────────────── + + #[test] + fn table_filter_matches_only_when_both_sides_match() { + let table = ident("T1", "ALPHA"); + + let alpha_t1: TableIdentifierFilter = "ALPHA.T1".parse().unwrap(); + let alpha_star: TableIdentifierFilter = "ALPHA.*".parse().unwrap(); + let star_t1: TableIdentifierFilter = "*.T1".parse().unwrap(); + let star_star: TableIdentifierFilter = "*.*".parse().unwrap(); + let beta_t1: TableIdentifierFilter = "BETA.T1".parse().unwrap(); + let alpha_other: TableIdentifierFilter = "ALPHA.OTHER".parse().unwrap(); + + assert!(alpha_t1.matches(&table)); + assert!(alpha_star.matches(&table)); + assert!(star_t1.matches(&table)); + assert!(star_star.matches(&table)); + assert!(!beta_t1.matches(&table)); + assert!(!alpha_other.matches(&table)); + } + + // ── TableIdentifierFilter::FromStr ─────────────────────────────── + + #[test] + fn table_filter_parses_all_four_shapes() { + let full: TableIdentifierFilter = "ns.name".parse().unwrap(); + assert!(matches!(full.namespace_filter, IdentFilter::Ident(ref s) if s == "NS")); + assert!(matches!(full.name_filter, IdentFilter::Ident(ref s) if s == "NAME")); + + let ns_star: TableIdentifierFilter = "ns.*".parse().unwrap(); + assert!(matches!(ns_star.namespace_filter, IdentFilter::Ident(ref s) if s == "NS")); + assert!(matches!(ns_star.name_filter, IdentFilter::Wildcard)); + + let star_name: TableIdentifierFilter = "*.name".parse().unwrap(); + assert!(matches!(star_name.namespace_filter, IdentFilter::Wildcard)); + assert!(matches!(star_name.name_filter, IdentFilter::Ident(ref s) if s == "NAME")); + + let star_star: TableIdentifierFilter = "*.*".parse().unwrap(); + assert!(matches!(star_star.namespace_filter, IdentFilter::Wildcard)); + assert!(matches!(star_star.name_filter, IdentFilter::Wildcard)); + } + + #[test] + fn table_filter_rejects_missing_dot() { + assert!(matches!( + "namename".parse::(), + Err(TableIdentifierFilterParseError::MalformedShape { .. }), + )); + } + + #[test] + fn table_filter_rejects_more_than_one_dot() { + assert!(matches!( + "a.b.c".parse::(), + Err(TableIdentifierFilterParseError::MalformedShape { .. }), + )); + } + + #[test] + fn table_filter_propagates_empty_side_error() { + assert!(matches!( + ".name".parse::(), + Err(TableIdentifierFilterParseError::Namespace { + source: IdentFilterParseError::Empty, + }), + )); + assert!(matches!( + "ns.".parse::(), + Err(TableIdentifierFilterParseError::Name { + source: IdentFilterParseError::Empty, + }), + )); + } + + // ── table_matches_filters ──────────────────────────────────────── + + #[test] + fn empty_filter_set_matches_everything() { + let table = ident("T1", "ALPHA"); + assert!(table_matches_filters(&table, &[])); + } + + #[test] + fn non_empty_filter_set_requires_at_least_one_match() { + let table = ident("T1", "ALPHA"); + let filters = vec![ + "BETA.*".parse().unwrap(), + "GAMMA.T1".parse().unwrap(), + "ALPHA.T1".parse().unwrap(), // this one matches + ]; + assert!(table_matches_filters(&table, &filters)); + + let no_match = vec![ + "BETA.*".parse().unwrap(), + "GAMMA.T1".parse().unwrap(), + ]; + assert!(!table_matches_filters(&table, &no_match)); + } +} From 9f2413fbc8f33aa7fce7a7f3ff4c4258d461c5c5 Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Tue, 9 Jun 2026 10:33:34 +0530 Subject: [PATCH 8/9] refactor(prover-db-indexer): empty include set now matches nothing Per review: split "no filters configured" from "match-all configured" so they're distinct, addressable states. table_matches_filters now returns false on an empty list; callers wanting match-all must pass an explicit *.* filter (the CLI default already does this). Pallet tests: setup_with_url seeds *.* instead of an empty vec; a new ocw_with_empty_include_set_forwards_nothing asserts the new empty-list-rejects-everything behaviour. --- pallets/prover_db_indexer/src/tests.rs | 57 ++++++++++++++++++++++---- sxt-core/src/prover_db_indexer.rs | 24 ++++++++--- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/pallets/prover_db_indexer/src/tests.rs b/pallets/prover_db_indexer/src/tests.rs index 00036a1a..32923ecc 100644 --- a/pallets/prover_db_indexer/src/tests.rs +++ b/pallets/prover_db_indexer/src/tests.rs @@ -73,11 +73,11 @@ fn no_checkpoint_response() -> Vec { .encode_to_vec() } -/// Seed an empty include set — equivalent to the operator running -/// without `--prover-db-include`, which is what every existing test -/// expects. +/// Seed a `*.*` filter — what the operator gets when omitting +/// `--prover-db-include` (clap default). Every captured event should +/// reach the indexer. fn setup_with_url() -> (polkadot_sdk::sp_io::TestExternalities, StateArc) { - setup_with_config(Vec::new()) + setup_with_config(vec!["*.*".parse().unwrap()]) } /// Seed a non-empty include set under the unified config key. Used by @@ -554,13 +554,13 @@ fn ocw_forwards_only_events_matching_include_set() { assert!(s.persistent_storage.get(&key_for_high_water(1)).is_none()); } -/// Empty include set in offchain storage ⇒ forward every event (default). +/// Explicit `*.*` filter in offchain storage ⇒ forward every event — +/// this is what the operator gets when omitting `--prover-db-include` +/// because the CLI default is `*.*`. #[test] -fn ocw_with_empty_include_set_forwards_everything() { +fn ocw_with_wildcard_filter_forwards_everything() { let (mut ext, state) = setup_with_url(); - // `setup_with_url` writes a config whose `include` is empty — - // same effect as the operator not passing `--prover-db-include`. let ddl = b"CREATE TABLE ANY.ANY (ID BIGINT NOT NULL)"; seed_block_events( &state, @@ -602,3 +602,44 @@ fn ocw_with_empty_include_set_forwards_everything() { ProverDbIndexer::offchain_worker(1); }); } + +/// Empty include set in offchain storage ⇒ forward nothing. The +/// checkpoint still advances (so the OCW makes server-side progress +/// past empty-as-far-as-this-node-is-concerned blocks), but no +/// `/v1/create_table` request is emitted. `TestOffchainExt` would +/// panic on an unexpected request, so the absence of that expectation +/// is the assertion. +#[test] +fn ocw_with_empty_include_set_forwards_nothing() { + let (mut ext, state) = setup_with_config(Vec::new()); + + let ddl = b"CREATE TABLE ANY.ANY (ID BIGINT NOT NULL)"; + seed_block_events( + &state, + 1, + 0, + vec![BlockEvent::Create(CreateEntry { + ident: Cow::Owned(TableIdentifier::from_str_unchecked("ANY", "ANY")), + ddl: ddl.to_vec().into(), + })], + ); + + { + let mut s = state.write(); + s.expect_request(expected_request( + "/v1/get_last_checkpoint", + vec![], + no_checkpoint_response(), + )); + s.expect_request(expected_request( + "/v1/checkpoint", + proto::CheckpointRequest { sequence_number: 1 }.encode_to_vec(), + vec![], + )); + } + + ext.execute_with(|| { + System::set_block_number(1); + ProverDbIndexer::offchain_worker(1); + }); +} diff --git a/sxt-core/src/prover_db_indexer.rs b/sxt-core/src/prover_db_indexer.rs index be21ff57..1639de85 100644 --- a/sxt-core/src/prover_db_indexer.rs +++ b/sxt-core/src/prover_db_indexer.rs @@ -39,7 +39,10 @@ pub struct ProverDbConsumerConfig { /// node start; stored as raw bytes so the pallet doesn't need to /// pull in the `url` crate's full parsing surface. pub url: String, - /// Per-node include set. Empty ⇒ forward every captured event. + /// Per-node include set. An event is forwarded iff its + /// `TableIdentifier` matches at least one filter. Empty ⇒ forward + /// nothing; callers that want "forward everything" must include an + /// explicit `*.*` filter. pub include: Vec, } @@ -189,7 +192,7 @@ impl FromStr for IdentFilter { /// /// Stored only in the indexer node's offchain local storage (as the /// `include` field of [`ProverDbConsumerConfig`]); not part of on-chain -/// state. An empty list is treated as "forward every event". +/// state. An empty list matches nothing — see [`table_matches_filters`]. #[derive(Encode, Decode, Debug, Clone, Eq, PartialEq)] pub struct TableIdentifierFilter { /// Filter applied to `ident.namespace`. @@ -256,9 +259,11 @@ impl FromStr for TableIdentifierFilter { } /// Returns true if `table` matches at least one filter in `filters`. An -/// empty filter set is treated as "match all". +/// empty filter set matches nothing — callers that want "match all" +/// must pass an explicit `*.*` filter. This makes "no filters +/// configured" and "match-all configured" distinct, addressable states. pub fn table_matches_filters(table: &TableIdentifier, filters: &[TableIdentifierFilter]) -> bool { - filters.is_empty() || filters.iter().any(|f| f.matches(table)) + filters.iter().any(|f| f.matches(table)) } #[cfg(test)] @@ -405,9 +410,16 @@ mod tests { // ── table_matches_filters ──────────────────────────────────────── #[test] - fn empty_filter_set_matches_everything() { + fn empty_filter_set_matches_nothing() { let table = ident("T1", "ALPHA"); - assert!(table_matches_filters(&table, &[])); + assert!(!table_matches_filters(&table, &[])); + } + + #[test] + fn explicit_wildcard_filter_matches_everything() { + let table = ident("T1", "ALPHA"); + let star: TableIdentifierFilter = "*.*".parse().unwrap(); + assert!(table_matches_filters(&table, core::slice::from_ref(&star))); } #[test] From f0debeae2e3c6553162f172b6060a9d7964237fc Mon Sep 17 00:00:00 2001 From: "rishitesh.mishra@spaceandtime.io" Date: Tue, 9 Jun 2026 11:26:31 +0530 Subject: [PATCH 9/9] fix(prover-db-indexer): collapse no_match vec to one line for cargo f --- sxt-core/src/prover_db_indexer.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sxt-core/src/prover_db_indexer.rs b/sxt-core/src/prover_db_indexer.rs index 1639de85..79e650de 100644 --- a/sxt-core/src/prover_db_indexer.rs +++ b/sxt-core/src/prover_db_indexer.rs @@ -432,10 +432,7 @@ mod tests { ]; assert!(table_matches_filters(&table, &filters)); - let no_match = vec![ - "BETA.*".parse().unwrap(), - "GAMMA.T1".parse().unwrap(), - ]; + let no_match = vec!["BETA.*".parse().unwrap(), "GAMMA.T1".parse().unwrap()]; assert!(!table_matches_filters(&table, &no_match)); } }