feat: add pallet-block-forwarder for offchain HTTP indexer forwarding

Cargo.toml

@@ -31,14 +31,15 @@ members = [
     "pallets/rewards",
     "pallets/keystore",
     "pallets/smartcontracts",
-    "event-forwarder", 
+    "event-forwarder",
     "chain-utils",
     "pallets/system-contracts",
     "canaries",
     "pallets/zkpay",
     "proof-of-sql/commitment-column-mapping",
     "eth-ecdsa",
     "arrow-ipc-no-std",
+    "pallets/prover_db_indexer",
 ]
 exclude = [
     "utoipa",  
@@ -47,7 +48,7 @@ resolver = "2"
 
 [workspace.dependencies]
 polkadot-sdk = { version = "0.9.0", default-features = false }
-sc-cli = { version = "0.47.0", default-features = false }
+sc-cli = { version = "0.47.0", default-features = false }
 sc-client-db = { version = "0.44.1", default-features = false }
 
 # Required for the runtime_interface macro in our native modules
@@ -79,6 +80,7 @@ pallet-smartcontracts = { path = "pallets/smartcontracts", default-features = fa
 sxt-core = { path = "sxt-core", default-features = false }
 pallet-tables = { path = "pallets/tables", default-features = false }
 pallet-indexing = { path = "pallets/indexing", default-features = false }
+pallet-prover-db-indexer = { path = "pallets/prover_db_indexer", default-features = false }
 native = { path = "native", default-features = false }
 native-api = { path = "native-api", default-features = false }
 num-bigint = { version = "0.4", default-features = false }

node/Cargo.toml

@@ -46,6 +46,7 @@ rand.workspace = true
 node-rpc.workspace = true
 serde.workspace = true
 codec = { workspace = true, default-features = true }
+url = { workspace = true }
 
 [build-dependencies]
 substrate-build-script-utils.workspace = true

node/src/cli.rs

@@ -19,6 +19,17 @@ pub struct Cli {
     #[clap(long)]
     pub event_forwarder_rpc: Option<String>,
 
+    /// If set, writes the URL into OCW persistent local storage under
+    /// `prover_db_indexer::prover_db_url` at startup, telling the
+    /// prover-db-indexer OCW where to POST forwarded events. Seeds the
+    /// storage before the first block is authored, so no events are
+    /// missed between node-up and URL-configured.
+    ///
+    /// Parsed as a `url::Url` so an invalid value rejects at startup
+    /// rather than silently failing on the first OCW HTTP request.
+    #[clap(long)]
+    pub prover_db_url: Option<url::Url>,
+
     #[allow(missing_docs)]
     #[clap(flatten)]
     pub storage_monitor: sc_storage_monitor::StorageMonitorParams,

node/src/service.rs

@@ -14,6 +14,7 @@ use polkadot_sdk::sc_service::error::Error as ServiceError;
 use polkadot_sdk::sc_service::TaskManager;
 use polkadot_sdk::sc_telemetry::{Telemetry, TelemetryWorker};
 use polkadot_sdk::sc_transaction_pool_api::OffchainTransactionPoolFactory;
+use polkadot_sdk::sp_core::offchain::{OffchainStorage, STORAGE_PREFIX};
 use polkadot_sdk::sp_runtime::traits::Block as BlockT;
 use polkadot_sdk::{
     sc_authority_discovery,
@@ -75,6 +76,32 @@ pub type TransactionPool = sc_transaction_pool::FullPool<Block, FullClient>;
 /// imported and generated.
 const GRANDPA_JUSTIFICATION_PERIOD: u32 = 512;
 
+/// Seed the prover-db-indexer OCW's persistent local storage with the
+/// prover-db URL given on the command line, before the first block is
+/// authored.
+///
+/// The storage key is
+/// `pallet_prover_db_indexer::PROVER_DB_URL_KEY = "prover_db_indexer::prover_db_url"`;
+/// the value is a SCALE-encoded `Vec<u8>` of the URL bytes.
+#[expect(
+    clippy::result_large_err,
+    reason = "ServiceError is from substrate and cannot be modified"
+)]
+fn configure_prover_db_url(backend: &FullBackend, url: &url::Url) -> Result<(), ServiceError> {
+    use codec::Encode;
+    let Some(mut storage) = backend.offchain_storage() else {
+        return Err(ServiceError::Other(
+            "backend did not expose an offchain storage handle; \
+             cannot apply --prover-db-url"
+                .into(),
+        ));
+    };
+    let key = sxt_runtime::pallet_prover_db_indexer::PROVER_DB_URL_KEY;
+    let encoded = url.as_str().as_bytes().to_vec().encode();
+    storage.set(STORAGE_PREFIX, key, &encoded);
+    Ok(())
+}
+
 #[allow(clippy::type_complexity)]
 #[expect(
     clippy::result_large_err,
@@ -290,6 +317,7 @@ pub struct NewFullBase {
 )]
 pub fn new_full_base<N: NetworkBackend<Block, <Block as BlockT>::Hash>>(
     config: Configuration,
+    prover_db_url: Option<url::Url>,
 ) -> Result<NewFullBase, ServiceError> {
     let role = config.role;
     let force_authoring = config.force_authoring;
@@ -322,6 +350,34 @@ pub fn new_full_base<N: NetworkBackend<Block, <Block as BlockT>::Hash>>(
         other: (rpc_builder, import_setup, rpc_setup, mut telemetry, statement_store),
     } = new_partial(&config)?;
 
+    // The prover-db-indexer pallet's `on_finalize` calls
+    // `sp_io::offchain_index::set` to hand events to the OCW. That call is a
+    // silent no-op unless `--enable-offchain-indexing=true` was passed. We
+    // surface that loudly rather than letting operators discover it by
+    // watching zero rows arrive at their indexer.
+    if let Some(url) = prover_db_url.as_ref() {
+        if !config.offchain_worker.indexing_enabled {
+            return Err(ServiceError::Other(
+                "--prover-db-url was set but --enable-offchain-indexing is not \
+                 true: prover-db-indexer's on_finalize writes would be silently \
+                 dropped. Restart with --enable-offchain-indexing=true, or \
+                 omit --prover-db-url."
+                    .into(),
+            ));
+        }
+        configure_prover_db_url(backend.as_ref(), url)?;
+    } else if !config.offchain_worker.indexing_enabled {
+        // Prover-db-indexer is in the runtime but offchain indexing is off.
+        // Forwarding can't work even if someone later writes the URL via RPC.
+        // Loud eprintln! so it shows up even without log filter setup.
+        eprintln!(
+            "warning: --enable-offchain-indexing is not true; the \
+             prover-db-indexer OCW cannot forward events even if \
+             prover_db_indexer::prover_db_url is set via RPC. Pass \
+             --enable-offchain-indexing=true to enable forwarding."
+        );
+    }
+
     let metrics = N::register_notification_metrics(
         config.prometheus_config.as_ref().map(|cfg| &cfg.registry),
     );
@@ -603,13 +659,14 @@ pub fn new_full(config: Configuration, cli: Cli) -> Result<TaskManager, ServiceE
     futures::executor::block_on(initialize_from_config(&cli.proof_of_sql_public_setup_args))
         .map_err(|e| ServiceError::Other(e.to_string()))?;
 
+    let prover_db_url = cli.prover_db_url.clone();
     let task_manager = match config.network.network_backend {
         sc_network::config::NetworkBackendType::Libp2p => {
-            new_full_base::<sc_network::NetworkWorker<_, _>>(config)
+            new_full_base::<sc_network::NetworkWorker<_, _>>(config, prover_db_url)
                 .map(|NewFullBase { task_manager, .. }| task_manager)?
         }
         sc_network::config::NetworkBackendType::Litep2p => {
-            new_full_base::<sc_network::Litep2pNetworkBackend>(config)
+            new_full_base::<sc_network::Litep2pNetworkBackend>(config, prover_db_url)
                 .map(|NewFullBase { task_manager, .. }| task_manager)?
         }
     };

pallets/prover_db_indexer/Cargo.toml

@@ -0,0 +1,67 @@
+[package]
+name = "pallet-prover-db-indexer"
+description = "Offchain worker that forwards table lifecycle and data events to an external prover-db indexer (HTTP)."
+version = "0.1.0"
+license = "Unlicense"
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+edition.workspace = true
+publish = false
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+codec = { features = ["derive"], workspace = true }
+scale-info = { features = ["derive"], workspace = true }
+
+polkadot-sdk = { workspace = true, features = [
+    "frame-support", "frame-system", "sp-core", "sp-io", "sp-runtime",
+]}
+
+sxt-core.workspace = true
+pallet-tables.workspace = true
+pallet-indexing.workspace = true
+native-api.workspace = true
+log = { workspace = true, default-features = false }
+prost = { version = "0.13", default-features = false, features = ["prost-derive"] }
+
+[build-dependencies]
+prost-build = "0.13"
+
+[dev-dependencies]
+polkadot-sdk = { workspace = true, features = [
+    "sp-core", "sp-io", "sp-runtime", "sp-staking",
+    "pallet-balances", "pallet-staking", "pallet-staking-reward-curve",
+    "pallet-session", "pallet-timestamp",
+    "frame-election-provider-support",
+]}
+pallet-permissions = { workspace = true, features = ["std"] }
+pallet-commitments = { workspace = true, features = ["std"] }
+pallet-tables = { workspace = true, features = ["std"] }
+pallet-indexing = { workspace = true, features = ["std"] }
+pallet-system-tables = { workspace = true, features = ["std"] }
+pallet-zkpay = { workspace = true, features = ["std"] }
+proof-of-sql-commitment-map.workspace = true
+parking_lot = "0.12"
+
+[features]
+default = ["std"]
+std = [
+    "polkadot-sdk/std",
+    "codec/std",
+    "scale-info/std",
+    "sxt-core/std",
+    "pallet-tables/std",
+    "pallet-indexing/std",
+    "native-api/std",
+    "log/std",
+    "prost/std",
+]
+try-runtime = [
+    "polkadot-sdk/try-runtime",
+]
+
+[lints]
+workspace = true