diff --git a/packages/cli/src/evm_hypersync_source/decode.rs b/packages/cli/src/evm_hypersync_source/decode.rs index 44a9a7ecd5..078b40587b 100644 --- a/packages/cli/src/evm_hypersync_source/decode.rs +++ b/packages/cli/src/evm_hypersync_source/decode.rs @@ -467,9 +467,7 @@ mod tests { fn registration_rejects_zero_topics() { let mut reg = value_reg(0, "C", false, VALID_SIGHASH); reg.topic_count = 0; - let err = Decoder::from_registrations(&[reg], false) - .err() - .unwrap(); + let err = Decoder::from_registrations(&[reg], false).err().unwrap(); assert!(format!("{err:#}").contains("topic_count must be 1..=4")); } @@ -477,9 +475,7 @@ mod tests { fn registration_rejects_five_topics() { let mut reg = value_reg(0, "C", false, VALID_SIGHASH); reg.topic_count = 5; - let err = Decoder::from_registrations(&[reg], false) - .err() - .unwrap(); + let err = Decoder::from_registrations(&[reg], false).err().unwrap(); assert!(format!("{err:#}").contains("topic_count must be 1..=4")); } diff --git a/packages/cli/src/evm_hypersync_source/mod.rs b/packages/cli/src/evm_hypersync_source/mod.rs index 6a351eee0c..29a2ce5e85 100644 --- a/packages/cli/src/evm_hypersync_source/mod.rs +++ b/packages/cli/src/evm_hypersync_source/mod.rs @@ -43,7 +43,7 @@ fn make_rate_limit_err(info: &hypersync_client::RateLimitInfo) -> napi::Error { } #[napi] -pub struct EvmHypersyncClient { +pub struct EvmHyperSyncClient { inner: hypersync_client::Client, enable_checksum_addresses: bool, decoder: Decoder, @@ -51,21 +51,20 @@ pub struct EvmHypersyncClient { } #[napi] -impl EvmHypersyncClient { +impl EvmHyperSyncClient { #[napi(factory)] pub fn new( cfg: ClientConfig, user_agent: String, event_registrations: Vec, - ) -> napi::Result { + ) -> napi::Result { init_logger(cfg.log_level.as_deref()); let enable_checksum_addresses = cfg.enable_checksum_addresses.unwrap_or_default(); - let decoder = - Decoder::from_registrations(&event_registrations, enable_checksum_addresses) - .context("build decoder") - .map_err(map_err)?; + let decoder = Decoder::from_registrations(&event_registrations, enable_checksum_addresses) + .context("build decoder") + .map_err(map_err)?; let selection_builder = SelectionBuilder::from_registrations(&event_registrations) .context("build selection builder") @@ -75,7 +74,7 @@ impl EvmHypersyncClient { .context("build client") .map_err(map_err)?; - Ok(EvmHypersyncClient { + Ok(EvmHyperSyncClient { inner, enable_checksum_addresses, decoder, diff --git a/packages/cli/src/fuel_hypersync_source/mod.rs b/packages/cli/src/fuel_hypersync_source/mod.rs index 62a2c42381..ed0926d818 100644 --- a/packages/cli/src/fuel_hypersync_source/mod.rs +++ b/packages/cli/src/fuel_hypersync_source/mod.rs @@ -1,29 +1,44 @@ +use std::collections::{HashMap, HashSet}; + use anyhow::Context; +use napi::bindgen_prelude::BigInt; use napi_derive::napi; mod config; -mod query; +mod selection; mod types; use config::ClientConfig; -use query::Query; -use types::{convert_response, ConvertError, QueryResponse}; +use hyperfuel_client::net_types; +use selection::{BuiltSelection, FuelOnEventRegistrationInput, RegistrationKind, SelectionBuilder}; +use types::{convert_response, Block, ConvertError, RawReceipt}; #[napi] -pub struct HyperfuelClient { +pub struct FuelHyperSyncClient { inner: hyperfuel_client::Client, + selection_builder: SelectionBuilder, } #[napi] -impl HyperfuelClient { +impl FuelHyperSyncClient { #[napi(factory)] - pub fn new(cfg: ClientConfig, user_agent: String) -> napi::Result { + pub fn new( + cfg: ClientConfig, + user_agent: String, + event_registrations: Vec, + ) -> napi::Result { + let selection_builder = SelectionBuilder::from_registrations(&event_registrations) + .context("build selection builder") + .map_err(map_err)?; let client_config: hyperfuel_client::ClientConfig = cfg.try_into().context("build config").map_err(map_err)?; let inner = hyperfuel_client::Client::new_with_agent(client_config, user_agent) .context("build client") .map_err(map_err)?; - Ok(HyperfuelClient { inner }) + Ok(FuelHyperSyncClient { + inner, + selection_builder, + }) } #[napi] @@ -37,16 +52,246 @@ impl HyperfuelClient { } #[napi] - pub async fn get_selected_data(&self, query: Query) -> napi::Result { - let query: hyperfuel_client::net_types::Query = - query.try_into().context("parse query").map_err(map_err)?; + pub async fn get_event_items( + &self, + params: EventItemsQuery, + ) -> napi::Result { + let built = self + .selection_builder + .build( + ¶ms.registration_indexes, + ¶ms.addresses_by_contract_name, + ) + .map_err(map_err)?; + + let query = build_query(¶ms, &built) + .context("build query") + .map_err(map_err)?; let res = self .inner .get_arrow(&query) .await .map_err(|e| request_err("Failed to get data from HyperFuel", e))?; - convert_response(res).map_err(convert_error_to_napi) + let raw = convert_response(res).map_err(convert_error_to_napi)?; + + let items = + route_receipts(raw.receipts, &raw.blocks, &built).map_err(convert_error_to_napi)?; + + Ok(EventItemsResponse { + archive_height: raw.archive_height, + next_block: raw.next_block, + blocks: raw.blocks, + items, + }) + } +} + +/// The whole per-query input for `get_event_items`: the block range, the +/// partition's registration selection (by index), and its current addresses. +/// Receipt selections, field selection, and routing are all derived internally +/// from the registrations passed at construction. +#[napi(object)] +pub struct EventItemsQuery { + pub from_block: i64, + /// Inclusive; `None` queries to the end of available data. + pub to_block: Option, + pub registration_indexes: Vec, + pub addresses_by_contract_name: HashMap>, +} + +/// One routed receipt. The receipt's kind-specific columns are flattened so +/// JS builds params without a tagged receipt union: LogData carries `data` +/// (decoded in JS against the contract ABI), Mint/Burn carry `val`/`subId`, +/// Transfer/TransferOut/Call carry `amount`/`assetId`/`to` — with +/// TransferOut's wallet recipient normalised into `to`. +#[napi(object)] +pub struct EventItem { + /// The registration this receipt routed to, as passed to the client + /// constructor. Receipts that route nowhere never cross the boundary. + pub on_event_registration_index: i64, + pub receipt_index: i64, + pub tx_id: String, + /// Height of the block this receipt belongs to. The block itself is + /// carried once, deduplicated, in `EventItemsResponse.blocks`. + pub block_height: i64, + pub src_address: String, + pub data: Option, + pub sub_id: Option, + pub val: Option, + pub amount: Option, + pub asset_id: Option, + pub to: Option, +} + +#[napi(object)] +pub struct EventItemsResponse { + pub archive_height: Option, + pub next_block: i64, + /// The page's blocks, one per height. Items reference them by + /// `block_height`; presence for every routed item is validated here. + pub blocks: Vec, + pub items: Vec, +} + +fn build_query( + params: &EventItemsQuery, + built: &BuiltSelection, +) -> anyhow::Result { + let from_block = u64::try_from(params.from_block).context("from_block must be >= 0")?; + let to_block = params + .to_block + // Inclusive on the boundary, exclusive on the wire. + .map(|b| { + u64::try_from(b) + .context("to_block must be >= 0")? + .checked_add(1) + .context("to_block overflow") + }) + .transpose()?; + + let mut receipt_fields = vec![ + "receipt_index", + "receipt_type", + "root_contract_id", + "tx_id", + "block_height", + ]; + if built.needs_log_data { + receipt_fields.extend(["data", "rb"]); + } + if built.needs_supply { + receipt_fields.extend(["val", "sub_id"]); + } + if built.needs_transfer || built.needs_call { + receipt_fields.extend(["amount", "asset_id", "to"]); + } + if built.needs_transfer { + receipt_fields.push("to_address"); + } + + Ok(net_types::Query { + from_block, + to_block, + receipts: built.receipt_selections.clone(), + field_selection: net_types::FieldSelection { + block: ["id", "height", "time"].map(str::to_string).into(), + receipt: receipt_fields.into_iter().map(str::to_string).collect(), + ..Default::default() + }, + ..Default::default() + }) +} + +fn push_unique(missing: &mut Vec, name: &str) { + if !missing.iter().any(|m| m == name) { + missing.push(name.to_string()); + } +} + +/// Fans each receipt out to every registration of the selection it matches +/// and flattens the kind-specific columns onto the items. A receipt without a +/// `root_contract_id` (no contract context for `srcAddress`) is dropped, as is +/// one that routes to no registration. Kind-required columns the source +/// omitted surface as `MissingFields` — never as garbage params. +fn route_receipts( + receipts: Vec, + blocks: &[Block], + built: &BuiltSelection, +) -> Result, ConvertError> { + let present_block_heights: HashSet = blocks.iter().map(|b| b.height).collect(); + let mut items = Vec::with_capacity(receipts.len()); + let mut missing: Vec = Vec::new(); + + for receipt in receipts { + let src_address = match &receipt.root_contract_id { + Some(address) => address.clone(), + None => continue, + }; + let contract_name = built + .contract_name_by_address + .get(&src_address) + .map(String::as_str); + + for reg in &built.registrations { + if !reg.matches(receipt.receipt_type, receipt.rb, contract_name) { + continue; + } + // Every routed item's block must resolve — JS reads it + // unconditionally onto the payload. + if !present_block_heights.contains(&receipt.block_height) { + push_unique(&mut missing, "block"); + } + let require_hex = |value: &Option, name: &str, missing: &mut Vec| { + if value.is_none() { + push_unique(missing, name); + } + value.clone() + }; + let require_u64 = |value: Option, name: &str, missing: &mut Vec| { + if value.is_none() { + push_unique(missing, name); + } + value.map(BigInt::from) + }; + let item = match reg.kind { + RegistrationKind::LogData { .. } => EventItem { + on_event_registration_index: reg.index, + receipt_index: receipt.receipt_index, + tx_id: receipt.tx_id.clone(), + block_height: receipt.block_height, + src_address: src_address.clone(), + data: require_hex(&receipt.data, "receipt.data", &mut missing), + sub_id: None, + val: None, + amount: None, + asset_id: None, + to: None, + }, + RegistrationKind::Mint | RegistrationKind::Burn => EventItem { + on_event_registration_index: reg.index, + receipt_index: receipt.receipt_index, + tx_id: receipt.tx_id.clone(), + block_height: receipt.block_height, + src_address: src_address.clone(), + data: None, + sub_id: require_hex(&receipt.sub_id, "receipt.subId", &mut missing), + val: require_u64(receipt.val, "receipt.val", &mut missing), + amount: None, + asset_id: None, + to: None, + }, + RegistrationKind::Transfer | RegistrationKind::Call => { + // TransferOut receipts carry the wallet recipient in + // `to_address`; everything else uses `to`. + let (recipient, recipient_name) = + if receipt.receipt_type == selection::RECEIPT_TRANSFER_OUT { + (&receipt.to_address, "receipt.toAddress") + } else { + (&receipt.to, "receipt.to") + }; + EventItem { + on_event_registration_index: reg.index, + receipt_index: receipt.receipt_index, + tx_id: receipt.tx_id.clone(), + block_height: receipt.block_height, + src_address: src_address.clone(), + data: None, + sub_id: None, + val: None, + amount: require_u64(receipt.amount, "receipt.amount", &mut missing), + asset_id: require_hex(&receipt.asset_id, "receipt.assetId", &mut missing), + to: require_hex(recipient, recipient_name, &mut missing), + } + } + }; + items.push(item); + } + } + + if !missing.is_empty() { + return Err(ConvertError::MissingFields(missing)); } + Ok(items) } /// The client embeds a `{:?}` debug dump in its error message; keep only the @@ -80,6 +325,7 @@ fn map_err(e: anyhow::Error) -> napi::Error { #[cfg(test)] mod tests { + use super::selection::FuelEventKind; use super::*; #[test] @@ -93,4 +339,152 @@ mod tests { assert_eq!(parsed["fields"][0], "receipt.txId"); assert_eq!(parsed["fields"][1], "block.time"); } + + const ADDR: &str = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcde1"; + + fn reg_input( + index: i64, + contract_name: &str, + kind: FuelEventKind, + is_wildcard: bool, + log_id: Option<&str>, + ) -> FuelOnEventRegistrationInput { + FuelOnEventRegistrationInput { + index, + event_name: format!("E{index}"), + contract_name: contract_name.to_string(), + is_wildcard, + kind, + log_id: log_id.map(str::to_string), + } + } + + fn raw_receipt(receipt_type: u8) -> RawReceipt { + RawReceipt { + receipt_index: 3, + root_contract_id: Some(ADDR.to_string()), + tx_id: "0xtx".to_string(), + block_height: 42, + receipt_type, + data: Some("0x01".to_string()), + rb: Some(7), + val: Some(100), + sub_id: Some("0xsub".to_string()), + amount: Some(5), + asset_id: Some("0xasset".to_string()), + to: Some("0xto".to_string()), + to_address: Some("0xwallet".to_string()), + } + } + + fn block_42() -> Block { + Block { + id: "0xblock".to_string(), + height: 42, + time: 1745179292, + } + } + + fn build( + registrations: &[FuelOnEventRegistrationInput], + indexes: &[i64], + addresses: &[(&str, &[&str])], + ) -> BuiltSelection { + let addresses: HashMap> = addresses + .iter() + .map(|(name, addrs)| { + ( + name.to_string(), + addrs.iter().map(|a| a.to_string()).collect(), + ) + }) + .collect(); + SelectionBuilder::from_registrations(registrations) + .unwrap() + .build(indexes, &addresses) + .unwrap() + } + + #[test] + fn routes_transfer_out_recipient_into_to() { + let built = build( + &[reg_input(0, "C", FuelEventKind::Transfer, true, None)], + &[0], + &[], + ); + let items = route_receipts(vec![raw_receipt(8)], &[block_42()], &built).unwrap(); + assert_eq!( + items + .iter() + .map(|i| (i.on_event_registration_index, i.to.as_deref())) + .collect::>(), + vec![(0, Some("0xwallet"))] + ); + } + + #[test] + fn fans_out_to_wildcard_and_owned_registration() { + let built = build( + &[ + reg_input(0, "Owned", FuelEventKind::Mint, false, None), + reg_input(1, "W", FuelEventKind::Mint, true, None), + ], + &[0, 1], + &[("Owned", &[ADDR])], + ); + let items = route_receipts(vec![raw_receipt(11)], &[block_42()], &built).unwrap(); + assert_eq!( + items + .iter() + .map(|i| i.on_event_registration_index) + .collect::>(), + vec![0, 1] + ); + } + + #[test] + fn unrouted_receipt_is_dropped() { + let built = build( + &[reg_input(0, "C", FuelEventKind::LogData, true, Some("9"))], + &[0], + &[], + ); + // rb 7 doesn't match the registration's logId 9. + let items = route_receipts(vec![raw_receipt(6)], &[block_42()], &built).unwrap(); + assert!(items.is_empty()); + } + + #[test] + fn missing_kind_required_column_is_typed_error() { + let built = build( + &[reg_input(0, "C", FuelEventKind::Mint, true, None)], + &[0], + &[], + ); + let mut receipt = raw_receipt(11); + receipt.val = None; + match route_receipts(vec![receipt], &[block_42()], &built) { + Err(ConvertError::MissingFields(fields)) => { + assert_eq!(fields, vec!["receipt.val".to_string()]) + } + Err(ConvertError::Other(e)) => panic!("unexpected ConvertError::Other: {e:?}"), + Ok(_) => panic!("expected MissingFields, got Ok"), + } + } + + #[test] + fn missing_block_for_routed_receipt_is_typed_error() { + let built = build( + &[reg_input(0, "C", FuelEventKind::Mint, true, None)], + &[0], + &[], + ); + match route_receipts(vec![raw_receipt(11)], &[], &built) { + Err(ConvertError::MissingFields(fields)) => { + assert_eq!(fields, vec!["block".to_string()]) + } + Err(ConvertError::Other(e)) => panic!("unexpected ConvertError::Other: {e:?}"), + Ok(_) => panic!("expected MissingFields, got Ok"), + } + } } diff --git a/packages/cli/src/fuel_hypersync_source/query.rs b/packages/cli/src/fuel_hypersync_source/query.rs deleted file mode 100644 index 997be45c26..0000000000 --- a/packages/cli/src/fuel_hypersync_source/query.rs +++ /dev/null @@ -1,102 +0,0 @@ -use anyhow::{anyhow, Context, Result}; -use hyperfuel_client::format::{Hash, Hex}; -use hyperfuel_client::net_types; -use napi::bindgen_prelude::BigInt; -use napi_derive::napi; - -/// Query for retrieving Fuel receipts and their blocks. -#[napi(object)] -#[derive(Default)] -pub struct Query { - pub from_block: i64, - #[napi(js_name = "toBlock")] - pub to_block_exclusive: Option, - pub receipts: Option>, - pub field_selection: FieldSelection, -} - -#[napi(object)] -#[derive(Default)] -pub struct ReceiptSelection { - pub root_contract_id: Option>, - pub receipt_type: Option>, - pub rb: Option>, - pub tx_status: Option>, -} - -#[napi(object)] -#[derive(Default)] -pub struct FieldSelection { - pub block: Option>, - pub receipt: Option>, -} - -fn parse_hashes(v: Option>) -> Result> { - v.unwrap_or_default() - .into_iter() - .map(|s| Hash::decode_hex(&s).map_err(|e| anyhow!("failed to parse hash {s}: {e:?}"))) - .collect() -} - -fn bigints_to_u64(v: Option>) -> Result> { - v.unwrap_or_default() - .into_iter() - .map(|b| { - let (sign_bit, value, lossless) = b.get_u64(); - if sign_bit || !lossless { - anyhow::bail!("rb filter value must be an unsigned 64-bit integer"); - } - Ok(value) - }) - .collect() -} - -impl TryFrom for net_types::ReceiptSelection { - type Error = anyhow::Error; - - fn try_from(s: ReceiptSelection) -> Result { - Ok(net_types::ReceiptSelection { - root_contract_id: parse_hashes(s.root_contract_id)?, - receipt_type: s.receipt_type.unwrap_or_default(), - rb: bigints_to_u64(s.rb).context("parse rb filter")?, - tx_status: s.tx_status.unwrap_or_default(), - ..Default::default() - }) - } -} - -impl From for net_types::FieldSelection { - fn from(f: FieldSelection) -> Self { - net_types::FieldSelection { - block: f.block.unwrap_or_default().into_iter().collect(), - receipt: f.receipt.unwrap_or_default().into_iter().collect(), - ..Default::default() - } - } -} - -impl TryFrom for net_types::Query { - type Error = anyhow::Error; - - fn try_from(q: Query) -> Result { - let from_block = u64::try_from(q.from_block).context("from_block must be >= 0")?; - let to_block = q - .to_block_exclusive - .map(|b| u64::try_from(b).context("toBlock must be >= 0")) - .transpose()?; - let receipts = q - .receipts - .unwrap_or_default() - .into_iter() - .map(TryInto::try_into) - .collect::>>()?; - - Ok(net_types::Query { - from_block, - to_block, - receipts, - field_selection: q.field_selection.into(), - ..Default::default() - }) - } -} diff --git a/packages/cli/src/fuel_hypersync_source/selection.rs b/packages/cli/src/fuel_hypersync_source/selection.rs new file mode 100644 index 0000000000..3aa883e3ce --- /dev/null +++ b/packages/cli/src/fuel_hypersync_source/selection.rs @@ -0,0 +1,584 @@ +use std::collections::HashMap; + +use anyhow::{anyhow, Context, Result}; +use hyperfuel_client::format::{Hash, Hex}; +use hyperfuel_client::net_types; +use napi_derive::napi; + +// FuelVM receipt type codes (see FuelSDK.receiptType on the JS side). +const RECEIPT_CALL: u8 = 0; +const RECEIPT_LOG_DATA: u8 = 6; +const RECEIPT_TRANSFER: u8 = 7; +pub(crate) const RECEIPT_TRANSFER_OUT: u8 = 8; +const RECEIPT_MINT: u8 = 11; +const RECEIPT_BURN: u8 = 12; + +// Only receipts from successful transactions are indexed. +const TX_STATUS_SUCCESS: u8 = 1; + +/// Receipt kind of a registration, mirroring `Internal.fuelEventKind`. +/// `Transfer` covers both `Transfer` (to a contract) and `TransferOut` +/// (to a wallet address) receipts. +#[napi(string_enum)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum FuelEventKind { + LogData, + Mint, + Burn, + Transfer, + Call, +} + +/// Internal per-registration kind. Unlike the `FuelEventKind` boundary enum, +/// the `LogData` variant carries its parsed `rb`, so a LogData registration +/// can't exist without one and no other kind can carry a stray rb — the +/// invalid states the napi input's `kind`+`log_id` pair could express are +/// resolved once, at construction. +#[derive(Clone, Copy)] +pub(crate) enum RegistrationKind { + LogData { rb: u64 }, + Mint, + Burn, + Transfer, + Call, +} + +impl RegistrationKind { + fn receipt_types(&self) -> &'static [u8] { + match self { + RegistrationKind::LogData { .. } => &[RECEIPT_LOG_DATA], + RegistrationKind::Mint => &[RECEIPT_MINT], + RegistrationKind::Burn => &[RECEIPT_BURN], + RegistrationKind::Transfer => &[RECEIPT_TRANSFER, RECEIPT_TRANSFER_OUT], + RegistrationKind::Call => &[RECEIPT_CALL], + } + } +} + +/// The full per-(event, chain) registration crossing the boundary once at +/// client construction: routing identity plus the receipt-selection state +/// queries are built from. +#[napi(object)] +pub struct FuelOnEventRegistrationInput { + /// Chain-scoped sequential registration index; returned on every routed + /// item so JS resolves the registration by array index. + pub index: i64, + pub event_name: String, + pub contract_name: String, + pub is_wildcard: bool, + pub kind: FuelEventKind, + /// The LogData `rb` value as a decimal string (u64). Required for + /// `LogData`, ignored otherwise. + pub log_id: Option, +} + +pub(crate) struct Registration { + pub index: i64, + pub contract_name: String, + pub is_wildcard: bool, + pub kind: RegistrationKind, +} + +impl Registration { + /// Whether a receipt belongs to this registration: matching receipt type + /// (and `rb` for LogData), emitted by an allowed contract — wildcard + /// registrations accept any contract, contract-bound ones only their own. + pub(crate) fn matches( + &self, + receipt_type: u8, + rb: Option, + contract_name: Option<&str>, + ) -> bool { + let kind_matches = match self.kind { + RegistrationKind::LogData { rb: reg_rb } => { + receipt_type == RECEIPT_LOG_DATA && rb == Some(reg_rb) + } + kind => kind.receipt_types().contains(&receipt_type), + }; + kind_matches && (self.is_wildcard || contract_name == Some(self.contract_name.as_str())) + } +} + +/// Everything a source query needs that depends on the partition's selection +/// and current addresses. +pub(crate) struct BuiltSelection { + pub receipt_selections: Vec, + /// Inverted address index for routing (1:1 — each address belongs to one + /// contract). + pub contract_name_by_address: HashMap, + /// The selection's registrations sorted by index, for routing. + pub registrations: Vec>, + /// Which receipt columns the selection's kinds read, so the field + /// selection only requests what item building needs. + pub needs_log_data: bool, + pub needs_supply: bool, + pub needs_transfer: bool, + pub needs_call: bool, +} + +fn parse_root_contract_ids(addresses: &[String]) -> Result> { + addresses + .iter() + .map(|a| Hash::decode_hex(a).map_err(|e| anyhow!("failed to parse contract id {a}: {e:?}"))) + .collect() +} + +fn push_unique(values: &mut Vec, value: T) { + if !values.contains(&value) { + values.push(value); + } +} + +/// Builds per-query receipt selections from the registrations passed at client +/// construction. Registrations are keyed by their chain-scoped sequential +/// index; a query names the indexes of its partition's selection plus the +/// partition's current addresses per contract. +pub(crate) struct SelectionBuilder { + registrations: HashMap>, +} + +impl SelectionBuilder { + pub(crate) fn from_registrations( + registrations: &[FuelOnEventRegistrationInput], + ) -> Result { + let mut map = HashMap::new(); + for reg in registrations { + let kind = match reg.kind { + FuelEventKind::LogData => { + let log_id = reg.log_id.as_ref().with_context(|| { + format!("LogData registration {} is missing logId", reg.event_name) + })?; + let rb = log_id.parse::().with_context(|| { + format!("parse logId {} for event {}", log_id, reg.event_name) + })?; + RegistrationKind::LogData { rb } + } + FuelEventKind::Call => { + anyhow::ensure!( + reg.is_wildcard, + "Call receipt indexing currently supported only in wildcard mode" + ); + RegistrationKind::Call + } + FuelEventKind::Mint => RegistrationKind::Mint, + FuelEventKind::Burn => RegistrationKind::Burn, + FuelEventKind::Transfer => RegistrationKind::Transfer, + }; + let parsed = Registration { + index: reg.index, + contract_name: reg.contract_name.clone(), + is_wildcard: reg.is_wildcard, + kind, + }; + anyhow::ensure!( + map.insert(reg.index, std::sync::Arc::new(parsed)).is_none(), + "Duplicate registration index {} for event {}", + reg.index, + reg.event_name, + ); + } + Ok(Self { registrations: map }) + } + + pub(crate) fn build( + &self, + registration_indexes: &[i64], + addresses_by_contract_name: &HashMap>, + ) -> Result { + // Wildcard registrations pool into address-free selections; the rest + // group per contract so one contract's query can't fetch a sibling's + // receipts. + let mut wildcard_receipt_types: Vec = Vec::new(); + let mut wildcard_rbs: Vec = Vec::new(); + let mut receipt_types_by_contract: HashMap<&str, Vec> = HashMap::new(); + let mut rbs_by_contract: HashMap<&str, Vec> = HashMap::new(); + // First-appearance order of address-bound contracts, so the built + // query is stable across calls. + let mut ordered_contracts: Vec<&str> = Vec::new(); + + let mut registrations = Vec::with_capacity(registration_indexes.len()); + let mut needs_log_data = false; + let mut needs_supply = false; + let mut needs_transfer = false; + let mut needs_call = false; + + for id in registration_indexes { + let reg = self + .registrations + .get(id) + .with_context(|| format!("Unknown registration index {id} in query selection"))?; + registrations.push(reg.clone()); + match reg.kind { + RegistrationKind::LogData { .. } => needs_log_data = true, + RegistrationKind::Mint | RegistrationKind::Burn => needs_supply = true, + RegistrationKind::Transfer => needs_transfer = true, + RegistrationKind::Call => needs_call = true, + } + match (reg.kind, reg.is_wildcard) { + (RegistrationKind::LogData { rb }, true) => push_unique(&mut wildcard_rbs, rb), + (RegistrationKind::LogData { rb }, false) => push_unique( + rbs_by_contract + .entry(reg.contract_name.as_str()) + .or_default(), + rb, + ), + (kind, true) => { + for &receipt_type in kind.receipt_types() { + push_unique(&mut wildcard_receipt_types, receipt_type); + } + } + (kind, false) => { + let bucket = receipt_types_by_contract + .entry(reg.contract_name.as_str()) + .or_default(); + for &receipt_type in kind.receipt_types() { + push_unique(bucket, receipt_type); + } + } + } + if !reg.is_wildcard && !ordered_contracts.contains(®.contract_name.as_str()) { + ordered_contracts.push(reg.contract_name.as_str()); + } + } + // Deterministic item order per receipt, independent of the selection's + // index order. + registrations.sort_unstable_by_key(|reg| reg.index); + + let mut receipt_selections: Vec = Vec::new(); + if !wildcard_receipt_types.is_empty() { + receipt_selections.push(net_types::ReceiptSelection { + receipt_type: wildcard_receipt_types, + tx_status: vec![TX_STATUS_SUCCESS], + ..Default::default() + }); + } + if !wildcard_rbs.is_empty() { + receipt_selections.push(net_types::ReceiptSelection { + receipt_type: vec![RECEIPT_LOG_DATA], + rb: wildcard_rbs, + tx_status: vec![TX_STATUS_SUCCESS], + ..Default::default() + }); + } + for contract_name in ordered_contracts { + let addresses = match addresses_by_contract_name.get(contract_name) { + None => continue, + Some(addresses) if addresses.is_empty() => continue, + Some(addresses) => parse_root_contract_ids(addresses)?, + }; + if let Some(receipt_types) = receipt_types_by_contract.remove(contract_name) { + receipt_selections.push(net_types::ReceiptSelection { + root_contract_id: addresses.clone(), + receipt_type: receipt_types, + tx_status: vec![TX_STATUS_SUCCESS], + ..Default::default() + }); + } + if let Some(rbs) = rbs_by_contract.remove(contract_name) { + receipt_selections.push(net_types::ReceiptSelection { + root_contract_id: addresses, + receipt_type: vec![RECEIPT_LOG_DATA], + rb: rbs, + tx_status: vec![TX_STATUS_SUCCESS], + ..Default::default() + }); + } + } + + // Routing needs the whole partition index, including contracts with no + // selection in this query (their receipts still fall back to wildcards). + let mut contract_name_by_address = HashMap::new(); + for (contract_name, addresses) in addresses_by_contract_name { + for address in addresses { + contract_name_by_address.insert(address.clone(), contract_name.clone()); + } + } + + Ok(BuiltSelection { + receipt_selections, + contract_name_by_address, + registrations, + needs_log_data, + needs_supply, + needs_transfer, + needs_call, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ADDR_1: &str = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcde1"; + const ADDR_2: &str = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcde2"; + const ADDR_3: &str = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcde3"; + + fn reg( + index: i64, + contract_name: &str, + kind: FuelEventKind, + is_wildcard: bool, + log_id: Option<&str>, + ) -> FuelOnEventRegistrationInput { + FuelOnEventRegistrationInput { + index, + event_name: format!("E{index}"), + contract_name: contract_name.to_string(), + is_wildcard, + kind, + log_id: log_id.map(str::to_string), + } + } + + fn addresses(entries: &[(&str, &[&str])]) -> HashMap> { + entries + .iter() + .map(|(name, addrs)| { + ( + name.to_string(), + addrs.iter().map(|a| a.to_string()).collect(), + ) + }) + .collect() + } + + fn selection_view( + s: &net_types::ReceiptSelection, + ) -> (Vec, Vec, Vec, Vec) { + ( + s.root_contract_id.iter().map(Hex::encode_hex).collect(), + s.receipt_type.clone(), + s.rb.clone(), + s.tx_status.clone(), + ) + } + + #[test] + fn groups_receipt_types_and_rbs_per_contract() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, "C1", FuelEventKind::LogData, false, Some("1")), + reg(1, "C1", FuelEventKind::Mint, false, None), + reg(2, "C1", FuelEventKind::Burn, false, None), + reg(3, "C1", FuelEventKind::Transfer, false, None), + reg(4, "C2", FuelEventKind::LogData, false, Some("3")), + reg(5, "C2", FuelEventKind::Burn, false, None), + ]) + .unwrap(); + let built = builder + .build( + &[0, 1, 2, 3, 4, 5], + &addresses(&[("C1", &[ADDR_1, ADDR_2]), ("C2", &[ADDR_3])]), + ) + .unwrap(); + assert_eq!( + built + .receipt_selections + .iter() + .map(selection_view) + .collect::>(), + vec![ + ( + vec![ADDR_1.to_string(), ADDR_2.to_string()], + vec![ + RECEIPT_MINT, + RECEIPT_BURN, + RECEIPT_TRANSFER, + RECEIPT_TRANSFER_OUT + ], + vec![], + vec![TX_STATUS_SUCCESS], + ), + ( + vec![ADDR_1.to_string(), ADDR_2.to_string()], + vec![RECEIPT_LOG_DATA], + vec![1], + vec![TX_STATUS_SUCCESS], + ), + ( + vec![ADDR_3.to_string()], + vec![RECEIPT_BURN], + vec![], + vec![TX_STATUS_SUCCESS], + ), + ( + vec![ADDR_3.to_string()], + vec![RECEIPT_LOG_DATA], + vec![3], + vec![TX_STATUS_SUCCESS], + ), + ] + ); + assert_eq!( + built.contract_name_by_address.get(ADDR_3), + Some(&"C2".to_string()) + ); + } + + #[test] + fn wildcard_selections_stay_address_free() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, "C1", FuelEventKind::Call, true, None), + reg(1, "C1", FuelEventKind::LogData, true, Some("2")), + reg(2, "C2", FuelEventKind::LogData, true, Some("3")), + ]) + .unwrap(); + let built = builder.build(&[0, 1, 2], &HashMap::new()).unwrap(); + assert_eq!( + built + .receipt_selections + .iter() + .map(selection_view) + .collect::>(), + vec![ + (vec![], vec![RECEIPT_CALL], vec![], vec![TX_STATUS_SUCCESS]), + ( + vec![], + vec![RECEIPT_LOG_DATA], + vec![2, 3], + vec![TX_STATUS_SUCCESS], + ), + ] + ); + } + + #[test] + fn contract_without_addresses_is_skipped() { + let builder = + SelectionBuilder::from_registrations(&[reg(0, "C1", FuelEventKind::Mint, false, None)]) + .unwrap(); + let built = builder.build(&[0], &addresses(&[("C1", &[])])).unwrap(); + assert!(built.receipt_selections.is_empty()); + } + + #[test] + fn selection_subset_excludes_other_registrations() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, "C1", FuelEventKind::Mint, false, None), + reg(1, "C1", FuelEventKind::Burn, false, None), + ]) + .unwrap(); + let built = builder + .build(&[1], &addresses(&[("C1", &[ADDR_1])])) + .unwrap(); + assert_eq!( + built + .receipt_selections + .iter() + .map(selection_view) + .collect::>(), + vec![( + vec![ADDR_1.to_string()], + vec![RECEIPT_BURN], + vec![], + vec![TX_STATUS_SUCCESS] + )] + ); + } + + #[test] + fn non_wildcard_call_errors() { + let err = + SelectionBuilder::from_registrations(&[reg(0, "C1", FuelEventKind::Call, false, None)]) + .err() + .unwrap(); + assert!(format!("{err:#}") + .contains("Call receipt indexing currently supported only in wildcard mode")); + } + + #[test] + fn log_data_without_log_id_errors() { + let err = SelectionBuilder::from_registrations(&[reg( + 0, + "C1", + FuelEventKind::LogData, + false, + None, + )]) + .err() + .unwrap(); + assert!(format!("{err:#}").contains("missing logId")); + } + + #[test] + fn routing_fans_out_to_wildcards_and_owned_contract() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, "Owned", FuelEventKind::Mint, false, None), + reg(1, "W", FuelEventKind::Mint, true, None), + reg(2, "Other", FuelEventKind::Mint, false, None), + ]) + .unwrap(); + let built = builder + .build(&[0, 1, 2], &addresses(&[("Owned", &[ADDR_1])])) + .unwrap(); + let route = |contract_name: Option<&str>| -> Vec { + built + .registrations + .iter() + .filter(|reg| reg.matches(RECEIPT_MINT, None, contract_name)) + .map(|reg| reg.index) + .collect() + }; + assert_eq!((route(Some("Owned")), route(None)), (vec![0, 1], vec![1])); + } + + #[test] + fn transfer_matches_both_transfer_and_transfer_out() { + let builder = SelectionBuilder::from_registrations(&[reg( + 0, + "C", + FuelEventKind::Transfer, + true, + None, + )]) + .unwrap(); + let built = builder.build(&[0], &HashMap::new()).unwrap(); + let reg = &built.registrations[0]; + assert_eq!( + ( + reg.matches(RECEIPT_TRANSFER, None, None), + reg.matches(RECEIPT_TRANSFER_OUT, None, None), + reg.matches(RECEIPT_MINT, None, None), + ), + (true, true, false) + ); + } + + #[test] + fn log_data_routes_by_rb() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, "C", FuelEventKind::LogData, true, Some("1")), + reg(1, "C", FuelEventKind::LogData, true, Some("2")), + ]) + .unwrap(); + let built = builder.build(&[0, 1], &HashMap::new()).unwrap(); + let routed: Vec = built + .registrations + .iter() + .filter(|reg| reg.matches(RECEIPT_LOG_DATA, Some(2), None)) + .map(|reg| reg.index) + .collect(); + assert_eq!(routed, vec![1]); + } + + #[test] + fn every_selection_filters_to_successful_tx_status() { + // Only receipts from successful transactions are indexed, so every + // built selection — wildcard, contract-bound receipt types, and + // contract-bound rb — must carry `tx_status = [1]`. + let builder = SelectionBuilder::from_registrations(&[ + reg(0, "C1", FuelEventKind::Mint, false, None), + reg(1, "C1", FuelEventKind::LogData, false, Some("1")), + reg(2, "W", FuelEventKind::Call, true, None), + reg(3, "W", FuelEventKind::LogData, true, Some("2")), + ]) + .unwrap(); + let built = builder + .build(&[0, 1, 2, 3], &addresses(&[("C1", &[ADDR_1])])) + .unwrap(); + assert!(built + .receipt_selections + .iter() + .all(|s| s.tx_status == vec![TX_STATUS_SUCCESS])); + } +} diff --git a/packages/cli/src/fuel_hypersync_source/types.rs b/packages/cli/src/fuel_hypersync_source/types.rs index 9ad7185791..e81df65292 100644 --- a/packages/cli/src/fuel_hypersync_source/types.rs +++ b/packages/cli/src/fuel_hypersync_source/types.rs @@ -1,45 +1,40 @@ use anyhow::{Context, Result}; use hyperfuel_client::{ArrowBatch, ArrowResponse}; -use napi::bindgen_prelude::BigInt; use napi_derive::napi; use polars_arrow::array::{BinaryArray, Int64Array, StaticArray, UInt64Array, UInt8Array}; #[napi(object)] -pub struct QueryResponse { - pub archive_height: Option, - pub next_block: i64, - pub total_execution_time: i64, - pub data: QueryResponseData, -} - -#[napi(object)] -pub struct QueryResponseData { - pub receipts: Vec, - pub blocks: Vec, +#[derive(Clone)] +pub struct Block { + pub id: String, + pub height: i64, + pub time: i64, } -#[napi(object)] -pub struct Receipt { +/// The page's receipt rows, decoded from Arrow but not yet routed — `get_event_items` +/// turns each one into zero or more `EventItem`s. Numeric u64 columns stay raw +/// here; they become JS BigInts only on the routed items. +pub(crate) struct RawReceipt { pub receipt_index: i64, pub root_contract_id: Option, pub tx_id: String, pub block_height: i64, - pub receipt_type: i64, + pub receipt_type: u8, pub data: Option, - pub rb: Option, - pub val: Option, + pub rb: Option, + pub val: Option, pub sub_id: Option, - pub amount: Option, + pub amount: Option, pub asset_id: Option, pub to: Option, pub to_address: Option, } -#[napi(object)] -pub struct Block { - pub id: String, - pub height: i64, - pub time: i64, +pub(crate) struct RawResponse { + pub archive_height: Option, + pub next_block: i64, + pub receipts: Vec, + pub blocks: Vec, } /// `MissingFields` is the shape the JS side recognizes (via the JSON payload @@ -69,17 +64,13 @@ fn u64_at(arr: &Option<&UInt64Array>, idx: usize) -> Option { arr.and_then(|a| a.get(idx)) } -fn bigint_at(arr: &Option<&UInt64Array>, idx: usize) -> Option { - u64_at(arr, idx).map(BigInt::from) -} - fn i64_field(arr: &Option<&UInt64Array>, idx: usize, name: &str) -> Result> { u64_at(arr, idx) .map(|v| v.try_into().with_context(|| format!("{name} overflow"))) .transpose() } -fn receipts_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertError> { +pub(crate) fn receipts_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertError> { let mut out = Vec::new(); for batch in batches { let receipt_index = batch.column::("receipt_index").ok(); @@ -120,17 +111,17 @@ fn receipts_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertEr return Err(ConvertError::MissingFields(missing)); } - out.push(Receipt { + out.push(RawReceipt { receipt_index: receipt_index_val.unwrap(), root_contract_id: hex_at(&root_contract_id, idx), tx_id: tx_id_val.unwrap(), block_height: block_height_val.unwrap(), - receipt_type: receipt_type_val.unwrap() as i64, + receipt_type: receipt_type_val.unwrap(), data: hex_at(&data, idx), - rb: bigint_at(&rb, idx), - val: bigint_at(&val, idx), + rb: u64_at(&rb, idx), + val: u64_at(&val, idx), sub_id: hex_at(&sub_id, idx), - amount: bigint_at(&amount, idx), + amount: u64_at(&amount, idx), asset_id: hex_at(&asset_id, idx), to: hex_at(&to, idx), to_address: hex_at(&to_address, idx), @@ -140,7 +131,7 @@ fn receipts_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertEr Ok(out) } -fn blocks_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertError> { +pub(crate) fn blocks_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertError> { let mut out = Vec::new(); for batch in batches { let id = batch.column::>("id").ok(); @@ -175,8 +166,8 @@ fn blocks_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertError> Ok(out) } -pub(crate) fn convert_response(res: ArrowResponse) -> Result { - Ok(QueryResponse { +pub(crate) fn convert_response(res: ArrowResponse) -> Result { + Ok(RawResponse { archive_height: res .archive_height .map(i64::try_from) @@ -188,15 +179,8 @@ pub(crate) fn convert_response(res: ArrowResponse) -> Result, - pub instructions: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InstructionSchemaJson { +/// One instruction's schema piece, assembled from a registration's +/// `accounts`/`args` at client creation. +#[derive(Debug)] +pub(crate) struct InstructionSchemaInput { pub name: String, /// Hex (`0x`-prefixed or bare) — the bytes the dispatcher matches against /// the head of `instruction.data`. pub discriminator: String, - #[serde(default)] pub accounts: Vec, - #[serde(default)] pub args: Vec, } -/// Parse a JSON descriptor into an upstream `ProgramSchema`. The Solana client -/// calls this once per configured program at creation time; the resulting -/// schema's `program_id` is the key the client decodes against. -pub(crate) fn parse_program_schema(descriptor_json: &str) -> Result { - let descriptor: ProgramSchemaJson = - serde_json::from_str(descriptor_json).context("parse program schema descriptor")?; - build_program_schema(descriptor).context("build program schema") -} - /// Decode a raw instruction against a resolved schema. Called inline by the -/// Solana client's `get`, so decoded instructions ride back on the query -/// response instead of crossing the napi boundary one instruction at a time. +/// Solana client's `get_event_items`, so decoded instructions ride back on the +/// query response instead of crossing the napi boundary one instruction at a +/// time. /// /// POC policy: any decode failure (unknown discriminator, account-count /// mismatch, trailing bytes, unresolved type) yields `None` so the indexer @@ -112,9 +90,12 @@ impl From for DecodedInstructionJson { } } -fn build_program_schema(d: ProgramSchemaJson) -> Result { - let defined_types: BTreeMap = d - .defined_types +pub(crate) fn build_program_schema( + program_id: String, + defined_types: &BTreeMap, + instruction_inputs: Vec, +) -> Result { + let defined_types: BTreeMap = defined_types .iter() .map(|(name, ty)| { arg_type_to_field_type(ty) @@ -124,7 +105,7 @@ fn build_program_schema(d: ProgramSchemaJson) -> Result { .collect::>()?; let mut instructions: BTreeMap, UpstreamIxSchema> = BTreeMap::new(); - for ix in d.instructions { + for ix in instruction_inputs { let discriminator = hex_to_bytes(&ix.discriminator) .with_context(|| format!("instruction '{}' discriminator", ix.name))?; let accounts = ix @@ -165,7 +146,7 @@ fn build_program_schema(d: ProgramSchemaJson) -> Result { } Ok(UpstreamSchema::build( - d.program_id, + program_id, instructions, defined_types, )) diff --git a/packages/cli/src/svm_hypersync_source/config.rs b/packages/cli/src/svm_hypersync_source/config.rs index e233ec720c..726036ee6a 100644 --- a/packages/cli/src/svm_hypersync_source/config.rs +++ b/packages/cli/src/svm_hypersync_source/config.rs @@ -12,10 +12,6 @@ pub struct SvmClientConfig { pub max_num_retries: Option, pub retry_base_ms: Option, pub retry_ceiling_ms: Option, - /// Per-program Borsh schema descriptors (JSON, one per program). Built into - /// decoders at client creation; the client decodes each returned - /// instruction whose `program_id` matches one of them, attaching `decoded`. - pub program_schemas: Option>, } impl From for hypersync_client_solana::config::ClientConfig { diff --git a/packages/cli/src/svm_hypersync_source/mod.rs b/packages/cli/src/svm_hypersync_source/mod.rs index 21fe72796c..86ef3df8a6 100644 --- a/packages/cli/src/svm_hypersync_source/mod.rs +++ b/packages/cli/src/svm_hypersync_source/mod.rs @@ -1,12 +1,13 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use anyhow::Context; +use anyhow::{Context, Result}; use napi_derive::napi; mod borsh_decoder; mod config; mod query; +mod selection; pub(crate) mod types; /// Local hex helpers. Lives here so `decoder.rs` can pull them via @@ -22,8 +23,9 @@ pub(crate) mod mod_helpers { (0..s.len()) .step_by(2) .map(|i| { - u8::from_str_radix(&s[i..i + 2], 16) - .map_err(|_| anyhow!("invalid hex byte at offset {i} in '{input}'")) + s.get(i..i + 2) + .and_then(|byte| u8::from_str_radix(byte, 16).ok()) + .ok_or_else(|| anyhow!("invalid hex byte at offset {i} in '{input}'")) }) .collect() } @@ -31,12 +33,17 @@ pub(crate) mod mod_helpers { use hypersync_client_solana::decode::ProgramSchema as UpstreamSchema; use hypersync_client_solana::simple_types as simple; +use hypersync_solana_net_types::field_selection::SolanaFieldSelection; +use hypersync_solana_net_types::query::SolanaQuery; use crate::block_store::BlockStore; +use crate::config_parsing::human_config::svm::{ArgDef, ArgType}; use crate::transaction_store::TransactionStore; +use borsh_decoder::{DecodedInstructionJson, InstructionSchemaInput}; use config::SvmClientConfig; use query::SvmQuery; -use types::QueryResponse; +use selection::{route_instruction, SelectionBuilder, SvmOnEventRegistrationInput}; +use types::{opt_hex, to_hex, QueryResponse}; /// Move the response's transactions and token balances into a /// `TransactionStore`, keyed by `(slot, transactionIndex)`. Kept in Rust so @@ -54,20 +61,102 @@ fn build_svm_store( store } +/// Per-program Borsh schemas from the registrations that carry schema pieces +/// (`accounts`/`args`), keyed by base58 program id. A registration without a +/// schema (or without a discriminator to dispatch on) contributes nothing; +/// the program's `definedTypes` come from its first schema-carrying +/// registration — every registration of a program duplicates the same +/// registry. +fn build_schemas( + registrations: &[SvmOnEventRegistrationInput], +) -> Result> { + struct ProgramParts { + defined_types: BTreeMap, + instructions: Vec, + } + let mut parts_by_program: Vec<(String, ProgramParts)> = Vec::new(); + + for reg in registrations { + if reg.program_id.is_empty() { + continue; + } + let has_schema = !reg.accounts.is_empty() || reg.args_json.is_some(); + let discriminator = reg.discriminator.as_deref().unwrap_or_default(); + if !has_schema || discriminator.is_empty() { + continue; + } + let args: Vec = reg + .args_json + .as_deref() + .map(|json| { + serde_json::from_str(json) + .with_context(|| format!("parse args schema for {}", reg.instruction_name)) + }) + .transpose()? + .unwrap_or_default(); + let instruction = InstructionSchemaInput { + name: reg.instruction_name.clone(), + discriminator: discriminator.to_string(), + accounts: reg.accounts.clone(), + args, + }; + match parts_by_program + .iter_mut() + .find(|(program_id, _)| program_id == ®.program_id) + { + Some((_, parts)) => parts.instructions.push(instruction), + None => { + let defined_types: BTreeMap = reg + .defined_types_json + .as_deref() + .map(|json| { + serde_json::from_str(json).with_context(|| { + format!("parse defined types for {}", reg.instruction_name) + }) + }) + .transpose()? + .unwrap_or_default(); + parts_by_program.push(( + reg.program_id.clone(), + ProgramParts { + defined_types, + instructions: vec![instruction], + }, + )); + } + } + } + + parts_by_program + .into_iter() + .map(|(program_id, parts)| { + let schema = borsh_decoder::build_program_schema( + program_id.clone(), + &parts.defined_types, + parts.instructions, + ) + .with_context(|| format!("build program schema for {program_id}"))?; + Ok((program_id, schema)) + }) + .collect() +} + #[napi] -pub struct SvmHypersyncClient { +pub struct SvmHyperSyncClient { inner: Arc, - /// Per-program Borsh schemas, keyed by base58 program id, built once from - /// the config at client creation. `get` decodes matching instructions - /// against these. schemas: HashMap, + selection_builder: SelectionBuilder, } #[napi] -impl SvmHypersyncClient { +impl SvmHyperSyncClient { #[napi(constructor)] - pub fn new(cfg: SvmClientConfig, user_agent: String) -> napi::Result { - Self::from_config(cfg, user_agent) + pub fn new( + cfg: SvmClientConfig, + user_agent: String, + event_registrations: Vec, + ) -> napi::Result { + Self::from_config(cfg, user_agent, event_registrations) } /// Factory taking a custom user agent, mirroring EVM's `new_with_agent`. @@ -77,18 +166,21 @@ impl SvmHypersyncClient { pub fn from_config( cfg: SvmClientConfig, user_agent: String, - ) -> napi::Result { - let mut schemas = HashMap::new(); - for descriptor_json in cfg.program_schemas.clone().unwrap_or_default() { - let schema = borsh_decoder::parse_program_schema(&descriptor_json).map_err(map_err)?; - schemas.insert(schema.program_id.clone(), schema); - } + event_registrations: Vec, + ) -> napi::Result { + let schemas = build_schemas(&event_registrations) + .context("build program schemas") + .map_err(map_err)?; + let selection_builder = SelectionBuilder::from_registrations(&event_registrations) + .context("build selection builder") + .map_err(map_err)?; let inner = hypersync_client_solana::Client::new_with_agent(cfg.into(), user_agent) .context("build solana client") .map_err(map_err)?; - Ok(SvmHypersyncClient { + Ok(SvmHyperSyncClient { inner: Arc::new(inner), schemas, + selection_builder, }) } @@ -100,16 +192,17 @@ impl SvmHypersyncClient { .map_err(map_err) } - /// Single-window query (no client-side pagination). The hyperindex source - /// layer paginates by chunking the slot range itself, so the napi binding - /// must NOT call `collect` (which spins up parallel batched requests under - /// `StreamConfig::default()` and can DoS the server on multi-day windows). + /// Single-window query (no client-side pagination), used for block-hash + /// range queries. The hyperindex source layer paginates by chunking the + /// slot range itself, so the napi binding must NOT call `collect` (which + /// spins up parallel batched requests under `StreamConfig::default()` and + /// can DoS the server on multi-day windows). #[napi] pub async fn get( &self, query: SvmQuery, ) -> napi::Result<(QueryResponse, TransactionStore, BlockStore)> { - let q: hypersync_solana_net_types::query::SolanaQuery = query + let q: SolanaQuery = query .try_into() .context("parse solana query") .map_err(map_err)?; @@ -120,62 +213,304 @@ impl SvmHypersyncClient { .context("solana get") .map_err(map_err)?; - // Decode each matching instruction inline against the client's - // configured schemas — Borsh decoding happens here, in Rust, rather - // than per-instruction over the napi boundary, mirroring how the EVM - // client returns pre-decoded params. Skipped when no schemas exist. - let decoded: Vec> = if self.schemas.is_empty() - { - Vec::new() - } else { - resp.instructions - .iter() - .map(|ix| { - self.schemas.get(&ix.program_id).and_then(|schema| { - borsh_decoder::decode_with_schema( - schema, - ix.accounts.clone(), - ix.data.clone(), - ) - }) - }) - .collect() - }; - - // Retain raw transactions + token balances in Rust; ReScript builds - // items from instructions and the store materialises the parent - // transaction (selected fields only) at batch prep. + // Retain raw transactions + token balances in Rust; the store + // materialises the parent transaction (selected fields only) at batch + // prep. let store = build_svm_store( std::mem::take(&mut resp.transactions), std::mem::take(&mut resp.token_balances), ); - // Take the raw blocks out before the response conversion consumes them, - // build the lean per-slot header from a borrow, then move the owned raw - // blocks into the store — avoiding a full raw-`Block` clone per block - // (mirrors the EVM source's borrow-then-move header pattern). - let raw_blocks = std::mem::take(&mut resp.blocks); - let block_headers: Vec = raw_blocks - .iter() - .map(types::Block::from_raw) - .collect::>>() - .context("mapping solana block headers") - .map_err(map_err)?; - - // slot/time/hash decode from the store like any other field, so every - // response block needs a store entry. - let block_store = BlockStore::new_svm(); - block_store.insert_svm_blocks(raw_blocks); + let (block_headers, block_store) = take_blocks(&mut resp).map_err(map_err)?; let mut out = QueryResponse::try_from(resp) .context("convert solana response") .map_err(map_err)?; out.data.blocks = block_headers; - for (instr, d) in out.data.instructions.iter_mut().zip(decoded) { - instr.decoded = d; - } Ok((out, store, block_store)) } + + #[napi] + pub async fn get_event_items( + &self, + params: EventItemsQuery, + ) -> napi::Result<(EventItemsResponse, TransactionStore, BlockStore)> { + let built = self + .selection_builder + .build(¶ms.registration_indexes) + .map_err(map_err)?; + + let mut field_selection = SolanaFieldSelection { + block: parse_columns(&built.block_columns).map_err(map_err)?, + // Instructions keep the server's full column set — everything + // item building reads (data, accounts, dN, addresses, flags). + ..Default::default() + }; + // Under the server's default merge mode, requesting a table's columns + // is what opts the matched result set into that join — a table with an + // empty field list returns no rows (instructions and blocks are + // exempt), so each opted-into table needs its columns spelled out. + if !built.transaction_columns.is_empty() { + field_selection.transaction = + parse_columns(&built.transaction_columns).map_err(map_err)?; + } + if built.needs_logs { + field_selection.log = parse_columns(&[ + "slot", + "transaction_index", + "instruction_address", + "kind", + "message", + ]) + .map_err(map_err)?; + } + if built.needs_token_balances { + // The store keys balance rows by account regardless of what the + // consumer selected, so `account` always rides along. + field_selection.token_balance = parse_columns(&[ + "slot", + "transaction_index", + "account", + "mint", + "owner", + "pre_amount", + "post_amount", + ]) + .map_err(map_err)?; + } + + let query = SolanaQuery { + from_slot: u64::try_from(params.from_slot) + .context("from_slot must be non-negative") + .map_err(map_err)?, + // Inclusive on the boundary, exclusive on the wire. + to_slot: params + .to_slot + .map(|b| { + u64::try_from(b) + .context("to_slot must be non-negative")? + .checked_add(1) + .context("to_slot overflow") + }) + .transpose() + .map_err(map_err)?, + instructions: built.instruction_selections.clone(), + field_selection, + max_num_instructions: usize::try_from(params.max_num_instructions).ok(), + ..Default::default() + }; + + let mut resp = self + .inner + .get(&query) + .await + .context("solana get") + .map_err(map_err)?; + + let store = build_svm_store( + std::mem::take(&mut resp.transactions), + std::mem::take(&mut resp.token_balances), + ); + let (block_headers, block_store) = take_blocks(&mut resp).map_err(map_err)?; + + let mut contract_name_by_address: HashMap = HashMap::new(); + for (contract_name, addresses) in ¶ms.addresses_by_contract_name { + for address in addresses { + contract_name_by_address.insert(address.clone(), contract_name.clone()); + } + } + + let items = build_event_items( + &resp.instructions, + std::mem::take(&mut resp.logs), + &built, + &self.schemas, + &contract_name_by_address, + ) + .map_err(map_err)?; + + let response = EventItemsResponse { + next_slot: i64::try_from(resp.next_slot) + .context("next_slot overflow") + .map_err(map_err)?, + blocks: block_headers, + items, + }; + Ok((response, store, block_store)) + } +} + +/// The whole per-query input for `get_event_items`: the slot range, the +/// partition's registration selection (by index), and its current addresses +/// (program ids per program name). Instruction selections, field selection, +/// and routing are all derived internally from the registrations passed at +/// construction. +#[napi(object)] +pub struct EventItemsQuery { + pub from_slot: i64, + /// Inclusive; `None` queries to the end of available data. + pub to_slot: Option, + pub max_num_instructions: i64, + pub registration_indexes: Vec, + pub addresses_by_contract_name: HashMap>, +} + +#[napi(object)] +#[derive(Clone)] +pub struct LogItem { + pub kind: String, + pub message: String, +} + +/// One routed instruction. Carries everything JS needs to build the handler +/// payload; the parent transaction and block are materialised from the +/// per-chain stores at batch prep. +#[napi(object)] +pub struct EventItem { + /// The registration this instruction routed to, as passed to the client + /// constructor. Instructions that route nowhere never cross the boundary. + pub on_event_registration_index: i64, + pub slot: i64, + pub transaction_index: i64, + pub instruction_address: Vec, + pub program_id: String, + pub accounts: Vec, + /// Raw instruction data, `0x`-prefixed hex; decoded params ride on + /// `decoded` when the registration carries a Borsh schema. + pub data: String, + pub d1: Option, + pub d2: Option, + pub d4: Option, + pub d8: Option, + pub is_inner: bool, + pub decoded: Option, + /// Logs scoped to this instruction; `Some` only when the routed + /// registration opted in via `includeLogs`. + pub logs: Option>, +} + +#[napi(object)] +pub struct EventItemsResponse { + pub next_slot: i64, + /// The page's lean block headers, one per slot; used for reorg detection + /// and the batch's latest timestamp. The full blocks live in the + /// `BlockStore` returned alongside this response. + pub blocks: Vec, + pub items: Vec, +} + +/// Fans each committed instruction out to the registrations it routes to. +/// Logs group per (slot, transactionIndex, instructionAddress) and attach only +/// to items whose registration opted in via `includeLogs`; logs without an +/// instruction address attach to no instruction (rare; usually only system +/// messages). Borsh decoding runs once per instruction against its program's +/// schema, when one exists. +fn build_event_items( + instructions: &[simple::Instruction], + logs: Vec, + built: &selection::BuiltSelection, + schemas: &HashMap, + contract_name_by_address: &HashMap, +) -> Result> { + let mut logs_by_key: HashMap<(u64, u32, Vec), Vec> = HashMap::new(); + for log in logs { + if let (Some(transaction_index), Some(instruction_address)) = + (log.transaction_index, log.instruction_address) + { + logs_by_key + .entry((log.slot, transaction_index, instruction_address)) + .or_default() + .push(LogItem { + kind: log.kind.unwrap_or_default(), + message: log.message.unwrap_or_default(), + }); + } + } + + let mut items: Vec = Vec::with_capacity(instructions.len()); + for instr in instructions { + // Instructions from failed transactions are excluded. HyperSync has no + // server-side predicate to filter instructions by parent-transaction + // success, so the client filters on the `isCommitted` flag it already + // delivers on every instruction row. + if !instr.is_committed { + continue; + } + let contract_name = contract_name_by_address + .get(&instr.program_id) + .map(String::as_str); + let routed = route_instruction(&built.registrations, instr, contract_name); + if routed.is_empty() { + continue; + } + let decoded = schemas.get(&instr.program_id).and_then(|schema| { + borsh_decoder::decode_with_schema(schema, instr.accounts.clone(), instr.data.clone()) + }); + let logs = if routed.iter().any(|reg| reg.include_logs) { + logs_by_key + .get(&( + instr.slot, + instr.transaction_index, + instr.instruction_address.clone(), + )) + .cloned() + } else { + None + }; + let slot = i64::try_from(instr.slot).context("instruction.slot overflow")?; + for reg in routed { + items.push(EventItem { + on_event_registration_index: reg.index, + slot, + transaction_index: i64::from(instr.transaction_index), + instruction_address: instr + .instruction_address + .iter() + .map(|&v| i64::from(v)) + .collect(), + program_id: instr.program_id.clone(), + accounts: instr.accounts.clone(), + data: to_hex(&instr.data), + d1: opt_hex(&instr.d1), + d2: opt_hex(&instr.d2), + d4: opt_hex(&instr.d4), + d8: opt_hex(&instr.d8), + is_inner: instr.is_inner, + decoded: decoded.clone(), + logs: if reg.include_logs { logs.clone() } else { None }, + }); + } + } + Ok(items) +} + +fn parse_columns(columns: &[&str]) -> Result> +where + F: std::str::FromStr, +{ + columns + .iter() + .map(|name| { + name.parse::() + .map_err(|_| anyhow::anyhow!("unknown field name {name:?}")) + }) + .collect() +} + +/// Take the raw blocks out of the response, build the lean per-slot header +/// from a borrow, then move the owned raw blocks into the store — avoiding a +/// full raw-`Block` clone per block. slot/time/hash decode from the store like +/// any other field, so every response block needs a store entry. +fn take_blocks(resp: &mut simple::SolanaResponse) -> Result<(Vec, BlockStore)> { + let raw_blocks = std::mem::take(&mut resp.blocks); + let block_headers: Vec = raw_blocks + .iter() + .map(types::Block::from_raw) + .collect::>>() + .context("mapping solana block headers")?; + let block_store = BlockStore::new_svm(); + block_store.insert_svm_blocks(raw_blocks); + Ok((block_headers, block_store)) } pub(crate) fn map_err(e: anyhow::Error) -> napi::Error { @@ -194,12 +529,13 @@ mod tests { #[tokio::test] #[ignore] async fn live_query_token_metadata() { - let client = SvmHypersyncClient::new( + let client = SvmHyperSyncClient::new( SvmClientConfig { url: "https://solana.hypersync.xyz".into(), ..Default::default() }, "hyperindex-test".into(), + vec![], ) .expect("build client"); @@ -236,4 +572,164 @@ mod tests { assert!(ix.data.len() > 2, "data should not be empty hex"); } } + + fn reg_input( + index: i64, + discriminator: &str, + include_logs: bool, + ) -> SvmOnEventRegistrationInput { + SvmOnEventRegistrationInput { + index, + instruction_name: format!("I{index}"), + contract_name: "TokenMetadata".to_string(), + program_id: TOKEN_METADATA_PROGRAM.to_string(), + is_wildcard: true, + discriminator: Some(discriminator.to_string()), + discriminator_byte_len: 1, + is_inner: None, + include_logs, + account_filters: vec![], + transaction_fields: vec![], + block_fields: vec![], + accounts: vec![], + args_json: None, + defined_types_json: None, + } + } + + fn committed_instruction(data: &[u8]) -> simple::Instruction { + simple::Instruction { + program_id: TOKEN_METADATA_PROGRAM.to_string(), + data: data.to_vec(), + slot: 42, + transaction_index: 7, + instruction_address: vec![1], + is_committed: true, + ..Default::default() + } + } + + #[test] + fn uncommitted_instructions_are_dropped() { + let built = SelectionBuilder::from_registrations(&[reg_input(0, "0x21", false)]) + .unwrap() + .build(&[0]) + .unwrap(); + let committed = committed_instruction(&[0x21]); + let mut uncommitted = committed_instruction(&[0x21]); + uncommitted.is_committed = false; + uncommitted.transaction_index = 8; + let items = build_event_items( + &[committed, uncommitted], + vec![], + &built, + &HashMap::new(), + &HashMap::new(), + ) + .unwrap(); + assert_eq!( + items + .iter() + .map(|i| ( + i.on_event_registration_index, + i.transaction_index, + i.data.as_str() + )) + .collect::>(), + vec![(0, 7, "0x21")] + ); + } + + #[test] + fn logs_attach_only_to_opted_in_registrations() { + // Two registrations fan out from the same instruction; only the + // includeLogs one carries the instruction-scoped log. + let built = SelectionBuilder::from_registrations(&[reg_input(0, "0x21", true), { + let mut with_different_contract = reg_input(1, "0x21", false); + with_different_contract.contract_name = "Other".to_string(); + with_different_contract + }]) + .unwrap() + .build(&[0, 1]) + .unwrap(); + let instr = committed_instruction(&[0x21]); + let log = simple::Log { + slot: 42, + transaction_index: Some(7), + instruction_address: Some(vec![1]), + kind: Some("data".to_string()), + message: Some("hello".to_string()), + ..Default::default() + }; + let unscoped_log = simple::Log { + slot: 42, + transaction_index: Some(7), + instruction_address: None, + ..Default::default() + }; + let items = build_event_items( + &[instr], + vec![log, unscoped_log], + &built, + &HashMap::new(), + &HashMap::new(), + ) + .unwrap(); + let views: Vec<(i64, Option>)> = items + .iter() + .map(|i| { + ( + i.on_event_registration_index, + i.logs.as_ref().map(|logs| { + logs.iter() + .map(|l| (l.kind.clone(), l.message.clone())) + .collect() + }), + ) + }) + .collect(); + assert_eq!( + views, + vec![ + (0, Some(vec![("data".to_string(), "hello".to_string())])), + (1, None), + ] + ); + } + + #[test] + fn schemas_group_instructions_per_program_and_skip_schemaless() { + let with_schema = + |index: i64, name: &str, discriminator: &str| selection::SvmOnEventRegistrationInput { + index, + instruction_name: name.to_string(), + contract_name: "TokenMetadata".to_string(), + program_id: TOKEN_METADATA_PROGRAM.to_string(), + is_wildcard: false, + discriminator: Some(discriminator.to_string()), + discriminator_byte_len: 1, + is_inner: None, + include_logs: false, + account_filters: vec![], + transaction_fields: vec![], + block_fields: vec![], + accounts: vec!["metadata".to_string()], + args_json: Some(r#"[{"name":"amount","type":"u64"}]"#.to_string()), + defined_types_json: None, + }; + let mut schemaless = with_schema(2, "NoSchema", "0x03"); + schemaless.accounts = vec![]; + schemaless.args_json = None; + + let schemas = build_schemas(&[ + with_schema(0, "CreateV1", "0x21"), + with_schema(1, "UpdateV1", "0x0f"), + schemaless, + ]) + .unwrap(); + assert_eq!( + schemas.keys().collect::>(), + vec![TOKEN_METADATA_PROGRAM] + ); + } } diff --git a/packages/cli/src/svm_hypersync_source/selection.rs b/packages/cli/src/svm_hypersync_source/selection.rs new file mode 100644 index 0000000000..2e282ff544 --- /dev/null +++ b/packages/cli/src/svm_hypersync_source/selection.rs @@ -0,0 +1,722 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use hypersync_client_solana::simple_types as simple; +use hypersync_solana_net_types::query as net; +use napi_derive::napi; + +use super::mod_helpers::hex_to_bytes; + +#[napi(object)] +#[derive(Clone)] +pub struct SvmAccountFilterInput { + /// Positional account index (`a0`..`a9` on the wire). + pub position: i64, + /// Base58 pubkeys; the account at `position` must be one of them. + pub values: Vec, +} + +/// The full per-(instruction, chain) registration crossing the boundary once +/// at client construction: routing identity, the fetch state queries are +/// built from, and the Borsh schema used for inline decoding. +#[napi(object)] +pub struct SvmOnEventRegistrationInput { + /// Chain-scoped sequential registration index; returned on every routed + /// item so JS resolves the registration by array index. + pub index: i64, + pub instruction_name: String, + /// Program name (the config's contract name). + pub contract_name: String, + /// Base58 program id. Empty means the config carries no real program + /// (placeholder); such a registration is never fetched or routed. + pub program_id: String, + pub is_wildcard: bool, + /// Hex-encoded discriminator. `None` matches every instruction in the + /// program (lowest routing priority). + pub discriminator: Option, + /// Discriminator length in bytes (1/2/4/8); 0 when no discriminator. + pub discriminator_byte_len: i64, + /// `None` matches both outer and inner (CPI-invoked) instructions. + pub is_inner: Option, + pub include_logs: bool, + /// Disjunctive normal form: outer array is OR of AND-groups. + pub account_filters: Vec>, + /// Selected transaction fields, camelCase (`Internal.svmTransactionField`). + pub transaction_fields: Vec, + /// Selected block fields, camelCase (`Internal.svmBlockField`). + pub block_fields: Vec, + /// Positional account names from the Borsh schema, in declared order. + /// Empty (with `args_json` absent) means no schema for this instruction. + pub accounts: Vec, + /// Borsh args layout as `Vec` JSON. Absent means no schema. + pub args_json: Option, + /// Program-level nominal-type registry (`BTreeMap` JSON), + /// duplicated on every instruction of the program. + pub defined_types_json: Option, +} + +/// Maps a selected transaction field to the extra query column it needs. +/// `transactionIndex` is always fetched as the store key, and `tokenBalances` +/// lives in a separate table, so neither adds a transaction column. +fn transaction_field_column(field: &str) -> Result> { + Ok(match field { + "transactionIndex" | "tokenBalances" => None, + "signatures" => Some("signatures"), + "feePayer" => Some("fee_payer"), + "success" => Some("success"), + "err" => Some("err"), + "fee" => Some("fee"), + "computeUnitsConsumed" => Some("compute_units_consumed"), + "accountKeys" => Some("account_keys"), + "recentBlockhash" => Some("recent_blockhash"), + "version" => Some("version"), + other => anyhow::bail!("unknown transaction field {other:?}"), + }) +} + +/// Maps a selected block field to its query column. slot/time/hash are always +/// fetched (as slot/block_time/blockhash), so they add no extra column. +fn block_field_column(field: &str) -> Result> { + Ok(match field { + "slot" | "time" | "hash" => None, + "height" => Some("block_height"), + "parentSlot" => Some("parent_slot"), + "parentHash" => Some("parent_blockhash"), + other => anyhow::bail!("unknown block field {other:?}"), + }) +} + +pub(crate) struct Registration { + pub index: i64, + pub contract_name: String, + pub program_id: String, + pub is_wildcard: bool, + /// Decoded discriminator bytes; `None` = program-wide. + pub discriminator: Option>, + /// Original hex value for the query's `dN` filter. + pub discriminator_hex: Option, + pub byte_len: usize, + pub is_inner: Option, + pub include_logs: bool, + pub account_filters: Vec)>>, + pub transaction_columns: Vec<&'static str>, + pub needs_token_balances: bool, + pub block_columns: Vec<&'static str>, +} + +impl Registration { + fn parse(input: &SvmOnEventRegistrationInput) -> Result { + let discriminator = input + .discriminator + .as_deref() + .filter(|d| !d.is_empty()) + .map(|d| hex_to_bytes(d).context("decode discriminator hex")) + .transpose()?; + let byte_len = usize::try_from(input.discriminator_byte_len) + .context("discriminator_byte_len out of range")?; + if let Some(bytes) = &discriminator { + anyhow::ensure!( + matches!(byte_len, 1 | 2 | 4 | 8) && bytes.len() == byte_len, + "discriminator byte length must be 1/2/4/8 and match the value, got {} bytes declared as {}", + bytes.len(), + byte_len, + ); + } + let account_filters = input + .account_filters + .iter() + .map(|group| { + group + .iter() + .map(|filter| { + let position = usize::try_from(filter.position) + .ok() + .filter(|p| *p <= 9) + .with_context(|| { + format!("account filter position {} out of a0..a9", filter.position) + })?; + Ok((position, filter.values.clone())) + }) + .collect::>>() + }) + .collect::>>()?; + let mut transaction_columns = Vec::new(); + let mut needs_token_balances = false; + for field in &input.transaction_fields { + if field == "tokenBalances" { + needs_token_balances = true; + } + if let Some(column) = transaction_field_column(field)? { + if !transaction_columns.contains(&column) { + transaction_columns.push(column); + } + } + } + let mut block_columns = Vec::new(); + for field in &input.block_fields { + if let Some(column) = block_field_column(field)? { + if !block_columns.contains(&column) { + block_columns.push(column); + } + } + } + Ok(Self { + index: input.index, + contract_name: input.contract_name.clone(), + program_id: input.program_id.clone(), + is_wildcard: input.is_wildcard, + discriminator, + discriminator_hex: input.discriminator.clone().filter(|d| !d.is_empty()), + byte_len, + is_inner: input.is_inner, + include_logs: input.include_logs, + account_filters, + transaction_columns, + needs_token_balances, + block_columns, + }) + } + + /// Whether an instruction belongs to this registration, discriminator + /// aside: same program, an allowed owner (wildcard registrations accept + /// any program address, program-bound ones only their own contract), the + /// `isInner` constraint, and the registration's account filters — the + /// filters are re-applied here so an instruction fetched for a sibling + /// selection can't leak into a registration whose own filter rejects it. + fn matches_scope(&self, instr: &simple::Instruction, contract_name: Option<&str>) -> bool { + self.program_id == instr.program_id + && (self.is_wildcard || contract_name == Some(self.contract_name.as_str())) + && self + .is_inner + .is_none_or(|is_inner| is_inner == instr.is_inner) + && (self.account_filters.is_empty() + || self.account_filters.iter().any(|group| { + group.iter().all(|(position, values)| { + instr + .accounts + .get(*position) + .is_some_and(|account| values.contains(account)) + }) + })) + } + + fn matches_discriminator(&self, data: &[u8]) -> bool { + match &self.discriminator { + Some(bytes) => { + data.len() >= self.byte_len && &data[..self.byte_len] == bytes.as_slice() + } + None => false, + } + } +} + +/// One instruction selection of a built query, before conversion to the wire +/// type; `PartialEq` so identical selections from same-signature +/// registrations are deduplicated. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +struct BuiltInstructionSelection { + program_id: String, + discriminator_hex: Option, + byte_len: usize, + accounts: [Option>; 10], + is_inner: Option, +} + +impl BuiltInstructionSelection { + fn into_net(self) -> net::InstructionSelection { + let mut selection = net::InstructionSelection { + program_id: vec![self.program_id], + is_inner: self.is_inner, + ..Default::default() + }; + if let Some(d) = self.discriminator_hex { + match self.byte_len { + 1 => selection.d1 = vec![d], + 2 => selection.d2 = vec![d], + 4 => selection.d4 = vec![d], + 8 => selection.d8 = vec![d], + _ => {} + } + } + let [a0, a1, a2, a3, a4, a5, a6, a7, a8, a9] = self.accounts; + selection.a0 = a0.unwrap_or_default(); + selection.a1 = a1.unwrap_or_default(); + selection.a2 = a2.unwrap_or_default(); + selection.a3 = a3.unwrap_or_default(); + selection.a4 = a4.unwrap_or_default(); + selection.a5 = a5.unwrap_or_default(); + selection.a6 = a6.unwrap_or_default(); + selection.a7 = a7.unwrap_or_default(); + selection.a8 = a8.unwrap_or_default(); + selection.a9 = a9.unwrap_or_default(); + selection + } +} + +/// Everything a source query needs that depends on the partition's selection. +pub(crate) struct BuiltSelection { + pub instruction_selections: Vec, + /// Union over the selection's registrations; always contains the + /// slot/blockhash/block_time trio. + pub block_columns: Vec<&'static str>, + /// Union over the selection's registrations; non-empty only when a stored + /// transaction record is actually read (then it carries the + /// slot/transaction_index store key too). + pub transaction_columns: Vec<&'static str>, + pub needs_token_balances: bool, + pub needs_logs: bool, + /// The selection's registrations sorted by index, for routing. + pub registrations: Vec>, +} + +/// Builds per-query instruction selections and field unions from the +/// registrations passed at client construction, and routes returned +/// instructions back to them. +pub(crate) struct SelectionBuilder { + registrations: HashMap>, +} + +impl SelectionBuilder { + pub(crate) fn from_registrations( + registrations: &[SvmOnEventRegistrationInput], + ) -> Result { + let mut map = HashMap::new(); + for reg in registrations { + let parsed = Registration::parse(reg) + .with_context(|| format!("parse registration for {}", reg.instruction_name))?; + anyhow::ensure!( + map.insert(reg.index, Arc::new(parsed)).is_none(), + "Duplicate registration index {} for instruction {}", + reg.index, + reg.instruction_name, + ); + } + Ok(Self { registrations: map }) + } + + pub(crate) fn build(&self, registration_indexes: &[i64]) -> Result { + let mut selections: Vec = Vec::new(); + // The always-fetched trio: `slot` keys the page's blocks, and the + // consumer reads time/hash off every block (reorg detection, item + // timestamps). + let mut block_columns = vec!["slot", "blockhash", "block_time"]; + let mut transaction_columns: Vec<&'static str> = Vec::new(); + let mut needs_token_balances = false; + let mut needs_logs = false; + let mut registrations = Vec::with_capacity(registration_indexes.len()); + + for id in registration_indexes { + let reg = self + .registrations + .get(id) + .with_context(|| format!("Unknown registration index {id} in query selection"))?; + registrations.push(reg.clone()); + + for &column in ®.block_columns { + if !block_columns.contains(&column) { + block_columns.push(column); + } + } + for &column in ®.transaction_columns { + if !transaction_columns.contains(&column) { + transaction_columns.push(column); + } + } + needs_token_balances = needs_token_balances || reg.needs_token_balances; + needs_logs = needs_logs || reg.include_logs; + + // Placeholder configs carry no real program — skip rather than + // ship a degenerate match-all selection. + if reg.program_id.is_empty() { + continue; + } + // Each AND-group becomes its own selection; groups sharing the + // same `(programId, dN)` are OR-ed by the wire protocol. An empty + // outer array emits one selection with no account filtering. + let groups: &[Vec<(usize, Vec)>] = if reg.account_filters.is_empty() { + &[Vec::new()] + } else { + ®.account_filters + }; + for group in groups { + let mut selection = BuiltInstructionSelection { + program_id: reg.program_id.clone(), + discriminator_hex: reg.discriminator_hex.clone(), + byte_len: reg.byte_len, + is_inner: reg.is_inner, + ..Default::default() + }; + for (position, values) in group { + selection.accounts[*position] = Some(values.clone()); + } + if !selections.contains(&selection) { + selections.push(selection); + } + } + } + // The transaction table is fetched only when a selected field is read + // off a stored transaction record; the store key columns ride along. + if !transaction_columns.is_empty() { + transaction_columns.splice(0..0, ["slot", "transaction_index"]); + } + // Deterministic item order per instruction, independent of the + // selection's index order. + registrations.sort_unstable_by_key(|reg| reg.index); + + Ok(BuiltSelection { + instruction_selections: selections + .into_iter() + .map(BuiltInstructionSelection::into_net) + .collect(), + block_columns, + transaction_columns, + needs_token_balances, + needs_logs, + registrations, + }) + } +} + +/// Routes an instruction to the selection's registrations, probing declared +/// discriminator byte lengths longest-first (d8/d4/d2/d1): the first length +/// with any full match wins and the instruction fans out to every +/// registration matching at that length. Program-wide registrations (no +/// discriminator) are the final fallback when no discriminator-keyed +/// registration matched. +pub(crate) fn route_instruction( + registrations: &[Arc], + instr: &simple::Instruction, + contract_name: Option<&str>, +) -> Vec> { + for byte_len in [8usize, 4, 2, 1] { + let matched: Vec> = registrations + .iter() + .filter(|reg| { + reg.byte_len == byte_len + && reg.matches_discriminator(&instr.data) + && reg.matches_scope(instr, contract_name) + }) + .cloned() + .collect(); + if !matched.is_empty() { + return matched; + } + } + registrations + .iter() + .filter(|reg| reg.discriminator.is_none() && reg.matches_scope(instr, contract_name)) + .cloned() + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + const PROG_A: &str = "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s"; + const PROG_B: &str = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"; + const ACCOUNT_1: &str = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"; + const ACCOUNT_2: &str = "So11111111111111111111111111111111111111112"; + + fn reg( + index: i64, + program_id: &str, + discriminator: Option<&str>, + byte_len: i64, + is_wildcard: bool, + ) -> SvmOnEventRegistrationInput { + SvmOnEventRegistrationInput { + index, + instruction_name: format!("I{index}"), + contract_name: format!("P_{program_id}"), + program_id: program_id.to_string(), + is_wildcard, + discriminator: discriminator.map(str::to_string), + discriminator_byte_len: byte_len, + is_inner: None, + include_logs: false, + account_filters: vec![], + transaction_fields: vec![], + block_fields: vec![], + accounts: vec![], + args_json: None, + defined_types_json: None, + } + } + + fn instruction(program_id: &str, data: &[u8]) -> simple::Instruction { + simple::Instruction { + program_id: program_id.to_string(), + data: data.to_vec(), + is_committed: true, + ..Default::default() + } + } + + fn route_indexes( + built: &BuiltSelection, + instr: &simple::Instruction, + contract_name: Option<&str>, + ) -> Vec { + route_instruction(&built.registrations, instr, contract_name) + .iter() + .map(|reg| reg.index) + .collect() + } + + #[test] + fn discriminator_becomes_the_matching_dn_filter() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, PROG_A, Some("0x21"), 1, false), + reg(1, PROG_A, Some("0x0102030405060708"), 8, false), + ]) + .unwrap(); + let built = builder.build(&[0, 1]).unwrap(); + let views: Vec<(Vec, Vec, Vec)> = built + .instruction_selections + .iter() + .map(|s| (s.program_id.clone(), s.d1.clone(), s.d8.clone())) + .collect(); + assert_eq!( + views, + vec![ + (vec![PROG_A.to_string()], vec!["0x21".to_string()], vec![]), + ( + vec![PROG_A.to_string()], + vec![], + vec!["0x0102030405060708".to_string()] + ), + ] + ); + } + + #[test] + fn account_filter_groups_fan_out_to_separate_selections() { + let mut input = reg(0, PROG_A, Some("0x0c"), 1, false); + input.account_filters = vec![ + vec![SvmAccountFilterInput { + position: 1, + values: vec![ACCOUNT_1.to_string()], + }], + vec![SvmAccountFilterInput { + position: 2, + values: vec![ACCOUNT_2.to_string()], + }], + ]; + let builder = SelectionBuilder::from_registrations(&[input]).unwrap(); + let built = builder.build(&[0]).unwrap(); + let views: Vec<(Vec, Vec)> = built + .instruction_selections + .iter() + .map(|s| (s.a1.clone(), s.a2.clone())) + .collect(); + assert_eq!( + views, + vec![ + (vec![ACCOUNT_1.to_string()], vec![]), + (vec![], vec![ACCOUNT_2.to_string()]), + ] + ); + } + + #[test] + fn empty_program_id_emits_no_selection() { + let builder = + SelectionBuilder::from_registrations(&[reg(0, "", Some("0x21"), 1, false)]).unwrap(); + let built = builder.build(&[0]).unwrap(); + assert!(built.instruction_selections.is_empty()); + } + + #[test] + fn identical_selections_are_deduplicated() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, PROG_A, Some("0x21"), 1, false), + reg(1, PROG_A, Some("0x21"), 1, true), + ]) + .unwrap(); + let built = builder.build(&[0, 1]).unwrap(); + assert_eq!(built.instruction_selections.len(), 1); + } + + #[test] + fn field_unions_and_flags() { + let mut a = reg(0, PROG_A, Some("0x21"), 1, false); + a.transaction_fields = vec!["signatures".to_string(), "transactionIndex".to_string()]; + a.block_fields = vec!["height".to_string(), "slot".to_string()]; + a.include_logs = true; + let mut b = reg(1, PROG_A, Some("0x22"), 1, false); + b.transaction_fields = vec!["tokenBalances".to_string()]; + let builder = SelectionBuilder::from_registrations(&[a, b]).unwrap(); + let built = builder.build(&[0, 1]).unwrap(); + assert_eq!( + ( + built.block_columns.clone(), + built.transaction_columns.clone(), + built.needs_token_balances, + built.needs_logs, + ), + ( + vec!["slot", "blockhash", "block_time", "block_height"], + vec!["slot", "transaction_index", "signatures"], + true, + true, + ) + ); + } + + #[test] + fn token_balances_or_transaction_index_alone_fetch_no_transaction_columns() { + let mut input = reg(0, PROG_A, Some("0x21"), 1, false); + input.transaction_fields = + vec!["transactionIndex".to_string(), "tokenBalances".to_string()]; + let builder = SelectionBuilder::from_registrations(&[input]).unwrap(); + let built = builder.build(&[0]).unwrap(); + assert_eq!( + ( + built.transaction_columns.clone(), + built.needs_token_balances + ), + (vec![], true) + ); + } + + #[test] + fn routes_longest_discriminator_first() { + // A d1 registration (0x0f) and a d8 registration starting with 0x0f: + // an instruction carrying the full 8-byte prefix routes to the d8 + // registration only. + let builder = SelectionBuilder::from_registrations(&[ + reg(0, PROG_A, Some("0x0f"), 1, true), + reg(1, PROG_A, Some("0x0fffffffffffffff"), 8, true), + ]) + .unwrap(); + let built = builder.build(&[0, 1]).unwrap(); + let long = instruction(PROG_A, &[0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); + let short = instruction(PROG_A, &[0x0f, 0x00]); + assert_eq!( + ( + route_indexes(&built, &long, None), + route_indexes(&built, &short, None), + ), + (vec![1], vec![0]) + ); + } + + #[test] + fn program_wide_registration_is_the_fallback() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, PROG_A, Some("0x21"), 1, true), + reg(1, PROG_A, None, 0, true), + ]) + .unwrap(); + let built = builder.build(&[0, 1]).unwrap(); + let keyed = instruction(PROG_A, &[0x21]); + let other = instruction(PROG_A, &[0x22]); + assert_eq!( + ( + route_indexes(&built, &keyed, None), + route_indexes(&built, &other, None), + ), + (vec![0], vec![1]) + ); + } + + #[test] + fn fans_out_to_wildcard_and_owned_registration() { + let mut owned = reg(0, PROG_A, Some("0x21"), 1, false); + owned.contract_name = "Owned".to_string(); + let wildcard = reg(1, PROG_A, Some("0x21"), 1, true); + let mut other = reg(2, PROG_A, Some("0x21"), 1, false); + other.contract_name = "Other".to_string(); + let builder = SelectionBuilder::from_registrations(&[owned, wildcard, other]).unwrap(); + let built = builder.build(&[0, 1, 2]).unwrap(); + let instr = instruction(PROG_A, &[0x21]); + assert_eq!( + ( + route_indexes(&built, &instr, Some("Owned")), + route_indexes(&built, &instr, None), + ), + (vec![0, 1], vec![1]) + ); + } + + #[test] + fn routing_scoped_to_program() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, PROG_A, Some("0x21"), 1, true), + reg(1, PROG_B, Some("0x21"), 1, true), + ]) + .unwrap(); + let built = builder.build(&[0, 1]).unwrap(); + let instr = instruction(PROG_B, &[0x21]); + assert_eq!(route_indexes(&built, &instr, None), vec![1]); + } + + #[test] + fn account_filters_reapplied_in_routing() { + let mut filtered = reg(0, PROG_A, Some("0x21"), 1, true); + filtered.account_filters = vec![vec![SvmAccountFilterInput { + position: 1, + values: vec![ACCOUNT_1.to_string()], + }]]; + let builder = SelectionBuilder::from_registrations(&[filtered]).unwrap(); + let built = builder.build(&[0]).unwrap(); + let mut matching = instruction(PROG_A, &[0x21]); + matching.accounts = vec![ACCOUNT_2.to_string(), ACCOUNT_1.to_string()]; + let mut rejected = instruction(PROG_A, &[0x21]); + rejected.accounts = vec![ACCOUNT_1.to_string(), ACCOUNT_2.to_string()]; + assert_eq!( + ( + route_indexes(&built, &matching, None), + route_indexes(&built, &rejected, None), + ), + (vec![0], vec![]) + ); + } + + #[test] + fn is_inner_constraint_reapplied_in_routing() { + let mut outer_only = reg(0, PROG_A, Some("0x21"), 1, true); + outer_only.is_inner = Some(false); + let builder = SelectionBuilder::from_registrations(&[outer_only]).unwrap(); + let built = builder.build(&[0]).unwrap(); + let outer = instruction(PROG_A, &[0x21]); + let mut inner = instruction(PROG_A, &[0x21]); + inner.is_inner = true; + assert_eq!( + ( + route_indexes(&built, &outer, None), + route_indexes(&built, &inner, None), + ), + (vec![0], vec![]) + ); + } + + #[test] + fn selection_subset_excludes_other_registrations() { + let builder = SelectionBuilder::from_registrations(&[ + reg(0, PROG_A, Some("0x21"), 1, true), + reg(1, PROG_A, Some("0x22"), 1, true), + ]) + .unwrap(); + let built = builder.build(&[1]).unwrap(); + let instr = instruction(PROG_A, &[0x21]); + assert_eq!(route_indexes(&built, &instr, None), Vec::::new()); + } + + #[test] + fn unknown_registration_index_errors() { + let builder = SelectionBuilder::from_registrations(&[]).unwrap(); + let err = builder.build(&[7]).err().unwrap(); + assert!(format!("{err:#}").contains("Unknown registration index 7")); + } + + #[test] + fn mismatched_discriminator_byte_len_errors() { + let err = SelectionBuilder::from_registrations(&[reg(0, PROG_A, Some("0x2122"), 1, true)]) + .err() + .unwrap(); + assert!(format!("{err:#}").contains("discriminator byte length")); + } +} diff --git a/packages/cli/src/svm_hypersync_source/types.rs b/packages/cli/src/svm_hypersync_source/types.rs index a1f53f477d..4a6a3a09fc 100644 --- a/packages/cli/src/svm_hypersync_source/types.rs +++ b/packages/cli/src/svm_hypersync_source/types.rs @@ -100,7 +100,7 @@ pub struct QueryResponse { pub data: QueryResponseData, } -fn to_hex(bytes: &[u8]) -> String { +pub(crate) fn to_hex(bytes: &[u8]) -> String { let mut s = String::with_capacity(2 + bytes.len() * 2); s.push_str("0x"); for b in bytes { @@ -109,7 +109,7 @@ fn to_hex(bytes: &[u8]) -> String { s } -fn opt_hex(bytes: &Option>) -> Option { +pub(crate) fn opt_hex(bytes: &Option>) -> Option { bytes.as_deref().map(to_hex) } diff --git a/packages/envio-tests/test/EventRouter_svm_test.res b/packages/envio-tests/test/EventRouter_svm_test.res deleted file mode 100644 index 4e0d194396..0000000000 --- a/packages/envio-tests/test/EventRouter_svm_test.res +++ /dev/null @@ -1,95 +0,0 @@ -open Vitest - -let pubkey = SvmTypes.Pubkey.fromStringUnsafe - -describe("EventRouter SVM helpers", () => { - it("getSvmEventId builds discriminator-keyed tag and falls back to _none", t => { - let programId = pubkey("metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s") - let actual = { - "withDiscriminator": EventRouter.getSvmEventId( - ~programId, - ~discriminator=Some("0x0f"), - ), - "withoutDiscriminator": EventRouter.getSvmEventId( - ~programId, - ~discriminator=None, - ), - } - t.expect(actual).toEqual({ - "withDiscriminator": "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s_0x0f", - "withoutDiscriminator": "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s_none", - }) - }) - - it("fromSvmEventConfigsOrThrow precomputes per-program discriminator-length ordering desc", t => { - let chain = ChainMap.Chain.makeUnsafe(~chainId=0) - let progA = pubkey("metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s") - let progB = pubkey("11111111111111111111111111111111") - let mk = ( - ~name, - ~contractName, - ~programId, - ~discriminator, - ~discriminatorByteLen, - ): Internal.svmOnEventRegistration => { - let eventConfig: Internal.svmInstructionEventConfig = { - id: discriminator->Option.getOr("none"), - name, - contractName, - paramsRawEventSchema: %raw(`null`), - simulateParamsSchema: %raw(`null`), - programId, - discriminator, - discriminatorByteLen, - includeLogs: false, - selectedTransactionFields: Utils.Set.make(), - transactionFieldMask: 0., - selectedBlockFields: Utils.Set.make(), - blockFieldMask: 0., - accountFilters: [], - isInner: None, - accounts: [], - args: JSON.Null, - definedTypes: JSON.Null, - } - EventConfigBuilder.buildSvmOnEventRegistration( - ~eventConfig, - ~isWildcard=false, - ~handler=None, - ~contractRegister=None, - ) - } - let configs = [ - mk( - ~name="One", - ~contractName="ProgA", - ~programId=progA, - ~discriminator=Some("0x0f"), - ~discriminatorByteLen=1, - ), - mk( - ~name="Eight", - ~contractName="ProgA", - ~programId=progA, - ~discriminator=Some("0x0fffffffffffffff"), - ~discriminatorByteLen=8, - ), - mk( - ~name="Wide", - ~contractName="ProgB", - ~programId=progB, - ~discriminator=None, - ~discriminatorByteLen=0, - ), - ] - let (_router, ordering) = EventRouter.fromSvmEventConfigsOrThrow(configs, ~chain) - let byProgram = - ordering - ->Array.map(o => (o.programId->SvmTypes.Pubkey.toString, o.byteLengthsDesc)) - ->Array.toSorted(((a, _), (b, _)) => String.compare(a, b)) - t.expect(byProgram).toEqual([ - ("11111111111111111111111111111111", [0]), - ("metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", [8, 1]), - ]) - }) -}) diff --git a/packages/envio-tests/test/HandlerRegisterValidation_test.res b/packages/envio-tests/test/HandlerRegisterValidation_test.res new file mode 100644 index 0000000000..88e7b18f03 --- /dev/null +++ b/packages/envio-tests/test/HandlerRegisterValidation_test.res @@ -0,0 +1,54 @@ +open Vitest + +// Covers `HandlerRegister.validateEventIdOrThrow`, the per-chain dispatch-id +// guard `finishRegistration` runs so a single log/receipt/instruction can't fan +// out to two events on one contract, or to two wildcards sharing a signature — +// Rust-side routing dispatches by these ids, so a collision double-delivers. + +describe("HandlerRegister.validateEventIdOrThrow", () => { + let add = (validator, ~eventId, ~contractName, ~isWildcard) => + validator->HandlerRegister.validateEventIdOrThrow( + ~eventId, + ~contractName, + ~eventName="Event1", + ~isWildcard, + ~chainId=1, + ) + + it("accepts the same eventId across different contracts", t => { + let validator = HandlerRegister.makeEventIdValidator() + validator->add(~eventId="0xsig", ~contractName="Contract1", ~isWildcard=false) + t.expect( + () => validator->add(~eventId="0xsig", ~contractName="Contract2", ~isWildcard=false), + ).not.toThrow() + }) + + it("throws on a duplicate event for the same contract", t => { + let validator = HandlerRegister.makeEventIdValidator() + validator->add(~eventId="0xsig", ~contractName="Contract1", ~isWildcard=false) + t.expect( + () => validator->add(~eventId="0xsig", ~contractName="Contract1", ~isWildcard=false), + ).toThrowError("Duplicate event detected: Event1 for contract Contract1 on chain 1") + }) + + it("throws when a second wildcard claims the same eventId", t => { + let validator = HandlerRegister.makeEventIdValidator() + validator->add(~eventId="0xsig", ~contractName="Contract1", ~isWildcard=true) + t.expect( + () => validator->add(~eventId="0xsig", ~contractName="Contract2", ~isWildcard=true), + ).toThrowError( + "Another event is already registered with the same signature that would interfere with wildcard filtering: Event1 for contract Contract2 on chain 1", + ) + }) + + it("accepts two wildcards whose eventIds differ (e.g. SVM program-scoped ids)", t => { + // finishRegistration scopes the SVM key by programId (`${programId}_${id}`), + // so two wildcard instructions sharing a discriminator on different programs + // reach the validator as distinct eventIds and must not collide. + let validator = HandlerRegister.makeEventIdValidator() + validator->add(~eventId="progA_0x0f", ~contractName="ProgA", ~isWildcard=true) + t.expect( + () => validator->add(~eventId="progB_0x0f", ~contractName="ProgB", ~isWildcard=true), + ).not.toThrow() + }) +}) diff --git a/packages/envio-tests/test/HyperSyncClient_test.res b/packages/envio-tests/test/HyperSyncClient_test.res index d34e453fbc..e1376936a9 100644 --- a/packages/envio-tests/test/HyperSyncClient_test.res +++ b/packages/envio-tests/test/HyperSyncClient_test.res @@ -147,10 +147,10 @@ describe("HyperSync client getEventItems (live)", () => { describe("HyperSync client getHeight with corrupted token", () => { // A corrupted token makes the server reply 401, so getHeight throws. The error - // must keep matching HyperSyncSource.isUnauthorizedError, otherwise + // must keep matching EvmHyperSyncSource.isUnauthorizedError, otherwise // getHeightOrThrow's block-forever guard silently stops working. Async.it( - "is detected by HyperSyncSource.isUnauthorizedError", + "is detected by EvmHyperSyncSource.isUnauthorizedError", async t => { let client = HyperSyncClient.make( ~url="https://eth.hypersync.xyz", @@ -164,7 +164,7 @@ describe("HyperSync client getHeight with corrupted token", () => { let _ = await client.getHeight() false } catch { - | JsExn(e) => e->JsExn.message->Option.getOr("")->HyperSyncSource.isUnauthorizedError + | JsExn(e) => e->JsExn.message->Option.getOr("")->EvmHyperSyncSource.isUnauthorizedError | _ => false } diff --git a/packages/envio-tests/test/SvmHyperSyncSource_test.res b/packages/envio-tests/test/SvmHyperSyncSource_test.res index 7ca8a52201..547a97d8cf 100644 --- a/packages/envio-tests/test/SvmHyperSyncSource_test.res +++ b/packages/envio-tests/test/SvmHyperSyncSource_test.res @@ -1,13 +1,14 @@ open Vitest // Regression coverage for SvmHyperSyncSource.getItemsOrThrow response -// parsing, driven through a mocked napi client (no network): -// 1. `instruction.block` is omitted on the item; it's materialised from the -// block store at batch prep, which this test doesn't exercise — see -// BlockStore_test.res. -// 2. The query must spell out transaction/log columns when an event config -// opts in — the server returns no rows for a table whose field list is -// empty, so omitting them silently drops `instruction.transaction`. +// parsing, driven through a mocked napi client (no network). Query building, +// routing, and the isCommitted filter live in Rust now (covered by the +// svm_hypersync_source unit tests); this test asserts: +// 1. The per-query input passed to the client: registration indexes, +// addresses, and the inclusive slot range. +// 2. Item building: registrations resolved by index, `block` omitted on the +// payload (materialised from the block store at batch prep), synthesized +// logIndex, and Rust-decoded params parsed from JSON strings. let metaplexProgramId = "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s" let chain = ChainMap.Chain.makeUnsafe(~chainId=0) @@ -18,17 +19,7 @@ let blockHash = "99K5yyU2jLxLDeRCJ9YSSMy6VBJTNcnePWUH9uCHAWCB" let makeEventConfig = ( ~selectedBlockFields: array=[], - ~selectedTransactionFields=[ - Internal.Signatures, - FeePayer, - Success, - Err, - Fee, - ComputeUnitsConsumed, - AccountKeys, - RecentBlockhash, - Version, - ], + ~selectedTransactionFields: array=[], ): Internal.svmInstructionEventConfig => { let selectedTransactionFields = Utils.Set.fromArray(selectedTransactionFields)->( @@ -60,51 +51,58 @@ let makeEventConfig = ( } } -let makeReg = (~eventConfig=makeEventConfig()) => - EventConfigBuilder.buildSvmOnEventRegistration( +let makeReg = (~eventConfig=makeEventConfig(), ~index=0) => { + let reg = EventConfigBuilder.buildSvmOnEventRegistration( ~eventConfig, ~isWildcard=false, ~handler=None, ~contractRegister=None, ) + {...reg, Internal.index: index} +} -let mockResponse: SvmHyperSyncClient.ResponseTypes.queryResponse = { +let mockResponse: SvmHyperSyncClient.EventItems.response = { nextSlot: slot + 1, - responseBytes: 0, - data: { - blocks: [ - { - slot, - blockhash: blockHash, - blockTime, - }, - ], - instructions: [ - { - slot, - transactionIndex: 965, - instructionAddress: [1], - programId: metaplexProgramId, - accounts: [], - data: "0x21", - d1: "0x21", - isInner: false, - isCommitted: true, + blocks: [ + { + slot, + blockhash: blockHash, + blockTime, + }, + ], + items: [ + { + onEventRegistrationIndex: 0, + slot, + transactionIndex: 965, + instructionAddress: [1], + programId: metaplexProgramId, + accounts: [], + data: "0x21", + d1: "0x21", + isInner: false, + decoded: { + name: "CreateMetadataAccountV3", + argsJson: `{"amount":"1"}`, + accountsJson: `{"metadata":"${metaplexProgramId}"}`, + extraAccounts: [], }, - ], - logs: [], - }, + }, + ], } -let capturedQueries: array = [] +let capturedQueries: array = [] +let capturedRegistrationInputs: array> = [] let makeMockClient = (~response=mockResponse): SvmHyperSyncClient.t => { getHeight: () => Promise.resolve(slot + 1000), - get: (~query) => { + get: (~query as _) => + JsError.throwWithMessage("get should only be used for block-data queries in tests"), + getEventItems: (~query) => { capturedQueries->Array.push(query) // The real Rust client builds the stores from raw transactions/blocks; the // mock returns empty pages (materialisation is covered by the Rust unit - // tests). This test asserts the item shape and the query columns. + // tests). Promise.resolve(( response, TransactionStore.make(~ecosystem=Ecosystem.Svm, ~shouldChecksum=false), @@ -115,48 +113,6 @@ let makeMockClient = (~response=mockResponse): SvmHyperSyncClient.t => { let mockClient = makeMockClient() -// Two instructions matching the same config, one from a committed transaction -// and one from a failed (uncommitted) transaction. HyperSync delivers both; -// the runtime drops the failed one. -let mixedCommittedResponse: SvmHyperSyncClient.ResponseTypes.queryResponse = { - nextSlot: slot + 1, - responseBytes: 0, - data: { - blocks: [ - { - slot, - blockhash: blockHash, - blockTime, - }, - ], - instructions: [ - { - slot, - transactionIndex: 965, - instructionAddress: [1], - programId: metaplexProgramId, - accounts: [], - data: "0x21", - d1: "0x21", - isInner: false, - isCommitted: true, - }, - { - slot, - transactionIndex: 966, - instructionAddress: [2], - programId: metaplexProgramId, - accounts: [], - data: "0x21", - d1: "0x21", - isInner: false, - isCommitted: false, - }, - ], - logs: [], - }, -} - // The source captures its client at construction, so the mock addon only // needs to be in place for the `make` call; restore the previous addon right // after to avoid leaking the mock into other tests. @@ -165,8 +121,15 @@ let makeSource = (~onEventRegistrations=[makeReg()], ~client=mockClient) => { Core.addonRef := Some( { - "SvmHypersyncClient": { - "fromConfig": (_: SvmHyperSyncClient.cfg, _: string) => client, + "SvmHyperSyncClient": { + "fromConfig": ( + _: SvmHyperSyncClient.cfg, + _: string, + registrations: array, + ) => { + capturedRegistrationInputs->Array.push(registrations) + client + }, }, }->(Utils.magic: {..} => Core.addon), ) @@ -188,226 +151,151 @@ let makeSource = (~onEventRegistrations=[makeReg()], ~client=mockClient) => { let contractNameByAddress = Dict.fromArray([(metaplexProgramId, "TokenMetadata")]) describe("SvmHyperSyncSource.getItemsOrThrow (mocked client)", () => { - Async.it("omits block on the item and requests opted-in table columns", async t => { - let reg = makeReg() - let source = makeSource(~onEventRegistrations=[reg]) - - let response = await source.getItemsOrThrow( - ~fromBlock=slot - 10, - ~toBlock=Some(slot + 10), - ~addressesByContractName=Dict.fromArray([ - ("TokenMetadata", [metaplexProgramId->Address.unsafeFromString]), - ]), - ~contractNameByAddress, - ~knownHeight=slot + 1000, - ~partitionId="0", - ~itemsTarget=5000, - ~selection={ - onEventRegistrations: [reg], - dependsOnAddresses: true, - }, - ~retry=0, - ~logger=Logging.createChild(~params={"test": "SvmHyperSyncSource"}), - ) - - let item = switch response.parsedQueueItems { - | [Internal.Event({blockNumber, payload, onEventRegistration})] => - let instruction = payload->(Utils.magic: Internal.eventPayload => Envio.svmInstruction) - Some({ - "blockNumber": blockNumber, - "block": instruction.block, - "usesSourceRegistration": onEventRegistration === (reg :> Internal.onEventRegistration), - }) - | _ => None - } - - t.expect({ - "item": item, - "query": capturedQueries->Array.getUnsafe(0), - }).toEqual({ - "item": Some({ - "blockNumber": slot, - // `block` is omitted here; it's materialised from the store at batch - // prep, which this test doesn't run. - "block": None, - "usesSourceRegistration": true, - }), - // Default merge mode: requesting a table's columns opts the matched - // result set into that join, so selections carry no include flags. - "query": ( - { - fromSlot: slot - 10, - // Inclusive `toBlock` becomes exclusive `toSlot` on the wire (+1). - toSlot: slot + 11, - instructions: [{programId: [metaplexProgramId], d1: ["0x21"]}], - maxNumInstructions: 5000, - fields: { - block: [Slot, Blockhash, BlockTime], - transaction: [ - Slot, - TransactionIndex, - Signatures, - FeePayer, - Success, - Err, - Fee, - ComputeUnitsConsumed, - AccountKeys, - RecentBlockhash, - Version, - ], - }, - }: SvmHyperSyncClient.query - ), - }) - }) - - // `transactionIndex` is materialised from the store key (the instruction's - // own index), not from a stored transaction record, so selecting it alone does - // not need the transaction table fetched. Async.it( - "does not fetch the transaction table when only transactionIndex is selected", + "passes the selection to the client and builds items by registration index", async t => { - let reg = makeReg(~eventConfig=makeEventConfig(~selectedTransactionFields=[TransactionIndex])) + let reg = makeReg() let source = makeSource(~onEventRegistrations=[reg]) - let _ = await source.getItemsOrThrow( - ~fromBlock=slot - 10, - ~toBlock=Some(slot + 10), - ~addressesByContractName=Dict.fromArray([ - ("TokenMetadata", [metaplexProgramId->Address.unsafeFromString]), - ]), - ~contractNameByAddress, - ~knownHeight=slot + 1000, - ~partitionId="0", - ~selection={ - onEventRegistrations: [reg], - dependsOnAddresses: true, - }, - ~itemsTarget=5000, - ~retry=0, - ~logger=Logging.createChild(~params={"test": "SvmHyperSyncSource"}), - ) - - let query = capturedQueries->Array.getUnsafe(capturedQueries->Array.length - 1) - t.expect(query.fields->Option.flatMap(fields => fields.transaction)).toEqual(None) - }, - ) - - // Selected block fields are added to the query's block columns on top of the - // always-fetched slot/blockhash/blockTime trio. - Async.it("requests the selected block fields' columns", async t => { - let reg = makeReg(~eventConfig=makeEventConfig(~selectedBlockFields=[Height, ParentHash])) - let source = makeSource(~onEventRegistrations=[reg]) - - let _ = await source.getItemsOrThrow( - ~fromBlock=slot - 10, - ~toBlock=Some(slot + 10), - ~addressesByContractName=Dict.fromArray([ + let addressesByContractName = Dict.fromArray([ ("TokenMetadata", [metaplexProgramId->Address.unsafeFromString]), - ]), - ~contractNameByAddress, - ~knownHeight=slot + 1000, - ~partitionId="0", - ~selection={ - onEventRegistrations: [reg], - dependsOnAddresses: true, - }, - ~itemsTarget=5000, - ~retry=0, - ~logger=Logging.createChild(~params={"test": "SvmHyperSyncSource"}), - ) - - let query = capturedQueries->Array.getUnsafe(capturedQueries->Array.length - 1) - let fields: SvmHyperSyncClient.QueryTypes.fieldSelection = query.fields->Option.getUnsafe - t.expect(fields.block).toEqual(Some([Slot, Blockhash, BlockTime, BlockHeight, ParentBlockhash])) - }) - - // Token balances are keyed by account in the store, so the query must always - // carry `account` alongside whatever columns opted the table in — matches - // the Rust store's field_table.rs Table<(slot, txIndex, account)> key. - Async.it( - "requests tokenBalance columns with account included, and skips the transaction table", - async t => { - let reg = makeReg(~eventConfig=makeEventConfig(~selectedTransactionFields=[TokenBalances])) - let source = makeSource(~onEventRegistrations=[reg]) - - let _ = await source.getItemsOrThrow( + ]) + let response = await source.getItemsOrThrow( ~fromBlock=slot - 10, ~toBlock=Some(slot + 10), - ~addressesByContractName=Dict.fromArray([ - ("TokenMetadata", [metaplexProgramId->Address.unsafeFromString]), - ]), + ~addressesByContractName, ~contractNameByAddress, ~knownHeight=slot + 1000, ~partitionId="0", + ~itemsTarget=5000, ~selection={ onEventRegistrations: [reg], dependsOnAddresses: true, }, - ~itemsTarget=5000, ~retry=0, ~logger=Logging.createChild(~params={"test": "SvmHyperSyncSource"}), ) - let query = capturedQueries->Array.getUnsafe(capturedQueries->Array.length - 1) - let fields: SvmHyperSyncClient.QueryTypes.fieldSelection = query.fields->Option.getUnsafe - let expectedTokenBalance: option> = Some([ - Slot, - TransactionIndex, - Account, - Mint, - Owner, - PreAmount, - PostAmount, - ]) + let item = switch response.parsedQueueItems { + | [Internal.Event({blockNumber, logIndex, transactionIndex, payload, onEventRegistration})] => + let instruction = payload->(Utils.magic: Internal.eventPayload => Envio.svmInstruction) + Some({ + "blockNumber": blockNumber, + // tx * 65536 + depth-weighted instruction address offset. + "logIndex": logIndex, + "transactionIndex": transactionIndex, + // `block` is omitted here; it's materialised from the store at batch + // prep, which this test doesn't run. + "block": instruction.block, + "params": instruction.params, + "usesSourceRegistration": onEventRegistration === (reg :> Internal.onEventRegistration), + }) + | _ => None + } + t.expect({ - "tokenBalance": fields.tokenBalance, - "transaction": fields.transaction, + "item": item, + "query": capturedQueries->Array.getUnsafe(0), + "latestFetchedBlockTimestamp": response.latestFetchedBlockTimestamp, + "blockHashes": response.blockHashes, }).toEqual({ - "tokenBalance": expectedTokenBalance, - "transaction": None, + "item": Some({ + "blockNumber": slot, + "logIndex": 965 * 65536 + 2, + "transactionIndex": 965, + "block": None, + "params": Some( + ( + { + name: "CreateMetadataAccountV3", + args: %raw(`{"amount": "1"}`), + accounts: Dict.fromArray([("metadata", metaplexProgramId)]), + extraAccounts: [], + }: Envio.svmInstructionParams + ), + ), + "usesSourceRegistration": true, + }), + // The slot range stays inclusive on the boundary; Rust converts to the + // wire's exclusive `toSlot`. + "query": ( + { + fromSlot: slot - 10, + toSlot: Some(slot + 10), + maxNumInstructions: 5000, + registrationIndexes: [0], + addressesByContractName, + }: SvmHyperSyncClient.EventItems.query + ), + "latestFetchedBlockTimestamp": blockTime, + "blockHashes": [{ReorgDetection.blockNumber: slot, blockHash}], }) }, ) - let deliveredInstructionAddresses = (response: Source.blockRangeFetchResponse) => - response.parsedQueueItems->Array.filterMap(item => - switch item { - | Internal.Event({payload}) => - let instruction = payload->(Utils.magic: Internal.eventPayload => Envio.svmInstruction) - Some(instruction.instructionAddress) - | _ => None - } - ) - - // The failed-tx instruction (address [2], isCommitted: false) is dropped - // before dispatch; only the committed instruction (address [1]) is delivered. - Async.it("drops instructions from failed transactions", async t => { - let reg = makeReg() - let source = makeSource( - ~onEventRegistrations=[reg], - ~client=makeMockClient(~response=mixedCommittedResponse), - ) - - let response = await source.getItemsOrThrow( - ~fromBlock=slot - 10, - ~toBlock=Some(slot + 10), - ~addressesByContractName=Dict.fromArray([ - ("TokenMetadata", [metaplexProgramId->Address.unsafeFromString]), - ]), - ~contractNameByAddress, - ~knownHeight=slot + 1000, - ~partitionId="0", - ~selection={ - onEventRegistrations: [reg], - dependsOnAddresses: true, + // The whole registration set crosses the boundary once at construction — + // selections, field unions, decoders, and routing derive from it in Rust. + it("builds registration inputs from the event configs", t => { + let _ = makeSource(~onEventRegistrations=[makeReg()]) + let inputs = + capturedRegistrationInputs->Array.getUnsafe(capturedRegistrationInputs->Array.length - 1) + t.expect(inputs).toEqual([ + { + index: 0, + instructionName: "CreateMetadataAccountV3", + contractName: "TokenMetadata", + programId: metaplexProgramId, + isWildcard: false, + discriminator: "0x21", + discriminatorByteLen: 1, + includeLogs: false, + accountFilters: [], + transactionFields: [], + blockFields: [], + accounts: [], }, - ~itemsTarget=5000, - ~retry=0, - ~logger=Logging.createChild(~params={"test": "SvmHyperSyncSource"}), - ) + ]) + }) - t.expect(deliveredInstructionAddresses(response)).toEqual([[1]]) + it("stringifies schema pieces and field selections onto registration inputs", t => { + let eventConfig = makeEventConfig( + ~selectedBlockFields=[Height, ParentHash], + ~selectedTransactionFields=[Signatures, TransactionIndex], + ) + let eventConfig = { + ...eventConfig, + accounts: ["metadata", "mint"], + args: %raw(`[{"name": "amount", "type": "u64"}]`), + isInner: Some(false), + accountFilters: [ + [ + { + Internal.position: 1, + values: [metaplexProgramId->SvmTypes.Pubkey.fromStringUnsafe], + }, + ], + ], + } + let _ = makeSource(~onEventRegistrations=[makeReg(~eventConfig)]) + let inputs = + capturedRegistrationInputs->Array.getUnsafe(capturedRegistrationInputs->Array.length - 1) + let input = inputs->Array.getUnsafe(0) + t.expect({ + "accountFilters": input.accountFilters, + "isInner": input.isInner, + "transactionFields": input.transactionFields->Array.toSorted(String.compare), + "blockFields": input.blockFields->Array.toSorted(String.compare), + "accounts": input.accounts, + "argsJson": input.argsJson, + "definedTypesJson": input.definedTypesJson, + }).toEqual({ + "accountFilters": [[{SvmHyperSyncClient.Registration.position: 1, values: [metaplexProgramId]}]], + "isInner": Some(false), + "transactionFields": ["signatures", "transactionIndex"], + "blockFields": ["height", "parentHash"], + "accounts": ["metadata", "mint"], + "argsJson": Some(`[{"name":"amount","type":"u64"}]`), + "definedTypesJson": None, + }) }) }) diff --git a/packages/envio/src/ChainState.res b/packages/envio/src/ChainState.res index cb97dc0475..c6d5145e82 100644 --- a/packages/envio/src/ChainState.res +++ b/packages/envio/src/ChainState.res @@ -224,7 +224,12 @@ let makeInternal = ( ~lowercaseAddresses, ) | Config.FuelSourceConfig({hypersync}) => [ - HyperFuelSource.make({chain, endpointUrl: hypersync, apiToken: Env.envioApiToken}), + FuelHyperSyncSource.make({ + chain, + endpointUrl: hypersync, + apiToken: Env.envioApiToken, + onEventRegistrations, + }), ] | Config.SvmSourceConfig({hypersync, rpc}) => switch (hypersync, rpc) { diff --git a/packages/envio/src/Core.res b/packages/envio/src/Core.res index de98af457c..23d028a5af 100644 --- a/packages/envio/src/Core.res +++ b/packages/envio/src/Core.res @@ -4,10 +4,10 @@ // NAPI encodes Rust `Option` as `null | T` (never `undefined`), so the // tighter `Null.t` captures the exact boundary shape. -type evmHypersyncClientCtor +type evmHyperSyncClientCtor type evmRpcClientCtor -type svmHypersyncClientCtor -type hyperfuelClientCtor +type svmHyperSyncClientCtor +type fuelHyperSyncClientCtor type transactionStoreCtor type blockStoreCtor type parseConfigYamlOptions = { @@ -22,14 +22,14 @@ type addon = { encodeIndexedTopic: (~abiType: string, ~value: unknown) => EvmTypes.Hex.t, parseConfigYaml: (string, parseConfigYamlOptions) => string, runCli: (~args: array, ~envioPackageDir: Null.t) => promise>, - @as("EvmHypersyncClient") - evmHypersyncClient: evmHypersyncClientCtor, + @as("EvmHyperSyncClient") + evmHyperSyncClient: evmHyperSyncClientCtor, @as("EvmRpcClient") evmRpcClient: evmRpcClientCtor, - @as("SvmHypersyncClient") - svmHypersyncClient: svmHypersyncClientCtor, - @as("HyperfuelClient") - hyperfuelClient: hyperfuelClientCtor, + @as("SvmHyperSyncClient") + svmHyperSyncClient: svmHyperSyncClientCtor, + @as("FuelHyperSyncClient") + fuelHyperSyncClient: fuelHyperSyncClientCtor, @as("TransactionStore") transactionStore: transactionStoreCtor, @as("BlockStore") diff --git a/packages/envio/src/FetchState.res b/packages/envio/src/FetchState.res index 9b033e8dba..6482b13b87 100644 --- a/packages/envio/src/FetchState.res +++ b/packages/envio/src/FetchState.res @@ -1006,8 +1006,8 @@ OptimizedPartitions.t => { // Addresses with different start blocks within range share a partition; // events before each address's effectiveStartBlock are dropped on the - // client side (EventRouter for the srcAddress, the event's - // clientAddressFilter for address-valued params). + // client side (the event's clientAddressFilter for address-valued + // params). if shouldJoinCurrentStartBlock { addressesRef := addressesRef.contents->Array.concat(byStartBlock->Dict.getUnsafe(nextStartBlockKey)) @@ -1411,7 +1411,7 @@ let registerDynamicContracts = ( // Drop events an address-param filter rejects. A merged partition may // over-fetch a wildcard event whose indexed address param references an // address registered after the log's block; `clientAddressFilter` is the -// param-level analogue of EventRouter's srcAddress effectiveStartBlock check. +// param-level analogue of the srcAddress effectiveStartBlock check. let filterByClientAddress = ( items: array, ~indexingAddresses: IndexingAddresses.t, diff --git a/packages/envio/src/HandlerRegister.res b/packages/envio/src/HandlerRegister.res index 39b430878a..dfae00a37b 100644 --- a/packages/envio/src/HandlerRegister.res +++ b/packages/envio/src/HandlerRegister.res @@ -588,6 +588,55 @@ let registerOnBlock = ( }) } +// Per-eventId dispatch validation state: two events on one contract can't +// share a dispatch id, and only one wildcard may claim it — Rust-side routing +// fans a log/receipt/instruction out by these ids, so a collision would +// double-deliver. Scoped per chain (a fresh validator per chain iteration). +type eventIdValidator = { + contractNamesByEventId: dict>, + wildcardEventIds: Utils.Set.t, +} + +let makeEventIdValidator = (): eventIdValidator => { + contractNamesByEventId: Dict.make(), + wildcardEventIds: Utils.Set.make(), +} + +let validateEventIdOrThrow = ( + validator: eventIdValidator, + ~eventId, + ~contractName, + ~eventName, + ~isWildcard, + ~chainId, +) => { + let chainSuffix = `on chain ${chainId->Int.toString}` + let contractNames = switch validator.contractNamesByEventId->Utils.Dict.dangerouslyGetNonOption( + eventId, + ) { + | Some(contractNames) => contractNames + | None => { + let contractNames = Utils.Set.make() + validator.contractNamesByEventId->Dict.set(eventId, contractNames) + contractNames + } + } + if contractNames->Utils.Set.has(contractName) { + JsError.throwWithMessage( + `Duplicate event detected: ${eventName} for contract ${contractName} ${chainSuffix}`, + ) + } + if isWildcard && validator.wildcardEventIds->Utils.Set.has(eventId) { + JsError.throwWithMessage( + `Another event is already registered with the same signature that would interfere with wildcard filtering: ${eventName} for contract ${contractName} ${chainSuffix}`, + ) + } + if isWildcard { + validator.wildcardEventIds->Utils.Set.add(eventId)->ignore + } + contractNames->Utils.Set.add(contractName)->ignore +} + let finishRegistration = (~config: Config.t): registrationsByChainId => { switch getActiveRegistration() { | Some(r) => { @@ -600,11 +649,7 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => { let key = chainConfig.id->Int.toString let pending = r.registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(key) - // We don't need the router itself, but only validation logic, - // since now event router is created for selection of events - // and validation doesn't work correctly in routers. - // Ideally to split it into two different parts. - let eventRouter = EventRouter.empty() + let eventIdValidator = makeEventIdValidator() let onEventRegistrations: array = [] @@ -622,17 +667,30 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => { ), ) - // Should validate the events - eventRouter->EventRouter.addOrThrow( - eventConfig.id, - (), + // SVM dispatch is keyed by (programId, discriminator), not the + // discriminator alone — `eventConfig.id` is only the + // discriminator (or "none"). Scope the SVM validation key by + // programId so two wildcard handlers on different programs that + // share a discriminator aren't rejected as a false collision + // (Rust routing already scopes matches by program_id). + let eventId = switch config.ecosystem.name { + | Svm => + let svmEventConfig = + eventConfig->( + Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig + ) + `${svmEventConfig.programId->SvmTypes.Pubkey.toString}_${eventConfig.id}` + | Evm | Fuel => eventConfig.id + } + eventIdValidator->validateEventIdOrThrow( + ~eventId, ~contractName, - ~chain=ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id), ~eventName, ~isWildcard=switch registration { | Some(registration) => registration.isWildcard | None => isWildcard(~contractName, ~eventName) }, + ~chainId=chainConfig.id, ) let registration = switch registration { diff --git a/packages/envio/src/HandlerRegister.resi b/packages/envio/src/HandlerRegister.resi index 7014ebd42b..401050facf 100644 --- a/packages/envio/src/HandlerRegister.resi +++ b/packages/envio/src/HandlerRegister.resi @@ -9,6 +9,17 @@ let isPendingRegistration: unit => bool let finishRegistration: (~config: Config.t) => registrationsByChainId let throwIfFinishedRegistration: (~methodName: string) => unit +type eventIdValidator +let makeEventIdValidator: unit => eventIdValidator +let validateEventIdOrThrow: ( + eventIdValidator, + ~eventId: string, + ~contractName: string, + ~eventName: string, + ~isWildcard: bool, + ~chainId: int, +) => unit + let buildOnEventRegistration: ( ~config: Config.t, ~chainId: int, diff --git a/packages/envio/src/Internal.res b/packages/envio/src/Internal.res index c04efcc535..cd08462705 100644 --- a/packages/envio/src/Internal.res +++ b/packages/envio/src/Internal.res @@ -179,7 +179,7 @@ type svmBlockField = let allSvmBlockFields: array = [Height, ParentSlot, ParentHash] let svmBlockFieldSchema = S.enum(allSvmBlockFields) -// Static sets of nullable field names — used by RpcSource and HyperSyncSource to wrap schemas with S.nullable +// Static sets of field names whose source schemas must be wrapped with S.nullable. let evmNullableBlockFields = Utils.Set.fromArray( ( [ diff --git a/packages/envio/src/sources/EventRouter.res b/packages/envio/src/sources/EventRouter.res deleted file mode 100644 index 25b63c6821..0000000000 --- a/packages/envio/src/sources/EventRouter.res +++ /dev/null @@ -1,165 +0,0 @@ -exception EventDuplicate -exception WildcardCollision - -module Group = { - type t<'a> = { - mutable wildcard: option<'a>, - byContractName: dict<'a>, - } - - let empty = () => { - wildcard: None, - byContractName: Dict.make(), - } - - let addOrThrow = (group: t<'a>, event, ~contractName, ~isWildcard) => { - let {byContractName, wildcard} = group - switch byContractName->Utils.Dict.dangerouslyGetNonOption(contractName) { - | Some(_) => throw(EventDuplicate) - | None => - if isWildcard && wildcard->Option.isSome { - throw(WildcardCollision) - } else { - if isWildcard { - group.wildcard = Some(event) - } - byContractName->Dict.set(contractName, event) - } - } - } - - // Ownership only: resolve the owning contract from the partition's reverse - // index (the partition that fetched the log), not a chain-wide snapshot. The - // `effectiveStartBlock` temporal gate now lives in `clientAddressFilter`. The - // wildcard partition has an empty index → every log falls back to `wildcard`, - // so it can never claim an address-bound contract's logs. - let get = (group: t<'a>, ~contractAddress, ~contractNameByAddress: dict) => - switch group { - | {wildcard, byContractName} => - switch contractNameByAddress->Utils.Dict.dangerouslyGetNonOption( - contractAddress->Address.toString, - ) { - | Some(contractName) => - switch byContractName->Utils.Dict.dangerouslyGetNonOption(contractName) { - // Fall back to the wildcard handler when the owning contract has no - // matching event for this tag. This covers addresses registered for - // contracts without events (persisted for future config changes) as - // well as addresses whose contract has other events but not this one. - | None => wildcard - | Some(_) as event => event - } - | None => wildcard - } - } -} - -type t<'a> = dict> - -let empty = () => Dict.make() - -let addOrThrow = ( - router: t<'a>, - eventId, - event, - ~contractName, - ~isWildcard, - ~eventName, - ~chain, -) => { - let group = switch router->Utils.Dict.dangerouslyGetNonOption(eventId) { - | None => - let group = Group.empty() - router->Dict.set(eventId, group) - group - | Some(group) => group - } - try group->Group.addOrThrow(event, ~contractName, ~isWildcard) catch { - | EventDuplicate => - JsError.throwWithMessage( - `Duplicate event detected: ${eventName} for contract ${contractName} on chain ${chain->ChainMap.Chain.toString}`, - ) - | WildcardCollision => - JsError.throwWithMessage( - `Another event is already registered with the same signature that would interfer with wildcard filtering: ${eventName} for contract ${contractName} on chain ${chain->ChainMap.Chain.toString}`, - ) - } -} - -let get = (router: t<'a>, ~tag, ~contractAddress, ~contractNameByAddress) => { - switch router->Utils.Dict.dangerouslyGetNonOption(tag) { - | None => None - | Some(group) => group->Group.get(~contractAddress, ~contractNameByAddress) - } -} - -/** Dispatch key for SVM instructions. `None` matches any instruction in the - program (lowest priority). */ -let getSvmEventId = (~programId: SvmTypes.Pubkey.t, ~discriminator: option) => - switch discriminator { - | None => programId->SvmTypes.Pubkey.toString ++ "_none" - | Some(d) => programId->SvmTypes.Pubkey.toString ++ "_" ++ d - } - -/** Discriminator byte-lengths declared by a program, sorted descending. The - source uses this to probe `(programId, dN)` keys longest-first when routing - a returned instruction to a handler — matching the locked Q1 answer. */ -type svmProgramOrdering = { - programId: SvmTypes.Pubkey.t, - /** Byte lengths in descending order, deduplicated. Includes `0` only when - a handler is registered with no discriminator (program-wide match). */ - byteLengthsDesc: array, -} - -let fromSvmEventConfigsOrThrow = (events: array, ~chain): ( - t, - array, -) => { - let router = empty() - events->Array.forEach(config => { - let eventConfig = - config.eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig) - // The router tag must include the programId so two programs declaring the - // same discriminator coexist. The source-side lookup uses the same shape - // via `getSvmEventId(~programId, ~discriminator)`. - let routerTag = getSvmEventId( - ~programId=eventConfig.programId, - ~discriminator=eventConfig.discriminator, - ) - router->addOrThrow( - routerTag, - config, - ~contractName=eventConfig.contractName, - ~eventName=eventConfig.name, - ~chain, - ~isWildcard=config.isWildcard, - ) - }) - - // Per-program list of declared discriminator byte lengths, sorted desc. - let byProgram: dict> = Dict.make() - events->Array.forEach(config => { - let eventConfig = - config.eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig) - let key = eventConfig.programId->SvmTypes.Pubkey.toString - let set = switch byProgram->Utils.Dict.dangerouslyGetNonOption(key) { - | Some(s) => s - | None => - let s = Utils.Set.make() - byProgram->Dict.set(key, s) - s - } - let _ = set->Utils.Set.add(eventConfig.discriminatorByteLen) - }) - let ordering = - byProgram - ->Dict.toArray - ->Array.map(((programIdString, lens)) => { - let sorted = lens->Utils.Set.toArray->Array.toSorted((a, b) => (b - a)->Int.toFloat) - { - programId: programIdString->SvmTypes.Pubkey.fromStringUnsafe, - byteLengthsDesc: sorted, - } - }) - - (router, ordering) -} diff --git a/packages/envio/src/sources/EvmChain.res b/packages/envio/src/sources/EvmChain.res index b924576d7f..82a1c421f6 100644 --- a/packages/envio/src/sources/EvmChain.res +++ b/packages/envio/src/sources/EvmChain.res @@ -48,7 +48,7 @@ let makeSources = ( ) => { let sources = switch hyperSync { | Some(endpointUrl) => [ - HyperSyncSource.make({ + EvmHyperSyncSource.make({ chain, endpointUrl, onEventRegistrations, diff --git a/packages/envio/src/sources/HyperSyncSource.res b/packages/envio/src/sources/EvmHyperSyncSource.res similarity index 100% rename from packages/envio/src/sources/HyperSyncSource.res rename to packages/envio/src/sources/EvmHyperSyncSource.res diff --git a/packages/envio/src/sources/FuelHyperSync.res b/packages/envio/src/sources/FuelHyperSync.res new file mode 100644 index 0000000000..ae894cb8e8 --- /dev/null +++ b/packages/envio/src/sources/FuelHyperSync.res @@ -0,0 +1,74 @@ +type logsQueryPage = { + items: array, + // Blocks referenced by `items`, one per height. + blocks: array, + nextBlock: int, + archiveHeight: int, +} + +module GetLogs = { + type error = + | UnexpectedMissingParams({missingParams: array}) + | WrongInstance + + exception Error(error) + + // Rust encodes structured failures as a JSON payload in the napi error's + // message: `{"kind":"MissingFields","fields":["receipt.txId", ...]}`. + // JSON.parse + shape check is the recovery protocol — no string-grepping + // on anyhow's Debug format. + let extractMissingParams = (exn: exn): option> => { + let message = switch exn { + | JsExn(jsExn) => jsExn->JsExn.message + | _ => None + } + switch message { + | None => None + | Some(msg) => + switch msg->JSON.parseOrThrow->JSON.Decode.object { + | exception _ => None + | None => None + | Some(obj) => + switch (obj->Dict.get("kind"), obj->Dict.get("fields")) { + | (Some(String("MissingFields")), Some(Array(fields))) => + Some(fields->Array.filterMap(JSON.Decode.string)) + | _ => None + } + } + } + } + + let query = async ( + ~client: FuelHyperSyncClient.t, + ~fromBlock, + ~toBlock, + ~registrationIndexes, + ~addressesByContractName, + ): logsQueryPage => { + let query: FuelHyperSyncClient.EventItems.query = { + fromBlock, + toBlock, + registrationIndexes, + addressesByContractName, + } + + let res = switch await client->FuelHyperSyncClient.getEventItems(query) { + | res => res + | exception exn => + switch exn->extractMissingParams { + | Some(missingParams) => throw(Error(UnexpectedMissingParams({missingParams: missingParams}))) + | None => throw(exn) + } + } + if res.nextBlock <= fromBlock { + // Might happen when /height response was from another instance of HyperFuel + throw(Error(WrongInstance)) + } + { + items: res.items, + blocks: res.blocks, + nextBlock: res.nextBlock, + archiveHeight: res.archiveHeight->Option.getOr(0), // TODO: FIXME: Shouldn't have a default here + } + } +} diff --git a/packages/envio/src/sources/FuelHyperSync.resi b/packages/envio/src/sources/FuelHyperSync.resi new file mode 100644 index 0000000000..43dc92636f --- /dev/null +++ b/packages/envio/src/sources/FuelHyperSync.resi @@ -0,0 +1,22 @@ +type logsQueryPage = { + items: array, + blocks: array, + nextBlock: int, + archiveHeight: int, +} + +module GetLogs: { + type error = + | UnexpectedMissingParams({missingParams: array}) + | WrongInstance + + exception Error(error) + + let query: ( + ~client: FuelHyperSyncClient.t, + ~fromBlock: int, + ~toBlock: option, + ~registrationIndexes: array, + ~addressesByContractName: dict>, + ) => promise +} diff --git a/packages/envio/src/sources/FuelHyperSyncClient.res b/packages/envio/src/sources/FuelHyperSyncClient.res new file mode 100644 index 0000000000..bfc3a3f92d --- /dev/null +++ b/packages/envio/src/sources/FuelHyperSyncClient.res @@ -0,0 +1,120 @@ +type t + +type cfg = { + url: string, + apiToken: string, +} + +module Registration = { + type kind = + | @as("LogData") LogData + | @as("Mint") Mint + | @as("Burn") Burn + | @as("Transfer") Transfer + | @as("Call") Call + + // The full per-(event, chain) registration passed to the Rust client at + // construction: routing identity plus the receipt-selection state queries + // are built from. + type input = { + // Chain-scoped sequential registration index, echoed back on routed items. + index: int, + eventName: string, + contractName: string, + isWildcard: bool, + kind: kind, + // The LogData `rb` value as a decimal string; absent for other kinds. + logId?: string, + } + + let fromOnEventRegistrations = ( + onEventRegistrations: array, + ): array => + onEventRegistrations->Array.map(reg => { + let eventConfig = + reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.fuelEventConfig) + let (kind, logId) = switch eventConfig.kind { + | LogData({logId}) => (LogData, Some(logId)) + | Mint => (Mint, None) + | Burn => (Burn, None) + | Transfer => (Transfer, None) + | Call => (Call, None) + } + { + index: reg.index, + eventName: eventConfig.name, + contractName: eventConfig.contractName, + isWildcard: reg.isWildcard, + kind, + ?logId, + } + }) +} + +module EventItems = { + // The whole per-query input: block range, the partition's registration + // selection (by index), and its current addresses. Receipt selections, + // field selection, and routing are derived on the Rust side. + type query = { + fromBlock: int, + // Inclusive; None queries to the end of available data. + toBlock: option, + registrationIndexes: array, + addressesByContractName: dict>, + } + + // One routed receipt with its kind-specific columns flattened: LogData + // carries `data` (decoded here in JS), Mint/Burn carry `val`/`subId`, + // Transfer/TransferOut/Call carry `amount`/`assetId`/`to` (TransferOut's + // wallet recipient normalised into `to`). + type item = { + onEventRegistrationIndex: int, + receiptIndex: int, + txId: string, + blockHeight: int, + srcAddress: Address.t, + data?: string, + subId?: string, + val?: bigint, + amount?: bigint, + assetId?: string, + to?: string, + } + + type block = { + id: string, + height: int, + time: int, + } + + type response = { + archiveHeight: option, + nextBlock: int, + // One block per height; items reference them by `blockHeight`. + blocks: array, + items: array, + } +} + +@send +external classNew: ( + Core.fuelHyperSyncClientCtor, + cfg, + ~userAgent: string, + array, +) => t = "new" + +let make = (cfg: cfg, ~eventRegistrations) => { + let envioVersion = Utils.EnvioPackage.value.version + Core.getAddon().fuelHyperSyncClient->classNew( + cfg, + ~userAgent=`hyperindex/${envioVersion}`, + eventRegistrations, + ) +} + +@send +external getEventItems: (t, EventItems.query) => promise = "getEventItems" + +@send +external getHeight: t => promise = "getHeight" diff --git a/packages/envio/src/sources/FuelHyperSyncSource.res b/packages/envio/src/sources/FuelHyperSyncSource.res new file mode 100644 index 0000000000..d79be286f0 --- /dev/null +++ b/packages/envio/src/sources/FuelHyperSyncSource.res @@ -0,0 +1,257 @@ +open Source + +let isUnauthorizedError = (message: string) => message->String.includes("401 Unauthorized") + +type options = { + chain: ChainMap.Chain.t, + endpointUrl: string, + apiToken: option, + // The chain's registrations, indexed by their sequential `index`. + onEventRegistrations: array, +} + +let make = ({chain, endpointUrl, apiToken, onEventRegistrations}: options): t => { + let name = "HyperFuel" + + let apiToken = switch apiToken { + | Some(token) => token + | None => + JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source. +Set the ENVIO_API_TOKEN environment variable in your .env file. +Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`) + } + + let client = switch FuelHyperSyncClient.make( + {url: endpointUrl, apiToken}, + ~eventRegistrations=FuelHyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations), + ) { + | client => client + | exception exn => + exn->ErrorHandling.mkLogAndRaise(~msg="Failed to instantiate the HyperFuel client") + } + + let getItemsOrThrow = async ( + ~fromBlock, + ~toBlock, + ~addressesByContractName, + ~contractNameByAddress as _, + ~knownHeight, + ~partitionId as _, + ~selection: FetchState.selection, + ~itemsTarget as _, + ~retry, + ~logger, + ) => { + let totalTimeRef = Performance.now() + + let startFetchingBatchTimeRef = Performance.now() + + //fetch batch + let pageUnsafe = try await FuelHyperSync.GetLogs.query( + ~client, + ~fromBlock, + ~toBlock, + ~registrationIndexes=selection.onEventRegistrations->Array.map(reg => reg.index), + ~addressesByContractName, + ) catch { + | FuelHyperSync.GetLogs.Error(error) => + throw( + Source.GetItemsError( + Source.FailedGettingItems({ + exn: %raw(`null`), + attemptedToBlock: toBlock->Option.getOr(knownHeight), + retry: switch error { + | WrongInstance => + let backoffMillis = switch retry { + | 0 => 100 + | _ => 500 * retry + } + WithBackoff({ + message: `Block #${fromBlock->Int.toString} not found in FuelHyperSync. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${backoffMillis->Int.toString}ms.`, + backoffMillis, + }) + | UnexpectedMissingParams({missingParams}) => + ImpossibleForTheQuery({ + message: `Source returned invalid data with missing required fields: ${missingParams->Array.joinUnsafe( + ", ", + )}`, + }) + }, + }), + ), + ) + | exn => + throw( + Source.GetItemsError( + Source.FailedGettingItems({ + exn, + attemptedToBlock: toBlock->Option.getOr(knownHeight), + retry: WithBackoff({ + message: `Unexpected issue while fetching events from HyperFuel client. Attempt a retry.`, + backoffMillis: switch retry { + | 0 => 500 + | _ => 1000 * retry + }, + }), + }), + ), + ) + } + + let pageFetchTime = startFetchingBatchTimeRef->Performance.secondsSince + let requestStats = [{Source.method: "getLogs", seconds: pageFetchTime}] + + //set height and next from block + let knownHeight = pageUnsafe.archiveHeight + + //The heighest (biggest) blocknumber that was accounted for in + //Our query. Not necessarily the blocknumber of the last log returned + //In the query + let heighestBlockQueried = pageUnsafe.nextBlock - 1 + + let parsingTimeRef = Performance.now() + + // Blocks are returned once per height; items reference them by blockHeight. + let blocksByHeight = Utils.Map.make() + pageUnsafe.blocks->Array.forEach(block => { + blocksByHeight->Utils.Map.set(block.height, block)->ignore + }) + + let chainId = chain->ChainMap.Chain.toChainId + + let parsedQueueItems = pageUnsafe.items->Array.map(item => { + // Routing happened in Rust; the item references its registration by + // chain-scoped index. + let onEventRegistration = onEventRegistrations->Array.getUnsafe(item.onEventRegistrationIndex) + let eventConfig = + onEventRegistration.eventConfig->( + Utils.magic: Internal.eventConfig => Internal.fuelEventConfig + ) + // Presence of every routed item's block is validated in Rust. + let block = blocksByHeight->Utils.Map.unsafeGet(item.blockHeight) + + let params = switch eventConfig.kind { + | LogData({decode}) => + // Kind-required columns are validated present in Rust before the item + // crosses the boundary. + let data = item.data->Option.getOr("") + try decode(data) catch { + | exn => { + let params = { + "chainId": chainId, + "blockNumber": item.blockHeight, + "logIndex": item.receiptIndex, + } + let logger = Logging.createChildFrom(~logger, ~params) + exn->ErrorHandling.mkLogAndRaise( + ~msg="Failed to decode Fuel LogData receipt, please double check your ABI.", + ~logger, + ) + } + } + | Mint | Burn => + ( + { + subId: item.subId->Option.getOr(""), + amount: item.val->Option.getOr(0n), + }: Internal.fuelSupplyParams + )->Obj.magic + | Transfer | Call => + ( + { + to: item.to->Option.getOr("")->Address.unsafeFromString, + assetId: item.assetId->Option.getOr(""), + amount: item.amount->Option.getOr(0n), + }: Internal.fuelTransferParams + )->Obj.magic + } + + Internal.Event({ + onEventRegistration, + chain, + blockNumber: item.blockHeight, + logIndex: item.receiptIndex, + // Fuel carries the transaction inline on the payload; the store key is + // unused (Fuel identifies transactions by hash, kept on the payload). + transactionIndex: 0, + payload: { + contractName: eventConfig.contractName, + eventName: eventConfig.name, + chainId, + params, + transaction: { + "id": item.txId, + }->Obj.magic, // TODO: Obj.magic needed until the field selection types are not configurable for Fuel and Evm separately + block: block->Obj.magic, + srcAddress: item.srcAddress, + logIndex: item.receiptIndex, + }->Fuel.fromPayload, + }) + }) + + let parsingTimeElapsed = parsingTimeRef->Performance.secondsSince + + // Fuel never rolls back on reorg, so block hashes here are purely informational + // for detect-only logging via ReorgDetection. + let blockHashes = pageUnsafe.blocks->Array.map(block => { + ReorgDetection.blockNumber: block.height, + blockHash: block.id, + }) + + let latestFetchedBlockTimestamp = switch blocksByHeight->Utils.Map.get(heighestBlockQueried) { + | Some(block) => block.time + | None => 0 + } + + let totalTimeElapsed = totalTimeRef->Performance.secondsSince + + let stats = { + totalTimeElapsed, + parsingTimeElapsed, + pageFetchTime, + } + + { + latestFetchedBlockTimestamp, + parsedQueueItems, + // Fuel keeps transaction and block on the payload; no store pages. + transactionStore: None, + blockStore: None, + latestFetchedBlockNumber: heighestBlockQueried, + stats, + knownHeight, + blockHashes, + fromBlockQueried: fromBlock, + requestStats, + } + } + + let getBlockHashes = (~blockNumbers as _, ~logger as _) => + JsError.throwWithMessage("HyperFuel does not support getting block hashes") + + { + name, + sourceFor: Sync, + chain, + getBlockHashes, + pollingInterval: 100, + poweredByHyperSync: true, + getHeightOrThrow: async () => { + let timerRef = Performance.now() + let height = try await client->FuelHyperSyncClient.getHeight catch { + | JsExn(e) => + switch e->JsExn.message { + | Some(message) if message->isUnauthorizedError => + Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`) + // Retrying an unauthorized request can never succeed, so block forever + let _ = await Promise.make((_, _) => ()) + 0 + | _ => throw(JsExn(e)) + } + } + let seconds = timerRef->Performance.secondsSince + {height, requestStats: [{method: "getHeight", seconds}]} + }, + getItemsOrThrow, + } +} diff --git a/packages/envio/src/sources/HyperFuel.res b/packages/envio/src/sources/HyperFuel.res deleted file mode 100644 index a1a356deb5..0000000000 --- a/packages/envio/src/sources/HyperFuel.res +++ /dev/null @@ -1,179 +0,0 @@ -type hyperSyncPage<'item> = { - items: array<'item>, - nextBlock: int, - archiveHeight: int, -} - -type block = { - id: string, - time: int, - height: int, -} - -type item = { - transactionId: string, - contractId: Address.t, - receipt: FuelSDK.Receipt.t, - receiptIndex: int, - block: block, -} - -type logsQueryPage = hyperSyncPage - -module GetLogs = { - type error = - | UnexpectedMissingParams({missingParams: array}) - | WrongInstance - - exception Error(error) - - // Rust encodes structured failures as a JSON payload in the napi error's - // message: `{"kind":"MissingFields","fields":["receipt.txId", ...]}`. - // JSON.parse + shape check is the recovery protocol — no string-grepping - // on anyhow's Debug format. - let extractMissingParams = (exn: exn): option> => { - let message = switch exn { - | JsExn(jsExn) => jsExn->JsExn.message - | _ => None - } - switch message { - | None => None - | Some(msg) => - switch msg->JSON.parseOrThrow->JSON.Decode.object { - | exception _ => None - | None => None - | Some(obj) => - switch (obj->Dict.get("kind"), obj->Dict.get("fields")) { - | (Some(String("MissingFields")), Some(Array(fields))) => - Some(fields->Array.filterMap(JSON.Decode.string)) - | _ => None - } - } - } - } - - let makeRequestBody = ( - ~fromBlock, - ~toBlockInclusive, - ~recieptsSelection, - ): HyperFuelClient.QueryTypes.query => { - { - fromBlock, - toBlockExclusive: ?switch toBlockInclusive { - | Some(toBlockInclusive) => Some(toBlockInclusive + 1) - | None => None - }, - receipts: recieptsSelection, - fieldSelection: { - receipt: [ - TxId, - BlockHeight, - RootContractId, - Data, - ReceiptIndex, - ReceiptType, - Rb, - // TODO: Include them only when there's a mint/burn/transferOut receipt selection - SubId, - Val, - Amount, - ToAddress, - AssetId, - To, - ], - block: [Id, Height, Time], - }, - } - } - - let getParam = (param, name) => { - switch param { - | Some(v) => v - | None => - throw( - Error( - UnexpectedMissingParams({ - missingParams: [name], - }), - ), - ) - } - } - - //Note this function can throw an error - let decodeLogQueryPageItems = (response_data: HyperFuelClient.queryResponseDataTyped): array< - item, - > => { - let {receipts, blocks} = response_data - - let blocksDict = Dict.make() - blocks->Array.forEach(block => { - blocksDict->Dict.set(block.height->(Utils.magic: int => string), block) - }) - - let items = [] - - receipts->Array.forEach(receipt => { - switch receipt.rootContractId { - | None => () - | Some(contractId) => { - let block = - blocksDict - ->Utils.Dict.dangerouslyGetNonOption(receipt.blockHeight->(Utils.magic: int => string)) - ->getParam("Failed to find block associated to receipt") - items - ->Array.push({ - transactionId: receipt.txId, - block: { - height: block.height, - id: block.id, - time: block.time, - }, - contractId, - receipt: receipt->(Utils.magic: HyperFuelClient.FuelTypes.receipt => FuelSDK.Receipt.t), - receiptIndex: receipt.receiptIndex, - }) - ->ignore - } - } - }) - items - } - - let convertResponse = (res: HyperFuelClient.queryResponseTyped): logsQueryPage => { - let {nextBlock, ?archiveHeight} = res - let page: logsQueryPage = { - items: res.data->decodeLogQueryPageItems, - nextBlock, - archiveHeight: archiveHeight->Option.getOr(0), // TODO: FIXME: Shouldn't have a default here - } - page - } - - let query = async ( - ~client: HyperFuelClient.t, - ~fromBlock, - ~toBlock, - ~recieptsSelection, - ): logsQueryPage => { - let query: HyperFuelClient.QueryTypes.query = makeRequestBody( - ~fromBlock, - ~toBlockInclusive=toBlock, - ~recieptsSelection, - ) - - let res = switch await client->HyperFuelClient.getSelectedData(query) { - | res => res - | exception exn => - switch exn->extractMissingParams { - | Some(missingParams) => throw(Error(UnexpectedMissingParams({missingParams: missingParams}))) - | None => throw(exn) - } - } - if res.nextBlock <= fromBlock { - // Might happen when /height response was from another instance of HyperFuel - throw(Error(WrongInstance)) - } - res->convertResponse - } -} diff --git a/packages/envio/src/sources/HyperFuel.resi b/packages/envio/src/sources/HyperFuel.resi deleted file mode 100644 index 81f7d42320..0000000000 --- a/packages/envio/src/sources/HyperFuel.resi +++ /dev/null @@ -1,36 +0,0 @@ -type hyperSyncPage<'item> = { - items: array<'item>, - nextBlock: int, - archiveHeight: int, -} - -type block = { - id: string, - time: int, - height: int, -} - -type item = { - transactionId: string, - contractId: Address.t, - receipt: FuelSDK.Receipt.t, - receiptIndex: int, - block: block, -} - -type logsQueryPage = hyperSyncPage - -module GetLogs: { - type error = - | UnexpectedMissingParams({missingParams: array}) - | WrongInstance - - exception Error(error) - - let query: ( - ~client: HyperFuelClient.t, - ~fromBlock: int, - ~toBlock: option, - ~recieptsSelection: array, - ) => promise -} diff --git a/packages/envio/src/sources/HyperFuelClient.res b/packages/envio/src/sources/HyperFuelClient.res deleted file mode 100644 index 2fb7f8b961..0000000000 --- a/packages/envio/src/sources/HyperFuelClient.res +++ /dev/null @@ -1,127 +0,0 @@ -type t - -type cfg = { - url: string, - apiToken: string, -} -module QueryTypes = { - type blockFieldOptions = - | @as("id") Id - | @as("height") Height - | @as("time") Time - - type blockFieldSelection = array - - type receiptFieldOptions = - | @as("tx_id") TxId - | @as("block_height") BlockHeight - | @as("to_address") ToAddress - | @as("amount") Amount - | @as("asset_id") AssetId - | @as("val") Val - | @as("rb") Rb - | @as("receipt_type") ReceiptType - | @as("receipt_index") ReceiptIndex - | @as("data") Data - | @as("root_contract_id") RootContractId - | @as("sub_id") SubId - | @as("to") To - - type receiptFieldSelection = array - - type fieldSelection = { - block?: blockFieldSelection, - receipt?: receiptFieldSelection, - } - - type receiptSelection = { - rootContractId?: array, - receiptType?: array, - rb?: array, - txStatus?: array, - } - - type query = { - /** The block to start the query from */ - fromBlock: int, - /** - * The block to end the query at. If not specified, the query will go until the - * end of data. Exclusive, the returned range will be [from_block..to_block). - * - * The query will return before it reaches this target block if it hits the time limit - * configured on the server. The user should continue their query by putting the - * next_block field in the response into from_block field of their next query. This implements - * pagination. - */ - @as("toBlock") - toBlockExclusive?: int, - /** - * List of receipt selections, the query will return receipts that match any of these selections and - * it will return receipts that are related to the returned objects. - */ - receipts?: array, - /** - * Field selection. The user can select which fields they are interested in, requesting less fields will improve - * query execution time and reduce the payload size so the user should always use a minimal number of fields. - */ - fieldSelection: fieldSelection, - } -} - -module FuelTypes = { - type receipt = { - receiptIndex: int, - rootContractId?: Address.t, - txId: string, - blockHeight: int, - receiptType: FuelSDK.receiptType, - data?: string, - rb?: bigint, - val?: bigint, - subId?: string, - amount?: bigint, - assetId?: string, - to?: string, - toAddress?: string, - } - - type block = { - id: string, - height: int, - time: int, - } -} - -type queryResponseDataTyped = { - receipts: array, - blocks: array, -} - -type queryResponseTyped = { - /** Current height of the source HyperFuel instance */ - archiveHeight?: int, - /** - * Next block to query for, the responses are paginated so - * the caller should continue the query from this block if they - * didn't get responses up to the to_block they specified in the Query. - */ - nextBlock: int, - /** Total time it took the HyperFuel instance to execute the query. */ - totalExecutionTime: int, - /** Response data */ - data: queryResponseDataTyped, -} - -@send -external classNew: (Core.hyperfuelClientCtor, cfg, ~userAgent: string) => t = "new" - -let make = (cfg: cfg) => { - let envioVersion = Utils.EnvioPackage.value.version - Core.getAddon().hyperfuelClient->classNew(cfg, ~userAgent=`hyperindex/${envioVersion}`) -} - -@send -external getSelectedData: (t, QueryTypes.query) => promise = "getSelectedData" - -@send -external getHeight: t => promise = "getHeight" diff --git a/packages/envio/src/sources/HyperFuelSource.res b/packages/envio/src/sources/HyperFuelSource.res deleted file mode 100644 index d22f8bd0fb..0000000000 --- a/packages/envio/src/sources/HyperFuelSource.res +++ /dev/null @@ -1,502 +0,0 @@ -open Source - -exception EventRoutingFailed - -let isUnauthorizedError = (message: string) => message->String.includes("401 Unauthorized") - -let mintEventTag = "mint" -let burnEventTag = "burn" -let transferEventTag = "transfer" -let callEventTag = "call" - -type selectionConfig = { - getRecieptsSelection: ( - ~addressesByContractName: dict>, - ) => array, - eventRouter: EventRouter.t, -} - -let logDataReceiptTypeSelection: array = [LogData] - -// only transactions with status 1 (success) -let txStatusSelection = [1] - -let makeGetNormalRecieptsSelection = ( - ~nonWildcardLogDataRbsByContract, - ~nonLogDataReceiptTypesByContract, - ~contractNames, -) => { - (~addressesByContractName) => { - let selection: array = [] - - //Instantiate each time to add new registered contract addresses - contractNames->Utils.Set.forEach(contractName => { - switch addressesByContractName->Utils.Dict.dangerouslyGetNonOption(contractName) { - | None - | Some([]) => () - | Some(addresses) => { - switch nonLogDataReceiptTypesByContract->Utils.Dict.dangerouslyGetNonOption( - contractName, - ) { - | Some(receiptTypes) => - selection - ->Array.push({ - rootContractId: addresses, - receiptType: receiptTypes, - txStatus: txStatusSelection, - }) - ->ignore - | None => () - } - switch nonWildcardLogDataRbsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) { - | None - | Some([]) => () - | Some(nonWildcardLogDataRbs) => - selection - ->Array.push({ - rootContractId: addresses, - receiptType: logDataReceiptTypeSelection, - txStatus: txStatusSelection, - rb: nonWildcardLogDataRbs, - }) - ->ignore - } - } - } - }) - - selection - } -} - -let makeWildcardRecieptsSelection = (~wildcardLogDataRbs, ~nonLogDataWildcardReceiptTypes) => { - let selection: array = [] - - switch nonLogDataWildcardReceiptTypes { - | [] => () - | nonLogDataWildcardReceiptTypes => - selection - ->Array.push( - ( - { - receiptType: nonLogDataWildcardReceiptTypes, - txStatus: txStatusSelection, - }: HyperFuelClient.QueryTypes.receiptSelection - ), - ) - ->ignore - } - - switch wildcardLogDataRbs { - | [] => () - | wildcardLogDataRbs => - selection - ->Array.push( - ( - { - receiptType: logDataReceiptTypeSelection, - txStatus: txStatusSelection, - rb: wildcardLogDataRbs, - }: HyperFuelClient.QueryTypes.receiptSelection - ), - ) - ->ignore - } - - selection -} - -let getSelectionConfig = (selection: FetchState.selection, ~chain) => { - let eventRouter = EventRouter.empty() - let nonWildcardLogDataRbsByContract = Dict.make() - let wildcardLogDataRbs = [] - - // This is for non-LogData events, since they don't have rb filter and can be grouped - let nonLogDataReceiptTypesByContract = Dict.make() - let nonLogDataWildcardReceiptTypes = [] - - let addNonLogDataWildcardReceiptTypes = (receiptType: FuelSDK.receiptType) => { - nonLogDataWildcardReceiptTypes->Array.push(receiptType)->ignore - } - let addNonLogDataReceiptType = (contractName, receiptType: FuelSDK.receiptType) => { - switch nonLogDataReceiptTypesByContract->Utils.Dict.dangerouslyGetNonOption(contractName) { - | None => nonLogDataReceiptTypesByContract->Dict.set(contractName, [receiptType]) - | Some(receiptTypes) => receiptTypes->Array.push(receiptType)->ignore // Duplication prevented by EventRouter - } - } - - let contractNames = Utils.Set.make() - - selection.onEventRegistrations->Array.forEach(reg => { - let eventConfig = - reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.fuelEventConfig) - let contractName = eventConfig.contractName - let isWildcard = reg.isWildcard - if !isWildcard { - let _ = contractNames->Utils.Set.add(contractName) - } - eventRouter->EventRouter.addOrThrow( - eventConfig.id, - reg, - ~contractName, - ~eventName=eventConfig.name, - ~chain, - ~isWildcard, - ) - - switch (eventConfig.kind, isWildcard) { - | (Mint, true) => addNonLogDataWildcardReceiptTypes(Mint) - | (Mint, false) => addNonLogDataReceiptType(contractName, Mint) - | (Burn, true) => addNonLogDataWildcardReceiptTypes(Burn) - | (Burn, false) => addNonLogDataReceiptType(contractName, Burn) - | (Transfer, true) => { - addNonLogDataWildcardReceiptTypes(Transfer) - addNonLogDataWildcardReceiptTypes(TransferOut) - } - | (Transfer, false) => { - addNonLogDataReceiptType(contractName, Transfer) - addNonLogDataReceiptType(contractName, TransferOut) - } - | (Call, true) => addNonLogDataWildcardReceiptTypes(Call) - | (Call, false) => - JsError.throwWithMessage("Call receipt indexing currently supported only in wildcard mode") - | (LogData({logId}), _) => { - let rb = logId->BigInt.fromStringOrThrow - if isWildcard { - wildcardLogDataRbs->Array.push(rb)->ignore - } else { - switch nonWildcardLogDataRbsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) { - | Some(arr) => arr->Array.push(rb) - | None => nonWildcardLogDataRbsByContract->Dict.set(contractName, [rb]) - } - } - } - } - }) - - { - getRecieptsSelection: switch selection.dependsOnAddresses { - | false => { - let recieptsSelection = makeWildcardRecieptsSelection( - ~wildcardLogDataRbs, - ~nonLogDataWildcardReceiptTypes, - ) - (~addressesByContractName as _) => recieptsSelection - } - | true => - makeGetNormalRecieptsSelection( - ~nonWildcardLogDataRbsByContract, - ~nonLogDataReceiptTypesByContract, - ~contractNames, - ) - }, - eventRouter, - } -} - -let memoGetSelectionConfig = (~chain) => { - let cache = Utils.WeakMap.make() - selection => - switch cache->Utils.WeakMap.get(selection) { - | Some(c) => c - | None => { - let c = selection->getSelectionConfig(~chain) - let _ = cache->Utils.WeakMap.set(selection, c) - c - } - } -} - -type options = { - chain: ChainMap.Chain.t, - endpointUrl: string, - apiToken: option, -} - -let make = ({chain, endpointUrl, apiToken}: options): t => { - let name = "HyperFuel" - - let apiToken = switch apiToken { - | Some(token) => token - | None => - JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source. -Set the ENVIO_API_TOKEN environment variable in your .env file. -Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`) - } - - let client = switch HyperFuelClient.make({url: endpointUrl, apiToken}) { - | client => client - | exception exn => - exn->ErrorHandling.mkLogAndRaise(~msg="Failed to instantiate the HyperFuel client") - } - - let getSelectionConfig = memoGetSelectionConfig(~chain) - - let getItemsOrThrow = async ( - ~fromBlock, - ~toBlock, - ~addressesByContractName, - ~contractNameByAddress, - ~knownHeight, - ~partitionId as _, - ~selection: FetchState.selection, - ~itemsTarget as _, - ~retry, - ~logger, - ) => { - let totalTimeRef = Performance.now() - - let selectionConfig = getSelectionConfig(selection) - let recieptsSelection = selectionConfig.getRecieptsSelection(~addressesByContractName) - - let startFetchingBatchTimeRef = Performance.now() - - //fetch batch - let pageUnsafe = try await HyperFuel.GetLogs.query( - ~client, - ~fromBlock, - ~toBlock, - ~recieptsSelection, - ) catch { - | HyperFuel.GetLogs.Error(error) => - throw( - Source.GetItemsError( - Source.FailedGettingItems({ - exn: %raw(`null`), - attemptedToBlock: toBlock->Option.getOr(knownHeight), - retry: switch error { - | WrongInstance => - let backoffMillis = switch retry { - | 0 => 100 - | _ => 500 * retry - } - WithBackoff({ - message: `Block #${fromBlock->Int.toString} not found in HyperFuel. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${backoffMillis->Int.toString}ms.`, - backoffMillis, - }) - | UnexpectedMissingParams({missingParams}) => - ImpossibleForTheQuery({ - message: `Source returned invalid data with missing required fields: ${missingParams->Array.joinUnsafe( - ", ", - )}`, - }) - }, - }), - ), - ) - | exn => - throw( - Source.GetItemsError( - Source.FailedGettingItems({ - exn, - attemptedToBlock: toBlock->Option.getOr(knownHeight), - retry: WithBackoff({ - message: `Unexpected issue while fetching events from HyperFuel client. Attempt a retry.`, - backoffMillis: switch retry { - | 0 => 500 - | _ => 1000 * retry - }, - }), - }), - ), - ) - } - - let pageFetchTime = startFetchingBatchTimeRef->Performance.secondsSince - let requestStats = [{Source.method: "getLogs", seconds: pageFetchTime}] - - //set height and next from block - let knownHeight = pageUnsafe.archiveHeight - - //The heighest (biggest) blocknumber that was accounted for in - //Our query. Not necessarily the blocknumber of the last log returned - //In the query - let heighestBlockQueried = pageUnsafe.nextBlock - 1 - - let parsingTimeRef = Performance.now() - - let parsedQueueItems = pageUnsafe.items->Array.map(item => { - let {contractId: contractAddress, receipt, block, receiptIndex} = item - - let chainId = chain->ChainMap.Chain.toChainId - let eventId = switch receipt { - | LogData({rb}) => BigInt.toString(rb) - | Mint(_) => mintEventTag - | Burn(_) => burnEventTag - | Transfer(_) - | TransferOut(_) => transferEventTag - | Call(_) => callEventTag - } - - let onEventRegistration = switch selectionConfig.eventRouter->EventRouter.get( - ~tag=eventId, - ~contractNameByAddress, - ~contractAddress, - ) { - | None => { - let logger = Logging.createChildFrom( - ~logger, - ~params={ - "chainId": chainId, - "blockNumber": block.height, - "logIndex": receiptIndex, - "contractAddress": contractAddress, - "eventId": eventId, - }, - ) - EventRoutingFailed->ErrorHandling.mkLogAndRaise( - ~msg="Failed to route registered event", - ~logger, - ) - } - | Some(onEventRegistration) => onEventRegistration - } - - let eventConfig = - onEventRegistration.eventConfig->( - Utils.magic: Internal.eventConfig => Internal.fuelEventConfig - ) - - let params = switch (eventConfig, receipt) { - | ({kind: LogData({decode})}, LogData({data})) => - try decode(data) catch { - | exn => { - let params = { - "chainId": chainId, - "blockNumber": block.height, - "logIndex": receiptIndex, - } - let logger = Logging.createChildFrom(~logger, ~params) - exn->ErrorHandling.mkLogAndRaise( - ~msg="Failed to decode Fuel LogData receipt, please double check your ABI.", - ~logger, - ) - } - } - | (_, Mint({val, subId})) - | (_, Burn({val, subId})) => - ( - { - subId, - amount: val, - }: Internal.fuelSupplyParams - )->Obj.magic - | (_, Transfer({amount, assetId, to})) => - ( - { - to: to->Address.unsafeFromString, - assetId, - amount, - }: Internal.fuelTransferParams - )->Obj.magic - | (_, TransferOut({amount, assetId, toAddress})) => - ( - { - to: toAddress->Address.unsafeFromString, - assetId, - amount, - }: Internal.fuelTransferParams - )->Obj.magic - | (_, Call({amount, assetId, to})) => - ( - { - to: to->Address.unsafeFromString, - assetId, - amount, - }: Internal.fuelTransferParams - )->Obj.magic - // This should never happen unless there's a bug in the routing logic - | _ => JsError.throwWithMessage("Unexpected bug in the event routing logic") - } - - Internal.Event({ - onEventRegistration: (onEventRegistration :> Internal.onEventRegistration), - chain, - blockNumber: block.height, - logIndex: receiptIndex, - // Fuel carries the transaction inline on the payload; the store key is - // unused (Fuel identifies transactions by hash, kept on the payload). - transactionIndex: 0, - payload: { - contractName: eventConfig.contractName, - eventName: eventConfig.name, - chainId, - params, - transaction: { - "id": item.transactionId, - }->Obj.magic, // TODO: Obj.magic needed until the field selection types are not configurable for Fuel and Evm separately - block: block->Obj.magic, - srcAddress: contractAddress, - logIndex: receiptIndex, - }->Fuel.fromPayload, - }) - }) - - let parsingTimeElapsed = parsingTimeRef->Performance.secondsSince - - // Fuel never rolls back on reorg, so block hashes here are purely informational - // for detect-only logging via ReorgDetection. - let blockHashes = pageUnsafe.items->Array.map(({block}) => { - ReorgDetection.blockNumber: block.height, - blockHash: block.id, - }) - - let latestFetchedBlockTimestamp = switch pageUnsafe.items->Array.get( - pageUnsafe.items->Array.length - 1, - ) { - | Some({block}) if block.height == heighestBlockQueried => block.time - | _ => 0 - } - - let totalTimeElapsed = totalTimeRef->Performance.secondsSince - - let stats = { - totalTimeElapsed, - parsingTimeElapsed, - pageFetchTime, - } - - { - latestFetchedBlockTimestamp, - parsedQueueItems, - // Fuel keeps transaction and block on the payload; no store pages. - transactionStore: None, - blockStore: None, - latestFetchedBlockNumber: heighestBlockQueried, - stats, - knownHeight, - blockHashes, - fromBlockQueried: fromBlock, - requestStats, - } - } - - let getBlockHashes = (~blockNumbers as _, ~logger as _) => - JsError.throwWithMessage("HyperFuel does not support getting block hashes") - - { - name, - sourceFor: Sync, - chain, - getBlockHashes, - pollingInterval: 100, - poweredByHyperSync: true, - getHeightOrThrow: async () => { - let timerRef = Performance.now() - let height = try await client->HyperFuelClient.getHeight catch { - | JsExn(e) => - switch e->JsExn.message { - | Some(message) if message->isUnauthorizedError => - Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`) - // Retrying an unauthorized request can never succeed, so block forever - let _ = await Promise.make((_, _) => ()) - 0 - | _ => throw(JsExn(e)) - } - } - let seconds = timerRef->Performance.secondsSince - {height, requestStats: [{method: "getHeight", seconds}]} - }, - getItemsOrThrow, - } -} diff --git a/packages/envio/src/sources/HyperSyncClient.res b/packages/envio/src/sources/HyperSyncClient.res index 454a25afa1..a448ba2cd8 100644 --- a/packages/envio/src/sources/HyperSyncClient.res +++ b/packages/envio/src/sources/HyperSyncClient.res @@ -358,11 +358,11 @@ type t = { } @send -external classNew: (Core.evmHypersyncClientCtor, cfg, string, array) => t = +external classNew: (Core.evmHyperSyncClientCtor, cfg, string, array) => t = "new" let makeWithAgent = (cfg, ~userAgent, ~eventRegistrations) => - Core.getAddon().evmHypersyncClient->classNew(cfg, userAgent, eventRegistrations) + Core.getAddon().evmHyperSyncClient->classNew(cfg, userAgent, eventRegistrations) type logLevel = [#trace | #debug | #info | #warn | #error] let logLevelSchema: S.t = S.enum([#trace, #debug, #info, #warn, #error]) diff --git a/packages/envio/src/sources/SvmHyperSyncClient.res b/packages/envio/src/sources/SvmHyperSyncClient.res index 8d06c85a25..598dcfbbe8 100644 --- a/packages/envio/src/sources/SvmHyperSyncClient.res +++ b/packages/envio/src/sources/SvmHyperSyncClient.res @@ -7,10 +7,78 @@ type cfg = { maxNumRetries?: int, retryBaseMs?: int, retryCeilingMs?: int, - /// Per-program Borsh schema descriptors (JSON, one per program). The Rust - /// client builds these into decoders at creation and decodes matching - /// instructions inline on `get`. - programSchemas?: array, +} + +module Registration = { + type accountFilter = { + position: int, + values: array, + } + + // The full per-(instruction, chain) registration passed to the Rust client + // at construction: routing identity, the fetch state queries are built + // from, and the Borsh schema pieces the client builds decoders from. + type input = { + // Chain-scoped sequential registration index, echoed back on routed items. + index: int, + instructionName: string, + contractName: string, + programId: string, + isWildcard: bool, + discriminator?: string, + discriminatorByteLen: int, + isInner?: bool, + includeLogs: bool, + // DNF: outer array is OR of AND-groups. + accountFilters: array>, + // camelCase Internal.svmTransactionField / svmBlockField names. + transactionFields: array, + blockFields: array, + // Borsh schema pieces; empty accounts + absent argsJson = no schema. + accounts: array, + argsJson?: string, + definedTypesJson?: string, + } + + let fromOnEventRegistrations = ( + onEventRegistrations: array, + ): array => + onEventRegistrations->Array.map(reg => { + let eventConfig = + reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig) + { + index: reg.index, + instructionName: eventConfig.name, + contractName: eventConfig.contractName, + programId: eventConfig.programId->SvmTypes.Pubkey.toString, + isWildcard: reg.isWildcard, + discriminator: ?eventConfig.discriminator, + discriminatorByteLen: eventConfig.discriminatorByteLen, + isInner: ?eventConfig.isInner, + includeLogs: eventConfig.includeLogs, + accountFilters: eventConfig.accountFilters->Array.map(group => + group->Array.map( + (filter): accountFilter => { + position: filter.position, + values: filter.values->SvmTypes.Pubkey.toStrings, + }, + ) + ), + transactionFields: eventConfig.selectedTransactionFields->Utils.Set.toArray, + blockFields: eventConfig.selectedBlockFields + ->(Utils.magic: Utils.Set.t => Utils.Set.t) + ->Utils.Set.toArray, + accounts: eventConfig.accounts, + argsJson: ?switch eventConfig.args { + | JSON.Null => None + | args => Some(args->JSON.stringify) + }, + definedTypesJson: ?switch eventConfig.definedTypes { + | JSON.Null => None + | definedTypes => Some(definedTypes->JSON.stringify) + }, + } + }) } module QueryTypes = { @@ -37,54 +105,7 @@ module QueryTypes = { | @as("loaded_addresses_writable") LoadedAddressesWritable | @as("loaded_addresses_readonly") LoadedAddressesReadonly - type instructionField = - | @as("slot") Slot - | @as("transaction_index") TransactionIndex - | @as("instruction_address") InstructionAddress - | @as("program_id") ProgramId - | @as("accounts") Accounts - | @as("data") Data - | @as("d1") D1 - | @as("d2") D2 - | @as("d4") D4 - | @as("d8") D8 - | @as("a0") A0 - | @as("a1") A1 - | @as("a2") A2 - | @as("a3") A3 - | @as("a4") A4 - | @as("a5") A5 - | @as("a6") A6 - | @as("a7") A7 - | @as("a8") A8 - | @as("a9") A9 - | @as("is_inner") IsInner - | @as("is_committed") IsCommitted - - type logField = - | @as("slot") Slot - | @as("transaction_index") TransactionIndex - | @as("instruction_address") InstructionAddress - | @as("program_id") ProgramId - | @as("kind") Kind - | @as("message") Message - - type tokenBalanceField = - | @as("slot") Slot - | @as("transaction_index") TransactionIndex - | @as("account") Account - | @as("mint") Mint - | @as("owner") Owner - | @as("pre_amount") PreAmount - | @as("post_amount") PostAmount - - type fieldSelection = { - block?: array, - transaction?: array, - instruction?: array, - log?: array, - tokenBalance?: array, - } + type fieldSelection = {block?: array, transaction?: array} /** Filter for selecting instructions. All non-empty fields are AND-ed: an instruction must match at least one value in every non-empty field. @@ -97,43 +118,19 @@ module QueryTypes = { d2?: array, d4?: array, d8?: array, - a0?: array, - a1?: array, - a2?: array, - a3?: array, - a4?: array, - a5?: array, - a6?: array, - a7?: array, - a8?: array, - a9?: array, isInner?: bool, } - type transactionSelection = { - feePayer?: array, - success?: bool, - } - - type logSelection = { - programId?: array, - kind?: array, - } - + // The `get` query surface, used only for block-data range queries; event + // fetching goes through `getEventItems`, which builds its query in Rust. type query = { fromSlot: int, toSlot?: int, instructions?: array, - transactions?: array, - logs?: array, includeAllBlocks?: bool, - includeTokenBalances?: bool, fields?: fieldSelection, maxNumBlocks?: int, - maxNumTransactions?: int, maxNumInstructions?: int, - maxNumLogs?: int, - maxNumTokenBalances?: int, } } @@ -148,13 +145,6 @@ module ResponseTypes = { /// Borsh-decoded view attached by the Rust client. `argsJson`/`accountsJson` /// are stringified to side-step napi-rs's lack of native JSON passthrough. - /** Solana instruction record. - - `data` is the raw instruction byte buffer, hex-encoded with a `0x` prefix. - `d1`..`d8` are the same byte prefix as `data` but truncated to N bytes - (only `Some` when the instruction is at least that long), exposed for - handler-dispatch convenience. - `accounts` is the full positional account list in base58. */ type decodedInstruction = { name: string, argsJson: string, @@ -175,22 +165,11 @@ module ResponseTypes = { d8?: string, isInner: bool, isCommitted: bool, - decoded?: decodedInstruction, - } - - type log = { - slot: int, - transactionIndex?: int, - instructionAddress?: array, - programId?: string, - kind?: string, - message?: string, } type queryResponseData = { blocks: array, instructions: array, - logs: array, } type queryResponse = { @@ -200,18 +179,76 @@ module ResponseTypes = { } } +module EventItems = { + // The whole per-query input: slot range, the partition's registration + // selection (by index), and its current addresses (program ids per program + // name). Instruction selections, field selection, and routing are derived + // on the Rust side. + type query = { + fromSlot: int, + // Inclusive; None queries to the end of available data. + toSlot: option, + maxNumInstructions: int, + registrationIndexes: array, + addressesByContractName: dict>, + } + + type log = { + kind: string, + message: string, + } + + // One routed instruction; `block` and `transaction` are materialised from + // the per-chain stores at batch prep. + type item = { + onEventRegistrationIndex: int, + slot: int, + transactionIndex: int, + instructionAddress: array, + programId: string, + accounts: array, + data: string, + d1?: string, + d2?: string, + d4?: string, + d8?: string, + isInner: bool, + decoded?: ResponseTypes.decodedInstruction, + // Present only when the routed registration opted in via `includeLogs`. + logs?: array, + } + + type response = { + nextSlot: int, + // One lean header per slot referenced by `items`; the full blocks live in + // the block store returned alongside. + blocks: array, + items: array, + } +} + type query = QueryTypes.query type queryResponse = ResponseTypes.queryResponse type t = { getHeight: unit => promise, - // Returns the response plus pages of raw transactions and blocks (kept in - // Rust), keyed by (slot, transactionIndex) / slot, materialised at batch prep. + // Block-data range queries only; the store pages it returns are empty. get: (~query: query) => promise<(queryResponse, TransactionStore.t, BlockStore.t)>, + // Returns the routed items plus pages of raw transactions and blocks (kept + // in Rust), keyed by (slot, transactionIndex) / slot, materialised at batch + // prep. + getEventItems: ( + ~query: EventItems.query, + ) => promise<(EventItems.response, TransactionStore.t, BlockStore.t)>, } @send -external classFromConfig: (Core.svmHypersyncClientCtor, cfg, string) => t = "fromConfig" +external classFromConfig: ( + Core.svmHyperSyncClientCtor, + cfg, + string, + array, +) => t = "fromConfig" let make = ( ~url, @@ -220,10 +257,10 @@ let make = ( ~maxNumRetries=?, ~retryBaseMs=?, ~retryCeilingMs=?, - ~programSchemas=?, + ~eventRegistrations=[], ) => { let envioVersion = Utils.EnvioPackage.value.version - Core.getAddon().svmHypersyncClient->classFromConfig( + Core.getAddon().svmHyperSyncClient->classFromConfig( { url, ?apiToken, @@ -231,8 +268,8 @@ let make = ( ?maxNumRetries, ?retryBaseMs, ?retryCeilingMs, - ?programSchemas, }, `hyperindex/${envioVersion}`, + eventRegistrations, ) } diff --git a/packages/envio/src/sources/SvmHyperSyncSource.res b/packages/envio/src/sources/SvmHyperSyncSource.res index 420d6e094b..55315746f7 100644 --- a/packages/envio/src/sources/SvmHyperSyncSource.res +++ b/packages/envio/src/sources/SvmHyperSyncSource.res @@ -8,159 +8,15 @@ type options = { clientTimeoutMillis: int, } -// Build HyperSync InstructionSelections from event configs. Each AND-group in -// `cfg.accountFilters` becomes its own selection; selections sharing the same -// `(programId, dN)` are OR-ed by the wire protocol. Empty outer array emits -// one selection with no `aN` set (no account filtering). -// -// Empty programId means the config carries no real program (placeholder), in -// which case we skip — better to over-fetch nothing than ship a degenerate -// query. -let buildInstructionSelections = (eventConfigs: array): array< - SvmHyperSyncClient.QueryTypes.instructionSelection, -> => { - eventConfigs->Array.flatMap(cfg => { - let programIdString = cfg.programId->SvmTypes.Pubkey.toString - if programIdString === "" { - [] - } else { - // Each instruction owns exactly one dN field — the one matching its - // declared byte length. The server-side filter is `d{N} IN [..]`. - let (d1, d2, d4, d8) = switch (cfg.discriminator, cfg.discriminatorByteLen) { - | (Some(d), 1) => (Some([d]), None, None, None) - | (Some(d), 2) => (None, Some([d]), None, None) - | (Some(d), 4) => (None, None, Some([d]), None) - | (Some(d), 8) => (None, None, None, Some([d])) - | _ => (None, None, None, None) - } - let groups = switch cfg.accountFilters { - | [] => [[]] - | gs => gs - } - groups->Array.map(group => { - let pick = position => - group - ->Array.filterMap( - f => f.position == position ? Some(f.values->SvmTypes.Pubkey.toStrings) : None, - ) - ->Array.get(0) - - ( - { - programId: [programIdString], - ?d1, - ?d2, - ?d4, - ?d8, - a0: ?pick(0), - a1: ?pick(1), - a2: ?pick(2), - a3: ?pick(3), - a4: ?pick(4), - a5: ?pick(5), - isInner: ?cfg.isInner, - }: SvmHyperSyncClient.QueryTypes.instructionSelection - ) - }) - } - }) -} - // Synthesize a stable logIndex for an SVM instruction so the FetchState // ordering machinery (which compares by `(blockNumber, logIndex)`) sorts // instructions deterministically within a slot. The bit packing fits inside // JS's 53-bit safe-integer range: transactionIndex ≤ ~10k per slot, // instruction position ≤ 1000 per tx, depth ≤ ~10. Outer-only instructions // land at `tx * 65536`; inner ones append depth-weighted offsets. -let synthLogIndex = (instr: SvmHyperSyncClient.ResponseTypes.instruction) => { - let tx = instr.transactionIndex - let addrSum = instr.instructionAddress->Array.reduce(0, (acc, n) => acc * 1024 + n + 1) - tx * 65536 + addrSum -} - -let serializeInstructionAddress = (addr: array) => - addr->Array.map(n => n->Int.toString)->Array.joinUnsafe(",") - -// Build per-program schema descriptors by grouping eventConfigs by programId, -// returning one descriptor JSON per program. These are handed to the Solana -// client at creation; it builds them into decoders and decodes matching -// instructions inline on `get`. -let buildProgramSchemas = (eventConfigs: array): array< - string, -> => { - // Group by programId base58 string. Skip events that carry no schema - // (accounts == [] && args is JSON.Null && definedTypes is JSON.Null — - // the resolved-empty case from system_config.rs). - let descriptorsByProgram: dict<{ - "programId": string, - "definedTypes": JSON.t, - "instructions": array<{ - "name": string, - "discriminator": string, - "accounts": array, - "args": JSON.t, - }>, - }> = Dict.make() - - eventConfigs->Array.forEach(ec => { - let programIdString = ec.programId->SvmTypes.Pubkey.toString - if programIdString === "" { - // Stage 4 placeholder pattern: skip empty program ids. - () - } else { - let hasSchema = ec.accounts->Array.length > 0 || ec.args !== JSON.Null - let discriminator = ec.discriminator->Option.getOr("") - if hasSchema && discriminator !== "" { - // Inline-schema programs declare no custom types, so `definedTypes` - // arrives as JSON.Null; the Rust descriptor's `#[serde(default)]` only - // covers an absent field, not an explicit null, so coalesce here. - let definedTypes = switch ec.definedTypes { - | JSON.Null => JSON.Object(Dict.make()) - | other => other - } - let existing = descriptorsByProgram->Dict.get(programIdString) - let descriptor = switch existing { - | Some(d) => d - | None => { - "programId": programIdString, - "definedTypes": definedTypes, - "instructions": [], - } - } - let instruction = { - "name": ec.name, - "discriminator": discriminator, - "accounts": ec.accounts, - "args": ec.args, - } - descriptorsByProgram->Dict.set( - programIdString, - { - "programId": descriptor["programId"], - "definedTypes": descriptor["definedTypes"], - "instructions": descriptor["instructions"]->Array.concat([instruction]), - }, - ) - } - } - }) - - descriptorsByProgram - ->Dict.valuesToArray - ->Array.map(descriptor => - descriptor - ->(Utils.magic: { - "programId": string, - "definedTypes": JSON.t, - "instructions": array<{ - "name": string, - "discriminator": string, - "accounts": array, - "args": JSON.t, - }>, - } => JSON.t) - ->JSON.stringify - ) +let synthLogIndex = (~transactionIndex, ~instructionAddress) => { + let addrSum = instructionAddress->Array.reduce(0, (acc, n) => acc * 1024 + n + 1) + transactionIndex * 65536 + addrSum } // Parse the Rust-decoded instruction (args/accounts arrive as JSON strings to @@ -186,234 +42,70 @@ let parseDecoded = ( // `block` is omitted; it's materialised from the block store at batch prep. let toSvmInstruction = ( - instr: SvmHyperSyncClient.ResponseTypes.instruction, + item: SvmHyperSyncClient.EventItems.item, ~programName, ~instructionName, - ~logs, ): Envio.svmInstruction => { programName, instructionName, - programId: instr.programId->SvmTypes.Pubkey.fromStringUnsafe, - data: instr.data, - accounts: instr.accounts->SvmTypes.Pubkey.fromStringsUnsafe, - instructionAddress: instr.instructionAddress, - isInner: instr.isInner, - d1: ?instr.d1, - d2: ?instr.d2, - d4: ?instr.d4, - d8: ?instr.d8, - params: ?(instr.decoded->Option.map(parseDecoded)), - ?logs, -} - -// Probe the discriminator byte-length ordering longest-first. Stops at the -// first router hit. Falls back to the `_none` key (program-wide handler) when -// no discriminator-keyed handler matches. -let probeRouter = ( - router: EventRouter.t, - programId: SvmTypes.Pubkey.t, - instr: SvmHyperSyncClient.ResponseTypes.instruction, - byteLengthsDesc: array, - ~contractAddress, - ~contractNameByAddress, -) => { - let probe = (dN: option) => { - let tag = EventRouter.getSvmEventId(~programId, ~discriminator=dN) - router->EventRouter.get(~tag, ~contractAddress, ~contractNameByAddress) - } - - let result = byteLengthsDesc->Array.reduce(None, (acc, len) => - switch acc { - | Some(_) => acc - | None => - let candidate = switch len { - | 8 => instr.d8 - | 4 => instr.d4 - | 2 => instr.d2 - | 1 => instr.d1 - | _ => None - } - switch candidate { - | Some(_) as d => probe(d) - | None => None - } - } - ) - - switch result { - | Some(_) as hit => hit - | None => probe(None) // program-wide fallback - } + programId: item.programId->SvmTypes.Pubkey.fromStringUnsafe, + data: item.data, + accounts: item.accounts->SvmTypes.Pubkey.fromStringsUnsafe, + instructionAddress: item.instructionAddress, + isInner: item.isInner, + d1: ?item.d1, + d2: ?item.d2, + d4: ?item.d4, + d8: ?item.d8, + params: ?(item.decoded->Option.map(parseDecoded)), + logs: ?( + item.logs->Option.map(logs => + logs->Array.map((log): Envio.svmLog => {kind: log.kind, message: log.message}) + ) + ), } -// Map a selected transaction field to the extra query-side column it needs. -// `transactionIndex` is always fetched as the store key, and `tokenBalances` -// lives in a separate table (requested via `needsTokenBalances`), so neither -// adds a transaction column here. -let toQueryTxField = (field: Internal.svmTransactionField): option< - SvmHyperSyncClient.QueryTypes.transactionField, -> => - switch field { - | TransactionIndex => None - | Signatures => Some(Signatures) - | FeePayer => Some(FeePayer) - | Success => Some(Success) - | Err => Some(Err) - | Fee => Some(Fee) - | ComputeUnitsConsumed => Some(ComputeUnitsConsumed) - | AccountKeys => Some(AccountKeys) - | RecentBlockhash => Some(RecentBlockhash) - | Version => Some(Version) - | TokenBalances => None - } - -// Map a selected block field to its HyperSync query column. Slot/Blockhash/ -// BlockTime are requested unconditionally (needed for reorg detection and the -// item's slot/timestamp), so selecting `slot`/`time`/`hash` adds no extra column. -let toQueryBlockField = (field: Internal.svmBlockField): option< - SvmHyperSyncClient.QueryTypes.blockField, -> => - switch field { - | Slot | Time | Hash => None - | Height => Some(BlockHeight) - | ParentSlot => Some(ParentSlot) - | ParentHash => Some(ParentBlockhash) - } - let make = ( {chain, endpointUrl, apiToken, onEventRegistrations, clientTimeoutMillis}: options, ): t => { let name = "SvmHyperSync" - // Definitions drive query/decode building; registrations drive routing and - // are attached directly to decoded runtime items. - let eventConfigs = - onEventRegistrations->Array.map(reg => - reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig) - ) - - // Built once at startup and handed to the client so `get` decodes matching - // instructions in Rust rather than per-instruction over the napi boundary. - let programSchemas = buildProgramSchemas(eventConfigs) + // The whole per-(instruction, chain) registration set crosses the boundary + // once at construction; the client derives instruction selections, field + // selections, Borsh decoders, and the routing index from it. let client = SvmHyperSyncClient.make( ~url=endpointUrl, ~apiToken?, ~httpReqTimeoutMillis=clientTimeoutMillis, - ~programSchemas=?switch programSchemas { - | [] => None - | arr => Some(arr) - }, + ~eventRegistrations=SvmHyperSyncClient.Registration.fromOnEventRegistrations( + onEventRegistrations, + ), ) - let (eventRouter, programOrderings) = EventRouter.fromSvmEventConfigsOrThrow( - onEventRegistrations, - ~chain, - ) - - // programId.toString -> sorted-desc byte lengths - let orderingByProgram = Dict.make() - programOrderings->Array.forEach(o => - orderingByProgram->Dict.set(o.programId->SvmTypes.Pubkey.toString, o.byteLengthsDesc) - ) - - let needsLogs = eventConfigs->Array.some(cfg => cfg.includeLogs) - - // Union of selected transaction fields across the chain's events. Drives both - // the query column selection (fetch only what's used) and the materialisation - // mask. `slot`/`transactionIndex` are always fetched as the store key. - let selectedTxFields = Utils.Set.make() - eventConfigs->Array.forEach(cfg => - cfg.selectedTransactionFields - ->(Utils.magic: Utils.Set.t => Utils.Set.t) - ->Utils.Set.forEach(field => selectedTxFields->Utils.Set.add(field)->ignore) - ) - let needsTokenBalances = selectedTxFields->Utils.Set.has(Internal.TokenBalances) - let txQueryFields = { - // Slot + TransactionIndex are always fetched so the store can be keyed by - // (slot, transactionIndex). - let fields: array = [Slot, TransactionIndex] - selectedTxFields - ->Utils.Set.toArray - ->Array.forEach(field => - switch toQueryTxField(field) { - | Some(queryField) => fields->Array.push(queryField) - | None => () - } - ) - fields - } - // The transaction table is fetched only when a selected field is actually read - // off a stored transaction record. `transactionIndex` materialises from the - // store key and `tokenBalances` lives in its own table, so neither requires it. - let needsTransactions = - selectedTxFields - ->Utils.Set.toArray - ->Array.some(field => - switch field { - | Internal.TransactionIndex | Internal.TokenBalances => false - | _ => true - } - ) - - // Union of selected block fields across the chain's events. `slot`/`time`/ - // `hash` are always fetched (as Slot/BlockTime/Blockhash); the rest are added - // only when an instruction selected them. - let blockQueryFields = { - let fields: array = [Slot, Blockhash, BlockTime] - let selected = Utils.Set.make() - eventConfigs->Array.forEach(cfg => - cfg.selectedBlockFields->Utils.Set.forEach(field => selected->Utils.Set.add(field)->ignore) - ) - selected->Utils.Set.forEach(field => - switch field->toQueryBlockField { - | Some(queryField) => fields->Array.push(queryField)->ignore - | None => () - } - ) - fields - } - let getItemsOrThrow = async ( ~fromBlock, ~toBlock, - ~addressesByContractName as _, - ~contractNameByAddress, + ~addressesByContractName, + ~contractNameByAddress as _, ~knownHeight, ~partitionId as _, - ~selection as _, + ~selection: FetchState.selection, ~itemsTarget, ~retry, - ~logger, + ~logger as _, ) => { let totalTimeRef = Performance.now() let pageFetchRef = Performance.now() - let instructionSelections = buildInstructionSelections(eventConfigs) - // Under the server's default merge mode, requesting a table's columns is - // what opts the matched result set into that join — a table with an empty - // field list returns no rows (instructions and blocks are exempt), so each - // opted-into table needs its columns spelled out here. - let fields: SvmHyperSyncClient.QueryTypes.fieldSelection = { - block: blockQueryFields, - transaction: ?(needsTransactions ? Some(txQueryFields) : None), - log: ?(needsLogs ? Some([Slot, TransactionIndex, InstructionAddress, Kind, Message]) : None), - tokenBalance: ?( - needsTokenBalances - ? Some([Slot, TransactionIndex, Account, Mint, Owner, PreAmount, PostAmount]) - : None - ), - } - // `toBlock` is inclusive, but `toSlot` is exclusive on the wire — without - // the +1 a bounded range stalls one slot short of its end. - let query: SvmHyperSyncClient.query = { + let query: SvmHyperSyncClient.EventItems.query = { fromSlot: fromBlock, - toSlot: ?(toBlock->Option.map(toBlock => toBlock + 1)), - instructions: instructionSelections, - fields, + toSlot: toBlock, maxNumInstructions: itemsTarget, + registrationIndexes: selection.onEventRegistrations->Array.map(reg => reg.index), + addressesByContractName, } - let (resp, transactionStore, blockStore) = try await client.get(~query) catch { + let (resp, transactionStore, blockStore) = try await client.getEventItems(~query) catch { | exn => throw( Source.GetItemsError( @@ -440,110 +132,38 @@ let make = ( // batch's `latestFetchedBlockTimestamp`. Slots without a block row (rare; // usually skipped slots) fall back to `None`. let blockTimeBySlot = Dict.make() - resp.data.blocks->Array.forEach(b => { + resp.blocks->Array.forEach(b => { switch b.blockTime { | Some(t) => blockTimeBySlot->Dict.set(b.slot->Int.toString, t) | None => () } }) - // Per (slot, transaction_index, instruction_address) lookup for logs - // scoped to a single instruction. `instructionAddress: None` logs are - // attached to no instruction (rare; usually only system messages). - let logsByKey = Dict.make() - resp.data.logs->Array.forEach(log => { - switch (log.transactionIndex, log.instructionAddress) { - | (Some(txIdx), Some(addr)) => - let key = - log.slot->Int.toString ++ - ":" ++ - txIdx->Int.toString ++ - ":" ++ - serializeInstructionAddress(addr) - switch logsByKey->Dict.get(key) { - | Some(existing) => existing->Array.push(log) - | None => logsByKey->Dict.set(key, [log]) - } - | _ => () - } - }) - - let parsedQueueItems = [] - resp.data.instructions->Array.forEach(instr => { - let programId = instr.programId->SvmTypes.Pubkey.fromStringUnsafe - let byteLengths = - orderingByProgram - ->Utils.Dict.dangerouslyGetNonOption(instr.programId) - ->Option.getOr([]) - - let contractAddress = instr.programId->Address.unsafeFromString - let maybeConfig = probeRouter( - eventRouter, - programId, - instr, - byteLengths, - ~contractAddress, - ~contractNameByAddress, - ) - - switch maybeConfig { - | None => () - // Exclude instructions from failed transactions. HyperSync has no - // server-side predicate to filter instructions by parent-transaction - // success (`InstructionSelection` only exposes `is_inner`, and - // instruction/transaction selections union at block level rather than - // joining), so we filter client-side on the `isCommitted` flag HyperSync - // already delivers on every instruction row (a required column, zero extra - // bandwidth). When HyperSync adds a server-side `is_committed` predicate the - // query can push this down and the client-side check becomes a redundant - // safety net. - | Some(_) if !instr.isCommitted => () - | Some(onEventRegistration) => - let eventConfig = - onEventRegistration.eventConfig->( - Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig - ) - let logKey = - instr.slot->Int.toString ++ - ":" ++ - instr.transactionIndex->Int.toString ++ - ":" ++ - serializeInstructionAddress(instr.instructionAddress) - let maybeLogs = - logsByKey - ->Utils.Dict.dangerouslyGetNonOption(logKey) - ->Option.map(logs => - logs->Array.map( - (log): Envio.svmLog => { - kind: log.kind->Option.getOr(""), - message: log.message->Option.getOr(""), - }, - ) - ) - - let payload = toSvmInstruction( - instr, - ~programName=eventConfig.contractName, - ~instructionName=eventConfig.name, - ~logs=eventConfig.includeLogs ? maybeLogs : None, + let parsedQueueItems = resp.items->Array.map(item => { + // Routing happened in Rust; the item references its registration by + // chain-scoped index. + let onEventRegistration = onEventRegistrations->Array.getUnsafe(item.onEventRegistrationIndex) + let eventConfig = + onEventRegistration.eventConfig->( + Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig ) - - parsedQueueItems - ->Array.push( - Internal.Event({ - onEventRegistration: (onEventRegistration :> Internal.onEventRegistration), - chain, - blockNumber: instr.slot, - logIndex: synthLogIndex(instr), - // The parent transaction is materialised from the store at batch prep. - transactionIndex: instr.transactionIndex, - payload: payload->(Utils.magic: Envio.svmInstruction => Internal.eventPayload), - }), - ) - ->ignore - } - - let _ = logger + let payload = toSvmInstruction( + item, + ~programName=eventConfig.contractName, + ~instructionName=eventConfig.name, + ) + Internal.Event({ + onEventRegistration, + chain, + blockNumber: item.slot, + logIndex: synthLogIndex( + ~transactionIndex=item.transactionIndex, + ~instructionAddress=item.instructionAddress, + ), + // The parent transaction is materialised from the store at batch prep. + transactionIndex: item.transactionIndex, + payload: payload->(Utils.magic: Envio.svmInstruction => Internal.eventPayload), + }) }) let parsingTimeElapsed = parsingRef->Performance.secondsSince @@ -556,7 +176,7 @@ let make = ( // Best-effort (slot, blockhash) pairs from the blocks the server returned // for this range. Gaps (skipped slots, or slots without matched data) are // fine — reorg detection only compares hashes for slots it has observed. - let blockHashes = resp.data.blocks->Array.map((b): ReorgDetection.blockData => { + let blockHashes = resp.blocks->Array.map((b): ReorgDetection.blockData => { blockNumber: b.slot, blockHash: b.blockhash, }) diff --git a/scenarios/fuel_test/test/HyperFuelHeight_test.res b/scenarios/fuel_test/test/FuelHyperSyncSourceHeight_test.res similarity index 92% rename from scenarios/fuel_test/test/HyperFuelHeight_test.res rename to scenarios/fuel_test/test/FuelHyperSyncSourceHeight_test.res index cb986c07a5..f83df12553 100644 --- a/scenarios/fuel_test/test/HyperFuelHeight_test.res +++ b/scenarios/fuel_test/test/FuelHyperSyncSourceHeight_test.res @@ -37,7 +37,7 @@ let withServer = async (handler, body) => { } } -describe("HyperFuelSource - getHeightOrThrow", () => { +describe("FuelHyperSyncSource - getHeightOrThrow", () => { let chain = ChainMap.Chain.makeUnsafe(~chainId=0) // The native client validates that the token is a UUID before sending requests. @@ -50,10 +50,11 @@ describe("HyperFuelSource - getHeightOrThrow", () => { res->writeHead(200) res->endWith(`{"height": 123}`) }, async endpointUrl => { - let source = HyperFuelSource.make({ + let source = FuelHyperSyncSource.make({ chain, endpointUrl, apiToken: Some(apiToken), + onEventRegistrations: [], }) let {height} = await source.getHeightOrThrow() @@ -75,10 +76,11 @@ describe("HyperFuelSource - getHeightOrThrow", () => { res->writeHead(401) res->endWith("Unauthorized") }, async endpointUrl => { - let source = HyperFuelSource.make({ + let source = FuelHyperSyncSource.make({ chain, endpointUrl, apiToken: Some(apiToken), + onEventRegistrations: [], }) let result = await Promise.race([ source.getHeightOrThrow()->Promise.thenResolve(_ => "resolved"), diff --git a/scenarios/fuel_test/test/HyperFuelSource_test.res b/scenarios/fuel_test/test/HyperFuelSource_test.res deleted file mode 100644 index aa6157509d..0000000000 --- a/scenarios/fuel_test/test/HyperFuelSource_test.res +++ /dev/null @@ -1,814 +0,0 @@ -open Vitest - -// Test-only contract shape: `events` carries full registrations (definition + -// isWildcard/handler/etc.), unlike `Internal.fuelContractConfig.events` which -// is bare definitions. -type mockContract = {name: string, events: array} - -// Builds a per-event registration for the `mock` helpers below. All test -// cases here use filterByAddresses=false and startBlock/handler/contractRegister=None, -// and dependsOnAddresses always follows the shared `!isWildcard` formula since -// none of these mocked events filter by addresses. -let mkRegistration = ( - ~id, - ~name, - ~contractName, - ~kind, - ~isWildcard=false, -): Internal.fuelOnEventRegistration => { - index: -1, - eventConfig: ({ - id, - name, - contractName, - kind, - paramsRawEventSchema: %raw(`"Not relevat"`), - simulateParamsSchema: %raw(`"Not relevat"`), - selectedTransactionFields: Utils.Set.make(), - transactionFieldMask: 0., - blockFieldMask: 0., - }: Internal.fuelEventConfig :> Internal.eventConfig), - isWildcard, - filterByAddresses: false, - dependsOnAddresses: !isWildcard, - startBlock: None, - handler: None, - contractRegister: None, -} - -describe("HyperFuelSource - getNormalRecieptsSelection", () => { - let contractName1 = "TestContract" - let contractName2 = "TestContract2" - let chain = ChainMap.Chain.makeUnsafe(~chainId=0) - let address1 = Address.unsafeFromString("0x1234567890abcdef1234567890abcdef1234567890abcde1") - let address2 = Address.unsafeFromString("0x1234567890abcdef1234567890abcdef1234567890abcde2") - let address3 = Address.unsafeFromString("0x1234567890abcdef1234567890abcdef1234567890abcde3") - - let mock = (~contracts: array) => { - let selectionConfig = { - dependsOnAddresses: true, - onEventRegistrations: contracts->Array.flatMap(c => { - c.events->Array.filterMap( - e => { - if e.isWildcard { - None - } else { - Some((e :> Internal.onEventRegistration)) - } - }, - ) - }), - }->HyperFuelSource.getSelectionConfig(~chain) - selectionConfig.getRecieptsSelection - } - - let mockAddressesByContractName = () => { - Dict.fromArray([(contractName1, [address1, address2]), (contractName2, [address3])]) - } - - it("Receipts Selection with no contracts", t => { - let getNormalRecieptsSelection = mock(~contracts=[]) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([]) - }) - - it("Receipts Selection with no events", t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([]) - }) - - it("Receipts Selection with single non-wildcard log event", t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="1", - ~name="StrLog", - ~contractName="TestContract", - ~kind=LogData({ - logId: "1", - decode: _ => %raw(`null`), - }), - ), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - rb: [1n], - receiptType: [LogData], - rootContractId: [address1, address2], - txStatus: [1], - }, - ]) - }) - - it( - "Receipts Selection with non-wildcard transfer event - catches both TRANSFER and TRANSFER_OUT receipts", - t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Transfer", - ~name="Transfer", - ~contractName="TestContract", - ~kind=Transfer, - ), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration( - ~id="Transfer", - ~name="Transfer", - ~contractName="TestContract2", - ~kind=Transfer, - ), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Transfer, TransferOut], - rootContractId: [address1, address2], - txStatus: [1], - }, - { - receiptType: [Transfer, TransferOut], - rootContractId: [address3], - txStatus: [1], - }, - ]) - }, - ) - - it("Receipts Selection with non-wildcard mint event", t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract", ~kind=Mint), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract2", ~kind=Mint), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Mint], - rootContractId: [address1, address2], - txStatus: [1], - }, - { - receiptType: [Mint], - rootContractId: [address3], - txStatus: [1], - }, - ]) - }) - - it("Receipts Selection with non-wildcard burn event", t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract", ~kind=Burn), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract2", ~kind=Burn), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Burn], - rootContractId: [address1, address2], - txStatus: [1], - }, - { - receiptType: [Burn], - rootContractId: [address3], - txStatus: [1], - }, - ]) - }) - - it("Receipts Selection with all possible events together", t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="1", - ~name="StrLog", - ~contractName="TestContract", - ~kind=LogData({ - logId: "1", - decode: _ => %raw(`null`), - }), - ), - mkRegistration( - ~id="2", - ~name="BoolLog", - ~contractName="TestContract", - ~kind=LogData({ - logId: "2", - decode: _ => %raw(`null`), - }), - ~isWildcard=true, - ), - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract", ~kind=Mint), - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract", ~kind=Burn), - mkRegistration( - ~id="Transfer", - ~name="Transfer", - ~contractName="TestContract", - ~kind=Transfer, - ), - mkRegistration( - ~id="Call", - ~name="Call", - ~contractName="TestContract", - ~kind=Call, - ~isWildcard=true, - ), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration( - ~id="UnitLog", - ~name="UnitLog", - ~contractName="TestContract2", - ~kind=LogData({ - logId: "3", - decode: _ => %raw(`null`), - }), - ), - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract2", ~kind=Burn), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ~message=`Note that non-wildcard events should be skipped`, - ).toEqual([ - { - receiptType: [Mint, Burn, Transfer, TransferOut], - rootContractId: [address1, address2], - txStatus: [1], - }, - { - rb: [1n], - receiptType: [LogData], - rootContractId: [address1, address2], - txStatus: [1], - }, - { - receiptType: [Burn], - rootContractId: [address3], - txStatus: [1], - }, - { - rb: [3n], - receiptType: [LogData], - rootContractId: [address3], - txStatus: [1], - }, - ]) - }) - - it("Fails with non-wildcard Call event", t => { - t.expect( - () => { - mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration(~id="Call", ~name="Call", ~contractName="TestContract", ~kind=Call), - ], - }, - ], - ) - }, - ).toThrowError("Call receipt indexing currently supported only in wildcard mode") - }) - - it("Fails when contract has multiple mint events", t => { - t.expect( - () => { - mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Mint", - ~name="MyEvent", - ~contractName="TestContract", - ~kind=Mint, - ), - mkRegistration( - ~id="Mint", - ~name="MyEvent2", - ~contractName="TestContract", - ~kind=Mint, - ), - ], - }, - ], - ) - }, - ).toThrowError("Duplicate event detected: MyEvent2 for contract TestContract on chain 0") - }) - - it("Fails when contract has multiple burn events", t => { - t.expect( - () => { - mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Burn", - ~name="MyEvent", - ~contractName="TestContract", - ~kind=Burn, - ), - mkRegistration( - ~id="Burn", - ~name="MyEvent2", - ~contractName="TestContract", - ~kind=Burn, - ), - ], - }, - ], - ) - }, - ).toThrowError("Duplicate event detected: MyEvent2 for contract TestContract on chain 0") - }) - - it( - "Shouldn't fail with contracts having the same wildcard and non-wildcard event. This should be handled when we create FetchState", - t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Mint", - ~name="WildcardMint", - ~contractName="TestContract", - ~kind=Mint, - ~isWildcard=true, - ), - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract", ~kind=Mint), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Mint], - rootContractId: [address1, address2], - txStatus: [1], - }, - ]) - }, - ) - - it("Works with wildcard mint and non-wildcard mint together in different contract", t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract2", - events: [ - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract2", ~kind=Mint), - ], - }, - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Mint", - ~name="Mint", - ~contractName="TestContract", - ~kind=Mint, - ~isWildcard=true, - ), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Mint], - rootContractId: [address3], - txStatus: [1], - }, - ]) - - // The same but with different event registration order - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Mint", - ~name="Mint", - ~contractName="TestContract", - ~kind=Mint, - ~isWildcard=true, - ), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract2", ~kind=Mint), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Mint], - rootContractId: [address3], - txStatus: [1], - }, - ]) - }) - - it("Works with wildcard burn and non-wildcard burn together in different contract", t => { - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract2", - events: [ - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract2", ~kind=Burn), - ], - }, - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Burn", - ~name="Burn", - ~contractName="TestContract", - ~kind=Burn, - ~isWildcard=true, - ), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Burn], - rootContractId: [address3], - txStatus: [1], - }, - ]) - - // The same but with different event registration order - let getNormalRecieptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Burn", - ~name="Burn", - ~contractName="TestContract", - ~kind=Burn, - ~isWildcard=true, - ), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract2", ~kind=Burn), - ], - }, - ], - ) - t.expect( - getNormalRecieptsSelection(~addressesByContractName=mockAddressesByContractName()), - ).toEqual([ - { - receiptType: [Burn], - rootContractId: [address3], - txStatus: [1], - }, - ]) - }) -}) - -describe("HyperFuelSource - makeWildcardRecieptsSelection", () => { - let chain = ChainMap.Chain.makeUnsafe(~chainId=0) - - let mock = (~contracts: array) => { - let selectionConfig = { - dependsOnAddresses: false, - onEventRegistrations: contracts->Array.flatMap(c => { - c.events->Array.filterMap( - e => { - if e.isWildcard { - Some((e :> Internal.onEventRegistration)) - } else { - None - } - }, - ) - }), - }->HyperFuelSource.getSelectionConfig(~chain) - selectionConfig.getRecieptsSelection(~addressesByContractName=Dict.make()) - } - - it("Receipts Selection with no contracts", t => { - let wildcardReceiptsSelection = mock(~contracts=[]) - t.expect( - wildcardReceiptsSelection, - ~message=`It should never happen, since the partition like this wouldn't exist`, - ).toEqual([]) - }) - - it("Receipts Selection with no events", t => { - let wildcardReceiptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [], - }, - ], - ) - t.expect( - wildcardReceiptsSelection, - ~message=`It should never happen, since the partition like this wouldn't exist`, - ).toEqual([]) - }) - - it("Receipts Selection with all possible events together", t => { - let wildcardReceiptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="1", - ~name="StrLog", - ~contractName="TestContract", - ~kind=LogData({ - logId: "1", - decode: _ => %raw(`null`), - }), - ), - mkRegistration( - ~id="2", - ~name="BoolLog", - ~contractName="TestContract", - ~kind=LogData({ - logId: "2", - decode: _ => %raw(`null`), - }), - ~isWildcard=true, - ), - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract", ~kind=Mint), - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract", ~kind=Burn), - mkRegistration( - ~id="Transfer", - ~name="Transfer", - ~contractName="TestContract", - ~kind=Transfer, - ), - mkRegistration( - ~id="Call", - ~name="Call", - ~contractName="TestContract", - ~kind=Call, - ~isWildcard=true, - ), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration( - ~id="3", - ~name="UnitLog", - ~contractName="TestContract2", - ~kind=LogData({ - logId: "3", - decode: _ => %raw(`null`), - }), - ), - mkRegistration(~id="Burn", ~name="Burn", ~contractName="TestContract2", ~kind=Burn), - ], - }, - ], - ) - t.expect( - wildcardReceiptsSelection, - ~message=`Note that wildcard events should be skipped`, - ).toEqual([ - { - receiptType: [Call], - txStatus: [1], - }, - { - rb: [2n], - receiptType: [LogData], - txStatus: [1], - }, - ]) - }) - - it("Works with wildcard mint and non-wildcard mint together in different contract", t => { - let wildcardReceiptsSelection = mock( - ~contracts=[ - { - name: "TestContract2", - events: [ - mkRegistration(~id="Mint", ~name="Mint", ~contractName="TestContract2", ~kind=Mint), - ], - }, - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Mint", - ~name="Mint", - ~contractName="TestContract", - ~kind=Mint, - ~isWildcard=true, - ), - ], - }, - ], - ) - t.expect(wildcardReceiptsSelection).toEqual([ - { - receiptType: [Mint], - txStatus: [1], - }, - ]) - }) - - it("Receipts Selection with wildcard mint event", t => { - let wildcardReceiptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Mint", - ~name="Mint", - ~contractName="TestContract", - ~kind=Mint, - ~isWildcard=true, - ), - ], - }, - ], - ) - t.expect(wildcardReceiptsSelection).toEqual([ - { - receiptType: [Mint], - txStatus: [1], - }, - ]) - }) - - it("Receipts Selection with wildcard burn event", t => { - let wildcardReceiptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="Burn", - ~name="Burn", - ~contractName="TestContract", - ~kind=Burn, - ~isWildcard=true, - ), - ], - }, - ], - ) - t.expect(wildcardReceiptsSelection).toEqual([ - { - receiptType: [Burn], - txStatus: [1], - }, - ]) - }) - - it("Receipts Selection with multiple wildcard log event", t => { - let wildcardReceiptsSelection = mock( - ~contracts=[ - { - name: "TestContract", - events: [ - mkRegistration( - ~id="1", - ~name="StrLog", - ~contractName="TestContract", - ~kind=LogData({ - logId: "1", - decode: _ => %raw(`null`), - }), - ~isWildcard=true, - ), - mkRegistration( - ~id="2", - ~name="BoolLog", - ~contractName="TestContract", - ~kind=LogData({ - logId: "2", - decode: _ => %raw(`null`), - }), - ~isWildcard=true, - ), - ], - }, - { - name: "TestContract2", - events: [ - mkRegistration( - ~id="3", - ~name="UnitLog", - ~contractName="TestContract2", - ~kind=LogData({ - logId: "3", - decode: _ => %raw(`null`), - }), - ~isWildcard=true, - ), - ], - }, - ], - ) - t.expect(wildcardReceiptsSelection).toEqual([ - { - rb: [1n, 2n, 3n], - receiptType: [LogData], - txStatus: [1], - }, - ]) - }) -}) diff --git a/scenarios/svm_flow_xray/src/indexer.test.ts b/scenarios/svm_flow_xray/src/indexer.test.ts index 570695a9f8..d6041dd12a 100644 --- a/scenarios/svm_flow_xray/src/indexer.test.ts +++ b/scenarios/svm_flow_xray/src/indexer.test.ts @@ -1,5 +1,5 @@ // Live E2E test against solana.hypersync.xyz. Drives the SVM stack end-to-end: -// SvmHyperSyncSource -> EventRouter -> indexer.onInstruction dispatch -> +// SvmHyperSyncSource -> Rust-side routing -> indexer.onInstruction dispatch -> // entity writes. The slot window is pinned in config.test.yaml. process.env.ENVIO_CONFIG = "config.test.yaml"; diff --git a/scenarios/svm_metaplex_demo/src/indexer.test.ts b/scenarios/svm_metaplex_demo/src/indexer.test.ts index a50bc7d995..286a44a041 100644 --- a/scenarios/svm_metaplex_demo/src/indexer.test.ts +++ b/scenarios/svm_metaplex_demo/src/indexer.test.ts @@ -1,5 +1,5 @@ // Live E2E test against `solana.hypersync.xyz`. Drives the SVM stack -// end-to-end: SvmHyperSyncSource → EventRouter → `indexer.onInstruction` +// end-to-end: SvmHyperSyncSource → Rust-side routing → `indexer.onInstruction` // dispatch → entity writes. `config.yaml` interpolates `ENVIO_METAPLEX_END_BLOCK` // into `end_block` to pin a finite window here; the live demo leaves it unset // for continuous tailing. diff --git a/scenarios/test_codegen/test/SourceBlockHashes_test.res b/scenarios/test_codegen/test/SourceBlockHashes_test.res index b90e59aebc..0d17caf564 100644 --- a/scenarios/test_codegen/test/SourceBlockHashes_test.res +++ b/scenarios/test_codegen/test/SourceBlockHashes_test.res @@ -13,7 +13,7 @@ let chain = ChainMap.Chain.makeUnsafe(~chainId=1) let pairCreatedTopic0 = "0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9" let pairCreatedEventId = pairCreatedTopic0 ++ "_3" -// Lowercase address so EventRouter lookup matches regardless of whether the +// Lowercase address so routing lookups match regardless of whether the // source returns checksummed or lowercase addresses. let uniswapV2FactoryAddress = "0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f"->Address.unsafeFromString @@ -91,7 +91,7 @@ let makeSelection = (): FetchState.selection => { } let makeHyperSyncSource = () => - HyperSyncSource.make({ + EvmHyperSyncSource.make({ chain, endpointUrl: "https://eth.hypersync.xyz", onEventRegistrations: [pairCreatedRegistration], diff --git a/scenarios/test_codegen/test/lib_tests/EventRouter_test.res b/scenarios/test_codegen/test/lib_tests/EventRouter_test.res deleted file mode 100644 index d8b9e5988f..0000000000 --- a/scenarios/test_codegen/test/lib_tests/EventRouter_test.res +++ /dev/null @@ -1,207 +0,0 @@ -open Vitest - -let mockChain = ChainMap.Chain.makeUnsafe(~chainId=1) -let mockAddress1 = Envio.TestHelpers.Addresses.mockAddresses[0]->Option.getOrThrow -let mockAddress2 = Envio.TestHelpers.Addresses.mockAddresses[1]->Option.getOrThrow - -let mockFromArray = (array): EventRouter.t<'a> => { - Dict.fromArray(array) -} - -describe("EventRouter", () => { - it("Succeeds on unique insertions", t => { - let router: EventRouter.t = EventRouter.empty() - - router->EventRouter.addOrThrow( - "test-event-tag", - 1, - ~contractName="Contract1", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=false, - ) - router->EventRouter.addOrThrow( - "test-event-tag", - 2, - ~contractName="Contract2", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=false, - ) - - t.expect(router).toEqual( - mockFromArray([ - ( - "test-event-tag", - { - wildcard: None, - byContractName: Dict.fromArray([("Contract1", 1), ("Contract2", 2)]), - }, - ), - ]), - ) - }) - - it("Fails on duplicate insertions", t => { - let router = EventRouter.empty() - - router->EventRouter.addOrThrow( - "test-event-tag", - 1, - ~contractName="Contract1", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=false, - ) - - t.expect( - () => { - router->EventRouter.addOrThrow( - "test-event-tag", - 1, - ~contractName="Contract1", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=false, - ) - }, - ).toThrowError("Duplicate event detected: Event1 for contract Contract1 on chain 1") - }) - - it("Fails on duplicate wildcard insertions", t => { - let router = EventRouter.empty() - - router->EventRouter.addOrThrow( - "test-event-tag", - 1, - ~contractName="Contract1", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=true, - ) - - t.expect( - () => { - router->EventRouter.addOrThrow( - "test-event-tag", - 1, - ~contractName="Contract2", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=true, - ) - }, - ).toThrowError( - "Another event is already registered with the same signature that would interfer with wildcard filtering: Event1 for contract Contract2 on chain 1", - ) - }) - - it("get doesn't returns the correct eventMod without address in mapping if unique", t => { - let router = EventRouter.empty() - - router->EventRouter.addOrThrow( - "test-event-tag", - 1, - ~contractName="Contract1", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=false, - ) - - t.expect( - router->EventRouter.get( - ~tag="test-event-tag", - ~contractAddress=mockAddress1, - ~contractNameByAddress=Dict.make(), - ), - ~message=`The address isn't owned by this partition, so it falls back to wildcard (here None)`, - ).toEqual(None) - }) - - it( - "get returns correct event mod with multiple contracts for both wildcard and non wildcard", - t => { - let wildcardContractAddress = mockAddress1 - let nonWildcardContractAddress = mockAddress2 - let nonWildcardContractName = "Contract2" - - let router = EventRouter.empty() - - router->EventRouter.addOrThrow( - "test-event-tag", - "wildcard", - ~contractName="Contract1", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=true, - ) - router->EventRouter.addOrThrow( - "test-event-tag", - "non-wildcard", - ~contractName=nonWildcardContractName, - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=false, - ) - - let contractNameByAddress = Dict.make() - contractNameByAddress->Dict.set( - nonWildcardContractAddress->Address.toString, - nonWildcardContractName, - ) - - t.expect( - router->EventRouter.get( - ~tag="test-event-tag", - ~contractAddress=nonWildcardContractAddress, - ~contractNameByAddress, - ), - ~message="Should return the non wildcard event", - ).toEqual(Some("non-wildcard")) - - t.expect( - router->EventRouter.get( - ~tag="test-event-tag", - ~contractAddress=wildcardContractAddress, - ~contractNameByAddress, - ), - ~message="Should return the wildcard event", - ).toEqual(Some("wildcard")) - }, - ) - - it( - "get falls back to wildcard when the address is indexed for a contract without a matching event", - t => { - // Covers the case where contractNameByAddress maps an address to a - // contract that has no events (persisted for future config changes). - // A wildcard event at that address should still fire. - let indexedAddress = mockAddress1 - let noEventsContractName = "UnknownContract" - - let router = EventRouter.empty() - - router->EventRouter.addOrThrow( - "test-event-tag", - "wildcard", - ~contractName="WildcardContract", - ~eventName="Event1", - ~chain=mockChain, - ~isWildcard=true, - ) - - let contractNameByAddress = Dict.make() - contractNameByAddress->Dict.set(indexedAddress->Address.toString, noEventsContractName) - - t.expect( - router->EventRouter.get( - ~tag="test-event-tag", - ~contractAddress=indexedAddress, - ~contractNameByAddress, - ), - ~message="Should fall back to the wildcard handler when the contract has no registered event for this tag", - ).toEqual(Some("wildcard")) - }, - ) - -})