diff --git a/Cargo.lock b/Cargo.lock index d2bb2e30ba..5e097aba37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,34 +46,96 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "alloy-dyn-abi" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf69d3061e2e908a4370bda5d8d6529d5080232776975489eec0b49ce971027e" +dependencies = [ + "alloy-json-abi 0.8.26", + "alloy-primitives 0.8.26", + "alloy-sol-type-parser 0.8.26", + "alloy-sol-types 0.8.26", + "const-hex", + "itoa", + "serde", + "serde_json", + "winnow", +] + [[package]] name = "alloy-dyn-abi" version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc2db5c583aaef0255aa63a4fe827f826090142528bba48d1bf4119b62780cad" dependencies = [ - "alloy-json-abi", - "alloy-primitives", - "alloy-sol-type-parser", - "alloy-sol-types", + "alloy-json-abi 1.5.7", + "alloy-primitives 1.5.7", + "alloy-sol-type-parser 1.5.7", + "alloy-sol-types 1.5.7", "itoa", "serde", "serde_json", "winnow", ] +[[package]] +name = "alloy-json-abi" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4584e3641181ff073e9d5bec5b3b8f78f9749d9fb108a1cfbc4399a4a139c72a" +dependencies = [ + "alloy-primitives 0.8.26", + "alloy-sol-type-parser 0.8.26", + "serde", + "serde_json", +] + [[package]] name = "alloy-json-abi" version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9dbe713da0c737d9e5e387b0ba790eb98b14dd207fe53eef50e19a5a8ec3dac" dependencies = [ - "alloy-primitives", - "alloy-sol-type-parser", + "alloy-primitives 1.5.7", + "alloy-sol-type-parser 1.5.7", "serde", "serde_json", ] +[[package]] +name = "alloy-primitives" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777d58b30eb9a4db0e5f59bc30e8c2caef877fee7dc8734cf242a51a60f22e05" +dependencies = [ + "alloy-rlp", + "bytes", + "cfg-if", + "const-hex", + "derive_more 2.1.1", + "foldhash 0.1.5", + "hashbrown 0.15.5", + "indexmap", + "itoa", + "k256", + "keccak-asm", + "paste", + "proptest", + "rand 0.8.5", + "ruint", + "rustc-hash", + "serde", + "sha3", + "tiny-keccak", +] + [[package]] name = "alloy-primitives" version = "1.5.7" @@ -111,27 +173,59 @@ dependencies = [ "bytes", ] +[[package]] +name = "alloy-sol-macro" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e68b32b6fa0d09bb74b4cefe35ccc8269d711c26629bc7cd98a47eeb12fe353f" +dependencies = [ + "alloy-sol-macro-expander 0.8.26", + "alloy-sol-macro-input 0.8.26", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "alloy-sol-macro" version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab81bab693da9bb79f7a95b64b394718259fdd7e41dceeced4cad57cb71c4f6a" dependencies = [ - "alloy-sol-macro-expander", - "alloy-sol-macro-input", + "alloy-sol-macro-expander 1.5.7", + "alloy-sol-macro-input 1.5.7", "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.117", ] +[[package]] +name = "alloy-sol-macro-expander" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2afe6879ac373e58fd53581636f2cce843998ae0b058ebe1e4f649195e2bd23c" +dependencies = [ + "alloy-sol-macro-input 0.8.26", + "const-hex", + "heck", + "indexmap", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", + "syn-solidity 0.8.26", + "tiny-keccak", +] + [[package]] name = "alloy-sol-macro-expander" version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "489f1620bb7e2483fb5819ed01ab6edc1d2f93939dce35a5695085a1afd1d699" dependencies = [ - "alloy-sol-macro-input", + "alloy-sol-macro-input 1.5.7", "const-hex", "heck", "indexmap", @@ -140,7 +234,23 @@ dependencies = [ "quote", "sha3", "syn 2.0.117", - "syn-solidity", + "syn-solidity 1.5.7", +] + +[[package]] +name = "alloy-sol-macro-input" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ba01aee235a8c699d07e5be97ba215607564e71be72f433665329bec307d28" +dependencies = [ + "const-hex", + "dunce", + "heck", + "macro-string", + "proc-macro2", + "quote", + "syn 2.0.117", + "syn-solidity 0.8.26", ] [[package]] @@ -156,7 +266,17 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.117", - "syn-solidity", + "syn-solidity 1.5.7", +] + +[[package]] +name = "alloy-sol-type-parser" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c13fc168b97411e04465f03e632f31ef94cad1c7c8951bf799237fd7870d535" +dependencies = [ + "serde", + "winnow", ] [[package]] @@ -169,15 +289,28 @@ dependencies = [ "winnow", ] +[[package]] +name = "alloy-sol-types" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e960c4b52508ef2ae1e37cae5058e905e9ae099b107900067a503f8c454036f" +dependencies = [ + "alloy-json-abi 0.8.26", + "alloy-primitives 0.8.26", + "alloy-sol-macro 0.8.26", + "const-hex", + "serde", +] + [[package]] name = "alloy-sol-types" version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64612d29379782a5dde6f4b6570d9c756d734d760c0c94c254d361e678a6591f" dependencies = [ - "alloy-json-abi", - "alloy-primitives", - "alloy-sol-macro", + "alloy-json-abi 1.5.7", + "alloy-primitives 1.5.7", + "alloy-sol-macro 1.5.7", "serde", ] @@ -270,6 +403,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "ar_archive_writer" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" +dependencies = [ + "object", +] + [[package]] name = "ark-ff" version = "0.3.0" @@ -459,6 +601,12 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "array-init-cursor" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed51fe0f224d1d4ea768be38c51f9f831dee9d05c163c11fba0b8c44387b1fc3" + [[package]] name = "arrayvec" version = "0.7.6" @@ -696,6 +844,39 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "atoi" version = "2.0.0" @@ -705,6 +886,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi_simd" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae037714f313c1353189ead58ef9eec30a8e8dc101b2622d461418fd59e28a9" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -862,6 +1049,17 @@ dependencies = [ "cfg_aliases", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.3", +] + [[package]] name = "brotli" version = "8.0.2" @@ -870,7 +1068,17 @@ checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 5.0.0", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] @@ -904,6 +1112,26 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -1170,6 +1398,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1606,9 +1843,9 @@ dependencies = [ name = "envio" version = "0.0.1-dev" dependencies = [ - "alloy-dyn-abi", - "alloy-json-abi", - "alloy-primitives", + "alloy-dyn-abi 1.5.7", + "alloy-json-abi 1.5.7", + "alloy-primitives 1.5.7", "anyhow", "arrayvec", "async-recursion", @@ -1624,6 +1861,7 @@ dependencies = [ "futures-util", "graphql-parser", "handlebars", + "hyperfuel-client", "hypersync-client", "hypersync-client-solana", "hypersync-solana-net-types", @@ -1638,6 +1876,7 @@ dependencies = [ "openssl", "paste", "pathdiff", + "polars-arrow", "pretty_assertions", "regex", "reqwest 0.11.27", @@ -1673,6 +1912,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ethnum" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" + [[package]] name = "eventsource-stream" version = "0.2.3" @@ -1684,6 +1929,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fast-float" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" + [[package]] name = "faster-hex" version = "0.9.0" @@ -1781,6 +2038,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ + "crc32fast", "miniz_oxide", "zlib-rs", ] @@ -2088,6 +2346,18 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", + "rayon", + "serde", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -2095,6 +2365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "foldhash 0.1.5", + "serde", ] [[package]] @@ -2278,6 +2549,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -2316,6 +2588,77 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperfuel-client" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc89b6e970dc4a896a05735b76f4ef7afe0464e425568495eb94ab514dff2a5" +dependencies = [ + "alloy-dyn-abi 0.8.26", + "alloy-json-abi 0.8.26", + "anyhow", + "arrayvec", + "bincode", + "capnp", + "faster-hex", + "fastrange-rs", + "futures", + "hyperfuel-format", + "hyperfuel-net-types", + "hyperfuel-schema", + "log", + "nohash-hasher", + "num_cpus", + "polars-arrow", + "polars-parquet", + "rand 0.8.5", + "rayon", + "reqwest 0.12.28", + "ruint", + "serde", + "serde_json", + "tokio", + "tokio-util", + "url", + "uuid", + "xxhash-rust", +] + +[[package]] +name = "hyperfuel-format" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a07f3546b013d2d3a2083ffc580897eb67feaa79d04b66c31098b8d17f9e9c" +dependencies = [ + "arrayvec", + "derive_more 1.0.0", + "faster-hex", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "hyperfuel-net-types" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7283c780ec92a6f58ac2e5cdbd0492904b063d50361e4bc884d0d1b6b8af7882" +dependencies = [ + "arrayvec", + "capnp", + "hyperfuel-format", + "serde", +] + +[[package]] +name = "hyperfuel-schema" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aaa329f946f4508099a6ed6a8570a1d657735676bc084119c2af8b99ea69f09" +dependencies = [ + "anyhow", + "polars-arrow", +] + [[package]] name = "hyperlocal" version = "0.9.1" @@ -2337,9 +2680,9 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb904f3c4d4a03d9462a9e6be0a4b4f1872e466dbd36661e1ceed27485a54e03" dependencies = [ - "alloy-dyn-abi", - "alloy-json-abi", - "alloy-primitives", + "alloy-dyn-abi 1.5.7", + "alloy-json-abi 1.5.7", + "alloy-primitives 1.5.7", "anyhow", "arrayvec", "arrow", @@ -2398,7 +2741,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5a308029c5ea9c5dcd4bc1c76bbbec76ff25044fab79cba29dbc46b177f34e" dependencies = [ - "alloy-primitives", + "alloy-primitives 1.5.7", "arrayvec", "derive_more 1.0.0", "faster-hex", @@ -2944,6 +3287,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lz4_flex" version = "0.12.2" @@ -2970,6 +3332,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -3015,6 +3386,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multiversion" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4851161a11d3ad0bf9402d90ffc3967bf231768bfd7aeb61755ad06dbf1a142" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79a74ddee9e0c27d2578323c13905793e91622148f138ba29738f9dddb835e90" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "target-features", +] + [[package]] name = "napi" version = "3.8.5" @@ -3189,6 +3582,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3341,7 +3743,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli", + "brotli 8.0.2", "bytes", "chrono", "flate2", @@ -3362,6 +3764,16 @@ dependencies = [ "zstd", ] +[[package]] +name = "parquet-format-safe" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" +dependencies = [ + "async-trait", + "futures", +] + [[package]] name = "paste" version = "1.0.15" @@ -3451,6 +3863,133 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "planus" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f" +dependencies = [ + "array-init-cursor", +] + +[[package]] +name = "polars-arrow" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d19c6db79cb6a3c55af3b5a3976276edaab64cbf7f69b392617c2af30d7742" +dependencies = [ + "ahash", + "atoi_simd", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "ethnum", + "fast-float", + "getrandom 0.2.17", + "hashbrown 0.14.5", + "itoa", + "lz4", + "multiversion", + "num-traits", + "parking_lot", + "polars-arrow-format", + "polars-error", + "polars-utils", + "ryu", + "simdutf8", + "streaming-iterator", + "strength_reduce", + "version_check", + "zstd", +] + +[[package]] +name = "polars-arrow-format" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b0ef2474af9396b19025b189d96e992311e6a47f90c53cd998b36c4c64b84c" +dependencies = [ + "planus", + "serde", +] + +[[package]] +name = "polars-compute" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30194a5ff325f61d6fcb62dc215c9210f308fc4fc85a493ef777dbcd938cba24" +dependencies = [ + "bytemuck", + "either", + "num-traits", + "polars-arrow", + "polars-error", + "polars-utils", + "strength_reduce", + "version_check", +] + +[[package]] +name = "polars-error" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07101d1803ca2046cdb3a8adb1523ddcc879229860f0ac56a853034269dec1e1" +dependencies = [ + "polars-arrow-format", + "simdutf8", + "thiserror 1.0.69", +] + +[[package]] +name = "polars-parquet" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2993265079ffa07dd16277189444424f8d787b00b01c6f6e001f58bab543ce" +dependencies = [ + "ahash", + "async-stream", + "base64 0.22.1", + "brotli 6.0.0", + "bytemuck", + "ethnum", + "flate2", + "futures", + "lz4", + "num-traits", + "parquet-format-safe", + "polars-arrow", + "polars-compute", + "polars-error", + "polars-utils", + "simdutf8", + "snap", + "streaming-decompression", + "zstd", +] + +[[package]] +name = "polars-utils" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19dd73207bd15efb0ae5c9c3ece3227927ed6a16ad63578acec342378e6bdcb4" +dependencies = [ + "ahash", + "bytemuck", + "bytes", + "hashbrown 0.14.5", + "indexmap", + "memmap2", + "num-traits", + "once_cell", + "polars-error", + "raw-cpuid", + "rayon", + "smartstring", + "stacker", + "version_check", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -3574,6 +4113,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "psm" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -3678,6 +4227,7 @@ dependencies = [ "libc", "rand_chacha 0.3.1", "rand_core 0.6.4", + "serde", ] [[package]] @@ -3763,6 +4313,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "rayon" version = "1.12.0" @@ -3939,6 +4498,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", ] [[package]] @@ -4482,6 +5042,17 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "snap" version = "1.1.1" @@ -4524,12 +5095,46 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strsim" version = "0.11.1" @@ -4616,6 +5221,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn-solidity" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4e6eed052a117409a1a744c8bda9c3ea6934597cf7419f791cb7d590871c4c" +dependencies = [ + "paste", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "syn-solidity" version = "1.5.7" @@ -4692,6 +5309,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "tempdir" version = "0.3.7" @@ -4864,6 +5487,7 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -5301,6 +5925,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/packages/cli/Cargo.toml b/packages/cli/Cargo.toml index a97ba46005..f9e2135b8b 100644 --- a/packages/cli/Cargo.toml +++ b/packages/cli/Cargo.toml @@ -45,6 +45,8 @@ colored = "2.0.4" thiserror = "1.0.50" fuel-abi-types = "0.7.0" hypersync-client = "1.2.0" +hyperfuel-client = "4.0.0" +polars-arrow = "0.42" hypersync-client-solana = "0.0.7" hypersync-solana-net-types = "0.0.7" faster-hex = "0.9" diff --git a/packages/cli/src/fuel_hypersync_source/config.rs b/packages/cli/src/fuel_hypersync_source/config.rs new file mode 100644 index 0000000000..e7f6c2ec75 --- /dev/null +++ b/packages/cli/src/fuel_hypersync_source/config.rs @@ -0,0 +1,27 @@ +use anyhow::{Context, Result}; +use napi_derive::napi; + +/// Configuration for the HyperFuel client. +#[napi(object)] +#[derive(Default, Clone)] +pub struct ClientConfig { + pub url: String, + pub api_token: String, +} + +impl TryFrom for hyperfuel_client::ClientConfig { + type Error = anyhow::Error; + + fn try_from(config: ClientConfig) -> Result { + // hyperfuel_client::ClientConfig holds a `url::Url`; go through serde so + // it parses the (already validated) endpoint without us taking a direct + // dependency on the url crate. + let json = serde_json::json!({ + "url": config.url, + "api_token": config.api_token, + // Retries are handled by the indexer, not the binary client. + "max_num_retries": 0, + }); + serde_json::from_value(json).context("build hyperfuel client config") + } +} diff --git a/packages/cli/src/fuel_hypersync_source/mod.rs b/packages/cli/src/fuel_hypersync_source/mod.rs new file mode 100644 index 0000000000..62a2c42381 --- /dev/null +++ b/packages/cli/src/fuel_hypersync_source/mod.rs @@ -0,0 +1,96 @@ +use anyhow::Context; +use napi_derive::napi; + +mod config; +mod query; +mod types; + +use config::ClientConfig; +use query::Query; +use types::{convert_response, ConvertError, QueryResponse}; + +#[napi] +pub struct HyperfuelClient { + inner: hyperfuel_client::Client, +} + +#[napi] +impl HyperfuelClient { + #[napi(factory)] + pub fn new(cfg: ClientConfig, user_agent: String) -> napi::Result { + let client_config: hyperfuel_client::ClientConfig = + cfg.try_into().context("build config").map_err(map_err)?; + let inner = hyperfuel_client::Client::new_with_agent(client_config, user_agent) + .context("build client") + .map_err(map_err)?; + Ok(HyperfuelClient { inner }) + } + + #[napi] + pub async fn get_height(&self) -> napi::Result { + let height = self + .inner + .get_height() + .await + .map_err(|e| request_err("Failed to get HyperFuel height", e))?; + height.try_into().context("convert height").map_err(map_err) + } + + #[napi] + pub async fn get_selected_data(&self, query: Query) -> napi::Result { + let query: hyperfuel_client::net_types::Query = + query.try_into().context("parse query").map_err(map_err)?; + let res = self + .inner + .get_arrow(&query) + .await + .map_err(|e| request_err("Failed to get data from HyperFuel", e))?; + convert_response(res).map_err(convert_error_to_napi) + } +} + +/// The client embeds a `{:?}` debug dump in its error message; keep only the +/// first line so it stays readable when the indexer surfaces it on retries. +fn request_err(prefix: &str, e: anyhow::Error) -> napi::Error { + let message = format!("{e}"); + let summary = message.lines().next().unwrap_or(message.as_str()); + napi::Error::from_reason(format!("{prefix}: {summary}")) +} + +/// Encodes `ConvertError::MissingFields` as a JSON payload in the napi +/// error's message — the same protocol as hypersync_source, which the +/// ReScript side recovers via JSON.parse and a `kind` dispatch. +fn convert_error_to_napi(err: ConvertError) -> napi::Error { + match err { + ConvertError::MissingFields(fields) => { + let payload = serde_json::json!({ + "kind": "MissingFields", + "fields": fields, + }) + .to_string(); + napi::Error::new(napi::Status::InvalidArg, payload) + } + ConvertError::Other(e) => map_err(e), + } +} + +fn map_err(e: anyhow::Error) -> napi::Error { + napi::Error::from_reason(format!("{:?}", e)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn convert_error_serializes_as_expected_json() { + let err = + ConvertError::MissingFields(vec!["receipt.txId".to_string(), "block.time".to_string()]); + let napi_err = convert_error_to_napi(err); + let parsed: serde_json::Value = + serde_json::from_str(&napi_err.reason).expect("payload must be JSON"); + assert_eq!(parsed["kind"], "MissingFields"); + assert_eq!(parsed["fields"][0], "receipt.txId"); + assert_eq!(parsed["fields"][1], "block.time"); + } +} diff --git a/packages/cli/src/fuel_hypersync_source/query.rs b/packages/cli/src/fuel_hypersync_source/query.rs new file mode 100644 index 0000000000..997be45c26 --- /dev/null +++ b/packages/cli/src/fuel_hypersync_source/query.rs @@ -0,0 +1,102 @@ +use anyhow::{anyhow, Context, Result}; +use hyperfuel_client::format::{Hash, Hex}; +use hyperfuel_client::net_types; +use napi::bindgen_prelude::BigInt; +use napi_derive::napi; + +/// Query for retrieving Fuel receipts and their blocks. +#[napi(object)] +#[derive(Default)] +pub struct Query { + pub from_block: i64, + #[napi(js_name = "toBlock")] + pub to_block_exclusive: Option, + pub receipts: Option>, + pub field_selection: FieldSelection, +} + +#[napi(object)] +#[derive(Default)] +pub struct ReceiptSelection { + pub root_contract_id: Option>, + pub receipt_type: Option>, + pub rb: Option>, + pub tx_status: Option>, +} + +#[napi(object)] +#[derive(Default)] +pub struct FieldSelection { + pub block: Option>, + pub receipt: Option>, +} + +fn parse_hashes(v: Option>) -> Result> { + v.unwrap_or_default() + .into_iter() + .map(|s| Hash::decode_hex(&s).map_err(|e| anyhow!("failed to parse hash {s}: {e:?}"))) + .collect() +} + +fn bigints_to_u64(v: Option>) -> Result> { + v.unwrap_or_default() + .into_iter() + .map(|b| { + let (sign_bit, value, lossless) = b.get_u64(); + if sign_bit || !lossless { + anyhow::bail!("rb filter value must be an unsigned 64-bit integer"); + } + Ok(value) + }) + .collect() +} + +impl TryFrom for net_types::ReceiptSelection { + type Error = anyhow::Error; + + fn try_from(s: ReceiptSelection) -> Result { + Ok(net_types::ReceiptSelection { + root_contract_id: parse_hashes(s.root_contract_id)?, + receipt_type: s.receipt_type.unwrap_or_default(), + rb: bigints_to_u64(s.rb).context("parse rb filter")?, + tx_status: s.tx_status.unwrap_or_default(), + ..Default::default() + }) + } +} + +impl From for net_types::FieldSelection { + fn from(f: FieldSelection) -> Self { + net_types::FieldSelection { + block: f.block.unwrap_or_default().into_iter().collect(), + receipt: f.receipt.unwrap_or_default().into_iter().collect(), + ..Default::default() + } + } +} + +impl TryFrom for net_types::Query { + type Error = anyhow::Error; + + fn try_from(q: Query) -> Result { + let from_block = u64::try_from(q.from_block).context("from_block must be >= 0")?; + let to_block = q + .to_block_exclusive + .map(|b| u64::try_from(b).context("toBlock must be >= 0")) + .transpose()?; + let receipts = q + .receipts + .unwrap_or_default() + .into_iter() + .map(TryInto::try_into) + .collect::>>()?; + + Ok(net_types::Query { + from_block, + to_block, + receipts, + field_selection: q.field_selection.into(), + ..Default::default() + }) + } +} diff --git a/packages/cli/src/fuel_hypersync_source/types.rs b/packages/cli/src/fuel_hypersync_source/types.rs new file mode 100644 index 0000000000..9ad7185791 --- /dev/null +++ b/packages/cli/src/fuel_hypersync_source/types.rs @@ -0,0 +1,361 @@ +use anyhow::{Context, Result}; +use hyperfuel_client::{ArrowBatch, ArrowResponse}; +use napi::bindgen_prelude::BigInt; +use napi_derive::napi; +use polars_arrow::array::{BinaryArray, Int64Array, StaticArray, UInt64Array, UInt8Array}; + +#[napi(object)] +pub struct QueryResponse { + pub archive_height: Option, + pub next_block: i64, + pub total_execution_time: i64, + pub data: QueryResponseData, +} + +#[napi(object)] +pub struct QueryResponseData { + pub receipts: Vec, + pub blocks: Vec, +} + +#[napi(object)] +pub struct Receipt { + pub receipt_index: i64, + pub root_contract_id: Option, + pub tx_id: String, + pub block_height: i64, + pub receipt_type: i64, + pub data: Option, + pub rb: Option, + pub val: Option, + pub sub_id: Option, + pub amount: Option, + pub asset_id: Option, + pub to: Option, + pub to_address: Option, +} + +#[napi(object)] +pub struct Block { + pub id: String, + pub height: i64, + pub time: i64, +} + +/// `MissingFields` is the shape the JS side recognizes (via the JSON payload +/// protocol shared with hypersync_source) and converts to +/// `UnexpectedMissingParams`; `Other` falls through to the generic napi error. +#[derive(Debug)] +pub(crate) enum ConvertError { + MissingFields(Vec), + Other(anyhow::Error), +} + +impl From for ConvertError { + fn from(e: anyhow::Error) -> Self { + Self::Other(e) + } +} + +fn encode_hex(bytes: &[u8]) -> String { + format!("0x{}", faster_hex::hex_string(bytes)) +} + +fn hex_at(arr: &Option<&BinaryArray>, idx: usize) -> Option { + arr.and_then(|a| a.get(idx)).map(encode_hex) +} + +fn u64_at(arr: &Option<&UInt64Array>, idx: usize) -> Option { + arr.and_then(|a| a.get(idx)) +} + +fn bigint_at(arr: &Option<&UInt64Array>, idx: usize) -> Option { + u64_at(arr, idx).map(BigInt::from) +} + +fn i64_field(arr: &Option<&UInt64Array>, idx: usize, name: &str) -> Result> { + u64_at(arr, idx) + .map(|v| v.try_into().with_context(|| format!("{name} overflow"))) + .transpose() +} + +fn receipts_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertError> { + let mut out = Vec::new(); + for batch in batches { + let receipt_index = batch.column::("receipt_index").ok(); + let root_contract_id = batch.column::>("root_contract_id").ok(); + let tx_id = batch.column::>("tx_id").ok(); + let block_height = batch.column::("block_height").ok(); + let receipt_type = batch.column::("receipt_type").ok(); + let data = batch.column::>("data").ok(); + let rb = batch.column::("rb").ok(); + let val = batch.column::("val").ok(); + let sub_id = batch.column::>("sub_id").ok(); + let amount = batch.column::("amount").ok(); + let asset_id = batch.column::>("asset_id").ok(); + let to = batch.column::>("to").ok(); + let to_address = batch.column::>("to_address").ok(); + + for idx in 0..batch.chunk.len() { + let mut missing: Vec = Vec::new(); + let receipt_index_val = i64_field(&receipt_index, idx, "receipt.receiptIndex")? + .or_else(|| { + missing.push("receipt.receiptIndex".into()); + None + }); + let tx_id_val = hex_at(&tx_id, idx).or_else(|| { + missing.push("receipt.txId".into()); + None + }); + let block_height_val = + i64_field(&block_height, idx, "receipt.blockHeight")?.or_else(|| { + missing.push("receipt.blockHeight".into()); + None + }); + let receipt_type_val = receipt_type.and_then(|a| a.get(idx)).or_else(|| { + missing.push("receipt.receiptType".into()); + None + }); + if !missing.is_empty() { + return Err(ConvertError::MissingFields(missing)); + } + + out.push(Receipt { + receipt_index: receipt_index_val.unwrap(), + root_contract_id: hex_at(&root_contract_id, idx), + tx_id: tx_id_val.unwrap(), + block_height: block_height_val.unwrap(), + receipt_type: receipt_type_val.unwrap() as i64, + data: hex_at(&data, idx), + rb: bigint_at(&rb, idx), + val: bigint_at(&val, idx), + sub_id: hex_at(&sub_id, idx), + amount: bigint_at(&amount, idx), + asset_id: hex_at(&asset_id, idx), + to: hex_at(&to, idx), + to_address: hex_at(&to_address, idx), + }); + } + } + Ok(out) +} + +fn blocks_from_arrow(batches: &[ArrowBatch]) -> Result, ConvertError> { + let mut out = Vec::new(); + for batch in batches { + let id = batch.column::>("id").ok(); + let height = batch.column::("height").ok(); + let time = batch.column::("time").ok(); + + for idx in 0..batch.chunk.len() { + let mut missing: Vec = Vec::new(); + let id_val = hex_at(&id, idx).or_else(|| { + missing.push("block.id".into()); + None + }); + let height_val = i64_field(&height, idx, "block.height")?.or_else(|| { + missing.push("block.height".into()); + None + }); + let time_val = time.and_then(|a| a.get(idx)).or_else(|| { + missing.push("block.time".into()); + None + }); + if !missing.is_empty() { + return Err(ConvertError::MissingFields(missing)); + } + + out.push(Block { + id: id_val.unwrap(), + height: height_val.unwrap(), + time: time_val.unwrap(), + }); + } + } + Ok(out) +} + +pub(crate) fn convert_response(res: ArrowResponse) -> Result { + Ok(QueryResponse { + archive_height: res + .archive_height + .map(i64::try_from) + .transpose() + .context("convert archive_height") + .map_err(ConvertError::Other)?, + next_block: res + .next_block + .try_into() + .context("convert next_block") + .map_err(ConvertError::Other)?, + total_execution_time: res + .total_execution_time + .try_into() + .context("convert total_execution_time") + .map_err(ConvertError::Other)?, + data: QueryResponseData { + receipts: receipts_from_arrow(&res.data.receipts)?, + blocks: blocks_from_arrow(&res.data.blocks)?, + }, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use polars_arrow::array::Array; + use polars_arrow::datatypes::{ArrowDataType, ArrowSchema, Field}; + use polars_arrow::record_batch::RecordBatchT; + use std::sync::Arc; + + fn make_batch(fields: Vec<(Field, Box)>) -> ArrowBatch { + let (schema_fields, arrays): (Vec<_>, Vec<_>) = fields.into_iter().unzip(); + ArrowBatch { + chunk: Arc::new(RecordBatchT::new(arrays)), + schema: Arc::new(ArrowSchema::from(schema_fields)), + } + } + + fn binary_field(name: &str, values: Vec>) -> (Field, Box) { + ( + Field::new(name, ArrowDataType::Binary, true), + Box::new(BinaryArray::::from_iter(values.into_iter())), + ) + } + + fn u64_field(name: &str, values: Vec>) -> (Field, Box) { + ( + Field::new(name, ArrowDataType::UInt64, true), + Box::new(UInt64Array::from(values)), + ) + } + + fn u8_field(name: &str, values: Vec>) -> (Field, Box) { + ( + Field::new(name, ArrowDataType::UInt8, true), + Box::new(UInt8Array::from(values)), + ) + } + + fn i64_field(name: &str, values: Vec>) -> (Field, Box) { + ( + Field::new(name, ArrowDataType::Int64, true), + Box::new(Int64Array::from(values)), + ) + } + + fn full_receipt_batch() -> ArrowBatch { + make_batch(vec![ + u64_field("receipt_index", vec![Some(1)]), + binary_field("tx_id", vec![Some(&[0xab; 32])]), + u64_field("block_height", vec![Some(42)]), + u8_field("receipt_type", vec![Some(6)]), + binary_field("root_contract_id", vec![Some(&[0xcd; 32])]), + binary_field("data", vec![Some(&[0x01, 0x02])]), + u64_field("rb", vec![Some(7)]), + ]) + } + + #[test] + fn converts_receipts_with_optional_columns_absent() { + let receipts = receipts_from_arrow(&[full_receipt_batch()]).unwrap(); + assert_eq!(receipts.len(), 1); + let r = &receipts[0]; + assert_eq!( + ( + r.receipt_index, + r.tx_id.as_str(), + r.block_height, + r.receipt_type, + r.root_contract_id.as_deref(), + r.data.as_deref(), + r.rb.as_ref().map(|b| b.get_u64().1), + r.val.as_ref().map(|b| b.get_u64().1), + ), + ( + 1, + format!("0x{}", "ab".repeat(32)).as_str(), + 42, + 6, + Some(format!("0x{}", "cd".repeat(32)).as_str()), + Some("0x0102"), + Some(7), + None, + ) + ); + } + + #[test] + fn missing_required_receipt_column_is_typed_error() { + // tx_id column not in the response at all + let batch = make_batch(vec![ + u64_field("receipt_index", vec![Some(1)]), + u64_field("block_height", vec![Some(42)]), + u8_field("receipt_type", vec![Some(6)]), + ]); + match receipts_from_arrow(&[batch]) { + Err(ConvertError::MissingFields(fields)) => { + assert_eq!(fields, vec!["receipt.txId".to_string()]) + } + Err(ConvertError::Other(e)) => panic!("unexpected ConvertError::Other: {e:?}"), + Ok(_) => panic!("expected MissingFields, got Ok"), + } + } + + #[test] + fn null_required_receipt_value_is_typed_error() { + let batch = make_batch(vec![ + u64_field("receipt_index", vec![None]), + binary_field("tx_id", vec![None]), + u64_field("block_height", vec![Some(42)]), + u8_field("receipt_type", vec![Some(6)]), + ]); + match receipts_from_arrow(&[batch]) { + Err(ConvertError::MissingFields(fields)) => assert_eq!( + fields, + vec![ + "receipt.receiptIndex".to_string(), + "receipt.txId".to_string() + ] + ), + Err(ConvertError::Other(e)) => panic!("unexpected ConvertError::Other: {e:?}"), + Ok(_) => panic!("expected MissingFields, got Ok"), + } + } + + #[test] + fn missing_block_time_is_typed_error() { + let batch = make_batch(vec![ + binary_field("id", vec![Some(&[0xee; 32])]), + u64_field("height", vec![Some(42)]), + ]); + match blocks_from_arrow(&[batch]) { + Err(ConvertError::MissingFields(fields)) => { + assert_eq!(fields, vec!["block.time".to_string()]) + } + Err(ConvertError::Other(e)) => panic!("unexpected ConvertError::Other: {e:?}"), + Ok(_) => panic!("expected MissingFields, got Ok"), + } + } + + #[test] + fn converts_blocks() { + let batch = make_batch(vec![ + binary_field("id", vec![Some(&[0xee; 32])]), + u64_field("height", vec![Some(42)]), + i64_field("time", vec![Some(1745179292)]), + ]); + let blocks = blocks_from_arrow(&[batch]).unwrap(); + assert_eq!(blocks.len(), 1); + assert_eq!( + (blocks[0].id.as_str(), blocks[0].height, blocks[0].time), + (format!("0x{}", "ee".repeat(32)).as_str(), 42, 1745179292i64) + ); + } + + #[test] + fn empty_batches_convert_to_empty() { + assert_eq!(receipts_from_arrow(&[]).unwrap().len(), 0); + assert_eq!(blocks_from_arrow(&[]).unwrap().len(), 0); + } +} diff --git a/packages/cli/src/lib.rs b/packages/cli/src/lib.rs index fa13c47164..feca85dfd4 100644 --- a/packages/cli/src/lib.rs +++ b/packages/cli/src/lib.rs @@ -10,6 +10,7 @@ mod evm_hypersync_source; mod evm_rpc_source; pub mod executor; mod fuel; +mod fuel_hypersync_source; mod hbs_templating; #[cfg_attr(test, allow(dead_code))] mod napi; diff --git a/packages/envio/package.json b/packages/envio/package.json index 648f89fa4d..bdfa438ccd 100644 --- a/packages/envio/package.json +++ b/packages/envio/package.json @@ -47,7 +47,6 @@ "dependencies": { "@clickhouse/client": "1.17.0", "@elastic/ecs-pino-format": "1.4.0", - "@envio-dev/hyperfuel-client": "1.2.2", "@fuel-ts/crypto": "0.96.1", "@fuel-ts/errors": "0.96.1", "@fuel-ts/hasher": "0.96.1", diff --git a/packages/envio/src/ChainState.res b/packages/envio/src/ChainState.res index f28f88ebfb..f96e32a17a 100644 --- a/packages/envio/src/ChainState.res +++ b/packages/envio/src/ChainState.res @@ -248,7 +248,9 @@ let makeInternal = ( ~rpcs=evmRpcs, ~lowercaseAddresses, ) - | Config.FuelSourceConfig({hypersync}) => [HyperFuelSource.make({chain, endpointUrl: hypersync})] + | Config.FuelSourceConfig({hypersync}) => [ + HyperFuelSource.make({chain, endpointUrl: hypersync, apiToken: Env.envioApiToken}), + ] | Config.SvmSourceConfig({hypersync, rpc}) => switch (hypersync, rpc) { | (None, None) => diff --git a/packages/envio/src/Core.res b/packages/envio/src/Core.res index 836ffa3394..42d4167329 100644 --- a/packages/envio/src/Core.res +++ b/packages/envio/src/Core.res @@ -8,6 +8,7 @@ type evmHypersyncClientCtor type evmRpcClientCtor type evmDecoderCtor type svmHypersyncClientCtor +type hyperfuelClientCtor type addon = { getConfigJson: (~configPath: Null.t, ~directory: Null.t) => string, @@ -20,6 +21,8 @@ type addon = { evmDecoder: evmDecoderCtor, @as("SvmHypersyncClient") svmHypersyncClient: svmHypersyncClientCtor, + @as("HyperfuelClient") + hyperfuelClient: hyperfuelClientCtor, } @module("node:module") external createRequire: string => {..} = "createRequire" diff --git a/packages/envio/src/sources/EnvioApiClient.res b/packages/envio/src/sources/EnvioApiClient.res deleted file mode 100644 index 79626d8f39..0000000000 --- a/packages/envio/src/sources/EnvioApiClient.res +++ /dev/null @@ -1,15 +0,0 @@ -// Rest client for envio's own REST endpoints (e.g. the HyperSync/HyperFuel -// /height poll). Tags requests with the hyperindex User-Agent so they're -// attributable on the server, mirroring the SSE height stream and the Rust -// data-query client which already set it. -let make = (baseUrl: string): Rest.client => { - let userAgent = `hyperindex/${Utils.EnvioPackage.value.version}` - Rest.client(baseUrl, ~fetcher=(args: Rest.ApiFetcher.args) => { - let headers = switch args.headers { - | Some(headers) => headers - | None => Dict.make() - } - headers->Dict.set("User-Agent", userAgent->(Utils.magic: string => unknown)) - Rest.ApiFetcher.default({...args, headers: Some(headers)}) - }) -} diff --git a/packages/envio/src/sources/HyperFuel.res b/packages/envio/src/sources/HyperFuel.res index b64b5911c0..a1a356deb5 100644 --- a/packages/envio/src/sources/HyperFuel.res +++ b/packages/envio/src/sources/HyperFuel.res @@ -1,21 +1,3 @@ -//Manage clients in cache so we don't need to reinstantiate each time -//Ideally client should be passed in as a param to the functions but -//we are still sharing the same signature with eth archive query builder - -module CachedClients = { - let cache: dict = Dict.make() - - let getClient = url => { - switch cache->Utils.Dict.dangerouslyGetNonOption(url) { - | Some(client) => client - | None => - let newClient = HyperFuelClient.make({url: url}) - cache->Dict.set(url, newClient) - newClient - } - } -} - type hyperSyncPage<'item> = { items: array<'item>, nextBlock: int, @@ -36,29 +18,8 @@ type item = { block: block, } -type blockNumberAndHash = { - blockNumber: int, - hash: string, -} - type logsQueryPage = hyperSyncPage -type missingParams = { - queryName: string, - missingParams: array, -} -type queryError = UnexpectedMissingParams(missingParams) - -let queryErrorToMsq = (e: queryError): string => { - switch e { - | UnexpectedMissingParams({queryName, missingParams}) => - `${queryName} query failed due to unexpected missing params on response: - ${missingParams->Array.joinUnsafe(", ")}` - } -} - -type queryResponse<'a> = result<'a, queryError> - module GetLogs = { type error = | UnexpectedMissingParams({missingParams: array}) @@ -66,6 +27,31 @@ module GetLogs = { exception Error(error) + // Rust encodes structured failures as a JSON payload in the napi error's + // message: `{"kind":"MissingFields","fields":["receipt.txId", ...]}`. + // JSON.parse + shape check is the recovery protocol — no string-grepping + // on anyhow's Debug format. + let extractMissingParams = (exn: exn): option> => { + let message = switch exn { + | JsExn(jsExn) => jsExn->JsExn.message + | _ => None + } + switch message { + | None => None + | Some(msg) => + switch msg->JSON.parseOrThrow->JSON.Decode.object { + | exception _ => None + | None => None + | Some(obj) => + switch (obj->Dict.get("kind"), obj->Dict.get("fields")) { + | (Some(String("MissingFields")), Some(Array(fields))) => + Some(fields->Array.filterMap(JSON.Decode.string)) + | _ => None + } + } + } + } + let makeRequestBody = ( ~fromBlock, ~toBlockInclusive, @@ -121,9 +107,7 @@ module GetLogs = { let {receipts, blocks} = response_data let blocksDict = Dict.make() - blocks - ->(Utils.magic: option<'a> => 'a) - ->Array.forEach(block => { + blocks->Array.forEach(block => { blocksDict->Dict.set(block.height->(Utils.magic: int => string), block) }) @@ -166,93 +150,30 @@ module GetLogs = { page } - let query = async (~serverUrl, ~fromBlock, ~toBlock, ~recieptsSelection): logsQueryPage => { + let query = async ( + ~client: HyperFuelClient.t, + ~fromBlock, + ~toBlock, + ~recieptsSelection, + ): logsQueryPage => { let query: HyperFuelClient.QueryTypes.query = makeRequestBody( ~fromBlock, ~toBlockInclusive=toBlock, ~recieptsSelection, ) - let hyperFuelClient = CachedClients.getClient(serverUrl) - - let res = await hyperFuelClient->HyperFuelClient.getSelectedData(query) + let res = switch await client->HyperFuelClient.getSelectedData(query) { + | res => res + | exception exn => + switch exn->extractMissingParams { + | Some(missingParams) => throw(Error(UnexpectedMissingParams({missingParams: missingParams}))) + | None => throw(exn) + } + } if res.nextBlock <= fromBlock { - // Might happen when /height response was from another instance of HyperSync + // Might happen when /height response was from another instance of HyperFuel throw(Error(WrongInstance)) } res->convertResponse } } - -module BlockData = { - let convertResponse = (res: HyperFuelClient.queryResponseTyped): option< - ReorgDetection.blockDataWithTimestamp, - > => { - res.data.blocks->Option.flatMap(blocks => { - blocks - ->Array.get(0) - ->Option.map(block => { - switch block { - | {height: blockNumber, time: timestamp, id: blockHash} => - ( - { - blockTimestamp: timestamp, - blockNumber, - blockHash, - }: ReorgDetection.blockDataWithTimestamp - ) - } - }) - }) - } - - let rec queryBlockData = async (~serverUrl, ~blockNumber, ~logger): option< - ReorgDetection.blockDataWithTimestamp, - > => { - let query: HyperFuelClient.QueryTypes.query = { - fromBlock: blockNumber, - toBlockExclusive: blockNumber + 1, - // FIXME: Theoretically it should work without the outputs filter, but it doesn't for some reason - outputs: [%raw(`{}`)], - // FIXME: Had to add inputs {} as well, since it failed on block 1211599 during wildcard Call indexing - inputs: [%raw(`{}`)], - fieldSelection: { - block: [Height, Id, Time], - }, - includeAllBlocks: true, - } - - let hyperFuelClient = CachedClients.getClient(serverUrl) - - let logger = Logging.createChildFrom( - ~logger, - ~params={"logType": "hypersync get blockhash query", "blockNumber": blockNumber}, - ) - - let executeQuery = () => hyperFuelClient->HyperFuelClient.getSelectedData(query) - - let res = await executeQuery->Time.retryAsyncWithExponentialBackOff(~logger) - - // If the block is not found, retry the query. This can occur since replicas of hypersync might not hack caught up yet - if res.nextBlock <= blockNumber { - let logger = Logging.createChild(~params={"url": serverUrl}) - let delayMilliseconds = 100 - logger->Logging.childInfo( - `Block #${blockNumber->Int.toString} not found in HyperFuel. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`, - ) - await Time.resolvePromiseAfterDelay(~delayMilliseconds) - await queryBlockData(~serverUrl, ~blockNumber, ~logger) - } else { - res->convertResponse - } - } -} - -let queryBlockData = BlockData.queryBlockData - -let heightRoute = Rest.route(() => { - path: "/height", - method: Get, - input: _ => (), - responses: [s => s.field("height", S.int)], -}) diff --git a/packages/envio/src/sources/HyperFuel.resi b/packages/envio/src/sources/HyperFuel.resi index 0e6a6cbfbc..81f7d42320 100644 --- a/packages/envio/src/sources/HyperFuel.resi +++ b/packages/envio/src/sources/HyperFuel.resi @@ -18,23 +18,8 @@ type item = { block: block, } -type blockNumberAndHash = { - blockNumber: int, - hash: string, -} - type logsQueryPage = hyperSyncPage -type missingParams = { - queryName: string, - missingParams: array, -} -type queryError = UnexpectedMissingParams(missingParams) - -let queryErrorToMsq: queryError => string - -type queryResponse<'a> = result<'a, queryError> - module GetLogs: { type error = | UnexpectedMissingParams({missingParams: array}) @@ -43,17 +28,9 @@ module GetLogs: { exception Error(error) let query: ( - ~serverUrl: string, + ~client: HyperFuelClient.t, ~fromBlock: int, ~toBlock: option, ~recieptsSelection: array, ) => promise } - -let queryBlockData: ( - ~serverUrl: string, - ~blockNumber: int, - ~logger: Pino.t, -) => promise> - -let heightRoute: Rest.route diff --git a/packages/envio/src/sources/HyperFuelClient.res b/packages/envio/src/sources/HyperFuelClient.res index 3f77816d91..2fb7f8b961 100644 --- a/packages/envio/src/sources/HyperFuelClient.res +++ b/packages/envio/src/sources/HyperFuelClient.res @@ -1,119 +1,43 @@ -type unchecksummedEthAddress = string - type t type cfg = { url: string, - bearerToken?: string, - http_req_timeout_millis?: int, + apiToken: string, } module QueryTypes = { type blockFieldOptions = | @as("id") Id - | @as("da_height") DaHeight - | @as("transactions_count") TransactionsCount - | @as("message_receipt_count") MessageReceiptCount - | @as("transactions_root") TransactionsRoot - | @as("message_receipt_root") MessageReceiptRoot | @as("height") Height - | @as("prev_root") PrevRoot | @as("time") Time - | @as("application_hash") ApplicationHash type blockFieldSelection = array - type transactionFieldOptions = - | @as("id") Id - | @as("block_height") BlockHeight - | @as("input_asset_ids") InputAssetIds - | @as("input_contracts") InputContracts - | @as("input_contract_utxo_id") InputContractUtxoId - | @as("input_contract_balance_root") InputContractBalanceRoot - | @as("input_contract_state_root") InputContractStateRoot - | @as("input_contract_tx_pointer_block_height") InputContractTxPointerBlockHeight - | @as("input_contract_tx_pointer_tx_index") InputContractTxPointerTxIndex - | @as("input_contract") InputContract - | @as("gas_price") GasPrice - | @as("gas_limit") GasLimit - | @as("maturity") Maturity - | @as("mint_amount") MintAmount - | @as("mint_asset_id") MintAssetId - | @as("tx_pointer_block_height") TxPointerBlockHeight - | @as("tx_pointer_tx_index") TxPointerTxIndex - | @as("tx_type") TxType - | @as("output_contract_input_index") OutputContractInputIndex - | @as("output_contract_balance_root") OutputContractBalanceRoot - | @as("output_contract_state_root") OutputContractStateRoot - | @as("witnesses") Witnesses - | @as("receipts_root") ReceiptsRoot - | @as("status") Status - | @as("time") Time - | @as("reason") Reason - | @as("script") Script - | @as("script_data") ScriptData - | @as("bytecode_witness_index") BytecodeWitnessIndex - | @as("bytecode_length") BytecodeLength - | @as("salt") Salt - - type transactionFieldSelection = array - type receiptFieldOptions = | @as("tx_id") TxId - | @as("tx_status") TxStatus | @as("block_height") BlockHeight - | @as("pc") Pc - | @as("is") Is - | @as("to") To | @as("to_address") ToAddress | @as("amount") Amount | @as("asset_id") AssetId - | @as("gas") Gas - | @as("param1") Param1 - | @as("param2") Param2 | @as("val") Val - | @as("ptr") Ptr - | @as("digest") Digest - | @as("reason") Reason - | @as("ra") Ra | @as("rb") Rb - | @as("rc") Rc - | @as("rd") Rd - | @as("len") Len | @as("receipt_type") ReceiptType | @as("receipt_index") ReceiptIndex - | @as("result") Result - | @as("gas_used") GasUsed | @as("data") Data - | @as("sender") Sender - | @as("recipient") Recipient - | @as("nonce") Nonce - | @as("contract_id") ContractId | @as("root_contract_id") RootContractId | @as("sub_id") SubId + | @as("to") To type receiptFieldSelection = array type fieldSelection = { block?: blockFieldSelection, - transaction?: transactionFieldSelection, receipt?: receiptFieldSelection, } - type inputSelection - type outputSelection - type receiptSelection = { rootContractId?: array, - toAddress?: array, - assetId?: array, receiptType?: array, - sender?: array, - recipient?: array, - contractId?: array, - ra?: array, rb?: array, - rc?: array, - rd?: array, txStatus?: array, } @@ -137,254 +61,44 @@ module QueryTypes = { */ receipts?: array, /** - * List of input selections, the query will return inputs that match any of these selections and - * it will return inputs that are related to the returned objects. - */ - inputs?: array, - /** - * List of output selections, the query will return outputs that match any of these selections and - * it will return outputs that are related to the returned objects. - */ - outputs?: array, - /** - * Whether to include all blocks regardless of if they are related to a returned transaction or log. Normally - * the server will return only the blocks that are related to the transaction or logs in the response. But if this - * is set to true, the server will return data for all blocks in the requested range [from_block, to_block). - */ - includeAllBlocks?: bool, - /** * Field selection. The user can select which fields they are interested in, requesting less fields will improve * query execution time and reduce the payload size so the user should always use a minimal number of fields. */ fieldSelection: fieldSelection, - /** - * Maximum number of blocks that should be returned, the server might return more blocks than this number but - * it won't overshoot by too much. - */ - maxNumBlocks?: int, - /** - * Maximum number of transactions that should be returned, the server might return more transactions than this number but - * it won't overshoot by too much. - */ - maxNumTransactions?: int, } } module FuelTypes = { - /** An object containing information about a transaction. */ - type transaction = { - /** block the transaction is in. */ - blockHeight: int, - /** A unique transaction id. */ - id: string, - /** An array of asset ids used for the transaction inputs. */ - inputAssetIds?: array, - /** An array of contracts used for the transaction inputs. */ - inputContracts?: array, - /** - * A contract used for the transaction input. - * A unique 32 byte identifier for the UTXO for a contract used for the transaction input. - */ - inputContractUtxoId?: string, - /** The root of amount of coins owned by contract before transaction execution for a contract used for the transaction input. */ - inputContractBalanceRoot?: string, - /** The state root of contract before transaction execution for a contract used for the transaction input. */ - inputContractStateRoot?: string, - /** A pointer to the TX whose output is being spent for a contract used for the transaction input. */ - inputContractTxPointerBlockHeight?: int, - /** A pointer to the TX whose output is being spent for a contract used for the transaction input. */ - inputContractTxPointerTxIndex?: int, - /** The contract id for a contract used for the transaction input. */ - inputContract?: string, - /** The gas price for the transaction. */ - gasPrice?: bigint, - /** The gas limit for the transaction. */ - gasLimit?: bigint, - /** The minimum block height that the transaction can be included at. */ - maturity?: int, - /** The amount minted in the transaction. */ - mintAmount?: bigint, - /** The asset ID for coins minted in the transaction. */ - mintAssetId?: string, - /** The location of the transaction in the block. */ - txPointerBlockHeight?: int, - txPointerTxIndex?: int, - /** Script, creating a new contract, or minting new coins */ - txType: int, - /** The index of the input from a transaction that changed the state of a contract. */ - outputContractInputIndex?: int, - /** The root of amount of coins owned by contract after transaction execution from a transaction that changed the state of a contract. */ - outputContractBalanceRoot?: string, - /** The state root of contract after transaction execution from a transaction that changed the state of a contract. */ - outputContractStateRoot?: string, - /** An array of witnesses. */ - witnesses?: string, - /** The root of the receipts. */ - receiptsRoot?: string, - /** The status type of the transaction. */ - status: int, - /** for SubmittedStatus, SuccessStatus, and FailureStatus, the time a transaction was submitted, successful, or failed */ - time: int, - /** - * for SuccessStatus, the state of the program execution - * for SqueezedOutStatus & FailureStatus, the reason the transaction was squeezed out or failed - */ - reason?: string, - /** The script to execute. */ - script?: string, - /** The script input parameters. */ - scriptData?: string, - /** The witness index of contract bytecode. */ - bytecodeWitnessIndex?: int, - /** The length of the transaction bytecode. */ - bytecodeLength?: int, - /** The salt value for the transaction. */ - salt?: string, - } - /** An object representing all possible types of receipts. */ type receipt = { - /** Index of the receipt in the block */ receiptIndex: int, - /** Contract that produced the receipt */ rootContractId?: Address.t, - /** transaction that this receipt originated from */ txId: string, - /** The status type of the transaction this receipt originated from */ - txStatus: int, - /** block that the receipt originated in */ blockHeight: int, - /** The value of the program counter register $pc, which is the memory address of the current instruction. */ - pc?: int, - /** The value of register $is, which is the pointer to the start of the currently-executing code. */ - is?: int, - /** The recipient contract */ - to?: string, - /** The recipient address */ - toAddress?: string, - /** The amount of coins transferred. */ - amount?: bigint, - /** The asset id of the coins transferred. */ - assetId?: string, - /** The gas used for the transaction. */ - gas?: int, - /** The first parameter for a CALL receipt type, holds the function selector. */ - param1?: bigint, - /** The second parameter for a CALL receipt type, typically used for the user-specified input to the ABI function being selected. */ - param2?: bigint, - /** The value of registers at the end of execution, used for debugging. */ - val?: bigint, - /** The value of the pointer register, used for debugging. */ - ptr?: bigint, - /** A 32-byte String of MEM[$rC, $rD]. The syntax MEM[x, y] means the memory range starting at byte x, of length y bytes. */ - digest?: string, - /** The decimal string representation of an 8-bit unsigned integer for the panic reason. Only returned if the receipt type is PANIC. */ - reason?: int, - /** The value of register $rA. */ - ra?: bigint, - /** The value of register $rB. */ - rb?: bigint, - /** The value of register $rC. */ - rc?: bigint, - /** The value of register $rD. */ - rd?: bigint, - /** The length of the receipt. */ - len?: bigint, - /** The type of receipt. */ receiptType: FuelSDK.receiptType, - /** 0 if script exited successfully, any otherwise. */ - result?: int, - /** The amount of gas consumed by the script. */ - gasUsed?: int, - /** The receipt data. */ data?: string, - /** The address of the message sender. */ - sender?: string, - /** The address of the message recipient. */ - recipient?: string, - /** The nonce value for a message. */ - nonce?: string, - /** Current context if in an internal context. null otherwise */ - contractId?: Address.t, - /** The sub id. */ + rb?: bigint, + val?: bigint, subId?: string, - } - - // Unused - in indexer currently - type input = { - txId: string, - blockHeight: int, - inputType: int, - utxoId?: string, - owner?: string, amount?: bigint, assetId?: string, - txPointerBlockHeight?: int, - txPointerTxIndex?: int, - witnessIndex?: int, - predicateGasUsed?: int, - predicate?: string, - predicateData?: string, - balanceRoot?: string, - stateRoot?: string, - contract?: string, - sender?: string, - recipient?: string, - nonce?: string, - data?: string, - } - - // Unused in indexer currently - type output = { - txId: string, - blockHeight: int, - outputType: int, to?: string, - amount?: bigint, - assetId?: string, - inputIndex?: int, - balanceRoot?: string, - stateRoot?: string, - contract?: string, + toAddress?: string, } - // Unused in indexer currently - /** The block header contains metadata about a certain block. */ type block = { - /** String of the header */ id: string, - /** The block height for the data availability layer up to which (inclusive) input messages are processed. */ - daHeight: int, - consensusParametersVersion: int, - stateTransitionBytecodeVersion: int, - /** The number of transactions in the block. */ - transactionsCount: string, - /** The number of receipt messages in the block. */ - messageReceiptCount: string, - /** The merkle root of the transactions in the block. */ - transactionsRoot: string, - messageOutboxRoot: string, - eventInboxRoot: string, - /** The block height. */ height: int, - /** The merkle root of all previous consensus header Stringes (not including this block). */ - prevRoot: string, - /** The timestamp for the block. */ time: int, - /** The String of the serialized application header for this block. */ - applicationHash: string, } } type queryResponseDataTyped = { - transactions: array, receipts: array, - blocks: option>, - inputs: array, - outputs: array, + blocks: array, } type queryResponseTyped = { - /** Current height of the source hypersync instance */ + /** Current height of the source HyperFuel instance */ archiveHeight?: int, /** * Next block to query for, the responses are paginated so @@ -392,17 +106,22 @@ type queryResponseTyped = { * didn't get responses up to the to_block they specified in the Query. */ nextBlock: int, - /** Total time it took the hypersync instance to execute the query. */ + /** Total time it took the HyperFuel instance to execute the query. */ totalExecutionTime: int, /** Response data */ data: queryResponseDataTyped, } -@module("@envio-dev/hyperfuel-client") @scope("HyperfuelClient") -external make: cfg => t = "new" +@send +external classNew: (Core.hyperfuelClientCtor, cfg, ~userAgent: string) => t = "new" + let make = (cfg: cfg) => { - make({...cfg, bearerToken: "3dc856dd-b0ea-494f-b27e-017b8b6b7e07"}) + let envioVersion = Utils.EnvioPackage.value.version + Core.getAddon().hyperfuelClient->classNew(cfg, ~userAgent=`hyperindex/${envioVersion}`) } @send external getSelectedData: (t, QueryTypes.query) => promise = "getSelectedData" + +@send +external getHeight: t => promise = "getHeight" diff --git a/packages/envio/src/sources/HyperFuelSource.res b/packages/envio/src/sources/HyperFuelSource.res index 9d714108fc..a0e9d68a59 100644 --- a/packages/envio/src/sources/HyperFuelSource.res +++ b/packages/envio/src/sources/HyperFuelSource.res @@ -2,6 +2,8 @@ open Source exception EventRoutingFailed +let isUnauthorizedError = (message: string) => message->String.includes("401 Unauthorized") + let mintEventTag = "mint" let burnEventTag = "burn" let transferEventTag = "transfer" @@ -207,11 +209,26 @@ let memoGetSelectionConfig = (~chain) => { type options = { chain: ChainMap.Chain.t, endpointUrl: string, + apiToken: option, } -let make = ({chain, endpointUrl}: options): t => { +let make = ({chain, endpointUrl, apiToken}: options): t => { let name = "HyperFuel" + let apiToken = switch apiToken { + | Some(token) => token + | None => + JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source. +Set the ENVIO_API_TOKEN environment variable in your .env file. +Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`) + } + + let client = switch HyperFuelClient.make({url: endpointUrl, apiToken}) { + | client => client + | exception exn => + exn->ErrorHandling.mkLogAndRaise(~msg="Failed to instantiate the HyperFuel client") + } + let getSelectionConfig = memoGetSelectionConfig(~chain) let getItemsOrThrow = async ( @@ -239,12 +256,12 @@ let make = ({chain, endpointUrl}: options): t => { ~method="getLogs", ) let pageUnsafe = try await HyperFuel.GetLogs.query( - ~serverUrl=endpointUrl, + ~client, ~fromBlock, ~toBlock, ~recieptsSelection, ) catch { - | HyperSync.GetLogs.Error(error) => + | HyperFuel.GetLogs.Error(error) => throw( Source.GetItemsError( Source.FailedGettingItems({ @@ -450,8 +467,6 @@ let make = ({chain, endpointUrl}: options): t => { let getBlockHashes = (~blockNumbers as _, ~logger as _) => JsError.throwWithMessage("HyperFuel does not support getting block hashes") - let jsonApiClient = EnvioApiClient.make(endpointUrl) - { name, sourceFor: Sync, @@ -461,7 +476,17 @@ let make = ({chain, endpointUrl}: options): t => { poweredByHyperSync: true, getHeightOrThrow: async () => { let timerRef = Hrtime.makeTimer() - let height = await HyperFuel.heightRoute->Rest.fetch((), ~client=jsonApiClient) + let height = try await client->HyperFuelClient.getHeight catch { + | JsExn(e) => + switch e->JsExn.message { + | Some(message) if message->isUnauthorizedError => + Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`) + // Retrying an unauthorized request can never succeed, so block forever + let _ = await Promise.make((_, _) => ()) + 0 + | _ => throw(JsExn(e)) + } + } let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat Prometheus.SourceRequestCount.increment( ~sourceName=name, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17dc0721f6..ca53b08dea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,9 +48,6 @@ importers: '@elastic/ecs-pino-format': specifier: 1.4.0 version: 1.4.0 - '@envio-dev/hyperfuel-client': - specifier: 1.2.2 - version: 1.2.2 '@fuel-ts/crypto': specifier: 0.96.1 version: 0.96.1 @@ -449,49 +446,6 @@ packages: resolution: {integrity: sha512-eCSBUTgl8KbPyxky8cecDRLCYu2C1oFV4AZ72bEsI+TxXEvaljaL2kgttfzfu7gW+M89eCz55s49uF2t+YMTWA==} engines: {node: '>=10'} - '@envio-dev/hyperfuel-client-darwin-arm64@1.2.2': - resolution: {integrity: sha512-eQyd9kJCIz/4WCTjkjpQg80DA3pdneHP7qhJIVQ2ZG+Jew9o5XDG+uI0Y16AgGzZ6KGmJSJF6wyUaaAjJfbO1Q==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [darwin] - - '@envio-dev/hyperfuel-client-darwin-x64@1.2.2': - resolution: {integrity: sha512-l7lRMSoyIiIvKZgQPfgqg7H1xnrQ37A8yUp4S2ys47R8f/wSCSrmMaY1u7n6CxVYCpR9fajwy0/356UgwwhVKw==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] - - '@envio-dev/hyperfuel-client-linux-arm64-gnu@1.2.2': - resolution: {integrity: sha512-kNiC/1fKuXnoSxp8yEsloDw4Ot/mIcNoYYGLl2CipSIpBtSuiBH5nb6eBcxnRZdKOwf5dKZtZ7MVPL9qJocNJw==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@envio-dev/hyperfuel-client-linux-x64-gnu@1.2.2': - resolution: {integrity: sha512-XDkvkBG/frS+xiZkJdY4KqOaoAwyxPdi2MysDQgF8NmZdssi32SWch0r4LTqKWLLlCBg9/R55POeXL5UAjg2wQ==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@envio-dev/hyperfuel-client-linux-x64-musl@1.2.2': - resolution: {integrity: sha512-DKnKJJSwsYtA7YT0EFGhFB5Eqoo42X0l0vZBv4lDuxngEXiiNjeLemXoKQVDzhcbILD7eyXNa5jWUc+2hpmkEg==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - libc: [musl] - - '@envio-dev/hyperfuel-client-win32-x64-msvc@1.2.2': - resolution: {integrity: sha512-SwIgTAVM9QhCFPyHwL+e1yQ6o3paV6q25klESkXw+r/KW9QPhOOyA6Yr8nfnur3uqMTLJHAKHTLUnkyi/Nh7Aw==} - engines: {node: '>= 10'} - cpu: [x64] - os: [win32] - - '@envio-dev/hyperfuel-client@1.2.2': - resolution: {integrity: sha512-raKA6DshYSle0sAOHBV1OkSRFMN+Mkz8sFiMmS3k+m5nP6pP56E17CRRePBL5qmR6ZgSEvGOz/44QUiKNkK9Pg==} - engines: {node: '>= 10'} - '@esbuild/aix-ppc64@0.27.2': resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} engines: {node: '>=18'} @@ -3153,33 +3107,6 @@ snapshots: dependencies: '@elastic/ecs-helpers': 1.1.0 - '@envio-dev/hyperfuel-client-darwin-arm64@1.2.2': - optional: true - - '@envio-dev/hyperfuel-client-darwin-x64@1.2.2': - optional: true - - '@envio-dev/hyperfuel-client-linux-arm64-gnu@1.2.2': - optional: true - - '@envio-dev/hyperfuel-client-linux-x64-gnu@1.2.2': - optional: true - - '@envio-dev/hyperfuel-client-linux-x64-musl@1.2.2': - optional: true - - '@envio-dev/hyperfuel-client-win32-x64-msvc@1.2.2': - optional: true - - '@envio-dev/hyperfuel-client@1.2.2': - optionalDependencies: - '@envio-dev/hyperfuel-client-darwin-arm64': 1.2.2 - '@envio-dev/hyperfuel-client-darwin-x64': 1.2.2 - '@envio-dev/hyperfuel-client-linux-arm64-gnu': 1.2.2 - '@envio-dev/hyperfuel-client-linux-x64-gnu': 1.2.2 - '@envio-dev/hyperfuel-client-linux-x64-musl': 1.2.2 - '@envio-dev/hyperfuel-client-win32-x64-msvc': 1.2.2 - '@esbuild/aix-ppc64@0.27.2': optional: true diff --git a/scenarios/fuel_test/test/HyperFuelHeight_test.res b/scenarios/fuel_test/test/HyperFuelHeight_test.res new file mode 100644 index 0000000000..90a876614d --- /dev/null +++ b/scenarios/fuel_test/test/HyperFuelHeight_test.res @@ -0,0 +1,91 @@ +open Vitest + +// For Logging.setLogger call +let _ = Env.logStrategy + +type server +type req = {headers: dict} +type res + +@module("node:http") +external createServer: ((req, res) => unit) => server = "createServer" +@send external listen: (server, int, unit => unit) => unit = "listen" +@send external close: server => unit = "close" +@send external address: server => {"port": int} = "address" +@send external writeHead: (res, int) => unit = "writeHead" +@send external endWith: (res, string) => unit = "end" + +let startServer = async handler => { + let server = createServer(handler) + await Promise.make((resolve, _) => server->listen(0, () => resolve())) + let port = (server->address)["port"] + (server, `http://127.0.0.1:${port->Int.toString}`) +} + +// Guarantees the temp server is closed even if the body throws, so a failing +// assertion can't leak a listener and hang/cross-talk with later tests. +let withServer = async (handler, body) => { + let (server, endpointUrl) = await startServer(handler) + try { + let result = await body(endpointUrl) + server->close + result + } catch { + | exn => + server->close + throw(exn) + } +} + +describe("HyperFuelSource - getHeightOrThrow", () => { + let chain = ChainMap.Chain.makeUnsafe(~chainId=0) + + // The native client validates that the token is a UUID before sending requests. + let apiToken = "11111111-1111-1111-1111-111111111111" + + Async.it("Requests height via the client with auth and user agent headers", async t => { + let capturedHeaders = ref(None) + await withServer((req, res) => { + capturedHeaders := Some(req.headers) + res->writeHead(200) + res->endWith(`{"height": 123}`) + }, async endpointUrl => { + let source = HyperFuelSource.make({ + chain, + endpointUrl, + apiToken: Some(apiToken), + }) + let height = await source.getHeightOrThrow() + + let headers = capturedHeaders.contents->Option.getOrThrow + t.expect(( + height, + headers->Dict.get("authorization"), + headers->Dict.get("user-agent"), + )).toEqual(( + 123, + Some(`Bearer ${apiToken}`), + Some(`hyperindex/${Utils.EnvioPackage.value.version}`), + )) + }) + }) + + Async.it("Blocks forever on 401 instead of throwing for a retry", async t => { + await withServer((_req, res) => { + res->writeHead(401) + res->endWith("Unauthorized") + }, async endpointUrl => { + let source = HyperFuelSource.make({ + chain, + endpointUrl, + apiToken: Some(apiToken), + }) + let result = await Promise.race([ + source.getHeightOrThrow()->Promise.thenResolve(_ => "resolved"), + Time.resolvePromiseAfterDelay(~delayMilliseconds=300)->Promise.thenResolve(() => "blocked"), + ]) + + t.expect(result).toEqual("blocked") + }) + }) +})