diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 5d96d6d3d47..d29938f42e0 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -86,6 +86,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, title: `Bump version to ${process.env.NEW_VERSION}`, + draft: true, head: `version-bump-${process.env.NEW_VERSION}`, base: process.env.TARGET_BRANCH }) diff --git a/.github/workflows/publish-windows-tarball.yml b/.github/workflows/publish-windows-tarball.yml index ef6c3cf2c40..add365ec79c 100644 --- a/.github/workflows/publish-windows-tarball.yml +++ b/.github/workflows/publish-windows-tarball.yml @@ -99,7 +99,7 @@ jobs: path: ./windows-release/ - name: Release - uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0 + uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 with: tag_name: ${{ needs.windows-build.outputs.tag }} files: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f9741a7cba..a13fc62f43b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Trigger a Buildkite Build - uses: "buildkite/trigger-pipeline-action@909fed762c73d5ae2b5d555ab910d66b3fae2670" # v2.4.1 + uses: "buildkite/trigger-pipeline-action@41fd38b69189bf186cf69cf10ec807a850cae593" # v2.5.0 with: buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }} pipeline: "anza/agave-secondary" diff --git a/.github/workflows/trigger-buildkite-pipeline.yml b/.github/workflows/trigger-buildkite-pipeline.yml index a9080b10c3e..ba77f6c8354 100644 --- a/.github/workflows/trigger-buildkite-pipeline.yml +++ b/.github/workflows/trigger-buildkite-pipeline.yml @@ -103,7 +103,7 @@ jobs: echo "pr_number=$PR_NUMBER" | tee -a $GITHUB_OUTPUT - name: Trigger a Buildkite Build - uses: "buildkite/trigger-pipeline-action@909fed762c73d5ae2b5d555ab910d66b3fae2670" # v2.4.1 + uses: "buildkite/trigger-pipeline-action@41fd38b69189bf186cf69cf10ec807a850cae593" # v2.5.0 with: pipeline: ${{ steps.prepare.outputs.pipeline }} buildkite_api_access_token: ${{ secrets.BUILDKITE_API_ACCESS_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bec9a2cbe3..0f6fb9365d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,10 +40,12 @@ Release channels have their own copy of this changelog: * `--experimental-poh-pinned-cpu-core` is now deprecated. Use `--poh-pinned-cpu-core` instead. #### Changes * Turbine shred ingestion now rejects shreds more than half an epoch in the future (previously up to 2 full epochs ahead was accepted). +* When XDP is enabled, gossip egress does not support private and loopback addresses. Operators running with `--allow-private-addr` must also pass `--no-xdp`. ### CLI #### Breaking #### Changes * `vote-account` supports Alpenglow and as such `vote-account --output json` breaks compatibility with older versions. +* Support Keystone hardware wallets using `usb://keystone` ## 4.1.0 ### RPC diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb6aa1ddff8..a33880f52c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -381,12 +381,6 @@ confused with 3-letter acronyms. ## Design Proposals -This Agave validator client's architecture is described by docs generated from markdown files in the `docs/src/` -directory and viewable on the official [Agave Validator Client](https://docs.anza.xyz) documentation website. - -Current design proposals may be viewed on the docs site: - -1. [Accepted Proposals](https://docs.anza.xyz/proposals/accepted-design-proposals) -2. [Implemented Proposals](https://docs.anza.xyz/implemented-proposals/implemented-proposals) - -New design proposals should follow this guide on [how to submit a design proposal](./docs/src/proposals.md#submit-a-design-proposal). +Design proposals are now tracked as Solana Improvement Documents (SIMDs) in the +[solana-foundation/solana-improvement-documents](https://github.com/solana-foundation/solana-improvement-documents) +repository. diff --git a/Cargo.lock b/Cargo.lock index 8c4af395a9b..904ba3bc34e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aead" version = "0.5.2" @@ -542,6 +548,7 @@ dependencies = [ name = "agave-watchtower" version = "4.2.0-alpha.0" dependencies = [ + "agave-feature-set", "agave-logger", "clap 2.33.3", "humantime", @@ -557,6 +564,7 @@ dependencies = [ "solana-rpc-client", "solana-rpc-client-api", "solana-version", + "solana-vote-interface", ] [[package]] @@ -573,6 +581,7 @@ dependencies = [ "crossbeam-channel", "libc", "log", + "nix", "thiserror 2.0.18", ] @@ -1232,19 +1241,22 @@ dependencies = [ [[package]] name = "aya" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d18bc4e506fbb85ab7392ed993a7db4d1a452c71b75a246af4a80ab8c9d2dd50" +checksum = "66e644424fada9fff4fdc63848db1732fb69b626e8328202ef55c03df1f4d939" dependencies = [ + "anyhow", "assert_matches", "aya-obj", "bitflags 2.13.0", - "bytes", + "hashbrown 0.17.0", "libc", "log", + "nix", "object", "once_cell", - "thiserror 1.0.69", + "scopeguard", + "thiserror 2.0.18", ] [[package]] @@ -1288,16 +1300,14 @@ dependencies = [ [[package]] name = "aya-obj" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51b96c5a8ed8705b40d655273bc4212cbbf38d4e3be2788f36306f154523ec7" +checksum = "8c76b9c75d9cdc155ff8f6a06d61e873f67bf47be8cfa92a3b5aaea43f4b4077" dependencies = [ "bytes", - "core-error", - "hashbrown 0.15.1", "log", "object", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1363,9 +1373,9 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90dbd31c98227229239363921e60fcf5e558e43ec69094d46fc4996f08d1d5bc" dependencies = [ - "bitcoin_hashes", + "bitcoin_hashes 0.14.100", "rand 0.8.6", - "rand_core 0.5.1", + "rand_core 0.6.4", "serde", "unicode-normalization", ] @@ -1385,6 +1395,21 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitcoin-private" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73290177011694f38ec25e165d0387ab7ea749a4b81cd4c80dae5988229f7a57" + +[[package]] +name = "bitcoin_hashes" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d7066118b13d4b20b23645932dfb3a81ce7e29f95726c2036fa33cd7b092501" +dependencies = [ + "bitcoin-private", +] + [[package]] name = "bitcoin_hashes" version = "0.14.100" @@ -1573,6 +1598,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ + "sha2 0.10.9", "tinyvec", ] @@ -1884,7 +1910,7 @@ version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -2022,15 +2048,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "core-error" -version = "0.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efcdb2972eb64230b4c50646d8498ff73f5128d196a90c7236eec4cbe8619b8f" -dependencies = [ - "version_check", -] - [[package]] name = "core-foundation" version = "0.9.4" @@ -2075,6 +2092,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.2.1" @@ -2331,6 +2363,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + [[package]] name = "dashmap" version = "5.5.3" @@ -2718,7 +2756,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -2813,7 +2851,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2872,7 +2910,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if 1.0.4", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2951,6 +2989,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "059c31d7d36c43fe39d89e55711858b4da8be7eb6dabac23c7289b1a19489406" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -3354,10 +3398,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" dependencies = [ "allocator-api2", - "equivalent", "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.0" @@ -3369,6 +3423,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -3445,6 +3505,15 @@ dependencies = [ "hmac 0.8.1", ] +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.0", +] + [[package]] name = "http" version = "0.2.12" @@ -4231,6 +4300,19 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "keystone-ur" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a08a7e0ccd113dac19485c5c0fe87c70b663f896c48e4493814e446175078d4" +dependencies = [ + "bitcoin_hashes 0.12.0", + "crc", + "minicbor", + "phf", + "rand_xoshiro 0.6.0", +] + [[package]] name = "konst" version = "0.2.20" @@ -4273,6 +4355,30 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libflate" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f7ef5c7e3c2ed51f0fbc40e016c66558b699f16593521f30b98713bbb99cb8" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] + +[[package]] +name = "libflate_lz77" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown 0.16.1", + "no_std_io2", + "rle-decode-fast", +] + [[package]] name = "libloading" version = "0.9.0" @@ -4559,6 +4665,26 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2687e6cf9c00f48e9284cf9fd15f2ef341d03cc7743abf9df4c5f07fdee50b18" +[[package]] +name = "minicbor" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7005aaf257a59ff4de471a9d5538ec868a21586534fff7f85dd97d4043a6139" +dependencies = [ + "minicbor-derive", +] + +[[package]] +name = "minicbor-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1154809406efdb7982841adb6311b3d095b46f78342dd646736122fe6b19e267" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "minimal-lexical" version = "0.1.4" @@ -4681,6 +4807,15 @@ dependencies = [ "memoffset 0.9.1", ] +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.0.0" @@ -4850,12 +4985,12 @@ checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" [[package]] name = "object" -version = "0.36.7" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "crc32fast", - "hashbrown 0.15.1", + "hashbrown 0.17.0", "indexmap 2.14.0", "memchr", ] @@ -5129,17 +5264,69 @@ dependencies = [ "ucd-trie", ] +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + [[package]] name = "petgraph" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "hashbrown 0.15.1", "indexmap 2.14.0", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher 1.0.2", +] + [[package]] name = "pickledb" version = "0.5.1" @@ -5311,6 +5498,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -5420,20 +5617,42 @@ dependencies = [ "prost-derive 0.14.4", ] +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools 0.10.5", + "lazy_static", + "log", + "multimap", + "petgraph 0.6.5", + "prettyplease 0.1.25", + "prost 0.11.9", + "prost-types 0.11.9", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + [[package]] name = "prost-build" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", - "petgraph", - "prettyplease", + "petgraph 0.8.3", + "prettyplease 0.2.37", "prost 0.14.4", - "prost-types", + "prost-types 0.14.4", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", @@ -5467,6 +5686,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost 0.11.9", +] + [[package]] name = "prost-types" version = "0.14.4" @@ -5515,9 +5743,9 @@ dependencies = [ [[package]] name = "protosol" -version = "8.2.0" +version = "9.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87417d0270466f8324148ed75569b940464e70e0aa8aaac574be560f582421ad" +checksum = "e19fd23beba10243272008acfa690c6cda43efed0fd23e3fa9c9dc6e1a989848" dependencies = [ "prost 0.11.9", ] @@ -5570,9 +5798,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -5590,9 +5818,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "fastbloom", @@ -6009,6 +6237,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "rocksdb" version = "0.24.0" @@ -6131,7 +6365,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -6189,7 +6423,7 @@ dependencies = [ "security-framework 3.2.0", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7637,12 +7871,13 @@ dependencies = [ [[package]] name = "solana-clock" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea35d8f69b67daddb921a9da7f78ca591b533cf5e98833cd9ae62fdc2e4652c" +checksum = "f0acdace90d96e2c9e70d681465b4fe888b6bcf27c354ae9774e9f8a3b72923d" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -8135,12 +8370,13 @@ dependencies = [ [[package]] name = "solana-epoch-rewards" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cddf2388b28291210d9aa60690740733cab527531f06ed153c4d388951e407c" +checksum = "daf7eb4986b0b1d6f562b21f75a836f1a6df6e00c275efcef50aab5c144dc59e" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-hash 4.4.0", "solana-sdk-ids", "solana-sdk-macro", @@ -8160,14 +8396,16 @@ dependencies = [ [[package]] name = "solana-epoch-schedule" -version = "3.1.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad280b1ed803853f7b453cb3ea9a57e600ca5599a63e69f7be199b486c0ec93" +checksum = "8116e6ffa6002237d5ab5edcbda17f9ba66b6742c45a89c9fb40a94dbacd4c1d" dependencies = [ "serde", "serde_derive", "solana-frozen-abi", "solana-frozen-abi-macro", + "solana-get-sysvar", + "solana-program-error", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -8308,9 +8546,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47b47faa2099702dc4c764bd70806ac07166a2c81fd0c5154ed04c6e57e2b93" +checksum = "038494fd91f75199a233ff12bc922b8e02da04bd6dd8fed26e710572733d39d5" dependencies = [ "bincode", "boxcar", @@ -8438,6 +8676,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "solana-get-sysvar" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef3bc859fc036ed490146793557386cbfae614ebba4adc704c37d94350824ed4" +dependencies = [ + "solana-address 2.6.1", + "solana-define-syscall 5.1.0", + "solana-program-error", +] + [[package]] name = "solana-geyser-plugin-manager" version = "4.2.0-alpha.0" @@ -8483,6 +8732,7 @@ dependencies = [ "bincode", "bs58", "bv", + "bytes", "criterion", "crossbeam-channel", "ed25519-dalek 2.2.0", @@ -8636,9 +8886,9 @@ dependencies = [ [[package]] name = "solana-instruction-error" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b188842592fdf6cb96f55263ae1bf11713ab5114401d1d5a881ed7cc41bef6" +checksum = "3b7d34343838343a3755b7dfb1e438d94c6db2263b519cfe3c2257af932b6e93" dependencies = [ "num-traits", "serde", @@ -9008,9 +9258,9 @@ dependencies = [ [[package]] name = "solana-message" -version = "4.2.2" +version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee01edb797313c1c8e1961d8ac6befc7b7cd0f90d1e9cf8f784add2b08926a3" +checksum = "b94164f9740d40f41568f6f48140a0866251a79a7bce013eb4ffefe12d0e38cc" dependencies = [ "blake3", "serde", @@ -9646,12 +9896,15 @@ dependencies = [ "assert_matches", "console 0.16.3", "dialoguer", + "hex", "hidapi", "log", "num-derive", "num-traits", "parking_lot 0.12.3", + "rusb", "semver 1.0.28", + "serde_json", "serial_test", "solana-derivation-path", "solana-offchain-message", @@ -9660,19 +9913,22 @@ dependencies = [ "solana-signer", "thiserror 2.0.18", "trezor-client", + "ur-parse-lib", + "ur-registry", "uriparse", ] [[package]] name = "solana-rent" -version = "4.2.1" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f02fbe2669ebe5d851dbf29a02e91ed6244b051bb64fcc57e6644aba636063" +checksum = "39f0d780bf8e8a1fe8b5b5fce1acad6b209485b86dec246e7523d5e4a8b7c7fc" dependencies = [ "serde", "serde_derive", "solana-frozen-abi", "solana-frozen-abi-macro", + "solana-get-sysvar", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -10138,9 +10394,9 @@ checksum = "dcf09694a0fc14e5ffb18f9b7b7c0f15ecb6eac5b5610bf76a1853459d19daf9" [[package]] name = "solana-sbpf" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f84c593fa3d4131045b606dec5acf9d8eac73791bc786ca9911057aec8f43ec" +checksum = "d777d7a89267dd133e985113c7e7f820fb7cfd9123a4a350cf8b39ebae1920bc" dependencies = [ "byteorder", "combine 3.8.1", @@ -10384,12 +10640,13 @@ dependencies = [ [[package]] name = "solana-slot-hashes" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a57c158c35629f9e302ab385f16b15813f4927a31c27dda72f3df828bb08d93" +checksum = "5c7ce2b4b8911bf2db3de7b6266e67bfc21a6a9f8c566fb096d9782ca2ad16ee" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-hash 4.4.0", "solana-sdk-ids", "solana-sysvar-id", @@ -10490,7 +10747,7 @@ dependencies = [ "hyper-util", "log", "prost 0.14.4", - "prost-types", + "prost-types 0.14.4", "serde", "smpl_jwt", "solana-clock", @@ -10567,6 +10824,7 @@ dependencies = [ "rustls", "smallvec", "solana-keypair", + "solana-measure", "solana-message", "solana-metrics", "solana-net-utils", @@ -10626,6 +10884,7 @@ dependencies = [ "solana-native-token", "solana-nonce", "solana-nonce-account", + "solana-poseidon", "solana-precompile-error", "solana-program-binaries", "solana-program-entrypoint", @@ -11088,9 +11347,9 @@ dependencies = [ [[package]] name = "solana-transaction" -version = "4.1.3" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d105ecce084206697230226c6b2230401c220feb4dc63e1274d58b38969292" +checksum = "2509e70bdce879db3e0f56cf97e40edd53742e8f0e6f34d64c46e7900071b53f" dependencies = [ "serde", "serde_derive", @@ -11135,9 +11394,9 @@ dependencies = [ [[package]] name = "solana-transaction-error" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441d6dcd51100e7d97c3fb3b723e08aa701066ff7afc00026fd8d8e222cb95b" +checksum = "757a648388ab1e7350a806ffceb31ce656dc5b5fe607b9f8209aa56f63040179" dependencies = [ "serde", "serde_derive", @@ -11399,15 +11658,16 @@ dependencies = [ [[package]] name = "solana-vote-interface" -version = "6.0.1" +version = "6.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab4307b353cbfab0ca1666c969f91fd7ca6f592abee2d03f5fa6a32c0e1a42b" +checksum = "61843d7be827cac5e025c3a16c1101a34fcdd8cf593f6a82eafdd253bd55a26b" dependencies = [ "arbitrary", "bincode", "cfg_eval", "num-derive", "num-traits", + "rand 0.9.4", "serde", "serde_derive", "serde_with", @@ -11888,7 +12148,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -12034,7 +12294,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -12429,7 +12689,7 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "syn 2.0.117", @@ -12452,10 +12712,10 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", - "prost-build", - "prost-types", + "prost-build 0.14.4", + "prost-types 0.14.4", "quote", "syn 2.0.117", "tempfile", @@ -12733,6 +12993,37 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "461d0c5956fcc728ecc03a3a961e4adc9a7975d86f6f8371389a289517c02ca9" +[[package]] +name = "ur-parse-lib" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88434c87d748dfb765bc5debbbab61d020392d23b568fe73eb44eaca46daa04" +dependencies = [ + "hex", + "keystone-ur", + "ur-registry", +] + +[[package]] +name = "ur-registry" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf91aff789fea1c05116a47b11fbe0bea32ef66e0c1f77b46550e1407efc4c6e" +dependencies = [ + "bs58", + "hex", + "keystone-ur", + "libflate", + "minicbor", + "no_std_io2", + "paste", + "prost 0.11.9", + "prost-build 0.11.9", + "prost-types 0.11.9", + "serde", + "thiserror 1.0.69", +] + [[package]] name = "uriparse" version = "0.6.4" @@ -12969,6 +13260,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.39", +] + [[package]] name = "wide" version = "0.7.33" diff --git a/Cargo.toml b/Cargo.toml index a1e5a76a3aa..16603c725f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,6 +160,7 @@ check-cfg = [ arithmetic_side_effects = "deny" default_trait_access = "deny" manual_let_else = "deny" +uninlined_format_args = "deny" used_underscore_binding = "deny" # Allowed lints @@ -201,7 +202,7 @@ assert_cmd = "2.2.2" assert_matches = "1.5.0" async-lock = "3.4.2" async-trait = "0.1.89" -aya = "0.13" +aya = "0.14" aya-ebpf = "0.1.1" base64 = "0.22.1" bencher = "0.1.5" @@ -315,9 +316,9 @@ proptest = "1.11" prost = "0.14.4" prost-types = "0.14.4" protobuf-src = "1.1.0" -protosol = "=8.2.0" +protosol = "=9.0.1" qualifier_attr = { version = "0.2.2", default-features = false } -quinn = "0.11.9" +quinn = "0.11.11" rand = "0.9.4" rand_chacha = "0.9.0" rayon = "1.12.0" @@ -328,6 +329,7 @@ reqwest-middleware = "0.4.2" rolling-file = "0.2.0" rpassword = "7.5" rts-alloc = { version = "4.0.0" } +rusb = "0.9" rustls = { version = "0.23.40", features = ["std"], default-features = false } scopeguard = "1.2.0" semver = "1.0.28" @@ -393,16 +395,16 @@ solana-download-utils = { path = "download-utils", version = "=4.2.0-alpha.0", f solana-ed25519-program = "3.0.0" solana-entry = { path = "entry", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-epoch-info = "3.1.0" -solana-epoch-rewards = "3.0.2" +solana-epoch-rewards = "3.1.0" solana-epoch-rewards-hasher = "3.1.0" -solana-epoch-schedule = "3.1.1" +solana-epoch-schedule = "3.2.0" solana-faucet = { path = "faucet", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-feature-gate-interface = "4.0.0" solana-fee = { path = "fee", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-fee-calculator = "3.2.2" solana-fee-structure = "3.0.0" solana-file-download = "3.1.4" -solana-frozen-abi = "3.6.0" +solana-frozen-abi = "3.7.0" solana-frozen-abi-macro = "3.6.0" solana-genesis = { path = "genesis", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-genesis-config = "4.0.0" @@ -414,7 +416,7 @@ solana-hash = "4.4.0" solana-hash-512 = "1.1.0" solana-inflation = "3.1.1" solana-instruction = "3.4.0" -solana-instruction-error = "2.3.0" +solana-instruction-error = "2.4.0" solana-instructions-sysvar = "3.0.0" solana-keccak-hasher = "3.1.0" solana-keypair = "3.1.2" @@ -428,7 +430,7 @@ solana-loader-v4-interface = "3.1.0" solana-local-cluster = { path = "local-cluster", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-measure = { path = "measure", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-merkle-tree = { path = "merkle-tree", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } -solana-message = "4.2.2" +solana-message = "4.2.3" solana-metrics = { path = "metrics", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-msg = "3.1.0" solana-native-token = "3.0.0" @@ -468,7 +470,7 @@ solana-rpc-client-types = { path = "rpc-client-types", version = "=4.2.0-alpha.0 solana-runtime = { path = "runtime", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-runtime-transaction = { path = "runtime-transaction", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-sanitize = "3.0.1" -solana-sbpf = { version = "=0.21.0", default-features = false } +solana-sbpf = { version = "=0.21.1", default-features = false } solana-sdk-ids = "3.1.0" solana-secp256k1-program = "3.0.1" solana-secp256k1-recover = "3.1.1" @@ -486,7 +488,7 @@ solana-shred-version = "3.0.1" solana-signature = { version = "3.4.1", default-features = false } solana-signer = "3.0.1" solana-signer-store = "0.1.0" -solana-slot-hashes = "3.0.2" +solana-slot-hashes = "3.1.0" solana-slot-history = "3.0.1" solana-stable-layout = "3.0.1" solana-stake-interface = "4.2.0" @@ -512,9 +514,9 @@ solana-time-utils = "3.0.0" solana-tls-utils = { path = "tls-utils", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-tpu-client = { path = "tpu-client", version = "=4.2.0-alpha.0", default-features = false, features = ["agave-unstable-api"] } solana-tpu-client-next = { path = "tpu-client-next", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } -solana-transaction = "4.1.3" +solana-transaction = "4.1.4" solana-transaction-context = { path = "transaction-context", version = "=4.2.0-alpha.0", features = ["agave-unstable-api", "bincode"] } -solana-transaction-error = "3.2.1" +solana-transaction-error = "3.3.0" solana-transaction-status = { path = "transaction-status", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-transaction-status-client-types = { path = "transaction-status-client-types", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-turbine = { path = "turbine", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } @@ -524,7 +526,7 @@ solana-unified-scheduler-pool = { path = "unified-scheduler-pool", version = "=4 solana-validator-exit = "3.0.0" solana-version = { path = "version", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-vote = { path = "vote", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } -solana-vote-interface = "6.0.1" +solana-vote-interface = "6.0.2" solana-vote-program = { path = "programs/vote", version = "=4.2.0-alpha.0", default-features = false, features = ["agave-unstable-api"] } solana-wincode-varint = "1.0.0" solana-zk-elgamal-proof-program = { path = "programs/zk-elgamal-proof", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } @@ -565,6 +567,8 @@ trees = "0.4.2" trezor-client = { version = "0.1.5", default-features = false, features = ["solana"] } tungstenite = "0.28.0" unwrap_none = "0.1.2" +ur-parse-lib = { version = "1.0.3", default-features = false, features = ["std"] } +ur-registry = { version = "1.0.3", default-features = false, features = ["std"] } uriparse = "0.6.4" url = "2.5.8" winapi = "0.3.8" diff --git a/README.md b/README.md index 0bea493e7d3..e9b66d5e4b7 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ $ ./cargo build ``` > [!NOTE] -> Note that this builds a debug version that is **not suitable for running a testnet or mainnet validator**. Please read [`docs/src/cli/install.md`](docs/src/cli/install.md#build-from-source) for instructions to build a release version for test and production uses. +> Note that this builds a debug version that is **not suitable for running a testnet or mainnet validator**. Please read [the install guide](https://docs.anza.xyz/cli/install#build-from-source) for instructions to build a release version for test and production uses. ## **4. Grant capabilities for XDP (Linux-only).** diff --git a/account-decoder-client-types/src/lib.rs b/account-decoder-client-types/src/lib.rs index 9d91da26d75..a41d40ba2d8 100644 --- a/account-decoder-client-types/src/lib.rs +++ b/account-decoder-client-types/src/lib.rs @@ -1,6 +1,6 @@ #![cfg(feature = "agave-unstable-api")] //! Core RPC client types for solana-account-decoder -#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #[cfg(feature = "zstd")] use std::io::Read; use { diff --git a/account-decoder/Cargo.toml b/account-decoder/Cargo.toml index 98907ea0413..cdbb5769ace 100644 --- a/account-decoder/Cargo.toml +++ b/account-decoder/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml index 86f62663331..8a15a33a9e2 100644 --- a/accounts-db/Cargo.toml +++ b/accounts-db/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/accounts-db/src/account_storage.rs b/accounts-db/src/account_storage.rs index f567e7e2e70..45b89461710 100644 --- a/accounts-db/src/account_storage.rs +++ b/accounts-db/src/account_storage.rs @@ -128,6 +128,20 @@ impl AccountStorage { self.map.iter().map(|iter_item| *iter_item.key()).collect() } + /// All slots with a storage entry below `max_slot_exclusive`. + pub(crate) fn slots_less_than(&self, max_slot_exclusive: Slot) -> Vec { + assert!( + self.no_shrink_in_progress(), + "shrink is in progress! slots: {:?}", + self.shrink_in_progress_map.read().unwrap().keys(), + ); + self.map + .iter() + .map(|iter_item| *iter_item.key()) + .filter(|slot| *slot < max_slot_exclusive) + .collect() + } + /// returns true if there is no entry for 'slot' #[cfg(test)] pub(crate) fn is_empty_entry(&self, slot: Slot) -> bool { diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 57f6dfd1b58..aa2694f0797 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -108,7 +108,7 @@ const UNREF_ACCOUNTS_BATCH_SIZE: usize = 10_000; const DEFAULT_NUM_DIRS: u32 = 4; // This value reflects recommended memory lock limit documented in the validator's -// setup instructions at docs/src/operations/guides/validator-start.md allowing use of +// setup instructions at https://docs.anza.xyz/operations/guides/validator-start allowing use of // several io_uring instances with fixed buffers for large disk IO operations. pub const TOTAL_IO_URING_BUFFERS_SIZE_LIMIT: usize = 2_000_000_000; @@ -912,9 +912,6 @@ pub struct AccountsDb { /// Stats from storing accounts for shrink store_accounts_for_shrink_stats: StoreAccountsForShrinkStats, - /// Stats from storing accounts for flush - store_accounts_for_flush_stats: StoreAccountsForFlushStats, - clean_accounts_stats: CleanAccountsStats, // Stats for purges called outside of clean_accounts() @@ -989,6 +986,9 @@ pub struct AccountsDb { /// Members are Slot and capacity. If capacity is smaller, then /// that means the storage was already shrunk. pub(crate) best_ancient_slots_to_shrink: RwLock>, + + /// The largest slot that has been added as a root via `add_root`. + max_root: AtomicU64, } pub fn quarter_thread_count() -> usize { @@ -1137,7 +1137,6 @@ impl AccountsDb { load_account_stats: LoadAccountsStats::default(), store_accounts_unfrozen_stats: StoreAccountsUnfrozenStats::default(), store_accounts_for_shrink_stats: StoreAccountsForShrinkStats::default(), - store_accounts_for_flush_stats: StoreAccountsForFlushStats::default(), #[cfg(test)] load_delay: u64::default(), #[cfg(test)] @@ -1150,6 +1149,7 @@ impl AccountsDb { latest_full_snapshot_slot: SeqLock::new(None), last_swept_full_snapshot_slot: AtomicU64::new(0), best_ancient_slots_to_shrink: RwLock::default(), + max_root: AtomicU64::new(0), }; { @@ -1405,20 +1405,17 @@ impl AccountsDb { /// get the oldest slot that is within one epoch of the highest known root. /// The slot will have been offset by `self.ancient_append_vec_offset` fn get_oldest_non_ancient_slot(&self, epoch_schedule: &EpochSchedule) -> Slot { - self.get_oldest_non_ancient_slot_from_slot( - epoch_schedule, - self.accounts_index.max_root_inclusive(), - ) + self.get_oldest_non_ancient_slot_from_slot(epoch_schedule, self.max_root()) } - /// get the oldest slot that is within one epoch of `max_root_inclusive`. + /// get the oldest slot that is within one epoch of `max_root`. /// The slot will have been offset by `self.ancient_append_vec_offset` fn get_oldest_non_ancient_slot_from_slot( &self, epoch_schedule: &EpochSchedule, - max_root_inclusive: Slot, + max_root: Slot, ) -> Slot { - let mut result = max_root_inclusive; + let mut result = max_root; if let Some(offset) = self.ancient_append_vec_offset { result = Self::apply_offset_to_slot(result, offset); } @@ -1426,18 +1423,20 @@ impl AccountsDb { result, -((epoch_schedule.slots_per_epoch as i64).saturating_sub(1)), ); - result.min(max_root_inclusive) + result.min(max_root) } /// Collect all the uncleaned slots, up to a max slot /// /// Search through the uncleaned Pubkeys and return all the slots, up to a maximum slot. - fn collect_uncleaned_slots_up_to_slot(&self, max_slot_inclusive: Slot) -> Vec { + fn collect_uncleaned_slots_up_to_slot(&self, max_slot_inclusive: Option) -> Vec { self.uncleaned_pubkeys .iter() .filter_map(|entry| { let slot = *entry.key(); - (slot <= max_slot_inclusive).then_some(slot) + max_slot_inclusive + .is_none_or(|max_slot_inclusive| slot <= max_slot_inclusive) + .then_some(slot) }) .collect() } @@ -1447,7 +1446,7 @@ impl AccountsDb { /// pubkeys to `candidates` for cleaning. fn remove_uncleaned_slots_up_to_slot_and_move_pubkeys( &self, - max_slot_inclusive: Slot, + max_slot_inclusive: Option, candidates: &[RwLock>], ) { let uncleaned_slots = self.collect_uncleaned_slots_up_to_slot(max_slot_inclusive); @@ -1512,14 +1511,14 @@ impl AccountsDb { timings: &mut CleanKeyTimings, ) -> CleaningCandidates { let mut dirty_store_processing_time = Measure::start("dirty_store_processing"); - let max_root_inclusive = self.accounts_index.max_root_inclusive(); - let max_slot_inclusive = max_clean_root_inclusive.unwrap_or(max_root_inclusive); let mut dirty_stores = Vec::with_capacity(self.dirty_stores.len()); // find the oldest dirty slot // we'll add logging if that append vec cannot be marked dead let mut min_dirty_slot = None::; self.dirty_stores.retain(|slot, store| { - if *slot > max_slot_inclusive { + if max_clean_root_inclusive + .is_some_and(|max_clean_root_inclusive| *slot > max_clean_root_inclusive) + { true } else { min_dirty_slot = min_dirty_slot.map(|min| min.min(*slot)).or(Some(*slot)); @@ -1543,15 +1542,14 @@ impl AccountsDb { .might_contain_zero_lamport_entry |= is_zero_lamport; }; - let mut dirty_store_routine = || { + // `min_dirty_slot` (computed above) already holds the oldest dirty slot over this same set. + timings.oldest_dirty_slot = min_dirty_slot.unwrap_or_default(); + let dirty_store_routine = || { let chunk_size = 1.max(dirty_stores_len.saturating_div(rayon::current_num_threads())); - let oldest_dirty_slots: Vec = dirty_stores + dirty_stores .par_chunks(chunk_size) - .map(|dirty_store_chunk| { - let mut oldest_dirty_slot = max_slot_inclusive.saturating_add(1); - dirty_store_chunk.iter().for_each(|(slot, store)| { - oldest_dirty_slot = oldest_dirty_slot.min(*slot); - + .for_each(|dirty_store_chunk| { + dirty_store_chunk.iter().for_each(|(_slot, store)| { store .accounts .scan_accounts_without_data(|_offset, account| { @@ -1561,13 +1559,7 @@ impl AccountsDb { }) .expect("must scan accounts storage"); }); - oldest_dirty_slot - }) - .collect(); - timings.oldest_dirty_slot = *oldest_dirty_slots - .iter() - .min() - .unwrap_or(&max_slot_inclusive.saturating_add(1)); + }); }; if is_startup { @@ -1587,7 +1579,10 @@ impl AccountsDb { timings.dirty_store_processing_us += dirty_store_processing_time.as_us(); let mut collect_delta_keys = Measure::start("key_create"); - self.remove_uncleaned_slots_up_to_slot_and_move_pubkeys(max_slot_inclusive, &candidates); + self.remove_uncleaned_slots_up_to_slot_and_move_pubkeys( + max_clean_root_inclusive, + &candidates, + ); collect_delta_keys.stop(); timings.collect_delta_keys_us += collect_delta_keys.as_us(); @@ -1613,7 +1608,9 @@ impl AccountsDb { self.zero_lamport_accounts_to_purge_after_full_snapshot .retain(|(slot, pubkey)| { let is_candidate_for_clean = - max_slot_inclusive >= *slot && latest_full_snapshot_slot >= *slot; + max_clean_root_inclusive.is_none_or(|max_clean_root_inclusive| { + max_clean_root_inclusive >= *slot + }) && latest_full_snapshot_slot >= *slot; if is_candidate_for_clean { insert_candidate(*pubkey, true); } @@ -1676,20 +1673,33 @@ impl AccountsDb { /// this function will call Rayon par_iter, so you will want to have thread pool installed if /// you want to call this without consuming all the cores on the CPU. fn exhaustively_verify_refcounts(&self, max_slot_inclusive: Option) { - let max_slot_inclusive = - max_slot_inclusive.unwrap_or_else(|| self.accounts_index.max_root_inclusive()); - info!("exhaustively verifying refcounts as of slot: {max_slot_inclusive}"); + info!("exhaustively verifying refcounts as of slot: {max_slot_inclusive:?}"); let pubkey_refcount = DashMap::>::default(); let mut storages = self.storage.all_storages(); - storages.retain(|s| s.slot() <= max_slot_inclusive); + // Flush is not running while we verify, so storages are stable. With no slot bound we + // verify every storage; otherwise we drop storages newer than the bound. + if let Some(max_slot_inclusive) = max_slot_inclusive { + storages.retain(|s| s.slot() <= max_slot_inclusive); + } // populate storages.par_iter().for_each_init( || Box::new(append_vec::new_scan_accounts_reader()), |reader, storage| { let slot = storage.slot(); + // Obsolete accounts are skipped during index generation, so they do not + // contribute to the index refcount. Skip them here too, otherwise we would count + // a physical copy the index never tracked and report a spurious mismatch. + let obsolete_accounts: IntSet<_> = storage + .obsolete_accounts_read_lock() + .filter_obsolete_accounts(None) + .map(|(offset, _)| offset) + .collect(); storage .accounts - .scan_accounts(reader.as_mut(), |_offset, account| { + .scan_accounts(reader.as_mut(), |offset, account| { + if obsolete_accounts.contains(&offset) { + return; + } let pk = account.pubkey(); match pubkey_refcount.entry(*pk) { dashmap::mapref::entry::Entry::Occupied(mut occupied_entry) => { @@ -1733,7 +1743,11 @@ impl AccountsDb { let slot_list = index_entry.slot_list_read_lock(); let num_too_new = slot_list .iter() - .filter(|(slot, _)| slot > &max_slot_inclusive) + .filter(|(slot, _)| { + max_slot_inclusive.is_some_and( + |max_slot_inclusive| *slot > max_slot_inclusive, + ) + }) .count(); if ((index_entry.ref_count() as usize) - num_too_new) @@ -3164,20 +3178,13 @@ impl AccountsDb { (shrink_slots, shrink_slots_next_batch) } - fn get_roots_less_than(&self, slot: Slot) -> Vec { - self.accounts_index - .roots_tracker - .read() - .unwrap() - .alive_roots - .get_all_less_than(slot) - } - /// return all slots that are more than one epoch old and thus could already be an ancient append vec /// or which could need to be combined into a new or existing ancient append vec /// offset is used to combine newer slots than we normally would. This is designed to be used for testing. fn get_sorted_potential_ancient_slots(&self, oldest_non_ancient_slot: Slot) -> Vec { - let mut ancient_slots = self.get_roots_less_than(oldest_non_ancient_slot); + // Only storages can be combined into ancient append vecs, so the storage map is the + // source of truth here. + let mut ancient_slots = self.storage.slots_less_than(oldest_non_ancient_slot); ancient_slots.sort_unstable(); ancient_slots } @@ -3191,7 +3198,11 @@ impl AccountsDb { let oldest_non_ancient_slot = self.get_oldest_non_ancient_slot(epoch_schedule); let can_randomly_shrink = true; - let sorted_slots = self.get_sorted_potential_ancient_slots(oldest_non_ancient_slot); + let (sorted_slots, select_slots_us) = + measure_us!(self.get_sorted_potential_ancient_slots(oldest_non_ancient_slot)); + self.shrink_ancient_stats + .select_slots_us + .fetch_add(select_slots_us, Ordering::Relaxed); self.combine_ancient_slots_packed(sorted_slots, can_randomly_shrink); } @@ -3422,13 +3433,11 @@ impl AccountsDb { F: FnMut(Option<(&Pubkey, AccountSharedData, Slot)>), { // Register this scan so that slots needed by the scan are not cleaned out from under us. - let scan_guard = ScanGuard::try_new(&self.scan_tracker, bank_id, || { - self.accounts_index.max_root_inclusive() - }) - .ok_or(ScanError::SlotRemoved { - slot: ancestors.max_slot(), - bank_id, - })?; + let scan_guard = ScanGuard::try_new(&self.scan_tracker, bank_id, || self.max_root()) + .ok_or(ScanError::SlotRemoved { + slot: ancestors.max_slot(), + bank_id, + })?; // If the scan's ancestors are all rooted, drop them and scan roots only // Scan Guard max root must be used as the scan guard guarantees that @@ -3532,13 +3541,11 @@ impl AccountsDb { } // Register this scan so that slots needed by the scan are not cleaned out from under us. - let scan_guard = ScanGuard::try_new(&self.scan_tracker, bank_id, || { - self.accounts_index.max_root_inclusive() - }) - .ok_or(ScanError::SlotRemoved { - slot: ancestors.max_slot(), - bank_id, - })?; + let scan_guard = ScanGuard::try_new(&self.scan_tracker, bank_id, || self.max_root()) + .ok_or(ScanError::SlotRemoved { + slot: ancestors.max_slot(), + bank_id, + })?; // If the scan's ancestors are all rooted, drop them and scan roots only // Scan Guard max root must be used as the scan guard guarantees that @@ -3792,10 +3799,10 @@ impl AccountsDb { // P1 purge_slot() | N/A // | | // V | - // P2 purge_slots_from_cache_and_store() | map of caches/stores (removes old entry) + // P2 purge_slots_from_cache() | map of caches/stores (removes old entry) // | | // V | - // P3 purge_slots_from_cache_and_store()/ | index + // P3 purge_slots_from_cache()/ | index // remove_dead_slots_metadata() | (removes index roots metadata for cached slot) // purge_slot_storage()/ | // purge_keys_exact() | (removes accounts index entries) @@ -3948,7 +3955,7 @@ impl AccountsDb { load_hint: LoadHint, populate_read_cache: PopulateReadCache, ) -> Option<(AccountSharedData, Slot)> { - let starting_max_root = self.accounts_index.max_root_inclusive(); + let starting_max_root = self.max_root(); // Check the write cache first; a hit is the freshest version visible on this fork, // so return it @@ -4004,7 +4011,7 @@ impl AccountsDb { } if load_hint == LoadHint::FixedMaxRoot { // If the load hint is that the max root is fixed, the max root should be fixed. - let ending_max_root = self.accounts_index.max_root_inclusive(); + let ending_max_root = self.max_root(); if starting_max_root != ending_max_root { warn!( "do_load_with_populate_read_cache() scanning pubkey {pubkey} called with \ @@ -4102,13 +4109,16 @@ impl AccountsDb { self.purge_slots(std::iter::once(&slot)); } - /// Purges every slot in `removed_slots` from both the cache and storage. This includes - /// entries in the accounts index, cache entries, and any backing storage entries. - fn purge_slots_from_cache_and_store<'a>( + /// Purges each slot in `removed_slots` from the write cache (and the accounts index). Slots + /// no longer present in the cache are skipped. This never touches backing storage, so it + /// cannot delete a flushed (rooted) slot's data. Returns whether any slot was actually + /// removed from the cache. This allows the snapshot minimizer to determine whether + /// it should purge the storage as well + fn purge_slots_from_cache<'a>( &self, - removed_slots: impl Iterator + Clone, + removed_slots: impl Iterator, purge_stats: &PurgeStats, - ) { + ) -> bool { let mut remove_cache_elapsed_across_slots = 0; let mut num_cached_slots_removed = 0; let mut total_removed_cached_bytes = 0; @@ -4121,8 +4131,6 @@ impl AccountsDb { // holding the index lock, finding the index entry, and then looking up the entry // in the cache. If it fails to find that entry, it will panic in `get_loaded_account()` if let Some(slot_cache) = self.accounts_cache.slot_cache(*remove_slot) { - // If the slot is still in the cache, remove the backing storages for - // the slot and from the Accounts Index num_cached_slots_removed += 1; total_removed_cached_bytes += slot_cache.total_bytes(); self.remove_dead_slots_metadata(iter::once(remove_slot)); @@ -4150,12 +4158,7 @@ impl AccountsDb { .handle_dead_keys(&pubkeys_removed, &self.account_indexes); } self.accounts_index.write_through_pubkeys(pubkeys_removed); - } else { - self.purge_slot_storage(*remove_slot, purge_stats); } - // It should not be possible that a slot is neither in the cache or storage. Even in - // a slot with all ticks, `Bank::new_from_parent()` immediately stores some sysvars - // on bank creation. } purge_stats @@ -4167,6 +4170,8 @@ impl AccountsDb { purge_stats .total_removed_cached_bytes .fetch_add(total_removed_cached_bytes, Ordering::Relaxed); + + num_cached_slots_removed > 0 } /// Purges every slot in `removed_slots` from both the cache and storage. This includes @@ -4176,10 +4181,17 @@ impl AccountsDb { #[cfg(feature = "dev-context-only-utils")] pub fn purge_slots_for_snapshot_minimizer<'a>( &self, - removed_slots: impl Iterator + Clone, + removed_slots: impl Iterator, ) { - let stats = PurgeStats::default(); - self.purge_slots_from_cache_and_store(removed_slots, &stats); + let purge_stats = PurgeStats::default(); + for remove_slot in removed_slots { + // Unlike the consensus purge paths, minimization may purge slots that have already + // been flushed to storage, so fall back to purging storage for any slot that is no + // longer in the cache. + if !self.purge_slots_from_cache(iter::once(remove_slot), &purge_stats) { + self.purge_slot_storage(*remove_slot, &purge_stats); + } + } } /// Purge the backing storage entries for the given slot, does not purge from @@ -4235,9 +4247,6 @@ impl AccountsDb { purge_stats .num_stored_slots_removed .fetch_add(num_stored_slots_removed, Ordering::Relaxed); - purge_stats - .total_removed_storage_entries - .fetch_add(num_stored_slots_removed, Ordering::Relaxed); purge_stats .total_removed_stored_bytes .fetch_add(total_removed_stored_bytes, Ordering::Relaxed); @@ -4246,6 +4255,7 @@ impl AccountsDb { .fetch_add(num_stored_slots_removed as u64, Ordering::Relaxed); } + #[cfg(feature = "dev-context-only-utils")] fn purge_slot_storage(&self, remove_slot: Slot, purge_stats: &PurgeStats) { // Because AccountsBackgroundService synchronously flushes from the accounts cache // and handles all Bank::drop() (the cleanup function that leads to this @@ -4319,13 +4329,13 @@ impl AccountsDb { // Also note roots are never removed via `remove_unrooted_slot()`, so // it's safe to filter them out here as they won't need deletion from // self.scan_tracker.removed_bank_ids in - // `purge_slots_from_cache_and_store()`. + // `purge_slots_from_cache()`. .filter(|slot| !self.accounts_index.is_alive_root(**slot)); safety_checks_elapsed.stop(); self.external_purge_slots_stats .safety_checks_elapsed .fetch_add(safety_checks_elapsed.as_us(), Ordering::Relaxed); - self.purge_slots_from_cache_and_store(non_roots, &self.external_purge_slots_stats); + self.purge_slots_from_cache(non_roots, &self.external_purge_slots_stats); self.external_purge_slots_stats .report("external_purge_slots_stats", Some(1000)); } @@ -4350,7 +4360,7 @@ impl AccountsDb { } let remove_unrooted_purge_stats = PurgeStats::default(); - self.purge_slots_from_cache_and_store( + self.purge_slots_from_cache( remove_slots.iter().map(|(slot, _)| slot), &remove_unrooted_purge_stats, ); @@ -4485,19 +4495,28 @@ impl AccountsDb { flush_stats.store_accounts_total_us.0, i64 ), + ("write_accounts_us", flush_stats.write_accounts_us.0, i64), + ("update_index_us", flush_stats.update_index_us.0, i64), + ("handle_reclaims_us", flush_stats.handle_reclaims_us.0, i64), ( - "update_index_us", - flush_stats.store_accounts_timing.update_index_elapsed, + "mark_zero_lamport_single_ref_accounts_us", + flush_stats.mark_zero_lamport_single_ref_accounts_us.0, i64 ), ( - "store_accounts_elapsed_us", - flush_stats.store_accounts_timing.store_accounts_elapsed, + "num_zero_lamport_single_ref_accounts_marked", + flush_stats.num_zero_lamport_single_ref_accounts_marked.0, i64 ), + ("num_reclaims", flush_stats.num_reclaims.0, i64), ( - "handle_reclaims_elapsed_us", - flush_stats.store_accounts_timing.handle_reclaims_elapsed, + "num_obsolete_slots_removed", + flush_stats.num_obsolete_slots_removed.0, + i64 + ), + ( + "num_obsolete_bytes_removed", + flush_stats.num_obsolete_bytes_removed.0, i64 ), ("select_pubkeys_us", flush_stats.select_pubkeys_us.0, i64), @@ -4684,15 +4703,15 @@ impl AccountsDb { self.create_store(slot, flush_stats.num_bytes_flushed.0, "flush_slot_cache"); self.storage.insert(Arc::clone(&flushed_store)); - let (store_accounts_timing_inner, store_accounts_total_inner_us) = + let (store_accounts_for_flush_stats, store_accounts_for_flush_us) = measure_us!(self.store_accounts_for_flush( (slot, &accounts[..]), &flushed_store, reclaim_method, UpdateIndexThreadSelection::PoolWithThreshold, )); - flush_stats.store_accounts_timing = store_accounts_timing_inner; - flush_stats.store_accounts_total_us = Saturating(store_accounts_total_inner_us); + flush_stats.accumulate_store_accounts_for_flush(store_accounts_for_flush_stats); + flush_stats.store_accounts_total_us += Saturating(store_accounts_for_flush_us); // If the above sizing function is correct, just one AppendVec is enough to hold // all the data for the slot @@ -5066,14 +5085,6 @@ impl AccountsDb { let target_slot = accounts.target_slot(); let len = std::cmp::min(accounts.len(), infos.len()); - // If reclaiming old slots, ensure the target slot is a root - // Having an unrooted slot reclaim a rooted version of a slot - // could lead to index corruption if the unrooted version is - // discarded - if reclaim == UpsertReclaim::ReclaimOldSlots { - assert!(target_slot <= self.accounts_index.max_root_inclusive()); - } - let update = |start, end| { let mut reclaims = ReclaimsSlotList::with_capacity((end - start) / 2); @@ -5570,6 +5581,37 @@ impl AccountsDb { self.report_store_timings(); } + /// Store `accounts` into `storage`. + /// + /// This fn is to only be called by ancient squash. + pub(crate) fn store_accounts_for_squash<'a>( + &self, + accounts: impl StorableAccounts<'a>, + storage: &AccountStorageEntry, + ) -> StoreAccountsTiming { + let slot = accounts.target_slot(); + // Flush the read cache if necessary + if self.read_only_accounts_cache.can_slot_be_in_cache(slot) { + let flush_read_cache_time = Measure::start("flush_read_cache"); + (0..accounts.len()).for_each(|index| { + // Based on the patterns of how a validator writes accounts, it is almost always + // the case that there is no read only cache entry for this pubkey and slot. + // So, we can give that hint to the `remove` for performance. + self.read_only_accounts_cache + .remove_assume_not_present(accounts.pubkey(index)); + }); + self.store_accounts_for_shrink_stats + .flush_read_cache_us + .fetch_add(flush_read_cache_time.end_as_us(), Ordering::Relaxed); + } + + self.store_accounts_for_shrink( + accounts, + storage, + UpdateIndexThreadSelection::PoolWithThreshold, + ) + } + /// Stores accounts in the storage and updates the index. /// This function is intended for accounts that are being shrunk (moving from one store to another) /// - `UpsertReclaims` is set to `IgnoreReclaims`. If the slot in `accounts` differs from the new slot, @@ -5587,18 +5629,6 @@ impl AccountsDb { debug_assert!(self.accounts_index.is_alive_root(slot)); - // Flush the read cache if necessary - let flush_read_cache_time = Measure::start("flush_read_cache"); - if self.read_only_accounts_cache.can_slot_be_in_cache(slot) { - (0..accounts.len()).for_each(|index| { - // based on the patterns of how a validator writes accounts, it is almost always the case that there is no read only cache entry - // for this pubkey and slot. So, we can give that hint to the `remove` for performance. - self.read_only_accounts_cache - .remove_assume_not_present(accounts.pubkey(index)); - }); - } - let flush_read_cache_us = flush_read_cache_time.end_as_us(); - // Write the accounts to storage let write_accounts_time = Measure::start("write_accounts"); let infos = self.write_accounts_to_storage(slot, storage, &accounts); @@ -5618,9 +5648,6 @@ impl AccountsDb { ); let update_index_us = update_index_time.end_as_us(); - stats - .flush_read_cache_us - .fetch_add(flush_read_cache_us, Ordering::Relaxed); stats .write_to_storage_us .fetch_add(write_accounts_us, Ordering::Relaxed); @@ -5657,10 +5684,8 @@ impl AccountsDb { storage: &AccountStorageEntry, reclaim_handling: UpsertReclaim, update_index_thread_selection: UpdateIndexThreadSelection, - ) -> StoreAccountsTiming { + ) -> StoreAccountsForFlushStats { let slot = accounts.target_slot(); - let num_accounts_stored = accounts.len(); - let stats = &self.store_accounts_for_flush_stats; debug_assert!(self.accounts_index.is_alive_root(slot)); @@ -5690,8 +5715,11 @@ impl AccountsDb { // should skip handle_reclaims only when reclaims is empty. No need to // check the elements of reclaims are empty. let handle_reclaims_time = Measure::start("handle_reclaims"); + let mut num_reclaims = 0; + let mut num_obsolete_slots_removed = 0; + let mut num_obsolete_bytes_removed = 0; if !reclaims.is_empty() { - let reclaims_len = reclaims.iter().map(|r| r.len()).sum::(); + num_reclaims = reclaims.iter().map(|r| r.len() as u64).sum(); let purge_stats = PurgeStats::default(); self.handle_reclaims( reclaims.iter().flatten(), @@ -5700,47 +5728,23 @@ impl AccountsDb { &purge_stats, MarkAccountsObsolete::Yes(slot), ); - stats.num_obsolete_slots_removed.fetch_add( - purge_stats.num_stored_slots_removed.load(Ordering::Relaxed), - Ordering::Relaxed, - ); - stats.num_obsolete_bytes_removed.fetch_add( - purge_stats - .total_removed_stored_bytes - .load(Ordering::Relaxed), - Ordering::Relaxed, - ); - stats - .num_reclaims - .fetch_add(reclaims_len as u64, Ordering::Relaxed); + num_obsolete_slots_removed = + purge_stats.num_stored_slots_removed.load(Ordering::Relaxed) as u64; + num_obsolete_bytes_removed = purge_stats + .total_removed_stored_bytes + .load(Ordering::Relaxed); } let handle_reclaims_us = handle_reclaims_time.end_as_us(); - stats - .write_to_storage_us - .fetch_add(write_accounts_us, Ordering::Relaxed); - stats - .update_index_us - .fetch_add(update_index_us, Ordering::Relaxed); - stats - .mark_zero_lamport_single_ref_accounts_us - .fetch_add(mark_zero_lamport_us, Ordering::Relaxed); - stats - .handle_reclaims_us - .fetch_add(handle_reclaims_us, Ordering::Relaxed); - stats - .num_accounts_stored - .fetch_add(num_accounts_stored as u64, Ordering::Relaxed); - stats.num_zero_lamport_single_ref_accounts_marked.fetch_add( + StoreAccountsForFlushStats { + write_accounts_us, + update_index_us, + handle_reclaims_us, + mark_zero_lamport_single_ref_accounts_us: mark_zero_lamport_us, num_zero_lamport_single_ref_accounts_marked, - Ordering::Relaxed, - ); - stats.report(); - - StoreAccountsTiming { - store_accounts_elapsed: write_accounts_us, - update_index_elapsed: update_index_us, - handle_reclaims_elapsed: handle_reclaims_us, + num_reclaims, + num_obsolete_slots_removed, + num_obsolete_bytes_removed, } } @@ -6028,12 +6032,19 @@ impl AccountsDb { self.accounts_cache.add_root(slot); cache_time.stop(); + self.max_root.fetch_max(slot, Ordering::Relaxed); + AccountsAddRootTiming { index_us: index_time.as_us(), cache_us: cache_time.as_us(), } } + /// Returns the largest slot that has been added as a root via `add_root`. + pub fn max_root(&self) -> Slot { + self.max_root.load(Ordering::Relaxed) + } + /// Returns storages for `requested_slots` pub fn get_storages( &self, @@ -6251,6 +6262,11 @@ impl AccountsDb { } let num_storages = storages.len(); + // `storages` is sorted by slot, so the last one is the highest root. + if let Some(storage) = storages.last() { + self.max_root.fetch_max(storage.slot(), Ordering::Relaxed); + } + self.accounts_index.set_startup(Startup::Startup); let mut total_accum = IndexGenerationAccumulator::with_slots_capacity(num_storages); @@ -6613,8 +6629,7 @@ impl AccountsDb { } ObsoleteAccountsStats { accounts_marked_obsolete: reclaims.len() as u64, - slots_removed: stats.total_removed_storage_entries.load(Ordering::Relaxed) - as u64, + slots_removed: stats.num_stored_slots_removed.load(Ordering::Relaxed) as u64, } }) .sum(); diff --git a/accounts-db/src/accounts_db/stats.rs b/accounts-db/src/accounts_db/stats.rs index d9822992a80..43ab87056e9 100644 --- a/accounts-db/src/accounts_db/stats.rs +++ b/accounts-db/src/accounts_db/stats.rs @@ -163,78 +163,14 @@ impl StoreAccountsForShrinkStats { /// Stats from storing accounts for flush (i.e. flushing the write cache to a storage file) #[derive(Debug, Default)] pub struct StoreAccountsForFlushStats { - pub last_report: AtomicInterval, - pub write_to_storage_us: AtomicU64, - pub update_index_us: AtomicU64, - pub mark_zero_lamport_single_ref_accounts_us: AtomicU64, - pub handle_reclaims_us: AtomicU64, - pub num_accounts_stored: AtomicU64, - pub num_zero_lamport_single_ref_accounts_marked: AtomicU64, - pub num_reclaims: AtomicU64, - pub num_obsolete_slots_removed: AtomicUsize, - pub num_obsolete_bytes_removed: AtomicU64, -} - -impl StoreAccountsForFlushStats { - const REPORT_INTERVAL_MS: u64 = Duration::from_secs(1).as_millis() as u64; - - pub fn report(&self) { - let should_report = self.last_report.should_update(Self::REPORT_INTERVAL_MS); - if !should_report { - return; - } - - datapoint_info!( - "accounts_db_store_accounts_for_flush", - ( - "write_to_storage_us", - self.write_to_storage_us.swap(0, Ordering::Relaxed), - i64 - ), - ( - "update_index_us", - self.update_index_us.swap(0, Ordering::Relaxed), - i64 - ), - ( - "mark_zero_lamport_single_ref_accounts_us", - self.mark_zero_lamport_single_ref_accounts_us - .swap(0, Ordering::Relaxed), - i64 - ), - ( - "handle_reclaims_us", - self.handle_reclaims_us.swap(0, Ordering::Relaxed), - i64 - ), - ( - "num_accounts_stored", - self.num_accounts_stored.swap(0, Ordering::Relaxed), - i64 - ), - ( - "num_zero_lamport_single_ref_accounts_marked", - self.num_zero_lamport_single_ref_accounts_marked - .swap(0, Ordering::Relaxed), - i64 - ), - ( - "num_reclaims", - self.num_reclaims.swap(0, Ordering::Relaxed), - i64 - ), - ( - "num_obsolete_slots_removed", - self.num_obsolete_slots_removed.swap(0, Ordering::Relaxed), - i64 - ), - ( - "num_obsolete_bytes_removed", - self.num_obsolete_bytes_removed.swap(0, Ordering::Relaxed), - i64 - ), - ); - } + pub write_accounts_us: u64, + pub update_index_us: u64, + pub handle_reclaims_us: u64, + pub mark_zero_lamport_single_ref_accounts_us: u64, + pub num_zero_lamport_single_ref_accounts_marked: u64, + pub num_reclaims: u64, + pub num_obsolete_slots_removed: u64, + pub num_obsolete_bytes_removed: u64, } #[derive(Debug, Default)] @@ -246,7 +182,6 @@ pub struct PurgeStats { pub drop_storage_entries_elapsed: AtomicU64, pub num_cached_slots_removed: AtomicUsize, pub num_stored_slots_removed: AtomicUsize, - pub total_removed_storage_entries: AtomicUsize, pub total_removed_cached_bytes: AtomicU64, pub total_removed_stored_bytes: AtomicU64, pub scan_storages_elapsed: AtomicU64, @@ -294,12 +229,6 @@ impl PurgeStats { self.num_stored_slots_removed.swap(0, Ordering::Relaxed), i64 ), - ( - "total_removed_storage_entries", - self.total_removed_storage_entries - .swap(0, Ordering::Relaxed), - i64 - ), ( "total_removed_cached_bytes", self.total_removed_cached_bytes.swap(0, Ordering::Relaxed), @@ -351,21 +280,54 @@ pub struct FlushStats { pub num_bytes_flushed: Saturating, pub num_accounts_purged: Saturating, pub num_bytes_purged: Saturating, - pub store_accounts_timing: StoreAccountsTiming, pub store_accounts_total_us: Saturating, + pub write_accounts_us: Saturating, + pub update_index_us: Saturating, + pub handle_reclaims_us: Saturating, + pub mark_zero_lamport_single_ref_accounts_us: Saturating, + pub num_zero_lamport_single_ref_accounts_marked: Saturating, + pub num_reclaims: Saturating, + pub num_obsolete_slots_removed: Saturating, + pub num_obsolete_bytes_removed: Saturating, pub select_pubkeys_us: Saturating, pub disk_index_write_through_us: Saturating, } impl FlushStats { + pub fn accumulate_store_accounts_for_flush( + &mut self, + store_accounts_stats: StoreAccountsForFlushStats, + ) { + self.write_accounts_us += Saturating(store_accounts_stats.write_accounts_us); + self.update_index_us += Saturating(store_accounts_stats.update_index_us); + self.handle_reclaims_us += Saturating(store_accounts_stats.handle_reclaims_us); + self.mark_zero_lamport_single_ref_accounts_us += + Saturating(store_accounts_stats.mark_zero_lamport_single_ref_accounts_us); + self.num_zero_lamport_single_ref_accounts_marked += + Saturating(store_accounts_stats.num_zero_lamport_single_ref_accounts_marked); + self.num_reclaims += Saturating(store_accounts_stats.num_reclaims); + self.num_obsolete_slots_removed += + Saturating(store_accounts_stats.num_obsolete_slots_removed); + self.num_obsolete_bytes_removed += + Saturating(store_accounts_stats.num_obsolete_bytes_removed); + } + pub fn accumulate(&mut self, other: &Self) { self.num_accounts_flushed += other.num_accounts_flushed; self.num_bytes_flushed += other.num_bytes_flushed; self.num_accounts_purged += other.num_accounts_purged; self.num_bytes_purged += other.num_bytes_purged; - self.store_accounts_timing - .accumulate(&other.store_accounts_timing); self.store_accounts_total_us += other.store_accounts_total_us; + self.write_accounts_us += other.write_accounts_us; + self.update_index_us += other.update_index_us; + self.handle_reclaims_us += other.handle_reclaims_us; + self.mark_zero_lamport_single_ref_accounts_us += + other.mark_zero_lamport_single_ref_accounts_us; + self.num_zero_lamport_single_ref_accounts_marked += + other.num_zero_lamport_single_ref_accounts_marked; + self.num_reclaims += other.num_reclaims; + self.num_obsolete_slots_removed += other.num_obsolete_slots_removed; + self.num_obsolete_bytes_removed += other.num_obsolete_bytes_removed; self.select_pubkeys_us += other.select_pubkeys_us; self.disk_index_write_through_us += other.disk_index_write_through_us; } @@ -374,7 +336,6 @@ impl FlushStats { #[derive(Debug, Default)] pub struct LatestAccountsIndexRootsStats { pub roots_len: AtomicUsize, - pub uncleaned_roots_len: AtomicUsize, pub roots_range: AtomicU64, pub rooted_cleaned_count: AtomicUsize, pub unrooted_cleaned_count: AtomicUsize, @@ -387,9 +348,6 @@ impl LatestAccountsIndexRootsStats { if let Some(value) = accounts_index_roots_stats.roots_len { self.roots_len.store(value, Ordering::Relaxed); } - if let Some(value) = accounts_index_roots_stats.uncleaned_roots_len { - self.uncleaned_roots_len.store(value, Ordering::Relaxed); - } if let Some(value) = accounts_index_roots_stats.roots_range { self.roots_range.store(value, Ordering::Relaxed); } @@ -415,11 +373,6 @@ impl LatestAccountsIndexRootsStats { datapoint_info!( "accounts_index_roots_len", ("roots_len", self.roots_len.load(Ordering::Relaxed), i64), - ( - "uncleaned_roots_len", - self.uncleaned_roots_len.load(Ordering::Relaxed), - i64 - ), ( "roots_range_width", self.roots_range.load(Ordering::Relaxed), @@ -488,9 +441,9 @@ pub struct ShrinkAncientStats { pub shrink_stats: ShrinkStats, pub ancient_append_vecs_shrunk: AtomicU64, pub total_us: AtomicU64, + pub select_slots_us: AtomicU64, pub random_shrink: AtomicU64, pub slots_considered: AtomicU64, - pub ancient_scanned: AtomicU64, pub bytes_ancient_created: AtomicU64, pub bytes_from_must_shrink: AtomicU64, pub bytes_from_smallest_storages: AtomicU64, @@ -551,7 +504,6 @@ pub struct ShrinkStats { pub accounts_loaded: AtomicU64, pub initial_candidates_count: AtomicU64, pub purged_zero_lamports: AtomicU64, - pub accounts_not_found_in_index: AtomicU64, pub num_ancient_slots_shrunk: AtomicU64, pub ancient_slots_added_to_shrink: AtomicU64, pub ancient_bytes_added_to_shrink: AtomicU64, @@ -689,11 +641,6 @@ impl ShrinkStats { self.num_ancient_slots_shrunk.swap(0, Ordering::Relaxed), i64 ), - ( - "accounts_not_found_in_index", - self.accounts_not_found_in_index.swap(0, Ordering::Relaxed), - i64 - ), ( "initial_candidates_count", self.initial_candidates_count.swap(0, Ordering::Relaxed), @@ -886,12 +833,12 @@ impl ShrinkAncientStats { self.slots_considered.swap(0, Ordering::Relaxed), i64 ), + ("total_us", self.total_us.swap(0, Ordering::Relaxed), i64), ( - "ancient_scanned", - self.ancient_scanned.swap(0, Ordering::Relaxed), + "select_slots_us", + self.select_slots_us.swap(0, Ordering::Relaxed), i64 ), - ("total_us", self.total_us.swap(0, Ordering::Relaxed), i64), ( "bytes_ancient_created", self.bytes_ancient_created.swap(0, Ordering::Relaxed), @@ -934,13 +881,6 @@ impl ShrinkAncientStats { .swap(0, Ordering::Relaxed), i64 ), - ( - "accounts_not_found_in_index", - self.shrink_stats - .accounts_not_found_in_index - .swap(0, Ordering::Relaxed), - i64 - ), ("slot", self.slot.load(Ordering::Relaxed), i64), ( "ideal_storage_size", @@ -987,7 +927,6 @@ pub struct WriteAccountsToCacheStats { pub struct LoadAccountsStats { pub num_loaded_from_write_cache: AtomicU64, pub num_loaded_from_read_cache: AtomicU64, - pub num_loaded_from_index_cache: AtomicU64, pub num_loaded_from_index_storage: AtomicU64, } @@ -1005,11 +944,6 @@ impl LoadAccountsStats { self.num_loaded_from_read_cache.swap(0, Ordering::Relaxed), i64 ), - ( - "num_loaded_from_index_cache", - self.num_loaded_from_index_cache.swap(0, Ordering::Relaxed), - i64 - ), ( "num_loaded_from_index_storage", self.num_loaded_from_index_storage diff --git a/accounts-db/src/accounts_db/tests.rs b/accounts-db/src/accounts_db/tests.rs index 7660f9b0542..ac5d4654b36 100644 --- a/accounts-db/src/accounts_db/tests.rs +++ b/accounts-db/src/accounts_db/tests.rs @@ -1938,7 +1938,7 @@ fn test_accounts_db_purge_keep_live() { // since the store count will not be zero accounts.store_for_tests((current_slot, [(&pubkey2, &account2)].as_slice())); accounts.add_root_and_flush_write_cache(current_slot); - let ancestors = Ancestors::from(vec![accounts.accounts_index.max_root_inclusive()]); + let ancestors = Ancestors::from(vec![accounts.max_root()]); let (slot1, account_info1) = accounts .accounts_index .get_with_and_then(&pubkey, &ancestors, false, |(slot, account_info)| { @@ -4932,9 +4932,9 @@ fn test_collect_uncleaned_slots_up_to_slot() { db.uncleaned_pubkeys.insert(slot2, vec![pubkey2]); db.uncleaned_pubkeys.insert(slot3, vec![pubkey3]); - let mut uncleaned_slots1 = db.collect_uncleaned_slots_up_to_slot(slot1); - let mut uncleaned_slots2 = db.collect_uncleaned_slots_up_to_slot(slot2); - let mut uncleaned_slots3 = db.collect_uncleaned_slots_up_to_slot(slot3); + let mut uncleaned_slots1 = db.collect_uncleaned_slots_up_to_slot(Some(slot1)); + let mut uncleaned_slots2 = db.collect_uncleaned_slots_up_to_slot(Some(slot2)); + let mut uncleaned_slots3 = db.collect_uncleaned_slots_up_to_slot(Some(slot3)); uncleaned_slots1.sort_unstable(); uncleaned_slots2.sort_unstable(); @@ -4979,7 +4979,7 @@ fn test_remove_uncleaned_slots_and_collect_pubkeys_up_to_slot() { std::iter::repeat_with(|| RwLock::new(HashMap::::new())) .take(num_bins) .collect(); - db.remove_uncleaned_slots_up_to_slot_and_move_pubkeys(slot3, &candidates); + db.remove_uncleaned_slots_up_to_slot_and_move_pubkeys(Some(slot3), &candidates); let candidates_contain = |pubkey: &Pubkey| { candidates @@ -5898,15 +5898,17 @@ define_accounts_db_test!(test_get_sorted_potential_ancient_slots, |db| { ); let root1 = DEFAULT_MAX_ANCIENT_STORAGES as u64 + ancient_append_vec_offset as u64 + 1; db.add_root(root1); + db.create_and_insert_store(root1, 4096, "test"); let root2 = root1 + 1; db.add_root(root2); + db.create_and_insert_store(root2, 4096, "test"); let oldest_non_ancient_slot = db.get_oldest_non_ancient_slot(&epoch_schedule); assert!( db.get_sorted_potential_ancient_slots(oldest_non_ancient_slot) .is_empty() ); let completed_slot = epoch_schedule.slots_per_epoch; - db.accounts_index.add_root(AccountsDb::apply_offset_to_slot( + db.add_root(AccountsDb::apply_offset_to_slot( completed_slot, ancient_append_vec_offset, )); @@ -5918,7 +5920,7 @@ define_accounts_db_test!(test_get_sorted_potential_ancient_slots, |db| { .is_empty() ); let completed_slot = epoch_schedule.slots_per_epoch + root1; - db.accounts_index.add_root(AccountsDb::apply_offset_to_slot( + db.add_root(AccountsDb::apply_offset_to_slot( completed_slot, ancient_append_vec_offset, )); @@ -5928,7 +5930,7 @@ define_accounts_db_test!(test_get_sorted_potential_ancient_slots, |db| { vec![root1, root2] ); let completed_slot = epoch_schedule.slots_per_epoch + root2; - db.accounts_index.add_root(AccountsDb::apply_offset_to_slot( + db.add_root(AccountsDb::apply_offset_to_slot( completed_slot, ancient_append_vec_offset, )); @@ -5937,12 +5939,7 @@ define_accounts_db_test!(test_get_sorted_potential_ancient_slots, |db| { db.get_sorted_potential_ancient_slots(oldest_non_ancient_slot), vec![root1, root2] ); - db.accounts_index - .roots_tracker - .write() - .unwrap() - .alive_roots - .remove(&root1); + db.storage.remove(&root1, false); let oldest_non_ancient_slot = db.get_oldest_non_ancient_slot(&epoch_schedule); assert_eq!( db.get_sorted_potential_ancient_slots(oldest_non_ancient_slot), @@ -6190,7 +6187,7 @@ fn test_shrink_collect_with_obsolete_accounts() { db.add_root_and_flush_write_cache(slot); let storage = db.get_and_assert_single_storage(slot); - let ancestors = Ancestors::from(vec![db.accounts_index.max_root_inclusive()]); + let ancestors = Ancestors::from(vec![db.max_root()]); for (i, pubkey) in pubkeys.iter().enumerate() { // Mark Some accounts obsolete. These will include zero lamport and non zero lamport accounts diff --git a/accounts-db/src/accounts_index.rs b/accounts-db/src/accounts_index.rs index 49a4d21ce21..7dbe283e3aa 100644 --- a/accounts-db/src/accounts_index.rs +++ b/accounts-db/src/accounts_index.rs @@ -195,7 +195,6 @@ pub fn default_num_flush_threads() -> NonZeroUsize { #[derive(Debug, Default)] pub struct AccountsIndexRootsStats { pub roots_len: Option, - pub uncleaned_roots_len: Option, pub roots_range: Option, pub rooted_cleaned_count: usize, pub unrooted_cleaned_count: usize, @@ -1140,14 +1139,6 @@ impl + Into> AccountsIndex { w_roots_tracker.alive_roots.insert(slot); } - pub fn max_root_inclusive(&self) -> Slot { - self.roots_tracker - .read() - .unwrap() - .alive_roots - .max_inclusive() - } - pub(crate) fn clean_dead_slots<'a>( &'a self, dead_slots_iter: impl Iterator, @@ -1307,7 +1298,7 @@ mod tests { let mut num = 0; index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1375,7 +1366,7 @@ mod tests { let mut num = 0; index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1431,7 +1422,7 @@ mod tests { let mut num = 0; index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1441,7 +1432,7 @@ mod tests { assert_eq!(index.ref_count_from_storage(pubkey), 1); index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1463,7 +1454,7 @@ mod tests { let mut num = 0; index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1473,7 +1464,7 @@ mod tests { assert_eq!(index.ref_count_from_storage(pubkey), 1); index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1814,7 +1805,7 @@ mod tests { let mut num = 0; index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1823,7 +1814,7 @@ mod tests { assert!(index.contains_with(&key, &ancestors)); index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1844,7 +1835,7 @@ mod tests { let mut num = 0; index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |_pubkey, _index| num += 1, &ScanConfig::default(), ); @@ -1913,7 +1904,7 @@ mod tests { let mut found_key = false; index.scan_accounts( &ancestors, - index.max_root_inclusive(), + 0, |pubkey, _index| { if pubkey == &key { found_key = true @@ -1969,7 +1960,7 @@ mod tests { let mut scanned_keys = HashSet::new(); index.scan_accounts( &Ancestors::default(), - index.max_root_inclusive(), + 0, |pubkey, _index| { scanned_keys.insert(*pubkey); }, @@ -2004,7 +1995,7 @@ mod tests { assert!(gc.is_empty()); index.add_root(0); - let ancestors = Ancestors::from(vec![index.max_root_inclusive()]); + let ancestors = Ancestors::from(vec![0]); index .get_with_and_then(&key, &ancestors, false, |(slot, account_info)| { assert_eq!(slot, 0); @@ -2091,6 +2082,7 @@ mod tests { let key = solana_pubkey::new_rand(); let index = AccountsIndex::::default_for_tests(); let mut gc = ReclaimsSlotList::new(); + let max_root = 3; index.upsert(0, 0, &key, true, &mut gc, UpsertReclaim::PopulateReclaims); assert!(gc.is_empty()); index.upsert(1, 1, &key, false, &mut gc, UpsertReclaim::PopulateReclaims); @@ -2098,13 +2090,13 @@ mod tests { index.upsert(3, 3, &key, true, &mut gc, UpsertReclaim::PopulateReclaims); index.add_root(0); index.add_root(1); - index.add_root(3); + index.add_root(max_root); index.upsert(4, 4, &key, true, &mut gc, UpsertReclaim::PopulateReclaims); // Updating index should not purge older roots, only purges // previous updates within the same slot assert_eq!(gc, ReclaimsSlotList::new()); - let ancestors = Ancestors::from(vec![index.max_root_inclusive()]); + let ancestors = Ancestors::from(vec![max_root]); index .get_with_and_then(&key, &ancestors, false, |(slot, account_info)| { assert_eq!(slot, 3); @@ -2116,7 +2108,7 @@ mod tests { let mut found_key = false; index.scan_accounts( &Ancestors::default(), - index.max_root_inclusive(), + max_root, |pubkey, index| { if pubkey == &key { found_key = true; diff --git a/accounts-db/src/accounts_index/in_mem_accounts_index.rs b/accounts-db/src/accounts_index/in_mem_accounts_index.rs index 4490415b9d3..9bac7911b1f 100644 --- a/accounts-db/src/accounts_index/in_mem_accounts_index.rs +++ b/accounts-db/src/accounts_index/in_mem_accounts_index.rs @@ -1616,8 +1616,6 @@ struct DiskFlushStats { num_not_flushed_ref_count: u64, /// Number of entries not flushed because slot list len != 1 num_not_flushed_slot_list_len: u64, - /// Number of entries not flushed because slot list contained a cached entry - num_not_flushed_slot_list_cached: u64, } impl DiskFlushStats { @@ -1641,10 +1639,6 @@ impl DiskFlushStats { &stats.held_in_mem.slot_list_len, self.num_not_flushed_slot_list_len, ); - Self::update_stat( - &stats.held_in_mem.slot_list_cached, - self.num_not_flushed_slot_list_cached, - ); } fn update_stat(stat: &AtomicU64, value: u64) { diff --git a/accounts-db/src/accounts_index/stats.rs b/accounts-db/src/accounts_index/stats.rs index 80299d4a90c..645f13913f5 100644 --- a/accounts-db/src/accounts_index/stats.rs +++ b/accounts-db/src/accounts_index/stats.rs @@ -23,7 +23,6 @@ pub struct HeldInMemStats { pub age: AtomicU64, pub ref_count: AtomicU64, pub slot_list_len: AtomicU64, - pub slot_list_cached: AtomicU64, } #[derive(Debug, Default)] @@ -266,8 +265,6 @@ impl Stats { let held_in_mem_ref_count = self.held_in_mem.ref_count.swap(0, Ordering::Relaxed); let held_in_mem_slot_list_len = self.held_in_mem.slot_list_len.swap(0, Ordering::Relaxed); - let held_in_mem_slot_list_cached = - self.held_in_mem.slot_list_cached.swap(0, Ordering::Relaxed); // If an entry is held in-mem due to ref count or slot list length, // then assume it has two slot list entries. // Since `approx_size_of_one_entry()` assumes 'regular' entries @@ -320,11 +317,6 @@ impl Stats { held_in_mem_slot_list_len, i64 ), - ( - "num_not_flushed_slot_list_cached", - held_in_mem_slot_list_cached, - i64 - ), ("min_in_bin_disk", disk_stats.0, i64), ("max_in_bin_disk", disk_stats.1, i64), ("count_from_bins_disk", disk_stats.2, i64), diff --git a/accounts-db/src/ancient_append_vecs.rs b/accounts-db/src/ancient_append_vecs.rs index 21ca09ed3a4..77751f5185e 100644 --- a/accounts-db/src/ancient_append_vecs.rs +++ b/accounts-db/src/ancient_append_vecs.rs @@ -10,7 +10,7 @@ use { account_storage_entry::AccountStorageEntry, accounts_db::{ AccountFromStorage, AccountsDb, AliveAccounts, GetUniqueAccountsResult, ShrinkCollect, - ShrinkCollectAliveSeparatedByRefs, UpdateIndexThreadSelection, + ShrinkCollectAliveSeparatedByRefs, stats::{ShrinkAncientStats, ShrinkStatsSub}, }, active_stats::ActiveStatItem, @@ -563,12 +563,9 @@ impl AccountsDb { .expect("ancient shrink target slot must already have a storage"); let (shrink_in_progress, create_and_insert_store_elapsed_us) = measure_us!(self.get_store_for_shrink(target_slot, old_store, bytes)); - let (store_accounts_timing, rewrite_elapsed_us) = - measure_us!(self.store_accounts_for_shrink( - accounts_to_write, - shrink_in_progress.new_storage(), - UpdateIndexThreadSelection::PoolWithThreshold - )); + let (store_accounts_timing, rewrite_elapsed_us) = measure_us!( + self.store_accounts_for_squash(accounts_to_write, shrink_in_progress.new_storage()) + ); write_ancient_accounts.metrics.accumulate(&ShrinkStatsSub { store_accounts_timing, diff --git a/accounts-db/src/blockhash_queue.rs b/accounts-db/src/blockhash_queue.rs index 5ce2043122a..d5dcad30b9f 100644 --- a/accounts-db/src/blockhash_queue.rs +++ b/accounts-db/src/blockhash_queue.rs @@ -9,7 +9,7 @@ use { std::collections::HashMap, }; -#[cfg_attr(feature = "frozen-abi", derive(AbiExample))] +#[cfg_attr(feature = "frozen-abi", derive(AbiExample, StableAbi, StableAbiSample))] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct HashInfo { fee_calculator: FeeCalculator, @@ -26,10 +26,10 @@ impl HashInfo { /// Low memory overhead, so can be cloned for every checkpoint #[cfg_attr( feature = "frozen-abi", - derive(AbiExample, StableAbi), + derive(AbiExample, StableAbi, StableAbiSample), frozen_abi( api_digest = "DZVVXt4saSgH1CWGrzBcX2sq5yswCuRqGx1Y1ZehtWT6", - abi_digest = "CGD97vsYSQpPbYkzYnHmrwRZc4BbHqTEvP5vz4jg8jzU" + abi_digest = "5ojmBDhhu9AjKUc1LSHhZfXF6KeicvZpKP6XdLNaFAdy", ) )] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -158,37 +158,6 @@ impl BlockhashQueue { } } -#[cfg(feature = "frozen-abi")] -impl solana_frozen_abi::rand::prelude::Distribution - for solana_frozen_abi::rand::distr::StandardUniform -{ - fn sample(&self, rng: &mut R) -> BlockhashQueue { - let seed1: u64 = rng.random(); - let seed2: u64 = rng.random(); - let seed3: u64 = rng.random(); - let seed4: u64 = rng.random(); - - let mut hashes = - HashMap::with_hasher(ahash::RandomState::with_seeds(seed1, seed2, seed3, seed4)); - hashes.insert( - Hash::new_from_array(rng.random()), - HashInfo { - fee_calculator: FeeCalculator { - lamports_per_signature: rng.random(), - }, - hash_index: rng.random(), - timestamp: rng.random(), - }, - ); - - BlockhashQueue { - last_hash_index: rng.random(), - last_hash: Some(Hash::new_from_array(rng.random())), - hashes, - max_age: rng.random_range(0..MAX_RECENT_BLOCKHASHES), - } - } -} #[cfg(test)] mod tests { #[allow(deprecated)] diff --git a/accounts-db/src/rolling_bit_field.rs b/accounts-db/src/rolling_bit_field.rs index 9d299e12325..08e2f57cef5 100644 --- a/accounts-db/src/rolling_bit_field.rs +++ b/accounts-db/src/rolling_bit_field.rs @@ -295,26 +295,6 @@ impl RollingBitField { self.max_exclusive.saturating_sub(1) } - /// return all items < 'max_slot_exclusive' - pub fn get_all_less_than(&self, max_slot_exclusive: Slot) -> Vec { - let mut all = Vec::with_capacity(self.count); - self.excess.iter().for_each(|slot| { - if slot < &max_slot_exclusive { - all.push(*slot) - } - }); - for key in self.min..self.max_exclusive { - if key >= max_slot_exclusive { - break; - } - - if self.contains_assume_in_range(&key) { - all.push(key); - } - } - all - } - /// return highest item < 'max_slot_exclusive' pub fn get_prior(&self, max_slot_exclusive: Slot) -> Option { let mut slot = max_slot_exclusive.saturating_sub(1); @@ -362,59 +342,6 @@ mod tests { } } - #[test] - fn test_get_all_less_than() { - agave_logger::setup(); - let len = 16; - let mut bitfield = RollingBitField::new(len); - assert!(bitfield.get_all_less_than(0).is_empty()); - bitfield.insert(0); - assert!(bitfield.get_all_less_than(0).is_empty()); - assert_eq!(bitfield.get_all_less_than(1), vec![0]); - bitfield.insert(1); - assert_eq!(bitfield.get_all_less_than(1), vec![0]); - let last_item_not_in_excess = len - 1; - bitfield.insert(last_item_not_in_excess); - assert!(bitfield.excess.is_empty()); - assert_eq!( - bitfield.get_all_less_than(last_item_not_in_excess), - vec![0, 1] - ); - assert_eq!( - bitfield.get_all_less_than(last_item_not_in_excess + 1), - vec![0, 1, last_item_not_in_excess] - ); - let first_item_in_excess = last_item_not_in_excess + 1; - bitfield.insert(first_item_in_excess); - assert!(bitfield.excess.contains(&0)); - assert_eq!( - bitfield.get_all_less_than(last_item_not_in_excess), - vec![0, 1] - ); - assert_eq!( - bitfield.get_all_less_than(last_item_not_in_excess + 1), - vec![0, 1, last_item_not_in_excess] - ); - assert_eq!( - bitfield.get_all_less_than(first_item_in_excess + 1), - vec![0, 1, last_item_not_in_excess, first_item_in_excess] - ); - - bitfield.insert(len * 2); - let mut less = bitfield.get_all_less_than(len * 2); - less.sort_unstable(); - assert_eq!( - vec![0, 1, last_item_not_in_excess, first_item_in_excess], - less - ); - let mut less = bitfield.get_all_less_than(len * 2 + 1); - less.sort_unstable(); - assert_eq!( - vec![0, 1, last_item_not_in_excess, first_item_in_excess, len * 2], - less - ); - } - #[test] fn test_bitfield_delete_non_excess() { agave_logger::setup(); diff --git a/banking-stage-ingress-types/Cargo.toml b/banking-stage-ingress-types/Cargo.toml index 81f34ab2dc3..f532ab95456 100644 --- a/banking-stage-ingress-types/Cargo.toml +++ b/banking-stage-ingress-types/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/banks-client/Cargo.toml b/banks-client/Cargo.toml index 98059cc180f..4d0c03555cd 100644 --- a/banks-client/Cargo.toml +++ b/banks-client/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/banks-interface/Cargo.toml b/banks-interface/Cargo.toml index 0cfa7b6dd62..534bfae0d20 100644 --- a/banks-interface/Cargo.toml +++ b/banks-interface/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/banks-server/Cargo.toml b/banks-server/Cargo.toml index 13ea619c701..272d02893a9 100644 --- a/banks-server/Cargo.toml +++ b/banks-server/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/bloom/Cargo.toml b/bloom/Cargo.toml index 2756425e38b..cc0bc523cfd 100644 --- a/bloom/Cargo.toml +++ b/bloom/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/bls-cert-verify/Cargo.toml b/bls-cert-verify/Cargo.toml index 5e3f02152b1..633572e189f 100644 --- a/bls-cert-verify/Cargo.toml +++ b/bls-cert-verify/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [] diff --git a/bls-cert-verify/benches/cert_verify.rs b/bls-cert-verify/benches/cert_verify.rs index 0a7d0d221a0..d6de27f0579 100644 --- a/bls-cert-verify/benches/cert_verify.rs +++ b/bls-cert-verify/benches/cert_verify.rs @@ -32,7 +32,7 @@ fn create_signed_vote_message( vote: Vote, rank: usize, ) -> VoteMessage { - let payload = get_vote_payload_to_sign(&vote, shred_version); + let payload = get_vote_payload_to_sign(vote, shred_version); let signature: BlsSignature = bls_keypair.sign(&payload).into(); VoteMessage { vote, @@ -205,7 +205,7 @@ fn bench_verify_cert(c: &mut Criterion) { verify_certificate(cert_base2, total_validators, total_stake, |rank| { pubkeys_ref .get(rank) - .map(|bls_pubkey| (TEST_STAKE, *bls_pubkey)) + .map(|bls_pubkey| (NonZero::new(TEST_STAKE).unwrap(), *bls_pubkey)) }) .unwrap(); }, @@ -231,7 +231,7 @@ fn bench_verify_cert(c: &mut Criterion) { verify_certificate(cert_base3, total_validators, total_stake, |rank| { pubkeys_ref .get(rank) - .map(|bls_pubkey| (TEST_STAKE, *bls_pubkey)) + .map(|bls_pubkey| (NonZero::new(TEST_STAKE).unwrap(), *bls_pubkey)) }) .unwrap(); }, diff --git a/bls-cert-verify/src/cert_verify.rs b/bls-cert-verify/src/cert_verify.rs index aad400c0a41..66e065fda66 100644 --- a/bls-cert-verify/src/cert_verify.rs +++ b/bls-cert-verify/src/cert_verify.rs @@ -71,14 +71,14 @@ pub fn verify_certificate( cert: UnverifiedCertificate, max_validators: usize, total_stake: NonZero, - mut rank_map: impl FnMut(usize) -> Option<(u64, PopVerified)>, + mut rank_map: impl FnMut(usize) -> Option<(NonZero, PopVerified)>, ) -> Result { let mut aggregate_stake = 0u64; // Wrap the `rank_map` to accumulate stake as a side-effect let accumulating_rank_map = |ind: usize| { rank_map(ind).map(|(stake, pubkey)| { - aggregate_stake = aggregate_stake.saturating_add(stake); + aggregate_stake = aggregate_stake.saturating_add(stake.get()); pubkey }) }; @@ -297,7 +297,7 @@ mod test { rank: usize, ) -> VoteMessage { let bls_keypair = &bls_keypairs[rank]; - let payload = get_vote_payload_to_sign(&vote, shred_version); + let payload = get_vote_payload_to_sign(vote, shred_version); let signature: BLSSignature = bls_keypair.sign(&payload).into(); VoteMessage { vote, @@ -347,7 +347,9 @@ mod test { &(0..6).collect::>(), ); verify_certificate(cert, 10, NonZero::new(600).unwrap(), |rank| { - bls_keypairs.get(rank).map(|kp| (100, kp.public)) + bls_keypairs + .get(rank) + .map(|kp| (NonZero::new(100).unwrap(), kp.public)) }) .unwrap(); } @@ -372,7 +374,9 @@ mod test { &(0..6).collect::>(), ); verify_certificate(cert, 10, total_stake, |rank| { - bls_keypairs.get(rank).map(|kp| (100, kp.public)) + bls_keypairs + .get(rank) + .map(|kp| (NonZero::new(100).unwrap(), kp.public)) }) .unwrap(); @@ -384,7 +388,9 @@ mod test { &(0..5).collect::>(), ); let Err(err) = verify_certificate(cert, 10, total_stake, |rank| { - bls_keypairs.get(rank).map(|kp| (100, kp.public)) + bls_keypairs + .get(rank) + .map(|kp| (NonZero::new(100).unwrap(), kp.public)) }) else { panic!("should fail"); }; @@ -442,7 +448,9 @@ mod test { }; verify_certificate(cert, 10, NonZero::new(700).unwrap(), |rank| { - bls_keypairs.get(rank).map(|kp| (100, kp.public)) + bls_keypairs + .get(rank) + .map(|kp| (NonZero::new(100).unwrap(), kp.public)) }) .unwrap(); } @@ -473,7 +481,9 @@ mod test { }; assert_eq!( verify_certificate(cert, 10, NonZero::new(1000).unwrap(), |rank| { - bls_keypairs.get(rank).map(|kp| (100, kp.public)) + bls_keypairs + .get(rank) + .map(|kp| (NonZero::new(100).unwrap(), kp.public)) }) .unwrap_err(), Error::VerifySig(BlsError::PointConversion) @@ -511,7 +521,7 @@ mod test { |rank| { bls_keypairs .get(rank) - .map(|kp| (per_validator_stake, kp.public)) + .map(|kp| (NonZero::new(per_validator_stake).unwrap(), kp.public)) }, ) .unwrap(); @@ -555,7 +565,7 @@ mod test { // verification contract. // ---------------------------------------------------------------------------- - const STAKE_PER_VALIDATOR: u64 = 100; + const STAKE_PER_VALIDATOR: NonZero = NonZero::new(100).unwrap(); /// A `Block` for `slot` with a fresh, unique block id. fn fresh_block(slot: u64) -> Block { @@ -594,7 +604,7 @@ mod test { /// Uniform `rank_map` over `keypairs`, each with `STAKE_PER_VALIDATOR` stake. fn rank_map( keypairs: &[BLSKeypair], - ) -> impl FnMut(usize) -> Option<(u64, PopVerified)> + '_ { + ) -> impl FnMut(usize) -> Option<(NonZero, PopVerified)> + '_ { move |rank| { keypairs .get(rank) diff --git a/bls-sigverify/Cargo.toml b/bls-sigverify/Cargo.toml index 4b5609e67bc..1dd539f91c6 100644 --- a/bls-sigverify/Cargo.toml +++ b/bls-sigverify/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [ diff --git a/bls-sigverify/benches/bls_vote_sigverify.rs b/bls-sigverify/benches/bls_vote_sigverify.rs index e2abdd69ba4..ccc2c22ed45 100644 --- a/bls-sigverify/benches/bls_vote_sigverify.rs +++ b/bls-sigverify/benches/bls_vote_sigverify.rs @@ -12,8 +12,10 @@ use { stats::SigVerifyVoteStats, }, agave_votor_messages::{ - consensus_message::Block, unverified_vote_message::UnverifiedVoteMessage, vote::Vote, - wire::get_vote_payload_to_sign, + consensus_message::Block, + unverified_vote_message::UnverifiedVoteMessage, + vote::Vote, + wire::{VotePayloadToSign, get_vote_payload_to_sign}, }, criterion::{BatchSize, Criterion, criterion_group, criterion_main}, rayon::{ThreadPool, ThreadPoolBuilder}, @@ -21,10 +23,9 @@ use { solana_hash::Hash, solana_keypair::Keypair, solana_signer::Signer, - std::{hint::black_box, sync::Arc}, + std::hint::black_box, }; -static MESSAGE_COUNTS: &[usize] = &[1, 2, 4, 8, 16]; static BATCH_SIZES: &[usize] = &[8, 16, 32, 64, 128]; fn get_thread_pool() -> ThreadPool { @@ -35,67 +36,39 @@ fn get_thread_pool() -> ThreadPool { .unwrap() } -fn get_matrix_params() -> impl Iterator { - BATCH_SIZES.iter().flat_map(|&batch_size| { - MESSAGE_COUNTS.iter().filter_map(move |&num_distinct| { - if num_distinct > batch_size { - None - } else { - Some((batch_size, num_distinct)) - } - }) - }) -} - fn generate_test_data( shred_version: u16, - num_distinct_messages: usize, batch_size: usize, -) -> Vec { - assert!( - batch_size >= num_distinct_messages, - "Batch size must be >= distinct messages" - ); - +) -> (VotePayloadToSign, Vec) { // Pre-calculate the payloads to ensure exact distinctness - let base_payloads = (0..num_distinct_messages) - .map(|i| { - let slot = (i as u64).saturating_add(100); - let vote = Vote::new_notarization_vote(Block { - slot, - block_id: Hash::new_unique(), - }); - let payload = get_vote_payload_to_sign(&vote, shred_version); - (vote, Arc::new(payload)) - }) - .collect::>(); - - let mut votes_to_verify = Vec::with_capacity(batch_size); - - for i in 0..batch_size { - let (vote, payload) = &base_payloads[i.rem_euclid(num_distinct_messages)]; - - let bls_keypair = BLSKeypair::new(); - - let signature = bls_keypair.sign(payload); - - let vote_message = UnverifiedVoteMessage { - vote: *vote, - signature: signature.into(), - rank: 0, - shred_version, - }; - - votes_to_verify.push(UnverifiedVotePayload { - vote_message, - sender_bls_pubkey: bls_keypair.public, - sender_vote_account_pubkey: Keypair::new().pubkey(), - sender_identity_pubkey: Keypair::new().pubkey(), - prepared_payload: None, - }); - } - - votes_to_verify + let slot = 100; + let vote = Vote::new_notarization_vote(Block { + slot, + block_id: Hash::new_unique(), + }); + let payload = get_vote_payload_to_sign(vote, shred_version); + ( + VotePayloadToSign::new_from_vote(vote, shred_version), + (0..batch_size) + .map(|_| { + let bls_keypair = BLSKeypair::new(); + let signature = bls_keypair.sign(&payload); + let vote_message = UnverifiedVoteMessage { + vote, + signature: signature.into(), + rank: 0, + shred_version, + }; + UnverifiedVotePayload { + vote_message, + sender_bls_pubkey: bls_keypair.public, + sender_vote_account_pubkey: Keypair::new().pubkey(), + sender_identity_pubkey: Keypair::new().pubkey(), + prepared_payload: None, + } + }) + .collect(), + ) } // Single Signature Verification @@ -145,13 +118,18 @@ fn bench_verify_votes_optimistic(c: &mut Criterion) { let mut stats = SigVerifyVoteStats::default(); let thread_pool = get_thread_pool(); - for (batch_size, num_distinct) in get_matrix_params() { - let votes = generate_test_data(shred_version, num_distinct, batch_size); - let label = format!("msgs_{num_distinct}/batch_{batch_size}"); + for &batch_size in BATCH_SIZES { + let (vote, mut unverified_votes) = generate_test_data(shred_version, batch_size); + let label = format!("batch_{batch_size}"); group.bench_function(&label, |b| { b.iter(|| { - let res = verify_votes_optimistic(black_box(&votes), &mut stats, &thread_pool); + let res = verify_votes_optimistic( + vote, + black_box(&mut unverified_votes), + &mut stats, + &thread_pool, + ); black_box(res); }) }); @@ -165,21 +143,19 @@ fn bench_verify_votes_optimistic(c: &mut Criterion) { fn bench_aggregate_pubkeys(c: &mut Criterion) { let shred_version = 1234; let mut group = c.benchmark_group("aggregate_pubkeys"); - let mut stats = SigVerifyVoteStats::default(); - for (batch_size, num_distinct) in get_matrix_params() { - let votes = generate_test_data(shred_version, num_distinct, batch_size); - let label = format!("msgs_{num_distinct}/batch_{batch_size}"); + for &batch_size in BATCH_SIZES { + let (vote, unverified_votes) = generate_test_data(shred_version, batch_size); + let label = format!("batch_{batch_size}"); group.bench_function(&label, |b| { b.iter(|| { - let res = aggregate_pubkeys_by_payload(black_box(&votes), &mut stats); - black_box(res).2.unwrap(); + let res = aggregate_pubkeys_by_payload(vote, black_box(&unverified_votes)); + black_box(res).1.unwrap(); }) }); } group.finish(); - black_box(stats); } // Signature Aggregation @@ -191,12 +167,12 @@ fn bench_aggregate_signatures(c: &mut Criterion) { for &batch_size in BATCH_SIZES { // Use 1 distinct message just to generate valid data cheaply. // It doesn't affect signature aggregation performance. - let votes = generate_test_data(shred_version, 1, batch_size); + let (_, unverified_votes) = generate_test_data(shred_version, batch_size); let label = format!("batch_{batch_size}"); group.bench_function(&label, |b| { b.iter(|| { - let res = aggregate_signatures(black_box(&votes)); + let res = aggregate_signatures(black_box(&unverified_votes)); black_box(res).unwrap(); }) }); @@ -213,15 +189,14 @@ fn bench_verify_individual_votes(c: &mut Criterion) { for &batch_size in BATCH_SIZES { // Distinctness doesn't affect the cost of N individual verifications. - let votes = generate_test_data(shred_version, 1, batch_size); + let (_vote, unverified_votes) = generate_test_data(shred_version, batch_size); let label = format!("batch_{batch_size}"); group.bench_function(&label, |b| { b.iter_batched( - || votes.clone(), + || unverified_votes.clone(), |votes| { - let res = - verify_individual_votes(black_box(votes), vec![], vec![], &thread_pool); + let res = verify_individual_votes(black_box(votes), &thread_pool); black_box(res); }, BatchSize::SmallInput, diff --git a/bls-sigverify/src/bls_sigverifier.rs b/bls-sigverify/src/bls_sigverifier.rs index d23a3a76ce7..b751c6dfb20 100644 --- a/bls-sigverify/src/bls_sigverifier.rs +++ b/bls-sigverify/src/bls_sigverifier.rs @@ -17,7 +17,8 @@ use { migration::MigrationStatus, reward_certificate::AddVoteMessage, unverified_vote_message::{DecodedWireConsensusMessage, UnverifiedVoteMessage}, - wire::VersionedWireConsensusMessage, + vote::Vote, + wire::{VersionedWireConsensusMessage, VotePayloadToSign}, }, crossbeam_channel::{Receiver, RecvTimeoutError, Sender, TryRecvError}, log::error, @@ -32,7 +33,7 @@ use { solana_runtime::{bank::Bank, bank_forks::SharableBanks}, solana_streamer::{nonblocking::simple_qos::SimpleQosBanlist, packet::PacketBatch}, std::{ - collections::HashSet, + collections::{HashMap, HashSet}, sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -211,10 +212,13 @@ impl SigVerifier { &mut self, batches: Vec, root_bank: &Bank, - ) -> (Vec, Vec) { + ) -> ( + Vec, + HashMap>, + ) { let root_slot = root_bank.slot(); let mut certs = Vec::new(); - let mut votes = Vec::new(); + let mut votes: HashMap> = HashMap::new(); let mut num_pkts = 0u64; let my_shred_version = self.cluster_info.my_shred_version(); for packet in batches.iter().flatten() { @@ -243,17 +247,23 @@ impl SigVerifier { }; match decoded_msg { - DecodedWireConsensusMessage::Vote(vote) => { + DecodedWireConsensusMessage::Vote(unverified_vote) => { if let Some((sender_vote_account_pubkey, sender_bls_pubkey)) = - self.keep_vote(&vote, root_bank) + self.keep_vote(&unverified_vote.vote, &unverified_vote, root_bank) { - votes.push(UnverifiedVotePayload { - vote_message: vote, - sender_bls_pubkey, - sender_vote_account_pubkey, - sender_identity_pubkey, - prepared_payload: None, - }); + let vote_payload_to_sign = VotePayloadToSign::new_from_vote( + unverified_vote.vote, + unverified_vote.shred_version, + ); + votes.entry(vote_payload_to_sign).or_default().push( + UnverifiedVotePayload { + vote_message: unverified_vote, + sender_bls_pubkey, + sender_vote_account_pubkey, + sender_identity_pubkey, + prepared_payload: None, + }, + ); } } DecodedWireConsensusMessage::Certificate(cert) => { @@ -283,11 +293,12 @@ impl SigVerifier { /// If this vote should be verified, then returns the sender's Pubkey and BlsPubkey. fn keep_vote( &mut self, + vote: &Vote, msg: &UnverifiedVoteMessage, root_bank: &Bank, ) -> Option<(Pubkey, PopVerified)> { let root_slot = root_bank.slot(); - let Some(rank_map) = root_bank.get_rank_map(msg.vote.slot()) else { + let Some(rank_map) = root_bank.get_rank_map(vote.slot()) else { self.stats.discard_vote_no_epoch_stakes += 1; return None; }; @@ -298,15 +309,10 @@ impl SigVerifier { None })?; let ret = Some((entry.vote_account_pubkey, entry.bls_pubkey)); - if msg.vote.slot() > root_slot { + if vote.slot() > root_slot { return ret; } - if rewards_wants_vote( - &self.cluster_info, - &self.leader_schedule, - root_slot, - &msg.vote, - ) { + if rewards_wants_vote(&self.cluster_info, &self.leader_schedule, root_slot, vote) { return ret; } self.stats.num_old_votes_received += 1; @@ -482,7 +488,7 @@ mod tests { rank: usize, ) -> VoteMessage { let bls_keypair = &validator_keypairs[rank].bls_keypair; - let payload = get_vote_payload_to_sign(&vote, shred_version); + let payload = get_vote_payload_to_sign(vote, shred_version); let signature: Signature = bls_keypair.sign(&payload).into(); VoteMessage { vote, @@ -822,7 +828,7 @@ mod tests { let mut packets = Vec::with_capacity(num_votes); let vote = Vote::new_skip_vote(42); let vote_payload = - get_vote_payload_to_sign(&vote, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote, ctx.verifier.cluster_info.my_shred_version()); for (i, validator_keypair) in ctx.validator_keypairs.iter().enumerate().take(num_votes) { let rank = i as u16; @@ -915,13 +921,15 @@ mod tests { .verify_and_send_batches(packet_batches) .unwrap(); let batches = ctx.pool_receiver.try_iter().collect::>(); - assert_eq!(batches.len(), 1); - match &batches[0] { - SigVerifiedBatch::Votes(votes) => { - assert_eq!(votes.len(), num_votes); - } - rest => panic!("unexpected type: {rest:?}"), - } + assert_eq!(batches.len(), 2); + let total_votes_verified = batches + .into_iter() + .map(|batch| match batch { + SigVerifiedBatch::Votes(votes) => votes.len(), + rest => panic!("unexpected type: {rest:?}"), + }) + .sum::(); + assert_eq!(total_votes_verified, num_votes); assert_eq!( ctx.verifier.stats.vote_stats.distinct_votes_stats.count(), 1 @@ -947,12 +955,12 @@ mod tests { let vote1 = Vote::new_skip_vote(42); let vote1_payload = - get_vote_payload_to_sign(&vote1, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote1, ctx.verifier.cluster_info.my_shred_version()); let vote2 = Vote::new_skip_vote(43); let vote2_payload = - get_vote_payload_to_sign(&vote2, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote2, ctx.verifier.cluster_info.my_shred_version()); let invalid_payload = get_vote_payload_to_sign( - &Vote::new_skip_vote(99), + Vote::new_skip_vote(99), ctx.verifier.cluster_info.my_shred_version(), ); @@ -990,27 +998,22 @@ mod tests { .verify_and_send_batches(packet_batches) .unwrap(); let batches = ctx.pool_receiver.try_iter().collect::>(); - assert_eq!(batches.len(), 1); - match &batches[0] { - SigVerifiedBatch::Votes(votes) => { - assert_eq!(votes.len(), num_votes - 1); - } - rest => panic!("unexpected type: {rest:?}"), - } - - let mut found_msg = false; - match &batches[0] { - SigVerifiedBatch::Votes(votes) => { - for vote in votes { - if vote.vote == vote2 && vote.rank == invalid_rank { - found_msg = true; - break; + assert_eq!(batches.len(), 2); + let total_votes_verified = batches + .into_iter() + .map(|batch| match batch { + SigVerifiedBatch::Votes(votes) => { + for vote in &votes { + if vote.vote == vote2 && vote.rank == invalid_rank { + panic!("invalid vote verified"); + } } + votes.len() } - } - rest => panic!("unexpected type: {rest:?}"), - } - assert!(!found_msg); + rest => panic!("unexpected type: {rest:?}"), + }) + .sum::(); + assert_eq!(total_votes_verified, num_votes - 1); } #[test] @@ -1024,9 +1027,9 @@ mod tests { let vote = Vote::new_skip_vote(42); let valid_vote_payload = - get_vote_payload_to_sign(&vote, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote, ctx.verifier.cluster_info.my_shred_version()); let invalid_vote_payload = get_vote_payload_to_sign( - &Vote::new_skip_vote(99), + Vote::new_skip_vote(99), ctx.verifier.cluster_info.my_shred_version(), ); @@ -1303,7 +1306,7 @@ mod tests { let vote = Vote::new_skip_vote(42); let vote_payload = - get_vote_payload_to_sign(&vote, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote, ctx.verifier.cluster_info.my_shred_version()); for (i, validator_keypair) in ctx.validator_keypairs.iter().enumerate().take(num_votes) { let rank = i as u16; let bls_keypair = &validator_keypair.bls_keypair; @@ -1328,7 +1331,7 @@ mod tests { }); let cert_original_vote = Vote::new_notarization_vote(cert_type.to_block().unwrap()); let cert_payload = get_vote_payload_to_sign( - &cert_original_vote, + cert_original_vote, ctx.verifier.cluster_info.my_shred_version(), ); @@ -1393,7 +1396,7 @@ mod tests { let invalid_rank = 999; let vote = Vote::new_skip_vote(42); let vote_payload = - get_vote_payload_to_sign(&vote, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote, ctx.verifier.cluster_info.my_shred_version()); let bls_keypair = &ctx.validator_keypairs[0].bls_keypair; let signature: Signature = bls_keypair.sign(&vote_payload).into(); @@ -1469,7 +1472,7 @@ mod tests { let vote = Vote::new_skip_vote(2); let vote_payload = - get_vote_payload_to_sign(&vote, sig_verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote, sig_verifier.cluster_info.my_shred_version()); let bls_keypair = &validator_keypairs[0].bls_keypair; let signature: Signature = bls_keypair.sign(&vote_payload).into(); let consensus_message_vote = ConsensusMessage::Vote(VoteMessage { @@ -1523,7 +1526,7 @@ mod tests { let cert_type = CertificateType::Notarize(block); let original_vote = Vote::new_notarization_vote(block); let signed_payload = - get_vote_payload_to_sign(&original_vote, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(original_vote, ctx.verifier.cluster_info.my_shred_version()); let mut vote_messages: Vec = (0..num_signers) .map(|i| { let signature = ctx.validator_keypairs[i].bls_keypair.sign(&signed_payload); @@ -1615,9 +1618,9 @@ mod tests { let vote = Vote::new_skip_vote(42); let valid_payload = - get_vote_payload_to_sign(&vote, ctx.verifier.cluster_info.my_shred_version()); + get_vote_payload_to_sign(vote, ctx.verifier.cluster_info.my_shred_version()); let invalid_payload = get_vote_payload_to_sign( - &Vote::new_skip_vote(999), + Vote::new_skip_vote(999), ctx.verifier.cluster_info.my_shred_version(), ); let invalid_indexes = [1usize, 3usize]; diff --git a/bls-sigverify/src/bls_vote_sigverify.rs b/bls-sigverify/src/bls_vote_sigverify.rs index 645dce29cad..b3cce61367f 100644 --- a/bls-sigverify/src/bls_vote_sigverify.rs +++ b/bls-sigverify/src/bls_vote_sigverify.rs @@ -12,9 +12,12 @@ use { }, }, agave_votor_messages::{ - consensus_message::VoteMessage, metric_types::ConsensusMetricsEvent, - reward_certificate::AddVoteMessage, unverified_vote_message::UnverifiedVoteMessage, - vote::Vote, wire::get_vote_payload_to_sign, + consensus_message::VoteMessage, + metric_types::ConsensusMetricsEvent, + reward_certificate::AddVoteMessage, + unverified_vote_message::UnverifiedVoteMessage, + vote::Vote, + wire::{VotePayloadToSign, get_vote_payload_to_sign}, }, log::info, rayon::{ @@ -44,10 +47,6 @@ fn into_vote_msg(msg: UnverifiedVoteMessage) -> VoteMessage { } } -/// This is the percentage threshold of distinct votes among the total votes under which we will prepare a cache of -/// prepared payloads for individual verification. -const PREPARED_PAYLOAD_CACHE_DISTINCT_VOTE_THRESHOLD_PERCENT: usize = 90; - #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))] struct VerifiedVotePayload { vote_message: VoteMessage, @@ -73,7 +72,7 @@ impl UnverifiedVotePayload { .is_ok() } else { let payload = - get_vote_payload_to_sign(&self.vote_message.vote, self.vote_message.shred_version); + get_vote_payload_to_sign(self.vote_message.vote, self.vote_message.shred_version); self.sender_bls_pubkey .verify_signature(&self.vote_message.signature, &payload) .is_ok() @@ -90,7 +89,7 @@ impl UnverifiedVotePayload { /// /// Any vote that fails fallback individual signature verification will have its sender banlisted. pub(super) fn verify_and_send_votes( - votes_to_verify: Vec, + unverified_votes: HashMap>, root_bank: &Bank, cluster_info: &ClusterInfo, leader_schedule: &LeaderScheduleCache, @@ -100,20 +99,33 @@ pub(super) fn verify_and_send_votes( ) -> Result { let mut measure = Measure::start("verify_and_send_votes"); let mut stats = SigVerifyVoteStats::default(); - if votes_to_verify.is_empty() { + if unverified_votes.is_empty() { return Ok(stats); } - stats.votes_to_sig_verify += votes_to_verify.len() as u64; - let verified_votes = verify_votes(root_bank, votes_to_verify, &mut stats, banlist, thread_pool); - stats.sig_verified_votes += verified_votes.len() as u64; + stats + .distinct_votes_stats + .add_sample(unverified_votes.len() as u64); + + for (vote_payload_to_sign, unverified_votes) in unverified_votes { + stats.votes_to_sig_verify += unverified_votes.len() as u64; + let verified_votes = verify_votes( + root_bank, + vote_payload_to_sign, + unverified_votes, + &mut stats, + banlist, + thread_pool, + ); + stats.sig_verified_votes += verified_votes.len() as u64; - let (votes_for_pool, msgs_for_repair, msg_for_reward, msg_for_metrics) = - process_verified_votes(verified_votes, root_bank, cluster_info, leader_schedule); + let (votes_for_pool, msgs_for_repair, msg_for_reward, msg_for_metrics) = + process_verified_votes(verified_votes, root_bank, cluster_info, leader_schedule); - send_votes_to_pool(votes_for_pool, &channels.channel_to_pool, &mut stats)?; - send_votes_to_repair(msgs_for_repair, &channels.channel_to_repair, &mut stats)?; - send_votes_to_rewards(msg_for_reward, &channels.channel_to_reward, &mut stats)?; - send_votes_to_metrics(msg_for_metrics, &channels.channel_to_metrics, &mut stats)?; + send_votes_to_pool(votes_for_pool, &channels.channel_to_pool, &mut stats)?; + send_votes_to_repair(msgs_for_repair, &channels.channel_to_repair, &mut stats)?; + send_votes_to_rewards(msg_for_reward, &channels.channel_to_reward, &mut stats)?; + send_votes_to_metrics(msg_for_metrics, &channels.channel_to_metrics, &mut stats)?; + } measure.stop(); stats @@ -196,30 +208,30 @@ fn process_verified_votes( ) } -/// Sig verifies `votes_to_verify` and returns a `Vec` of votes that passed verification. +/// Sig verifies `unverified_votes` and returns a `Vec` of votes that passed verification. fn verify_votes( root_bank: &Bank, - votes_to_verify: Vec, + vote_payload_to_sign: VotePayloadToSign, + mut unverified_votes: Vec, stats: &mut SigVerifyVoteStats, banlist: &SimpleQosBanlist, thread_pool: &ThreadPool, ) -> Vec { // Filter votes too far in the future. - let len_before = votes_to_verify.len(); - let votes_to_verify = votes_to_verify - .into_iter() - .filter(|v| { - v.vote_message.vote.slot() <= root_bank.slot().saturating_add(NUM_SLOTS_FOR_VERIFY) - }) - .collect::>(); - let num_discarded = len_before.saturating_sub(votes_to_verify.len()); - stats.too_far_in_future += num_discarded as u64; + if vote_payload_to_sign.slot() > root_bank.slot().saturating_add(NUM_SLOTS_FOR_VERIFY) { + stats.too_far_in_future += unverified_votes.len() as u64; + return vec![]; + } // Try optimistic verification - fast to verify, but cannot identify invalid votes - let (optimistic_result, distinct_votes, distinct_payloads) = - verify_votes_optimistic(&votes_to_verify, stats, thread_pool); - if matches!(optimistic_result, OptimisticVerificationResult::Verified) { - return votes_to_verify + let is_verified = verify_votes_optimistic( + vote_payload_to_sign, + &mut unverified_votes, + stats, + thread_pool, + ); + if is_verified { + return unverified_votes .into_iter() .map(|v| VerifiedVotePayload { vote_message: into_vote_msg(v.vote_message), @@ -229,12 +241,8 @@ fn verify_votes( } // Fallback to individual verification - let ((verified_votes, invalid_remote_pubkeys), time_us) = measure_us!(verify_individual_votes( - votes_to_verify, - distinct_votes, - distinct_payloads, - thread_pool, - )); + let ((verified_votes, invalid_remote_pubkeys), time_us) = + measure_us!(verify_individual_votes(unverified_votes, thread_pool)); for sender_identity_pubkey in invalid_remote_pubkeys { if banlist.ban(sender_identity_pubkey, BAN_TIMEOUT) { stats.already_banned += 1; @@ -250,17 +258,6 @@ fn verify_votes( verified_votes } -/// Outcome of optimistic aggregate vote verification. -/// -/// `Failed` carries the number of distinct vote messages seen before falling -/// back to individual verification. -#[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum OptimisticVerificationResult { - Verified, - Failed { num_distinct_messages: usize }, -} - #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))] /// Attempts aggregate BLS verification across the full vote set. /// @@ -273,15 +270,13 @@ enum OptimisticVerificationResult { /// Returns the optimistic verification outcome together with the distinct vote /// messages and their prepared payloads, which can be reused by the fallback /// path. +#[must_use] fn verify_votes_optimistic( - votes: &[UnverifiedVotePayload], + vote_payload_to_sign: VotePayloadToSign, + unverified_votes: &mut Vec, stats: &mut SigVerifyVoteStats, thread_pool: &ThreadPool, -) -> ( - OptimisticVerificationResult, - Vec, - Vec, -) { +) -> bool { let mut measure = Measure::start("verify_votes_optimistic"); // For BLS verification, minimizing the expensive pairing operation is key. @@ -293,62 +288,34 @@ fn verify_votes_optimistic( // // By verifying the aggregated signature against the aggregated public keys, // the number of pairings required is reduced to (1 + number of distinct messages). - let (signature_result, (distinct_votes, distinct_payloads, pubkeys_result)) = thread_pool.join( - || aggregate_signatures(votes), - || aggregate_pubkeys_by_payload(votes, stats), + let (signature_result, (prepared_hash_msg, pubkey_result)) = thread_pool.join( + || aggregate_signatures(unverified_votes), + || aggregate_pubkeys_by_payload(vote_payload_to_sign, unverified_votes), ); let Ok(aggregate_signature) = signature_result else { - return ( - OptimisticVerificationResult::Failed { - num_distinct_messages: 0, - }, - Vec::new(), - Vec::new(), - ); + return false; }; - let Ok(aggregate_pubkeys) = pubkeys_result else { - return ( - OptimisticVerificationResult::Failed { - num_distinct_messages: distinct_payloads.len(), - }, - distinct_votes, - distinct_payloads, - ); + let Ok(aggregate_pubkey) = pubkey_result else { + return false; }; - let verified = if distinct_payloads.len() == 1 { - // if one unique payload, just verify the aggregate signature for the single payload - // this requires (2 pairings) - aggregate_pubkeys[0] - .verify_signature_prepared(&aggregate_signature, &distinct_payloads[0]) - .is_ok() - } else { - // if non-unique payload, we need to apply a pairing for each distinct message, - // which is done inside `par_verify_distinct_aggregated_prepared`. - thread_pool.install(|| { - SignatureProjective::par_verify_distinct_aggregated_prepared( - &aggregate_pubkeys, - &aggregate_signature, - &distinct_payloads, - ) - .is_ok() - }) - }; + let verified = aggregate_pubkey + .verify_signature_prepared(&aggregate_signature, &prepared_hash_msg) + .is_ok(); measure.stop(); stats .fn_verify_votes_optimistic_stats .add_sample(measure.as_us()); - let result = if verified { - OptimisticVerificationResult::Verified - } else { - OptimisticVerificationResult::Failed { - num_distinct_messages: distinct_payloads.len(), + if !verified { + let prepared_hash_msg = Arc::new(prepared_hash_msg); + for unverified_vote in unverified_votes { + unverified_vote.prepared_payload = Some(prepared_hash_msg.clone()); } - }; - (result, distinct_votes, distinct_payloads) + } + verified } #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))] @@ -364,57 +331,23 @@ fn aggregate_signatures(votes: &[UnverifiedVotePayload]) -> Result ( - Vec, - Vec, - Result>, BlsError>, + PreparedHashedMessage, + Result, BlsError>, ) { debug_assert!(current_thread_index().is_some()); - let mut grouped_votes: HashMap>)> = - HashMap::new(); - - for v in votes { - let shred_version = v.vote_message.shred_version; - grouped_votes - .entry(v.vote_message.vote) - .or_insert_with(|| (shred_version, Vec::new())) - .1 - .push(&v.sender_bls_pubkey); - } - - stats - .distinct_votes_stats - .add_sample(grouped_votes.len() as u64); - - let distinct_grouped_votes = grouped_votes - .into_par_iter() - .map(|(vote, (shred_version, pubkeys))| { - let serialized_vote = get_vote_payload_to_sign(&vote, shred_version); - // converting aggregate pubkey to `PopVerified` is safe here - // since the pubkeys are all PoP verified in the vote account - let pubkey = PubkeyProjective::par_aggregate(pubkeys.into_par_iter()) - .map(|agg| unsafe { PopVerified::new_unchecked(*agg) }); - (vote, PreparedHashedMessage::new(&serialized_vote), pubkey) - }) - .collect::>(); - let (distinct_votes, distinct_payloads, distinct_pubkeys_results): (Vec<_>, Vec<_>, Vec<_>) = - distinct_grouped_votes.into_iter().fold( - (Vec::new(), Vec::new(), Vec::new()), - |mut acc, (vote, payload, pubkey)| { - acc.0.push(vote); - acc.1.push(payload); - acc.2.push(pubkey); - acc - }, - ); - let aggregate_pubkeys_result = distinct_pubkeys_results.into_iter().collect(); - - (distinct_votes, distinct_payloads, aggregate_pubkeys_result) + let serialized_vote = wincode::serialize(&vote_payload_to_sign).unwrap(); + let prepared_hash_msg = PreparedHashedMessage::new(&serialized_vote); + // converting aggregate pubkey to `PopVerified` is safe here + // since the pubkeys are all PoP verified in the vote account + let pubkey = + PubkeyProjective::par_aggregate(votes.into_par_iter().map(|v| &v.sender_bls_pubkey)) + .map(|agg| unsafe { PopVerified::new_unchecked(*agg) }); + (prepared_hash_msg, pubkey) } /// Verifies votes individually on a thread pool. @@ -424,53 +357,27 @@ fn aggregate_pubkeys_by_payload( /// - `Vec`: senders' identity pubkeys for votes that failed verification. #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))] fn verify_individual_votes( - mut votes_to_verify: Vec, - distinct_votes: Vec, - distinct_payloads: Vec, + unverified_votes: Vec, thread_pool: &ThreadPool, ) -> (Vec, Vec) { - if should_prepare_payload_cache(distinct_votes.len(), votes_to_verify.len()) { - let prepared_payloads: HashMap<_, _> = distinct_votes - .into_iter() - .zip(distinct_payloads) - .map(|(vote, payload)| (vote, Arc::new(payload))) - .collect(); - for vote in &mut votes_to_verify { - vote.prepared_payload = prepared_payloads.get(&vote.vote_message.vote).cloned(); - } - } - thread_pool.install(|| { - votes_to_verify.into_par_iter().partition_map(|vote| { - let sender_identity_pubkey = vote.sender_identity_pubkey; - match vote.verify() { - Some(vote) => Either::Left(vote), - None => Either::Right(sender_identity_pubkey), - } - }) + unverified_votes + .into_par_iter() + .partition_map(|unverified_vote| { + let sender_identity_pubkey = unverified_vote.sender_identity_pubkey; + match unverified_vote.verify() { + Some(vote) => Either::Left(vote), + None => Either::Right(sender_identity_pubkey), + } + }) }) } -fn should_prepare_payload_cache(distinct_vote_count: usize, total_vote_count: usize) -> bool { - distinct_vote_count.saturating_mul(100) - <= total_vote_count.saturating_mul(PREPARED_PAYLOAD_CACHE_DISTINCT_VOTE_THRESHOLD_PERCENT) -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn test_should_prepare_payload_cache() { - assert!(should_prepare_payload_cache(9, 10)); - assert!(should_prepare_payload_cache(0, 0)); - assert!(!should_prepare_payload_cache(1, 1)); - assert!(!should_prepare_payload_cache(10, 10)); - assert!(!should_prepare_payload_cache(91, 100)); - assert!(should_prepare_payload_cache(90, 100)); - } - #[test] #[should_panic] fn ensure_aggregate_signatures_runs_on_thread_pool() { @@ -483,8 +390,12 @@ mod tests { #[should_panic] fn ensure_aggregate_pubkeys_by_payload_runs_on_thread_pool() { let votes = vec![]; - let mut stats = SigVerifyVoteStats::default(); + let shred_version = 1234; + let vote = Vote::new_skip_vote(1); + let vote_payload_to_sign = VotePayloadToSign::new_from_vote(vote, shred_version); // calling without a rayon thread pool should trigger a debug assert. - aggregate_pubkeys_by_payload(&votes, &mut stats).2.unwrap(); + aggregate_pubkeys_by_payload(vote_payload_to_sign, &votes) + .1 + .unwrap(); } } diff --git a/bucket_map/Cargo.toml b/bucket_map/Cargo.toml index 06a29badba9..3c3660fb543 100644 --- a/bucket_map/Cargo.toml +++ b/bucket_map/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [lib] crate-type = ["lib"] name = "solana_bucket_map" diff --git a/builtins-default-costs/Cargo.toml b/builtins-default-costs/Cargo.toml index 6e083605b2f..3a7cde4b23f 100644 --- a/builtins-default-costs/Cargo.toml +++ b/builtins-default-costs/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] # Add additional builtin programs here [lib] diff --git a/builtins/Cargo.toml b/builtins/Cargo.toml index 24db4d80d45..5988ae4d481 100644 --- a/builtins/Cargo.toml +++ b/builtins/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [] diff --git a/ci/coverage/part-1.sh b/ci/coverage/part-1.sh index 5440be089cf..95abc1124be 100755 --- a/ci/coverage/part-1.sh +++ b/ci/coverage/part-1.sh @@ -16,4 +16,5 @@ echo "--- coverage: root (part 1)" --features frozen-abi \ --features dev-context-only-utils \ --lib \ + --bins \ "${packages[@]}" diff --git a/ci/coverage/part-2.sh b/ci/coverage/part-2.sh index 105868e5690..840e3b3a705 100755 --- a/ci/coverage/part-2.sh +++ b/ci/coverage/part-2.sh @@ -15,4 +15,5 @@ echo "--- coverage: root (part 2)" "$git_root"/ci/test-coverage.sh \ --features dev-context-only-utils \ --lib \ + --bins \ "${packages[@]}" diff --git a/ci/coverage/part-3.sh b/ci/coverage/part-3.sh index c3410806e7f..45d09c07c1f 100755 --- a/ci/coverage/part-3.sh +++ b/ci/coverage/part-3.sh @@ -21,6 +21,7 @@ echo "--- coverage: coverage (part 3)" --features dev-context-only-utils \ --workspace \ --lib \ + --bins \ "${exclude_packages[@]}" # Clean up diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index 448170c6e9f..469d20eca4a 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -38,6 +38,7 @@ RUN \ git \ vim \ jq \ + iproute2 \ ca-certificates \ curl \ gnupg \ @@ -81,15 +82,11 @@ RUN \ rustup component add rustfmt --toolchain=$RUST_NIGHTLY_VERSION && \ rustup component add miri --toolchain=$RUST_NIGHTLY_VERSION && \ rustup component add llvm-tools-preview --toolchain=$RUST_NIGHTLY_VERSION && \ - rustup target add wasm32-unknown-unknown && \ cargo install cargo-audit && \ cargo install cargo-hack && \ cargo install cargo-sort@2.0.2 && \ - cargo install mdbook && \ - cargo install mdbook-linkcheck && \ - cargo install svgbob_cli && \ - cargo install wasm-pack && \ cargo install rustfilt && \ + cargo install cargo-build-sbf@4.1.0 --locked && \ rustup show && \ rustc --version && \ cargo --version && \ @@ -113,8 +110,6 @@ RUN \ chmod -R a+w /.cache && \ mkdir /.config && \ chmod -R a+w /.config && \ - mkdir /.npm && \ - chmod -R a+w /.npm && \ # grcov curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \ curl -LsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/checksums.sha256" | sha256sum -c --ignore-missing - && \ diff --git a/ci/test-sanity.sh b/ci/test-sanity.sh index 33ef66fec19..c5bfbb05e08 100755 --- a/ci/test-sanity.sh +++ b/ci/test-sanity.sh @@ -82,6 +82,22 @@ EOF fi ) +# Disallow adding new files under docs/ — docs now live in a separate repo +( + added_docs=$(git diff --diff-filter=AR --name-only "$target" -- 'docs/*') + if [ -n "$added_docs" ]; then + cat <&2 + +Error: new files added under docs/: +$added_docs + +Documentation has moved to https://github.com/anza-xyz/docs.anza.xyz +Please add your changes there instead. +EOF + exit 1 + fi +) + ./scripts/cargo-install-all.sh --dcou-check-only echo --- ok diff --git a/ci/test-xdp.sh b/ci/test-xdp.sh new file mode 100755 index 00000000000..2750a875a8e --- /dev/null +++ b/ci/test-xdp.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +cd "$(dirname "$0")/.." + +# The CI container runs as root for network namespace privileges. Keep +# root-owned Cargo artifacts out of the mounted checkout. +export CARGO_TARGET_DIR=/tmp/agave-xdp-target +cargo xtask xdp-test --release-with-debug diff --git a/ci/xtask/Cargo.lock b/ci/xtask/Cargo.lock index 27f27ee1b81..4e6dcd1937e 100644 --- a/ci/xtask/Cargo.lock +++ b/ci/xtask/Cargo.lock @@ -1473,9 +1473,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -1493,9 +1493,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "getrandom 0.3.4", diff --git a/ci/xtask/Cargo.toml b/ci/xtask/Cargo.toml index 55f72fb60a7..e948165411d 100644 --- a/ci/xtask/Cargo.toml +++ b/ci/xtask/Cargo.toml @@ -44,3 +44,15 @@ debug = "line-tables-only" [profile.full-dev] inherits = "dev" debug = "full" + +# Keep in sync with the root [workspace.lints.clippy]. +[lints.clippy] +# Denied lints +arithmetic_side_effects = "deny" +default_trait_access = "deny" +manual_let_else = "deny" +uninlined_format_args = "deny" +used_underscore_binding = "deny" + +# Allowed lints +new_without_default = "allow" diff --git a/ci/xtask/src/commands.rs b/ci/xtask/src/commands.rs index b3adf4fb07b..d8febbe2a50 100644 --- a/ci/xtask/src/commands.rs +++ b/ci/xtask/src/commands.rs @@ -1,3 +1,4 @@ pub mod channel_info; pub mod generate_pipeline; pub mod hello; +pub mod xdp_test; diff --git a/ci/xtask/src/commands/generate_pipeline.rs b/ci/xtask/src/commands/generate_pipeline.rs index 6a67716b6a4..aeca426ede0 100644 --- a/ci/xtask/src/commands/generate_pipeline.rs +++ b/ci/xtask/src/commands/generate_pipeline.rs @@ -143,6 +143,7 @@ fn generate_private_pipeline() -> Result { pipeline.add_step(default_local_cluster_step(10)); pipeline.add_step(default_docs_check_step()); pipeline.add_step(default_localnet_step()); + pipeline.add_step(default_xdp_test_step()); pipeline.add_step(buildkite::Step::Wait(buildkite::WaitStep {})); @@ -234,6 +235,7 @@ struct PullRequestPipelineFlags { stable_sbf: bool, shuttle: bool, coverage: bool, + xdp_tests: bool, } impl PullRequestPipelineFlags { @@ -340,6 +342,11 @@ impl PullRequestPipelineFlags { || file.ends_with("ci/test-coverage.sh") || file.starts_with("ci/coverage/") }), + xdp_tests: trigger_all + || rust_changed + || changed_files + .iter() + .any(|file| file.starts_with("xdp/") || file.ends_with("ci/test-xdp.sh")), } } } @@ -388,6 +395,9 @@ async fn generate_pull_request_pipeline( if flags.localnet { pipeline.add_step(default_localnet_step()); } + if flags.xdp_tests { + pipeline.add_step(default_xdp_test_step()); + } pipeline.add_step(buildkite::Step::Wait(buildkite::WaitStep {})); @@ -424,6 +434,7 @@ fn generate_full_pipeline() -> Result { pipeline.add_step(default_local_cluster_step(10)); pipeline.add_step(default_docs_check_step()); pipeline.add_step(default_localnet_step()); + pipeline.add_step(default_xdp_test_step()); pipeline.add_step(buildkite::Step::Wait(buildkite::WaitStep {})); @@ -661,6 +672,32 @@ fn default_localnet_step() -> buildkite::Step { }) } +fn default_xdp_test_step() -> buildkite::Step { + buildkite::Step::Command(buildkite::CommandStep { + name: String::from("xdp-test"), + command: String::from("ci/docker-run-default-image.sh ci/test-xdp.sh"), + agents: Some(HashMap::from([( + String::from("queue"), + String::from("default"), + )])), + timeout_in_minutes: Some(25), + env: Some(HashMap::from([ + ( + String::from("EXTRA_DOCKER_RUN_ARGS"), + String::from( + "--cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_ADMIN --security-opt \ + apparmor=unconfined", + ), + ), + ( + String::from("SOLANA_DOCKER_RUN_NOSETUID"), + String::from("1"), + ), + ])), + ..Default::default() + }) +} + fn default_stable_sbf_step() -> buildkite::Step { buildkite::Step::Command(buildkite::CommandStep { name: String::from("stable-sbf"), @@ -876,6 +913,7 @@ mod tests { assert!(!f.stable_sbf); assert!(!f.shuttle); assert!(!f.coverage); + assert!(!f.xdp_tests); } #[test] @@ -892,6 +930,7 @@ mod tests { assert!(f.stable_sbf); assert!(f.shuttle); assert!(f.coverage); + assert!(f.xdp_tests); } #[test] @@ -908,6 +947,20 @@ mod tests { assert!(f.stable_sbf); assert!(f.shuttle); assert!(f.coverage); + assert!(f.xdp_tests); + } + + #[test] + fn test_xdp_change_triggers_xdp_tests() { + let f = flags(&["xdp/tests/README.md"]); + assert!(f.xdp_tests); + } + + #[test] + fn test_test_xdp_sh_triggers_xdp_tests_and_shellcheck() { + let f = flags(&["ci/test-xdp.sh"]); + assert!(f.shellcheck); + assert!(f.xdp_tests); } #[test] @@ -925,6 +978,7 @@ mod tests { assert!(!f.stable_sbf); assert!(!f.shuttle); assert!(!f.coverage); + assert!(!f.xdp_tests); } #[test] @@ -942,5 +996,6 @@ mod tests { assert!(!f.stable_sbf); assert!(!f.shuttle); assert!(!f.coverage); + assert!(!f.xdp_tests); } } diff --git a/ci/xtask/src/commands/xdp_test.rs b/ci/xtask/src/commands/xdp_test.rs new file mode 100644 index 00000000000..d8af5c87daa --- /dev/null +++ b/ci/xtask/src/commands/xdp_test.rs @@ -0,0 +1,204 @@ +use { + anyhow::{Context, Result, bail, ensure}, + clap::Args, + log::info, + serde::Deserialize, + std::{ + collections::HashMap, + env, + ffi::OsString, + io::{self, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + }, +}; + +const DEFAULT_TESTS: &[&str] = &[ + "netlink_snapshot", + "route_monitor", + "router_snapshot", + "transmitter_smoke", +]; + +#[derive(Args)] +pub struct CommandArgs { + #[arg( + long, + help = "Build and run the tests with the release-with-debug profile" + )] + pub release_with_debug: bool, + + #[arg( + long, + help = "Optional command prefix used to run test executables with privileges, for \ + example: sudo -n -E" + )] + runner: Option, + + #[arg(long = "test", value_name = "TEST")] + tests: Vec, + + #[arg(last = true)] + run_args: Vec, +} + +pub fn run(args: CommandArgs) -> Result<()> { + let CommandArgs { + release_with_debug, + runner, + tests, + run_args, + } = args; + let repo_root = repo_root(); + + info!("building local xdp tests from {}", repo_root.display()); + if tests.is_empty() { + let executables = build_tests(&repo_root, DEFAULT_TESTS, release_with_debug)?; + test_executables(&repo_root, executables, runner, run_args) + } else { + let executables = build_tests(&repo_root, &tests, release_with_debug)?; + test_executables(&repo_root, executables, runner, run_args) + } +} + +fn build_tests<'a, S>( + repo_root: &Path, + tests: &'a [S], + release_with_debug: bool, +) -> Result> +where + S: AsRef, +{ + let mut cmd = Command::new(cargo_bin()); + cmd.current_dir(repo_root) + .arg("test") + .arg("-p") + .arg("agave-xdp") + .arg("--features") + .arg("agave-unstable-api") + .arg("--no-run") + .arg("--message-format=json-render-diagnostics") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if release_with_debug { + cmd.arg("--profile").arg("release-with-debug"); + } + for test in tests { + cmd.arg("--test").arg(test.as_ref()); + } + + let output = cmd.output().context("failed to build xdp tests")?; + io::stderr() + .write_all(&output.stderr) + .context("failed to write cargo stderr")?; + if !output.status.success() { + bail!("failed to build xdp tests with {}", output.status); + } + + test_executables_from_cargo_stdout(&output.stdout, tests) +} + +#[derive(Deserialize)] +struct CargoMessage { + reason: String, + target: Option, + executable: Option, +} + +#[derive(Deserialize)] +struct CargoTarget { + name: String, + kind: Vec, + test: bool, +} + +fn test_executables_from_cargo_stdout<'a, S>( + stdout: &[u8], + tests: &'a [S], +) -> Result> +where + S: AsRef, +{ + let mut executables = HashMap::new(); + let stdout = std::str::from_utf8(stdout).context("cargo output is not valid UTF-8")?; + for line in stdout.lines() { + let Ok(message) = serde_json::from_str::(line) else { + continue; + }; + if message.reason != "compiler-artifact" { + continue; + } + let Some(target) = message.target else { + continue; + }; + if !target.test || !target.kind.iter().any(|kind| kind == "test") { + continue; + } + let Some(executable) = message.executable else { + continue; + }; + executables.insert(target.name, executable); + } + + tests + .iter() + .map(|test| { + let executable = executables.remove(test.as_ref()).with_context(|| { + format!("cargo did not report executable for {}", test.as_ref()) + })?; + Ok((test, executable)) + }) + .collect() +} + +fn test_executables( + repo_root: &Path, + executables: I, + runner: Option, + run_args: Vec, +) -> Result<()> +where + I: IntoIterator, + S: AsRef, +{ + for (test, executable) in executables { + info!("running {} from {}", test.as_ref(), executable.display()); + let mut cmd = command_with_runner(runner.as_deref(), &executable)?; + cmd.current_dir(repo_root) + .arg("--include-ignored") + .arg("--test-threads=1"); + for arg in &run_args { + cmd.arg(arg); + } + let status = cmd + .status() + .with_context(|| format!("failed to run {}", test.as_ref()))?; + ensure!(status.success(), "{} failed with {status}", test.as_ref()); + } + + Ok(()) +} + +fn repo_root() -> PathBuf { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + root.canonicalize().unwrap_or(root) +} + +fn command_with_runner(runner: Option<&str>, program: &Path) -> Result { + let Some(runner) = runner else { + return Ok(Command::new(program)); + }; + let mut parts = runner.split_whitespace(); + let Some(runner_program) = parts.next() else { + bail!("runner cannot be empty"); + }; + let mut cmd = Command::new(runner_program); + cmd.args(parts).arg(program); + Ok(cmd) +} + +fn cargo_bin() -> PathBuf { + env::var_os("CARGO") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("cargo")) +} diff --git a/ci/xtask/src/main.rs b/ci/xtask/src/main.rs index dfa758628a0..2dfd6a1020a 100644 --- a/ci/xtask/src/main.rs +++ b/ci/xtask/src/main.rs @@ -30,6 +30,8 @@ enum Commands { GeneratePipeline(commands::generate_pipeline::CommandArgs), #[command(about = "Print release channel info")] ChannelInfo, + #[command(about = "Run XDP integration tests")] + XdpTest(commands::xdp_test::CommandArgs), } #[derive(Args, Debug)] @@ -83,6 +85,9 @@ async fn try_main(xtask: Xtask) -> Result<()> { Commands::ChannelInfo => { commands::channel_info::run().await?; } + Commands::XdpTest(args) => { + commands::xdp_test::run(args)?; + } } Ok(()) diff --git a/clap-utils/Cargo.toml b/clap-utils/Cargo.toml index 0c7b32c12b2..80bf76c39ef 100644 --- a/clap-utils/Cargo.toml +++ b/clap-utils/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_clap_utils" diff --git a/clap-v3-utils/Cargo.toml b/clap-v3-utils/Cargo.toml index efddfb47a0f..081bb0fffc5 100644 --- a/clap-v3-utils/Cargo.toml +++ b/clap-v3-utils/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_clap_v3_utils" diff --git a/cli-config/Cargo.toml b/cli-config/Cargo.toml index 59d223cf7a7..eff06275ece 100644 --- a/cli-config/Cargo.toml +++ b/cli-config/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/cli-output/Cargo.toml b/cli-output/Cargo.toml index 0e7f2324222..6a91b700510 100644 --- a/cli-output/Cargo.toml +++ b/cli-output/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/cli-output/src/cli_output.rs b/cli-output/src/cli_output.rs index d551bc314fa..1f7258927ab 100644 --- a/cli-output/src/cli_output.rs +++ b/cli-output/src/cli_output.rs @@ -46,7 +46,7 @@ use { EncodedConfirmedBlock, EncodedTransaction, TransactionConfirmationStatus, UiTransactionStatusMeta, }, - solana_transaction_status_client_types::UiTransactionError, + solana_transaction_status_client_types::{Rewards, UiTransactionError}, solana_vote_program::{ authorized_voters::AuthorizedVoters, vote_state::{ @@ -3106,35 +3106,33 @@ pub struct CliBlock { pub slot: Slot, } -impl QuietDisplay for CliBlock {} -impl VerboseDisplay for CliBlock {} - -impl fmt::Display for CliBlock { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - writeln!(f, "Slot: {}", self.slot)?; - writeln!( - f, - "Parent Slot: {}", - self.encoded_confirmed_block.parent_slot - )?; - writeln!(f, "Blockhash: {}", self.encoded_confirmed_block.blockhash)?; - writeln!( - f, - "Previous Blockhash: {}", - self.encoded_confirmed_block.previous_blockhash - )?; - if let Some(block_time) = self.encoded_confirmed_block.block_time { +impl CliBlock { + pub fn display_block_meta( + f: &mut fmt::Formatter, + slot: Slot, + parent_slot: Slot, + blockhash: &str, + previous_blockhash: &str, + block_time: Option, + block_height: Option, + rewards: &Rewards, + ) -> fmt::Result { + writeln!(f, "Slot: {slot}")?; + writeln!(f, "Parent Slot: {parent_slot}")?; + writeln!(f, "Blockhash: {blockhash}")?; + writeln!(f, "Previous Blockhash: {previous_blockhash}")?; + if let Some(block_time) = block_time { writeln!( f, "Block Time: {:?}", Local.timestamp_opt(block_time, 0).unwrap() )?; } - if let Some(block_height) = self.encoded_confirmed_block.block_height { + if let Some(block_height) = block_height { writeln!(f, "Block Height: {block_height:?}")?; } - if !self.encoded_confirmed_block.rewards.is_empty() { - let mut rewards = self.encoded_confirmed_block.rewards.clone(); + if !rewards.is_empty() { + let mut rewards = rewards.clone(); rewards.sort_by(|a, b| a.pubkey.cmp(&b.pubkey)); let mut total_rewards = 0; writeln!(f, "Rewards:")?; @@ -3189,6 +3187,26 @@ impl fmt::Display for CliBlock { build_balance_message(total_rewards.unsigned_abs(), false, false) )?; } + Ok(()) + } +} + +impl QuietDisplay for CliBlock {} +impl VerboseDisplay for CliBlock {} + +impl fmt::Display for CliBlock { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Self::display_block_meta( + f, + self.slot, + self.encoded_confirmed_block.parent_slot, + &self.encoded_confirmed_block.blockhash, + &self.encoded_confirmed_block.previous_blockhash, + self.encoded_confirmed_block.block_time, + self.encoded_confirmed_block.block_height, + &self.encoded_confirmed_block.rewards, + )?; + for (index, transaction_with_meta) in self.encoded_confirmed_block.transactions.iter().enumerate() { diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a88ea538e65..25484740e6a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -11,6 +11,11 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +# `all-features` can't be used: remote-wallet-hidraw and remote-wallet-libusb +# forward to hidapi's mutually-exclusive linux backends. Document the default +# (hidraw) backend plus the unstable API and dev-only utils. +features = ["agave-unstable-api", "dev-context-only-utils"] +rustdoc-args = ["--cfg=docsrs"] [[bin]] name = "solana" @@ -72,7 +77,7 @@ solana-packet = { workspace = true } solana-program-runtime = { workspace = true } solana-pubkey = { workspace = true } solana-pubsub-client = { workspace = true } -solana-remote-wallet = { workspace = true } +solana-remote-wallet = { workspace = true, features = ["keystone"] } solana-rent = { workspace = true } solana-rpc-client = { workspace = true, features = ["default"] } solana-rpc-client-api = { workspace = true } diff --git a/client/Cargo.toml b/client/Cargo.toml index 8aac0c7b200..0ac76cc726d 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/compute-budget-instruction/Cargo.toml b/compute-budget-instruction/Cargo.toml index 4fe0d3cc512..28cd3036f73 100644 --- a/compute-budget-instruction/Cargo.toml +++ b/compute-budget-instruction/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/compute-budget/Cargo.toml b/compute-budget/Cargo.toml index c1e2a982605..d3391b8d2e2 100644 --- a/compute-budget/Cargo.toml +++ b/compute-budget/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] conf-stack-frame-size = ["solana-program-runtime/conf-stack-frame-size"] diff --git a/connection-cache/Cargo.toml b/connection-cache/Cargo.toml index f31d7672953..2cf013b1dc8 100644 --- a/connection-cache/Cargo.toml +++ b/connection-cache/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/core/Cargo.toml b/core/Cargo.toml index 3a8b2524d70..5ad3bf4fef6 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/core/src/block_creation_loop.rs b/core/src/block_creation_loop.rs index b404986df48..b33c2970092 100644 --- a/core/src/block_creation_loop.rs +++ b/core/src/block_creation_loop.rs @@ -60,6 +60,17 @@ use { pub(crate) mod rewards; mod stats; +// Empirically derived value estimating the time to +// - drain and record the final batch of transactions, +// - produce the block footer, +// - produce the 'alpentick', +// - freeze the bank, +// - shred the final batches of the block, +// - broadcast. +// Recording stops this much before the slot timeout so block completion has time to finish before +// the leader window deadline. +const TIME_TO_COMPLETE_BLOCK_BROADCAST: Duration = Duration::from_millis(6); + /// Source of a leader-window notification consumed by BCL. enum ParentSource { /// Parent from ParentReady event for this leader window is already known. @@ -138,7 +149,6 @@ pub struct BlockCreationLoopConfig { struct LeaderContext { exit: Arc, - my_shred_version: u16, my_pubkey: Pubkey, /// Finalized leader-window notifications from Votor. leader_window_info_receiver: Receiver, @@ -273,7 +283,6 @@ fn start_loop(config: BlockCreationLoopConfig, reward_certs_requestor: CertsRequ let mut ctx = LeaderContext { exit, - my_shred_version: cluster_info.my_shred_version(), my_pubkey, highest_parent_ready, leader_window_info_receiver, @@ -424,6 +433,7 @@ fn reset_poh_recorder(bank: &Arc, ctx: &LeaderContext) { fn block_timeout(bank: &Bank, slot: Slot) -> Duration { Duration::from_nanos_u128(bank.ns_per_slot_at_slot(slot)) .saturating_mul((leader_slot_index(slot) as u32).saturating_add(1)) + .saturating_sub(TIME_TO_COMPLETE_BLOCK_BROADCAST) } /// Select the freshest leader-window notification within one source. @@ -763,9 +773,10 @@ fn record_and_complete_block( let RewardRespSucc { skip, notar, - validators: _, + validators, } = reward_certs; - let reward_cert = ValidatedRewardCert::try_new(&bank, ctx.my_shred_version, &skip, ¬ar)?; + let reward_cert = + ValidatedRewardCert::try_new_for_leader(bank.slot(), &skip, ¬ar, validators)?; let guard = ctx.highest_finalized.read().unwrap(); let footer = produce_block_footer(&bank, skip, notar, guard.as_ref()); let final_cert_input = guard.as_ref().map(|c| c.vote_rewards_input()); @@ -1321,9 +1332,8 @@ fn maybe_include_genesis_certificate( let bank = poh_recorder.bank().expect("Bank cannot have been cleared"); let processor = bank.block_component_processor.read().unwrap(); processor - .on_genesis_cert_block_marker( + .on_genesis_cert_block_marker_leader( bank.clone(), - ctx.my_shred_version, ctx.genesis_cert_block_marker.clone(), &ctx.bank_forks.read().unwrap().migration_status(), ) @@ -1347,11 +1357,9 @@ mod tests { crossbeam_channel::bounded, solana_bls_signatures::{BLS_SIGNATURE_AFFINE_SIZE, Signature as BLSSignature}, solana_entry::{block_component::VersionedUpdateParent, entry_or_marker::EntryOrMarker}, - solana_gossip::node::Node, solana_keypair::Keypair, solana_leader_schedule::{FixedSchedule, LeaderSchedule, SlotLeader}, solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path_auto_delete}, - solana_net_utils::SocketAddrSpace, solana_poh::{ poh_recorder::{PohRecorder, Record, WorkingBankEntryOrMarker}, record_channels::record_channels, @@ -1361,7 +1369,6 @@ mod tests { bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config_with_leader, installed_scheduler_pool::BankWithScheduler, }, - solana_signer::Signer, solana_system_transaction as system_transaction, std::num::NonZeroUsize, }; @@ -1398,20 +1405,6 @@ mod tests { Arc::new(leader_schedule_cache) } - fn test_cluster_info() -> (Pubkey, Arc) { - let keypair = Arc::new(Keypair::new()); - let my_pubkey = keypair.pubkey(); - let contact_info = Node::new_localhost_with_pubkey(&my_pubkey).info; - ( - my_pubkey, - Arc::new(ClusterInfo::new( - contact_info, - keypair, - SocketAddrSpace::Unspecified, - )), - ) - } - struct TestBankForksController { bank_forks: Arc>, } @@ -1530,7 +1523,7 @@ mod tests { fn test_abort_failed_working_bank() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); - let (my_pubkey, cluster_info) = test_cluster_info(); + let my_pubkey = Pubkey::new_unique(); let genesis = create_genesis_config_with_leader(10_000, &my_pubkey, 1_000); let root_bank = Bank::new_for_tests(&genesis.genesis_config); root_bank.freeze(); @@ -1562,7 +1555,6 @@ mod tests { let mut ctx = LeaderContext { exit, - my_shred_version: cluster_info.my_shred_version(), my_pubkey, leader_window_info_receiver, pending_parent_ready: None, @@ -1635,7 +1627,7 @@ mod tests { fn test_marker_send_clears_bank() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); - let (my_pubkey, cluster_info) = test_cluster_info(); + let my_pubkey = Pubkey::new_unique(); let genesis = create_genesis_config_with_leader(10_000, &my_pubkey, 1_000); let root_bank = Bank::new_for_tests(&genesis.genesis_config); root_bank.freeze(); @@ -1681,7 +1673,6 @@ mod tests { let mut ctx = LeaderContext { exit, - my_shred_version: cluster_info.my_shred_version(), my_pubkey, leader_window_info_receiver, pending_parent_ready: None, @@ -1725,7 +1716,7 @@ mod tests { fn test_moved_on_aborts() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); - let (my_pubkey, cluster_info) = test_cluster_info(); + let my_pubkey = Pubkey::new_unique(); let genesis = create_genesis_config_with_leader(10_000, &my_pubkey, 1_000); let root_bank = Bank::new_for_tests(&genesis.genesis_config); root_bank.freeze(); @@ -1757,7 +1748,6 @@ mod tests { let mut ctx = LeaderContext { exit, - my_shred_version: cluster_info.my_shred_version(), my_pubkey, leader_window_info_receiver, pending_parent_ready: None, @@ -1811,7 +1801,7 @@ mod tests { fn test_sad_leader_handover() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); - let (my_pubkey, cluster_info) = test_cluster_info(); + let my_pubkey = Pubkey::new_unique(); let genesis = create_genesis_config_with_leader(10_000, &my_pubkey, 1_000); let root_bank = Bank::new_for_tests(&genesis.genesis_config); root_bank.set_block_id(Some(Hash::new_unique())); @@ -1867,7 +1857,6 @@ mod tests { let mut ctx = LeaderContext { exit, - my_shred_version: cluster_info.my_shred_version(), my_pubkey, leader_window_info_receiver, pending_parent_ready: None, diff --git a/core/src/block_creation_loop/rewards/certs_builder/entry.rs b/core/src/block_creation_loop/rewards/certs_builder/entry.rs index aebeeb173a5..aa891a17817 100644 --- a/core/src/block_creation_loop/rewards/certs_builder/entry.rs +++ b/core/src/block_creation_loop/rewards/certs_builder/entry.rs @@ -149,7 +149,7 @@ mod tests { keypairs: &[BlsKeypair], shred_version: u16, ) -> VoteMessage { - let serialized = get_vote_payload_to_sign(&vote, shred_version); + let serialized = get_vote_payload_to_sign(vote, shred_version); let signature = keypairs[rank].sign(&serialized).into(); VoteMessage { vote, diff --git a/core/src/block_creation_loop/rewards/certs_builder/entry/partial_cert.rs b/core/src/block_creation_loop/rewards/certs_builder/entry/partial_cert.rs index e89c50581e6..0d4a055fd86 100644 --- a/core/src/block_creation_loop/rewards/certs_builder/entry/partial_cert.rs +++ b/core/src/block_creation_loop/rewards/certs_builder/entry/partial_cert.rs @@ -69,7 +69,7 @@ impl PartialCert { .ok_or(AddVoteError::InvalidRank)?; self.signature.aggregate_with(std::iter::once(signature))?; self.validators.push(entry.vote_account_pubkey); - self.stake = self.stake.saturating_add(entry.stake); + self.stake = self.stake.saturating_add(entry.stake.get()); *ind = true; } } @@ -106,13 +106,15 @@ mod tests { crate::block_creation_loop::rewards::certs_builder::entry::tests::{ get_rank_map_keypairs, new_vote, validate_bitmap, }, - agave_votor_messages::{consensus_message::VoteMessage, vote::Vote}, + agave_votor_messages::{ + consensus_message::VoteMessage, vote::Vote, wire::get_vote_payload_to_sign, + }, rand::Rng, solana_bls_signatures::Keypair as BlsKeypair, }; fn new_invalid_vote(vote: Vote, rank: usize) -> VoteMessage { - let serialized = wincode::serialize(&vote).unwrap(); + let serialized = get_vote_payload_to_sign(vote, 0); let keypair = BlsKeypair::new(); let signature = keypair.sign(&serialized).into(); VoteMessage { diff --git a/core/src/consensus.rs b/core/src/consensus.rs index 670353faced..1acd4753544 100644 --- a/core/src/consensus.rs +++ b/core/src/consensus.rs @@ -498,14 +498,14 @@ impl Tower { vote_slots.insert(slot); } - if start_root != vote_state.root_slot { - if let Some(root) = start_root { - // The account's prior root can be older than this fork's root; clamp to - // the same range for the same reason as above. - if root > root_slot { - trace!("ROOT: {root}"); - vote_slots.insert(root); - } + if start_root != vote_state.root_slot + && let Some(root) = start_root + { + // The account's prior root can be older than this fork's root; clamp to + // the same range for the same reason as above. + if root > root_slot { + trace!("ROOT: {root}"); + vote_slots.insert(root); } } if let Some(root) = vote_state.root_slot { @@ -703,15 +703,15 @@ impl Tower { vote_hash: Hash, block_id: Hash, ) -> Option { - if let Some(last_voted_slot) = self.vote_state.last_voted_slot() { - if vote_slot <= last_voted_slot { - panic!( - "Error while recording vote {} {} in local tower {:?}", - vote_slot, - vote_hash, - VoteError::VoteTooOld - ); - } + if let Some(last_voted_slot) = self.vote_state.last_voted_slot() + && vote_slot <= last_voted_slot + { + panic!( + "Error while recording vote {} {} in local tower {:?}", + vote_slot, + vote_hash, + VoteError::VoteTooOld + ); } trace!("{} record_vote for {}", self.node_pubkey, vote_slot); @@ -807,10 +807,10 @@ impl Tower { if slot <= last_voted_slot { return false; } - } else if let Some(root) = self.vote_state.root_slot { - if slot <= root { - return false; - } + } else if let Some(root) = self.vote_state.root_slot + && slot <= root + { + return false; } true } @@ -841,15 +841,15 @@ impl Tower { } } - if let Some(root_slot) = vote_state.root_slot { - if slot != root_slot { - // This case should never happen because bank forks purges all - // non-descendants of the root every time root is set - assert!( - ancestors.contains(&root_slot), - "ancestors: {ancestors:?}, slot: {slot} root: {root_slot}" - ); - } + if let Some(root_slot) = vote_state.root_slot + && slot != root_slot + { + // This case should never happen because bank forks purges all + // non-descendants of the root every time root is set + assert!( + ancestors.contains(&root_slot), + "ancestors: {ancestors:?}, slot: {slot} root: {root_slot}" + ); } false @@ -1061,9 +1061,11 @@ impl Tower { .unwrap_or(true) { // Our last vote slot was purged because it was on a duplicate fork, don't continue below - // where checks may panic. We allow a freebie vote here that may violate switching - // thresholds - // TODO: Properly handle this case + // where checks may panic. We allow a freebie vote here without checking the switch + // threshold as it is trivially satisfied: + // - Freebie can only occur because our last vote block was dumped & repaired + // - Dump & repair only triggers due to another version reaching duplicate confirmation (52%) + // - 52% > 38% so the switching threshold is implicitely satisifed info!( "Allowing switch vote on {:?} because last vote {:?} was rolled back", (switch_slot, switch_hash), @@ -1998,7 +2000,7 @@ pub mod test { // Fill the BankForks according to the above fork structure vote_simulator.fill_bank_forks(forks, &HashMap::new(), true); - for (_, fork_progress) in vote_simulator.progress.iter_mut() { + for fork_progress in vote_simulator.progress.values_mut() { fork_progress.fork_stats.computed = true; } @@ -3093,7 +3095,7 @@ pub mod test { // Fill the BankForks according to the above fork structure vote_simulator.fill_bank_forks(forks, &HashMap::new(), true); - for (_, fork_progress) in vote_simulator.progress.iter_mut() { + for fork_progress in vote_simulator.progress.values_mut() { fork_progress.fork_stats.computed = true; } @@ -3182,7 +3184,7 @@ pub mod test { // Fill the BankForks according to the above fork structure vote_simulator.fill_bank_forks(forks, &HashMap::new(), true); - for (_, fork_progress) in vote_simulator.progress.iter_mut() { + for fork_progress in vote_simulator.progress.values_mut() { fork_progress.fork_stats.computed = true; } @@ -3820,7 +3822,7 @@ pub mod test { // Fill the BankForks according to the above fork structure vote_simulator.fill_bank_forks(forks, &HashMap::new(), true); - for (_, fork_progress) in vote_simulator.progress.iter_mut() { + for fork_progress in vote_simulator.progress.values_mut() { fork_progress.fork_stats.computed = true; } diff --git a/core/src/consensus/heaviest_subtree_fork_choice.rs b/core/src/consensus/heaviest_subtree_fork_choice.rs index 641597a74cd..ae04153cb0c 100644 --- a/core/src/consensus/heaviest_subtree_fork_choice.rs +++ b/core/src/consensus/heaviest_subtree_fork_choice.rs @@ -141,15 +141,15 @@ impl ForkInfo { my_key: &SlotHashKey, newly_valid_ancestor: Slot, ) { - if let Some(latest_invalid_ancestor) = self.latest_invalid_ancestor { - if latest_invalid_ancestor <= newly_valid_ancestor { - info!( - "Fork choice for {my_key:?} clearing latest invalid ancestor \ - {latest_invalid_ancestor:?} because {newly_valid_ancestor:?} was duplicate \ - confirmed" - ); - self.latest_invalid_ancestor = None; - } + if let Some(latest_invalid_ancestor) = self.latest_invalid_ancestor + && latest_invalid_ancestor <= newly_valid_ancestor + { + info!( + "Fork choice for {my_key:?} clearing latest invalid ancestor \ + {latest_invalid_ancestor:?} because {newly_valid_ancestor:?} was duplicate \ + confirmed" + ); + self.latest_invalid_ancestor = None; } } diff --git a/core/src/repair/cluster_slot_state_verifier.rs b/core/src/repair/cluster_slot_state_verifier.rs index 6e7764b32ed..63f8ebcd751 100644 --- a/core/src/repair/cluster_slot_state_verifier.rs +++ b/core/src/repair/cluster_slot_state_verifier.rs @@ -640,17 +640,14 @@ fn on_epoch_slots_frozen( // // Thus if we have a duplicate confirmation, but `slot` is pruned, we continue // processing it as `epoch_slots_frozen`. - if !is_popular_pruned { - if let Some(duplicate_confirmed_hash) = duplicate_confirmed_hash { - if epoch_slots_frozen_hash != duplicate_confirmed_hash { - warn!( - "EpochSlots sample returned slot {slot} with hash {epoch_slots_frozen_hash}, \ - but we already saw duplicate confirmation on hash: \ - {duplicate_confirmed_hash:?}", - ); - } - return vec![]; + if !is_popular_pruned && let Some(duplicate_confirmed_hash) = duplicate_confirmed_hash { + if epoch_slots_frozen_hash != duplicate_confirmed_hash { + warn!( + "EpochSlots sample returned slot {slot} with hash {epoch_slots_frozen_hash}, but \ + we already saw duplicate confirmation on hash: {duplicate_confirmed_hash:?}", + ); } + return vec![]; } match bank_status { @@ -900,10 +897,10 @@ pub(crate) fn check_slot_agrees_with_cluster( // Avoid duplicate work from multiple of the same DuplicateConfirmed signal. This can // happen if we get duplicate confirmed from gossip and from local replay. if let SlotStateUpdate::DuplicateConfirmed(state) = &slot_state_update { - if let Some(bank_hash) = state.bank_status.bank_hash() { - if let Some(true) = fork_choice.is_duplicate_confirmed(&(slot, bank_hash)) { - return; - } + if let Some(bank_hash) = state.bank_status.bank_hash() + && let Some(true) = fork_choice.is_duplicate_confirmed(&(slot, bank_hash)) + { + return; } datapoint_info!( @@ -926,15 +923,13 @@ pub(crate) fn check_slot_agrees_with_cluster( ); } - if let SlotStateUpdate::EpochSlotsFrozen(epoch_slots_frozen_state) = &slot_state_update { - if let Some(old_epoch_slots_frozen_hash) = + if let SlotStateUpdate::EpochSlotsFrozen(epoch_slots_frozen_state) = &slot_state_update + && let Some(old_epoch_slots_frozen_hash) = epoch_slots_frozen_slots.insert(slot, epoch_slots_frozen_state.epoch_slots_frozen_hash) - { - if old_epoch_slots_frozen_hash == epoch_slots_frozen_state.epoch_slots_frozen_hash { - // If EpochSlots has already told us this same hash was frozen, return - return; - } - } + && old_epoch_slots_frozen_hash == epoch_slots_frozen_state.epoch_slots_frozen_hash + { + // If EpochSlots has already told us this same hash was frozen, return + return; } let state_changes = slot_state_update.into_state_changes(slot); diff --git a/core/src/repair/duplicate_repair_status.rs b/core/src/repair/duplicate_repair_status.rs index 5eff267d4a3..60ce6450c96 100644 --- a/core/src/repair/duplicate_repair_status.rs +++ b/core/src/repair/duplicate_repair_status.rs @@ -206,18 +206,15 @@ impl AncestorRequestStatus { response_slot_hashes: Vec<(Slot, Hash)>, blockstore: &Blockstore, ) -> Option { - if let Some(did_get_response) = self.sampled_validators.get_mut(from_addr) { - if *did_get_response { - // If we've already received a response from this validator, return. - return None; - } - // Mark we got a response from this validator already - *did_get_response = true; - self.num_responses += 1; - } else { - // If this is not a response from one of the sampled validators, return. + // If this is not a response from one of the sampled validators, return. + let did_get_response = self.sampled_validators.get_mut(from_addr)?; + if *did_get_response { + // If we've already received a response from this validator, return. return None; } + // Mark we got a response from this validator already + *did_get_response = true; + self.num_responses += 1; let validators_with_same_response = self .ancestor_request_responses diff --git a/core/src/repair/malicious_repair_handler.rs b/core/src/repair/malicious_repair_handler.rs index 27a225ed881..d0b878f739a 100644 --- a/core/src/repair/malicious_repair_handler.rs +++ b/core/src/repair/malicious_repair_handler.rs @@ -56,10 +56,10 @@ impl MaliciousRepairHandler { /// Check if we should respond maliciously for this slot and shred index fn should_respond_maliciously(&self, slot: Slot, shred_index: u64) -> bool { - if let Some((start, end)) = self.config.slot_range { - if slot < start || slot > end { - return false; - } + if let Some((start, end)) = self.config.slot_range + && (slot < start || slot > end) + { + return false; } let slot_matches = self @@ -162,16 +162,14 @@ impl RepairHandler for MaliciousRepairHandler { // Parse the original shred to get its metadata if let Ok(original_shred) = Shred::new_from_serialized_shred(original_shred_bytes.clone()) - { - if let Some(equivocating_shred) = + && let Some(equivocating_shred) = self.generate_equivocating_shred(&original_shred, shred_index) - { - info!( - "Responding with equivocating shred in slot {slot} index {shred_index} to \ - {dest}" - ); - return repair_response_packet_from_bytes(equivocating_shred, dest, nonce); - } + { + info!( + "Responding with equivocating shred in slot {slot} index {shred_index} to \ + {dest}" + ); + return repair_response_packet_from_bytes(equivocating_shred, dest, nonce); } } diff --git a/core/src/repair/repair_generic_traversal.rs b/core/src/repair/repair_generic_traversal.rs index c8e0eb6afe6..a34266facb4 100644 --- a/core/src/repair/repair_generic_traversal.rs +++ b/core/src/repair/repair_generic_traversal.rs @@ -66,17 +66,17 @@ pub fn get_unknown_last_index( let slot_meta = slot_meta_cache .entry(slot) .or_insert_with(|| blockstore.meta_repair(slot).unwrap()); - if let Some(slot_meta) = slot_meta { - if slot_meta.last_index.is_none() { - let shred_index = blockstore.get_index(slot).unwrap(); - let num_processed_shreds = if let Some(shred_index) = shred_index { - shred_index.data().num_shreds() as u64 - } else { - slot_meta.consumed - }; - unknown_last.push((slot, slot_meta.received, num_processed_shreds)); - processed_slots.insert(slot); - } + if let Some(slot_meta) = slot_meta + && slot_meta.last_index.is_none() + { + let shred_index = blockstore.get_index(slot).unwrap(); + let num_processed_shreds = if let Some(shred_index) = shred_index { + shred_index.data().num_shreds() as u64 + } else { + slot_meta.consumed + }; + unknown_last.push((slot, slot_meta.received, num_processed_shreds)); + processed_slots.insert(slot); } } // Prioritize slots with more data shreds currently present in blockstore. @@ -107,12 +107,12 @@ fn get_unrepaired_path( let slot_meta = slot_meta_cache .entry(slot) .or_insert_with(|| blockstore.meta_repair(slot).unwrap()); - if let Some(slot_meta) = slot_meta { - if !slot_meta.is_full() { - path.push(slot); - if let Some(parent_slot) = slot_meta.parent_slot { - slot = parent_slot - } + if let Some(slot_meta) = slot_meta + && !slot_meta.is_full() + { + path.push(slot); + if let Some(parent_slot) = slot_meta.parent_slot { + slot = parent_slot } } } diff --git a/core/src/repair/repair_weight.rs b/core/src/repair/repair_weight.rs index be1f44ccce0..5af565f7b72 100644 --- a/core/src/repair/repair_weight.rs +++ b/core/src/repair/repair_weight.rs @@ -563,17 +563,16 @@ impl RepairWeight { epoch_stakes, epoch_schedule, ); - if let Some(new_orphan_root) = new_orphan_root { - if new_orphan_root != self.root { - if let Some(repair_request) = RepairService::request_repair_if_needed( - outstanding_repairs, - ShredRepairType::Orphan(new_orphan_root), - ) { - repairs.push(repair_request); - processed_slots.insert(new_orphan_root); - new_best_orphan_requests += 1; - } - } + if let Some(new_orphan_root) = new_orphan_root + && new_orphan_root != self.root + && let Some(repair_request) = RepairService::request_repair_if_needed( + outstanding_repairs, + ShredRepairType::Orphan(new_orphan_root), + ) + { + repairs.push(repair_request); + processed_slots.insert(new_orphan_root); + new_best_orphan_requests += 1; } } } @@ -606,7 +605,7 @@ impl RepairWeight { outstanding_repairs: &mut HashMap, ) -> Vec { let mut repairs = Vec::default(); - for (_slot, tree) in self.trees.iter() { + for tree in self.trees.values() { if repairs.len() >= max_new_repairs { break; } @@ -638,7 +637,7 @@ impl RepairWeight { ) -> (Vec, /* processed slots */ usize) { let mut repairs = Vec::default(); let mut total_processed_slots = 0; - for (_slot, tree) in self.trees.iter() { + for tree in self.trees.values() { if repairs.len() >= max_new_repairs { break; } diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 9943f0ebac2..9c5dfd2b59b 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -93,7 +93,6 @@ use { vote_sender_types::{ReplayVoteMessage, ReplayVoteSender}, }, solana_signer::Signer, - solana_svm_timings::ExecuteTimings, solana_time_utils::timestamp, solana_transaction::Transaction, solana_vote::vote_transaction::VoteTransaction, @@ -1605,8 +1604,11 @@ impl ReplayStage { let bank_forks_r = bank_forks.read().unwrap(); new_frozen_slots .iter() - .filter(|slot| migration_status.should_allow_fast_leader_handover(**slot)) .filter_map(|slot| bank_forks_r.get(*slot)) + .filter(|bank| { + bank.feature_set.snapshot().alpenglow_fast_leader_handover + && migration_status.should_allow_block_markers(bank.slot()) + }) .collect_vec() }; for bank in flh_candidate_banks { @@ -2656,6 +2658,8 @@ impl ReplayStage { } else if let Some(prev_hash) = duplicate_confirmed_slots.insert(confirmed_slot, duplicate_confirmed_hash) { + // This assertion is intentional - it is not possible to split the cluster to get 52% on two versions + // without a massive turbine failure assert_eq!( prev_hash, duplicate_confirmed_hash, "Additional duplicate confirmed notification for slot {confirmed_slot} \ @@ -3817,8 +3821,6 @@ impl ReplayStage { let bank_forks = &process_active_banks_context.bank_forks; // TODO: See if processing of blockstore replay results and bank completion can be made thread safe. - let mut tx_count = 0; - let mut execute_timings = ExecuteTimings::default(); let mut new_frozen_slots = vec![]; for replay_result in replay_result_vec { if replay_result.is_slot_dead { @@ -3832,7 +3834,7 @@ impl ReplayStage { }; if let Some(replay_result) = &replay_result.replay_result { match replay_result { - Ok(replay_tx_count) => tx_count += replay_tx_count, + Ok(_) => {} Err(BlockstoreProcessorError::BlockComponentProcessor( BlockComponentProcessorError::AbandonedBank(update_parent), )) => { @@ -4202,7 +4204,6 @@ impl ReplayStage { bank_complete_time.as_us(), is_unified_scheduler_enabled, ); - execute_timings.accumulate(&r_replay_stats.batch_execute.totals); } else { trace!( "bank {} not completed tick_height: {}, max_tick_height: {}", diff --git a/core/src/replay_stage/dead_slots.rs b/core/src/replay_stage/dead_slots.rs index a33f98ec875..f9c2a79d181 100644 --- a/core/src/replay_stage/dead_slots.rs +++ b/core/src/replay_stage/dead_slots.rs @@ -146,7 +146,8 @@ fn should_mark_soft_dead( migration_status: &MigrationStatus, ) -> bool { let slot = bank.slot(); - if !migration_status.should_allow_fast_leader_handover(slot) + if !bank.feature_set.snapshot().alpenglow_fast_leader_handover + || !migration_status.should_allow_block_markers(slot) || blockstore.is_dead(slot) || !is_update_parent_recoverable_replay_error(err) { diff --git a/core/src/replay_stage/tests.rs b/core/src/replay_stage/tests.rs index 210994cd977..047e227482a 100644 --- a/core/src/replay_stage/tests.rs +++ b/core/src/replay_stage/tests.rs @@ -65,6 +65,7 @@ use { solana_sha256_hasher::hash, solana_shred_version::compute_shred_version, solana_signature::Signature, + solana_svm_timings::ExecuteTimings, solana_system_transaction as system_transaction, solana_tpu_client::tpu_client::{DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_VOTE_USE_QUIC}, solana_transaction_error::TransactionError, diff --git a/core/src/replay_stage/update_parent.rs b/core/src/replay_stage/update_parent.rs index 5580914e1c3..fcdde3e4800 100644 --- a/core/src/replay_stage/update_parent.rs +++ b/core/src/replay_stage/update_parent.rs @@ -113,7 +113,7 @@ fn try_restart_slot_from_update_parent( migration_status: &MigrationStatus, source: &str, ) -> bool { - if !migration_status.should_allow_fast_leader_handover(slot) { + if !migration_status.should_allow_block_markers(slot) { return false; } if blockstore.is_dead(slot) { @@ -135,10 +135,10 @@ fn try_restart_slot_from_update_parent( if bank.is_none() && !progress.contains_key(&slot) { return false; } - if bank - .as_ref() - .is_some_and(|bank| ReplayStage::leader_is_me(bank.leader_id(), my_pubkey)) - { + if bank.as_ref().is_some_and(|bank| { + ReplayStage::leader_is_me(bank.leader_id(), my_pubkey) + || !bank.feature_set.snapshot().alpenglow_fast_leader_handover + }) { return false; } if progress.get(&slot).is_some_and(|progress| { diff --git a/core/src/tpu_entry_notifier.rs b/core/src/tpu_entry_notifier.rs index 6e37c6e716f..d95f799d025 100644 --- a/core/src/tpu_entry_notifier.rs +++ b/core/src/tpu_entry_notifier.rs @@ -64,15 +64,12 @@ impl TpuEntryNotifier { let (bank, (entry_or_marker, tick_height)) = entry_receiver.recv_timeout(Duration::from_secs(1))?; let slot = bank.slot(); - let index = if slot != *current_slot { + if slot != *current_slot { *current_index = 0; *current_transaction_index = 0; *current_slot = slot; - 0 - } else { - *current_index += 1; - *current_index }; + let index = *current_index; if let EntryOrMarker::Entry(ref entry) = entry_or_marker { let entry_summary = EntrySummary { @@ -91,11 +88,11 @@ impl TpuEntryNotifier { EntryNotifierService, error {err:?}", ); } + *current_index += 1; *current_transaction_index += entry.transactions.len(); }; if let Err(err) = broadcast_entry_sender.send((bank, (entry_or_marker, tick_height))) { - let index = *current_index; warn!( "Failed to send slot {slot:?} entry/marker {index:?} from Tpu to BroadcastStage, \ error {err:?}", diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 71c6d338877..684e802f113 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -29,7 +29,6 @@ use { agave_bls_sigverify::{ bls_sigverifier::{self, SigVerifierChannels, SigVerifierContext}, generated_cert_types::GeneratedCertTypes, - sig_verified_messages::SigVerifiedBatch, }, agave_votor::{ event::{LatestSwitchRequest, LeaderWindowInfo, VotorEventReceiver, VotorEventSender}, @@ -110,13 +109,6 @@ pub(crate) const MAX_ALPENGLOW_PACKET_NUM: usize = 10_000; /// of votes / certificate need to be refreshed. const MAX_BLS_MESSAGES_TO_SEND: usize = 1000; -enum BlsSigVerifyThreadsOrChannel { - /// Alpenglow is active so handlers to the threads related to the bls sigverify. - Threads(JoinHandle<()>, JoinHandle<()>), - /// Alpenglow is not active so hold on to the send side to prevent the channel from disconnecting. - Channel { _sender: Sender }, -} - pub struct Tvu { fetch_stage: ShredFetchStage, shred_sigverify: JoinHandle<()>, @@ -131,7 +123,7 @@ pub struct Tvu { warm_quic_cache_service: Option, drop_bank_service: DropBankService, duplicate_shred_listener: DuplicateShredListener, - bls_sigverify_threads_or_channel: BlsSigVerifyThreadsOrChannel, + bls_sigverify_threads: (JoinHandle<()>, JoinHandle<()>), votor: Votor, commitment_service: AggregateCommitmentService, } @@ -141,7 +133,7 @@ pub struct TvuSockets { pub repair: UdpSocket, pub retransmit: Vec, pub ancestor_hashes_requests: UdpSocket, - pub alpenglow: Option, + pub alpenglow: UdpSocket, pub block_id_repair: UdpSocket, } @@ -295,9 +287,7 @@ impl Tvu { bounded(MAX_IN_FLIGHT_CONSENSUS_EVENTS); let generated_cert_types = Arc::new(GeneratedCertTypes::default()); - // The BLS socket is currently only available on Testnet and Development clusters. - // Closer to release we will enable this for all clusters. - let bls_sigverify_threads_or_channel = if let Some(bls_socket) = bls_socket { + let bls_sigverify_threads = { let (bls_packet_sender, bls_packet_receiver) = bounded(MAX_ALPENGLOW_PACKET_NUM); let ( @@ -357,11 +347,7 @@ impl Tvu { let mut key_notifiers = key_notifiers.write().unwrap(); key_notifiers.add(KeyUpdaterType::Bls, bls_key_updater); - BlsSigVerifyThreadsOrChannel::Threads(bls_streamer_t, bls_sigverifier_t) - } else { - BlsSigVerifyThreadsOrChannel::Channel { - _sender: consensus_message_sender, - } + (bls_streamer_t, bls_sigverifier_t) }; let (fetch_sender, fetch_receiver) = EvictingSender::new_bounded(SHRED_FETCH_CHANNEL_SIZE); @@ -670,7 +656,7 @@ impl Tvu { warm_quic_cache_service, drop_bank_service, duplicate_shred_listener, - bls_sigverify_threads_or_channel, + bls_sigverify_threads, votor, commitment_service, }) @@ -692,12 +678,9 @@ impl Tvu { } self.drop_bank_service.join()?; self.duplicate_shred_listener.join()?; - if let BlsSigVerifyThreadsOrChannel::Threads(streamer, sigverifier) = - self.bls_sigverify_threads_or_channel - { - streamer.join()?; - sigverifier.join()?; - } + let (streamer, sigverifier) = self.bls_sigverify_threads; + streamer.join()?; + sigverifier.join()?; self.votor.join()?; self.commitment_service.join()?; Ok(()) @@ -867,7 +850,7 @@ pub mod tests { retransmit: target1.sockets.retransmit_sockets, fetch: target1.sockets.tvu, ancestor_hashes_requests: target1.sockets.ancestor_hashes_requests, - alpenglow: Some(target1.sockets.alpenglow), + alpenglow: target1.sockets.alpenglow, block_id_repair: target1.sockets.block_id_repair, }, blockstore, diff --git a/core/src/validator.rs b/core/src/validator.rs index 7aade94c2f5..bb53ae5fb0f 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -51,7 +51,6 @@ use { }, solana_client::connection_cache::{ConnectionCache, Protocol}, solana_clock::Slot, - solana_cluster_type::ClusterType, solana_entry::poh::compute_hash_time, solana_epoch_schedule::MAX_LEADER_SCHEDULE_EPOCH_OFFSET, solana_genesis_config::GenesisConfig, @@ -145,7 +144,7 @@ use { }, solana_time_utils::timestamp, solana_tpu_client::tpu_client::{DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_VOTE_USE_QUIC}, - solana_turbine::{self, XdpSender as TurbineXdpSender, broadcast_stage::BroadcastStageType}, + solana_turbine::{self, broadcast_stage::BroadcastStageType}, solana_unified_scheduler_pool::DefaultSchedulerPool, solana_validator_exit::Exit, solana_vote_program::vote_state::{VoteStateV4, handler::VoteStateHandler}, @@ -1427,10 +1426,60 @@ impl Validator { let epoch_specs: Box = Box::new(crate::epoch_specs::EpochSpecs::from(bank_forks.clone())); + let ( + xdp_transmitter, + turbine_xdp_sender, + quic_xdp_sender, + repair_xdp_sender, + gossip_xdp_sender, + ) = if let Some(XdpTransmitSetup { + transmitter_builder, + src_ip, + }) = xdp_transmit_setup + { + let turbine_src_port = node.sockets.retransmit_sockets[0] + .local_addr() + .expect("retransmit socket should have local address") + .port(); + + let repair_src_port = node + .sockets + .repair + .local_addr() + .expect("repair socket should have local address") + .port(); + + let gossip_src_port = node.sockets.gossip[0] + .local_addr() + .expect("gossip socket should have local address") + .port(); + + let (transmitter, sender) = transmitter_builder.build(); + ( + Some(transmitter), + Some(PinnedXdpSender::new( + sender.clone(), + SocketAddrV4::new(src_ip, turbine_src_port), + )), + Some((sender.clone(), src_ip)), + Some(PinnedXdpSender::new( + sender.clone(), + SocketAddrV4::new(src_ip, repair_src_port), + )), + Some(PinnedXdpSender::new( + sender, + SocketAddrV4::new(src_ip, gossip_src_port), + )), + ) + } else { + (None, None, None, None, None) + }; + let gossip_service = GossipService::new( &cluster_info, Some(epoch_specs), node.sockets.gossip.clone(), + gossip_xdp_sender, config.gossip_validators.clone(), config.should_check_duplicate_instance, Some(stats_reporter_sender.clone()), @@ -1567,49 +1616,6 @@ impl Validator { // This channel backing up indicates a serious problem in votor let (votor_event_sender, votor_event_receiver) = bounded(1000); - let (xdp_transmitter, turbine_xdp_sender, quic_xdp_sender, repair_xdp_sender) = - if let Some(XdpTransmitSetup { - transmitter_builder, - src_ip, - }) = xdp_transmit_setup - { - let turbine_src_port = node.sockets.retransmit_sockets[0] - .local_addr() - .expect("retransmit socket should have local address") - .port(); - let repair_src_port = node - .sockets - .repair - .local_addr() - .expect("repair socket should have local address") - .port(); - - let (transmitter, sender) = transmitter_builder.build(); - ( - Some(transmitter), - Some(TurbineXdpSender::new( - sender.clone(), - SocketAddrV4::new(src_ip, turbine_src_port), - )), - Some((sender.clone(), src_ip)), - Some(PinnedXdpSender::new( - sender, - SocketAddrV4::new(src_ip, repair_src_port), - )), - ) - } else { - (None, None, None, None) - }; - - // disable Alpenglow votor networking if not allowed for cluster type - let alpenglow_socket = if genesis_config.cluster_type == ClusterType::Testnet - || genesis_config.cluster_type == ClusterType::Development - { - Some(node.sockets.alpenglow) - } else { - None - }; - let tvu = Tvu::new( vote_account, authorized_voter_keypairs, @@ -1620,7 +1626,7 @@ impl Validator { retransmit: node.sockets.retransmit_sockets, fetch: node.sockets.tvu, ancestor_hashes_requests: node.sockets.ancestor_hashes_requests, - alpenglow: alpenglow_socket, + alpenglow: node.sockets.alpenglow, block_id_repair: node.sockets.block_id_repair, }, blockstore.clone(), @@ -1919,6 +1925,10 @@ impl Validator { "local retransmit address: {}", node.sockets.retransmit_sockets[0].local_addr().unwrap() ); + info!( + "local alpenglow address: {}", + node.sockets.alpenglow.local_addr().unwrap() + ); } pub fn join(self) { diff --git a/core/src/window_service.rs b/core/src/window_service.rs index 3d6367e1126..389fd326c49 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -136,12 +136,22 @@ fn run_check_duplicate( shred_slot, &root_bank, ); + let no_verify_chained_merkle_root = shred::filter::check_feature_activation_from_bank( + &feature_set::alpenglow::id(), + shred_slot, + &root_bank, + ); let (shred1, shred2) = match shred { PossibleDuplicateShred::LastIndexConflict(shred, conflict) | PossibleDuplicateShred::ErasureConflict(shred, conflict) | PossibleDuplicateShred::MerkleRootConflict(shred, conflict) => (shred, conflict), PossibleDuplicateShred::ChainedMerkleRootConflict(_slot) => { + if no_verify_chained_merkle_root { + // If we're in the full alpenglow epoch, we stop validating the chained merkle root. + // In Alpenglow we only use the double merkle root + return Ok(()); + } if validate_chained_block_id || validate_chained_block_id_2 { // Although chained merkle roots are not necessary for agave duplicate resolution protocols, // We still need to mark the block as dead for other client teams. @@ -150,6 +160,11 @@ fn run_check_duplicate( return Ok(()); } PossibleDuplicateShred::FixedFECChainedMerkleRootConflict(_slot) => { + if no_verify_chained_merkle_root { + // If we're in the full alpenglow epoch, we stop validating the chained merkle root. + // In Alpenglow we only use the double merkle root + return Ok(()); + } if validate_chained_block_id_2 { blockstore.set_dead_slot(shred_slot)?; } diff --git a/cost-model/Cargo.toml b/cost-model/Cargo.toml index 29d85f78828..7e9ceaf14fa 100644 --- a/cost-model/Cargo.toml +++ b/cost-model/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/cpu-utils/Cargo.toml b/cpu-utils/Cargo.toml index 3958efbf076..734a299a55b 100644 --- a/cpu-utils/Cargo.toml +++ b/cpu-utils/Cargo.toml @@ -8,6 +8,11 @@ license = { workspace = true } edition = { workspace = true } publish = true +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/dev-bins/Cargo.lock b/dev-bins/Cargo.lock index cbfcc18c589..5ace732d76a 100644 --- a/dev-bins/Cargo.lock +++ b/dev-bins/Cargo.lock @@ -1048,19 +1048,20 @@ dependencies = [ [[package]] name = "aya" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d18bc4e506fbb85ab7392ed993a7db4d1a452c71b75a246af4a80ab8c9d2dd50" +checksum = "66e644424fada9fff4fdc63848db1732fb69b626e8328202ef55c03df1f4d939" dependencies = [ "assert_matches", "aya-obj", "bitflags 2.13.0", - "bytes", + "hashbrown 0.17.0", "libc", "log", "object", "once_cell", - "thiserror 1.0.69", + "scopeguard", + "thiserror 2.0.18", ] [[package]] @@ -1118,16 +1119,14 @@ dependencies = [ [[package]] name = "aya-obj" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51b96c5a8ed8705b40d655273bc4212cbbf38d4e3be2788f36306f154523ec7" +checksum = "8c76b9c75d9cdc155ff8f6a06d61e873f67bf47be8cfa92a3b5aaea43f4b4077" dependencies = [ "bytes", - "core-error", - "hashbrown 0.15.5", "log", "object", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1189,7 +1188,7 @@ checksum = "90dbd31c98227229239363921e60fcf5e558e43ec69094d46fc4996f08d1d5bc" dependencies = [ "bitcoin_hashes", "rand 0.8.6", - "rand_core 0.6.4", + "rand_core 0.5.1", "serde", "unicode-normalization", ] @@ -1715,15 +1714,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "core-error" -version = "0.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efcdb2972eb64230b4c50646d8498ff73f5128d196a90c7236eec4cbe8619b8f" -dependencies = [ - "version_check", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -4213,12 +4203,12 @@ dependencies = [ [[package]] name = "object" -version = "0.36.7" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "crc32fast", - "hashbrown 0.15.5", + "hashbrown 0.17.0", "indexmap", "memchr", ] @@ -4760,9 +4750,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -4780,9 +4770,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "fastbloom", @@ -6292,12 +6282,13 @@ dependencies = [ [[package]] name = "solana-clock" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea35d8f69b67daddb921a9da7f78ca591b533cf5e98833cd9ae62fdc2e4652c" +checksum = "f0acdace90d96e2c9e70d681465b4fe888b6bcf27c354ae9774e9f8a3b72923d" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -6697,12 +6688,13 @@ dependencies = [ [[package]] name = "solana-epoch-rewards" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cddf2388b28291210d9aa60690740733cab527531f06ed153c4d388951e407c" +checksum = "daf7eb4986b0b1d6f562b21f75a836f1a6df6e00c275efcef50aab5c144dc59e" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-hash 4.4.0", "solana-sdk-ids", "solana-sdk-macro", @@ -6722,12 +6714,14 @@ dependencies = [ [[package]] name = "solana-epoch-schedule" -version = "3.1.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad280b1ed803853f7b453cb3ea9a57e600ca5599a63e69f7be199b486c0ec93" +checksum = "8116e6ffa6002237d5ab5edcbda17f9ba66b6742c45a89c9fb40a94dbacd4c1d" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", + "solana-program-error", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -6862,6 +6856,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "solana-get-sysvar" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef3bc859fc036ed490146793557386cbfae614ebba4adc704c37d94350824ed4" +dependencies = [ + "solana-address 2.6.1", + "solana-define-syscall 5.1.0", + "solana-program-error", +] + [[package]] name = "solana-geyser-plugin-manager" version = "4.2.0-alpha.0" @@ -6904,6 +6909,7 @@ dependencies = [ "arc-swap", "assert_matches", "bv", + "bytes", "crossbeam-channel", "ed25519-dalek 2.2.0", "flate2", @@ -7020,9 +7026,9 @@ dependencies = [ [[package]] name = "solana-instruction-error" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b188842592fdf6cb96f55263ae1bf11713ab5114401d1d5a881ed7cc41bef6" +checksum = "3b7d34343838343a3755b7dfb1e438d94c6db2263b519cfe3c2257af932b6e93" dependencies = [ "num-traits", "serde", @@ -7263,9 +7269,9 @@ dependencies = [ [[package]] name = "solana-message" -version = "4.2.2" +version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee01edb797313c1c8e1961d8ac6befc7b7cd0f90d1e9cf8f784add2b08926a3" +checksum = "b94164f9740d40f41568f6f48140a0866251a79a7bce013eb4ffefe12d0e38cc" dependencies = [ "blake3", "serde", @@ -7721,12 +7727,13 @@ dependencies = [ [[package]] name = "solana-rent" -version = "4.2.1" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f02fbe2669ebe5d851dbf29a02e91ed6244b051bb64fcc57e6644aba636063" +checksum = "39f0d780bf8e8a1fe8b5b5fce1acad6b209485b86dec246e7523d5e4a8b7c7fc" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -8081,9 +8088,9 @@ checksum = "dcf09694a0fc14e5ffb18f9b7b7c0f15ecb6eac5b5610bf76a1853459d19daf9" [[package]] name = "solana-sbpf" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f84c593fa3d4131045b606dec5acf9d8eac73791bc786ca9911057aec8f43ec" +checksum = "d777d7a89267dd133e985113c7e7f820fb7cfd9123a4a350cf8b39ebae1920bc" dependencies = [ "byteorder", "combine 3.8.1", @@ -8309,12 +8316,13 @@ dependencies = [ [[package]] name = "solana-slot-hashes" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a57c158c35629f9e302ab385f16b15813f4927a31c27dda72f3df828bb08d93" +checksum = "5c7ce2b4b8911bf2db3de7b6266e67bfc21a6a9f8c566fb096d9782ca2ad16ee" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-hash 4.4.0", "solana-sdk-ids", "solana-sysvar-id", @@ -8446,6 +8454,7 @@ dependencies = [ "rustls", "smallvec", "solana-keypair", + "solana-measure", "solana-metrics", "solana-net-utils", "solana-packet 4.1.0", @@ -8790,9 +8799,9 @@ dependencies = [ [[package]] name = "solana-transaction" -version = "4.1.3" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d105ecce084206697230226c6b2230401c220feb4dc63e1274d58b38969292" +checksum = "2509e70bdce879db3e0f56cf97e40edd53742e8f0e6f34d64c46e7900071b53f" dependencies = [ "serde", "serde_derive", @@ -8828,9 +8837,9 @@ dependencies = [ [[package]] name = "solana-transaction-error" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441d6dcd51100e7d97c3fb3b723e08aa701066ff7afc00026fd8d8e222cb95b" +checksum = "757a648388ab1e7350a806ffceb31ce656dc5b5fe607b9f8209aa56f63040179" dependencies = [ "serde", "serde_derive", @@ -9054,9 +9063,9 @@ dependencies = [ [[package]] name = "solana-vote-interface" -version = "6.0.1" +version = "6.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab4307b353cbfab0ca1666c969f91fd7ca6f592abee2d03f5fa6a32c0e1a42b" +checksum = "61843d7be827cac5e025c3a16c1101a34fcdd8cf593f6a82eafdd253bd55a26b" dependencies = [ "bincode", "cfg_eval", diff --git a/dev-bins/Cargo.toml b/dev-bins/Cargo.toml index 80a5a9a5547..37df2c3a548 100644 --- a/dev-bins/Cargo.toml +++ b/dev-bins/Cargo.toml @@ -25,6 +25,7 @@ check-cfg = [ arithmetic_side_effects = "deny" default_trait_access = "deny" manual_let_else = "deny" +uninlined_format_args = "deny" used_underscore_binding = "deny" # Allowed lints @@ -77,7 +78,7 @@ solana-clap-utils = { path = "../clap-utils", version = "=4.2.0-alpha.0", featur solana-cli-config = { path = "../cli-config", version = "=4.2.0-alpha.0" } solana-cli-output = { path = "../cli-output", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-client = { path = "../client", version = "=4.2.0-alpha.0" } -solana-clock = "3.0.1" +solana-clock = "3.1.1" solana-cluster-type = "3.1.0" solana-commitment-config = "3.0.0" solana-compute-budget = { path = "../compute-budget", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } @@ -97,7 +98,7 @@ solana-geyser-plugin-manager = { path = "../geyser-plugin-manager", version = "= solana-hash = "4.4.0" solana-inflation = "3.1.1" solana-instruction = "3.4.0" -solana-instruction-error = "2.3.0" +solana-instruction-error = "2.4.0" solana-keypair = "3.1.2" solana-ledger = { path = "../ledger", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-loader-v3-interface = "7.0.0" @@ -119,7 +120,7 @@ solana-rpc-client = { path = "../rpc-client", version = "=4.2.0-alpha.0", defaul solana-rpc-client-api = { path = "../rpc-client-api", version = "=4.2.0-alpha.0" } solana-runtime = { path = "../runtime", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } solana-runtime-transaction = { path = "../runtime-transaction", version = "=4.2.0-alpha.0", features = ["agave-unstable-api"] } -solana-sbpf = { version = "=0.21.0", default-features = false } +solana-sbpf = { version = "=0.21.1", default-features = false } solana-sdk-ids = "3.1.0" solana-shred-version = "3.0.1" solana-signature = { version = "3.4.1", default-features = false } diff --git a/docs/.eslintignore b/docs/.eslintignore deleted file mode 100644 index 869a07b8e36..00000000000 --- a/docs/.eslintignore +++ /dev/null @@ -1,8 +0,0 @@ -node_modules -build -static -html - -# FIXME -src/pages/index.js -src/theme/Footer/index.js diff --git a/docs/.eslintrc b/docs/.eslintrc deleted file mode 100644 index d2204b0b8d6..00000000000 --- a/docs/.eslintrc +++ /dev/null @@ -1,21 +0,0 @@ -{ - "env": { - "browser": true, - "node": true - }, - "parser": "babel-eslint", - "rules": { - "strict": 0, - "no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], - "no-trailing-spaces": ["error", { "skipBlankLines": true }] - }, - "settings": { - "react": { - "version": "detect", // React version. "detect" automatically picks the version you have installed. - } - }, - "extends": [ - "eslint:recommended", - "plugin:react/recommended" - ] - } \ No newline at end of file diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index dc83674a20e..00000000000 --- a/docs/.gitignore +++ /dev/null @@ -1,35 +0,0 @@ -# Dependencies -/node_modules - -# Production -/build - -# Generated files -.docusaurus -.cache-loader -.vercel -/static/img/*.svg -/static/img/*.png -vercel.json - -# use pnpm and pnpm-lock.yaml -yarn.lock -package-lock.json - -# Misc -.DS_Store -.env -.env.local -.env.development.local -.env.test.local -.env.production.local - -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# Documentation build artifacts -/html/ -/src/tests.ok -/src/cli/usage.md -/src/.gitbook/assets/*.svg diff --git a/docs/.npmrc b/docs/.npmrc deleted file mode 100644 index e595aad2e6b..00000000000 --- a/docs/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=10080 diff --git a/docs/.prettierignore b/docs/.prettierignore deleted file mode 100644 index 12ef0727eb2..00000000000 --- a/docs/.prettierignore +++ /dev/null @@ -1,4 +0,0 @@ -.docusaurus -build -html -static diff --git a/docs/.prettierrc.json b/docs/.prettierrc.json deleted file mode 100644 index d3e25d5eb89..00000000000 --- a/docs/.prettierrc.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "trailingComma": "all", - "tabWidth": 2, - "semi": true, - "singleQuote": false, - "proseWrap": "always", - "printWidth": 80 -} diff --git a/download-utils/Cargo.toml b/download-utils/Cargo.toml index de448541269..95791558249 100644 --- a/download-utils/Cargo.toml +++ b/download-utils/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/entry/Cargo.toml b/entry/Cargo.toml index 2af48e6765b..b7d0b8ccd53 100644 --- a/entry/Cargo.toml +++ b/entry/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/faucet-cli/Cargo.toml b/faucet-cli/Cargo.toml index 7b52feea980..efe827d8b07 100644 --- a/faucet-cli/Cargo.toml +++ b/faucet-cli/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [[bin]] name = "solana-faucet" diff --git a/faucet/Cargo.toml b/faucet/Cargo.toml index 3f9ccab0a8b..919186d1e64 100644 --- a/faucet/Cargo.toml +++ b/faucet/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/feature-set/Cargo.toml b/feature-set/Cargo.toml index 5e0de57dfd2..ef2c8917ccd 100644 --- a/feature-set/Cargo.toml +++ b/feature-set/Cargo.toml @@ -9,6 +9,11 @@ license = { workspace = true } edition = { workspace = true } readme = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] frozen-abi = ["dep:solana-frozen-abi", "dep:solana-frozen-abi-macro"] diff --git a/feature-set/src/lib.rs b/feature-set/src/lib.rs index df5cfc8d6aa..8d28c53ccd1 100644 --- a/feature-set/src/lib.rs +++ b/feature-set/src/lib.rs @@ -52,6 +52,7 @@ pub struct FeatureSnapshot { pub fix_alt_bn128_multiplication_input_length: bool, pub formalize_loaded_transaction_data_size: bool, pub alpenglow: bool, + pub alpenglow_fast_leader_handover: bool, pub disable_zk_elgamal_proof_program: bool, pub reenable_zk_elgamal_proof_program: bool, pub raise_block_limits_to_100m: bool, @@ -156,6 +157,7 @@ impl From<&AHashMap> for FeatureSnapshot { &formalize_loaded_transaction_data_size::ID, ), alpenglow: is_active(&alpenglow::ID), + alpenglow_fast_leader_handover: is_active(&alpenglow_fast_leader_handover::ID), disable_zk_elgamal_proof_program: is_active(&disable_zk_elgamal_proof_program::ID), reenable_zk_elgamal_proof_program: is_active(&reenable_zk_elgamal_proof_program::ID), raise_block_limits_to_100m: is_active(&raise_block_limits_to_100m::ID), @@ -1547,6 +1549,10 @@ pub mod upgrade_bpf_stake_program_to_v5_1 { } } +pub mod alpenglow_fast_leader_handover { + solana_pubkey::declare_id!("FastLeaderHandover1111111111111111111111111"); +} + pub static FEATURE_NAMES: LazyLock> = LazyLock::new(|| { [ (secp256k1_program_enabled::id(), "secp256k1 program"), @@ -2621,6 +2627,10 @@ pub static FEATURE_NAMES: LazyLock> = LazyLock::n upgrade_bpf_stake_program_to_v5_1::id(), "SIMD-0391: Upgrade BPF Stake Program to v5.1.0 (fixed-point warmup/cooldown)", ), + ( + alpenglow_fast_leader_handover::id(), + "SIMD-0326: Alpenglow fast leader handover", + ), /*************** ADD NEW FEATURES HERE ***************/ /***** ADD NEW FEATURE BOOL TO `FeatureSnapshot` *****/ ] diff --git a/fee/Cargo.toml b/fee/Cargo.toml index 06d9cf2057a..518446af098 100644 --- a/fee/Cargo.toml +++ b/fee/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/fs/Cargo.toml b/fs/Cargo.toml index 4b013ed614f..6ef2f418d44 100644 --- a/fs/Cargo.toml +++ b/fs/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/genesis-utils/Cargo.toml b/genesis-utils/Cargo.toml index e8d7f030f74..94fd2c241a9 100644 --- a/genesis-utils/Cargo.toml +++ b/genesis-utils/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/genesis/Cargo.toml b/genesis/Cargo.toml index 1acda478a8a..6db997f9160 100644 --- a/genesis/Cargo.toml +++ b/genesis/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_genesis" diff --git a/geyser-plugin-interface/Cargo.toml b/geyser-plugin-interface/Cargo.toml index 93bf3da7b6e..4fc2376bdeb 100644 --- a/geyser-plugin-interface/Cargo.toml +++ b/geyser-plugin-interface/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index 46fc359f46b..8a0ea97af83 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/gossip-cli/Cargo.toml b/gossip-cli/Cargo.toml index b92fd8fb31e..6aa93f32ca1 100644 --- a/gossip-cli/Cargo.toml +++ b/gossip-cli/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [[bin]] name = "solana-gossip" path = "src/main.rs" diff --git a/gossip/Cargo.toml b/gossip/Cargo.toml index 5e5b5a89e29..969d0c8c7a6 100644 --- a/gossip/Cargo.toml +++ b/gossip/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] bench = false @@ -42,6 +44,7 @@ agave-random = { workspace = true } arc-swap = { workspace = true } assert_matches = { workspace = true } bv = { workspace = true, features = ["serde"] } +bytes = { workspace = true } crossbeam-channel = { workspace = true } ed25519-dalek = { workspace = true } flate2 = { workspace = true } diff --git a/gossip/src/duplicate_shred.rs b/gossip/src/duplicate_shred.rs index 0b469bbd769..8f4e4202a10 100644 --- a/gossip/src/duplicate_shred.rs +++ b/gossip/src/duplicate_shred.rs @@ -401,7 +401,7 @@ pub(crate) mod tests { _unused_shred_type: rng.random(), num_chunks: rng.random(), chunk_index: rng.random(), - chunk: (0..chunk_len).map(|_| rng.random()).collect(), + chunk: (0..chunk_len).map(|_| rng.random::()).collect(), }; let bincode_bytes = bincode::serialize(&dup).unwrap(); diff --git a/gossip/src/gossip_service.rs b/gossip/src/gossip_service.rs index ae21169153e..08e71bb386b 100644 --- a/gossip/src/gossip_service.rs +++ b/gossip/src/gossip_service.rs @@ -2,24 +2,33 @@ use { crate::{ + XdpSender, cluster_info::{ClusterInfo, GOSSIP_CHANNEL_CAPACITY}, cluster_info_metrics::submit_gossip_stats, contact_info::ContactInfo, epoch_specs::EpochSpecs, }, - crossbeam_channel::Sender, + crossbeam_channel::{Sender, TrySendError}, solana_keypair::Keypair, - solana_net_utils::{DEFAULT_IP_ECHO_SERVER_THREADS, SocketAddrSpace}, - solana_perf::recycler::Recycler, + solana_net_utils::{ + DEFAULT_IP_ECHO_SERVER_THREADS, SocketAddrSpace, + multihomed_sockets::{BindIpAddrs, MultihomedSocketProvider, SocketProvider}, + }, + solana_perf::{packet::PacketBatch, recycler::Recycler}, solana_pubkey::Pubkey, solana_signer::Signer, solana_streamer::{ evicting_sender::EvictingSender, - streamer::{self, StreamerReceiveStats}, + sendmmsg::{SendPktsError, batch_send}, + streamer::{ + self, PacketBatchReceiver, ResponseSender, StreamerReceiveStats, + filter_packets_by_socket_addr_space, responder_loop, + }, }, std::{ collections::HashSet, - net::{SocketAddr, TcpListener, UdpSocket}, + io, + net::{IpAddr, SocketAddr, TcpListener, UdpSocket}, sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -40,6 +49,7 @@ impl GossipService { cluster_info: &Arc, mut epoch_specs: Option>, gossip_sockets: Arc<[UdpSocket]>, + xdp_sender: Option, gossip_validators: Option>, should_check_duplicate_instance: bool, stats_reporter_sender: Option>>, @@ -91,12 +101,18 @@ impl GossipService { gossip_validators, exit.clone(), ); - let t_responder = streamer::responder_atomic( + let gossip_responder_socket = match xdp_sender { + Some(xdp_sender) => GossipResponderSocket::Xdp(xdp_sender), + None => GossipResponderSocket::Udp { + sockets: gossip_sockets.clone(), + bind_ip_addrs: cluster_info.bind_ip_addrs(), + socket_addr_space, + }, + }; + let t_responder = run_responder( "Gossip", - gossip_sockets, - cluster_info.bind_ip_addrs(), + gossip_responder_socket, response_receiver, - socket_addr_space, stats_reporter_sender, ); let t_metrics = Builder::new() @@ -329,6 +345,7 @@ pub fn make_node( None, gossip_sockets, None, + None, should_check_duplicate_instance, None, exit, @@ -336,6 +353,129 @@ pub fn make_node( (gossip_service, ip_echo, cluster_info) } +enum GossipResponderSocket { + Udp { + sockets: Arc<[UdpSocket]>, + bind_ip_addrs: Arc, + socket_addr_space: SocketAddrSpace, + }, + Xdp(XdpSender), +} + +fn run_responder( + name: &'static str, + socket: GossipResponderSocket, + r: PacketBatchReceiver, + stats_reporter_sender: Option>>, +) -> JoinHandle<()> { + Builder::new() + .name(format!("solRspndr{name}")) + .spawn(move || match socket { + GossipResponderSocket::Udp { + sockets, + bind_ip_addrs, + socket_addr_space, + } => responder_loop( + name, + r, + GossipUdpSocketProvider::new(sockets, bind_ip_addrs, socket_addr_space), + stats_reporter_sender, + ), + GossipResponderSocket::Xdp(xdp_sender) => { + responder_loop(name, r, GossipXdpSender(xdp_sender), stats_reporter_sender) + } + }) + .unwrap() +} + +struct GossipUdpSocketProvider { + socket_provider: MultihomedSocketProvider, + socket_addr_space: SocketAddrSpace, +} + +impl GossipUdpSocketProvider { + pub fn new( + sockets: Arc<[UdpSocket]>, + bind_ip_addrs: Arc, + socket_addr_space: SocketAddrSpace, + ) -> Self { + Self { + socket_provider: MultihomedSocketProvider::new(sockets, bind_ip_addrs), + socket_addr_space, + } + } +} + +impl ResponseSender for GossipUdpSocketProvider { + fn send_batch(&self, batch: PacketBatch) -> std::result::Result<(), SendPktsError> { + let packets = filter_packets_by_socket_addr_space(batch.iter(), &self.socket_addr_space); + let sock = self.socket_provider.current_socket_ref(); + batch_send(sock, packets.collect::>()) + } +} + +struct GossipXdpSender(XdpSender); + +impl ResponseSender for GossipXdpSender { + fn send_batch(&self, batch: PacketBatch) -> std::result::Result<(), SendPktsError> { + let packets = batch.iter().filter_map(|pkt| { + let addr = pkt.meta().socket_addr(); + let data = pkt.data(..)?; + + // For XDP, we don't support IPv6 and no private or loopback IPv4 addresses. + match addr.ip() { + IpAddr::V4(ip) if !ip.is_private() && !ip.is_loopback() => Some((data, addr)), + _ => None, + } + }); + + let mut num_sent = 0; + let mut num_dropped_full = 0; + let mut num_dropped_disconnected = 0; + + for (idx, (payload, addr)) in packets.enumerate() { + match self + .0 + .try_send(idx, addr, bytes::Bytes::copy_from_slice(payload)) + { + Ok(()) => { + num_sent += 1; + } + Err(TrySendError::Full(_)) => { + num_dropped_full += 1; + continue; + } + Err(TrySendError::Disconnected(_)) => { + num_dropped_disconnected += 1; + continue; + } + } + } + + let num_failed = num_dropped_full + num_dropped_disconnected; + if num_failed > 0 { + let kind = if num_dropped_disconnected != 0 { + io::ErrorKind::BrokenPipe + } else { + io::ErrorKind::WouldBlock + }; + return Err(SendPktsError::IoError( + io::Error::new( + kind, + format!( + "XDP sender failed to enqueue {num_failed} out of {num_total} gossip \ + packets ({num_dropped_full} full queue, {num_dropped_disconnected} \ + disconnected)", + num_total = num_sent + num_failed + ), + ), + num_failed, + )); + } + Ok(()) + } +} + #[cfg(test)] mod tests { use { @@ -358,6 +498,7 @@ mod tests { None, tn.sockets.gossip, None, + None, true, // should_check_duplicate_instance None, exit.clone(), diff --git a/gossip/src/lib.rs b/gossip/src/lib.rs index ff197c8d020..4c46340c529 100644 --- a/gossip/src/lib.rs +++ b/gossip/src/lib.rs @@ -34,6 +34,8 @@ pub mod restart_crds_values; mod verifying_key_cache; pub mod weighted_shuffle; +pub use solana_net_utils::PinnedXdpSender as XdpSender; + #[macro_use] extern crate log; diff --git a/gossip/tests/gossip.rs b/gossip/tests/gossip.rs index c2e54e92ce7..9d6f3f0c7b1 100644 --- a/gossip/tests/gossip.rs +++ b/gossip/tests/gossip.rs @@ -46,6 +46,7 @@ fn test_node(exit: Arc) -> (Arc, GossipService, UdpSock None, test_node.sockets.gossip, None, + None, true, // should_check_duplicate_instance None, exit, @@ -74,6 +75,7 @@ fn test_node_with_bank( Some(epoch_specs), test_node.sockets.gossip, None, + None, true, // should_check_duplicate_instance None, exit, diff --git a/install/Cargo.toml b/install/Cargo.toml index 8b0d132b277..1e6ab35eaf4 100644 --- a/install/Cargo.toml +++ b/install/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/io-uring/Cargo.toml b/io-uring/Cargo.toml index 7cf92569a6b..2ae5cf2e08d 100644 --- a/io-uring/Cargo.toml +++ b/io-uring/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/keygen/Cargo.toml b/keygen/Cargo.toml index ec229e51c27..fa7f59e5a50 100644 --- a/keygen/Cargo.toml +++ b/keygen/Cargo.toml @@ -11,6 +11,11 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +# `all-features` can't be used: remote-wallet-hidraw and remote-wallet-libusb +# forward to hidapi's mutually-exclusive linux backends. Document the default +# (hidraw) backend plus the unstable API. +features = ["agave-unstable-api"] +rustdoc-args = ["--cfg=docsrs"] [[bin]] name = "solana-keygen" @@ -36,9 +41,9 @@ solana-cli-config = { workspace = true } solana-derivation-path = { workspace = true } solana-instruction = { version = "=3.4.0", features = ["bincode"] } solana-keypair = "=3.1.2" -solana-message = { version = "=4.2.2", features = ["wincode"] } +solana-message = { version = "=4.2.3", features = ["wincode"] } solana-pubkey = { version = "=4.2.0", default-features = false } -solana-remote-wallet = { workspace = true } +solana-remote-wallet = { workspace = true, features = ["keystone"] } solana-seed-derivable = { workspace = true } solana-signer = "=3.0.1" solana-version = { workspace = true } diff --git a/lattice-hash/Cargo.toml b/lattice-hash/Cargo.toml index 392bbe39d30..b100c4d76dc 100644 --- a/lattice-hash/Cargo.toml +++ b/lattice-hash/Cargo.toml @@ -8,6 +8,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/leader-schedule/Cargo.toml b/leader-schedule/Cargo.toml index 4c739f4bd71..03ea28319f5 100644 --- a/leader-schedule/Cargo.toml +++ b/leader-schedule/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 945ab2efa46..0763cd819a4 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1516,14 +1516,6 @@ fn main() { .long("enable-capitalization-change") .takes_value(false) .help("If snapshot creation should succeed with a capitalization delta."), - ) - .arg( - Arg::with_name("fix_testnet_ed25519_precompile_account") - .long("fix-testnet-ed25519-precompile-account") - .help( - "correct misassigned owner and data on testnet ed25519 precompile \ - account deployment", - ), ), ) .subcommand( @@ -2085,9 +2077,6 @@ fn main() { archive_format }; - let fix_testnet_ed25519_precompile_account = - arg_matches.is_present("fix_testnet_ed25519_precompile_account"); - let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let mut process_options = parse_process_options(&ledger_path, arg_matches); @@ -2199,8 +2188,7 @@ fn main() { || !feature_gates_to_deactivate.is_empty() || !vote_accounts_to_destake.is_empty() || faucet_pubkey.is_some() - || bootstrap_validator_pubkeys.is_some() - || fix_testnet_ed25519_precompile_account; + || bootstrap_validator_pubkeys.is_some(); if child_bank_required { let mut child_bank = @@ -2303,48 +2291,6 @@ fn main() { } } - if fix_testnet_ed25519_precompile_account { - use solana_sdk_ids::{ed25519_program, native_loader, system_program}; - - if bank.cluster_type() != ClusterType::Testnet { - eprintln!( - "--fix-testnet-ed25519-precompile-account is incompatible with \ - the supplied base snapshot" - ); - std::process::exit(1); - } - - let mut ed25519_program_account = - bank.get_account(&ed25519_program::id()).unwrap_or_else(|| { - eprintln!("Error: `{}` is not deployed", ed25519_program::id()); - exit(1); - }); - - if ed25519_program_account.owner() != &system_program::id() { - eprintln!( - "Error: expected `{}` to be owned by `{}`, found `{}`", - ed25519_program::id(), - system_program::id(), - ed25519_program_account.owner(), - ); - exit(1); - } - - if !ed25519_program_account.data().is_empty() { - eprintln!( - "Error: expected `{}` account data to be empty, found {} bytes", - ed25519_program::id(), - ed25519_program_account.data().len(), - ); - exit(1); - } - - ed25519_program_account.set_owner(native_loader::id()); - ed25519_program_account.set_data_from_slice(b"ed25519_program"); - - bank.store_account(&ed25519_program::id(), &ed25519_program_account); - } - if let Some(bootstrap_validator_pubkeys) = bootstrap_validator_pubkeys { assert_eq!(bootstrap_validator_pubkeys.len() % 3, 0); diff --git a/ledger-tool/src/output.rs b/ledger-tool/src/output.rs index 65c10fd524e..c88812012dd 100644 --- a/ledger-tool/src/output.rs +++ b/ledger-tool/src/output.rs @@ -3,7 +3,6 @@ use { error::{LedgerToolError, Result}, ledger_utils::get_program_ids, }, - chrono::{Local, TimeZone}, itertools::Either, pretty_hex::PrettyHex, serde::{ @@ -13,8 +12,8 @@ use { solana_account::{AccountSharedData, ReadableAccount}, solana_accounts_db::is_loadable::IsLoadable as _, solana_cli_output::{ - CliAccount, CliAccountNewConfig, OutputFormat, QuietDisplay, VerboseDisplay, - display::{build_balance_message, writeln_transaction}, + CliAccount, CliAccountNewConfig, CliBlock, OutputFormat, QuietDisplay, VerboseDisplay, + display::writeln_transaction, }, solana_clock::{Slot, UnixTimestamp}, solana_hash::Hash, @@ -208,84 +207,17 @@ impl VerboseDisplay for CliBlockWithEntries {} impl fmt::Display for CliBlockWithEntries { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - writeln!(f, "Slot: {}", self.slot)?; - writeln!( - f, - "Parent Slot: {}", - self.encoded_confirmed_block.parent_slot - )?; - writeln!(f, "Blockhash: {}", self.encoded_confirmed_block.blockhash)?; - writeln!( + CliBlock::display_block_meta( f, - "Previous Blockhash: {}", - self.encoded_confirmed_block.previous_blockhash + self.slot, + self.encoded_confirmed_block.parent_slot, + &self.encoded_confirmed_block.blockhash, + &self.encoded_confirmed_block.previous_blockhash, + self.encoded_confirmed_block.block_time, + self.encoded_confirmed_block.block_height, + &self.encoded_confirmed_block.rewards, )?; - if let Some(block_time) = self.encoded_confirmed_block.block_time { - writeln!( - f, - "Block Time: {:?}", - Local.timestamp_opt(block_time, 0).unwrap() - )?; - } - if let Some(block_height) = self.encoded_confirmed_block.block_height { - writeln!(f, "Block Height: {block_height:?}")?; - } - if !self.encoded_confirmed_block.rewards.is_empty() { - let mut rewards = self.encoded_confirmed_block.rewards.clone(); - rewards.sort_by(|a, b| a.pubkey.cmp(&b.pubkey)); - let mut total_rewards = 0; - writeln!(f, "Rewards:")?; - writeln!( - f, - " {:<44} {:^15} {:<15} {:<20} {:>14} {:>10}", - "Address", "Type", "Amount", "New Balance", "Percent Change", "Commission" - )?; - for reward in rewards { - let sign = if reward.lamports < 0 { "-" } else { "" }; - total_rewards += reward.lamports; - #[allow(clippy::format_in_format_args)] - writeln!( - f, - " {:<44} {:^15} {:>15} {} {}", - reward.pubkey, - if let Some(reward_type) = reward.reward_type { - format!("{reward_type}") - } else { - "-".to_string() - }, - format!( - "{}â—Ž{:<14.9}", - sign, - build_balance_message(reward.lamports.unsigned_abs(), false, false) - ), - if reward.post_balance == 0 { - " - -".to_string() - } else { - format!( - "â—Ž{:<19.9} {:>13.9}%", - build_balance_message(reward.post_balance, false, false), - (reward.lamports.abs() as f64 - / (reward.post_balance as f64 - reward.lamports as f64)) - * 100.0 - ) - }, - reward - .commission_bps - .map(|bps| format!("{:>8}.{:02}%", bps / 100, bps % 100)) - .or_else(|| reward.commission.map(|c| format!("{c:>9}%"))) - .unwrap_or_else(|| " -".to_string()) - )?; - } - - let sign = if total_rewards < 0 { "-" } else { "" }; - writeln!( - f, - "Total Rewards: {}â—Ž{:<12.9}", - sign, - build_balance_message(total_rewards.unsigned_abs(), false, false) - )?; - } for (index, entry) in self.encoded_confirmed_block.entries.iter().enumerate() { writeln_entry(f, index, &entry.into(), "")?; for (index, transaction_with_meta) in entry.transactions.iter().enumerate() { diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index b3dd98d7556..04050826dea 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index f02537224da..a5d45702277 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -1827,7 +1827,8 @@ fn confirm_slot_with_components( // Only replay that starts at the persisted UpdateParent FEC set may accept // UpdateParent as its first parent marker. From-shred-zero replay still // requires a block header before UpdateParent. - let replay_starts_at_update_parent = migration_status.should_allow_fast_leader_handover(slot) + let replay_starts_at_update_parent = bank.feature_set.snapshot().alpenglow_fast_leader_handover + && migration_status.should_allow_block_markers(slot) && leader_slot_index(slot) == 0 && blockstore .meta(slot) @@ -2490,7 +2491,8 @@ fn load_frozen_forks( // Live replay restarts UpdateParent slots from the marker's FEC set. // Startup replay must use the same offset or a restarted validator can // execute the obsolete optimistic-parent prefix. - if migration_status.should_allow_fast_leader_handover(slot) + if bank.feature_set.snapshot().alpenglow_fast_leader_handover + && migration_status.should_allow_block_markers(slot) && leader_slot_index(slot) == 0 && meta.has_update_parent() { @@ -2726,9 +2728,9 @@ fn supermajority_root_from_vote_accounts( /// Validates the chained block ID for a child slot against its parent. /// /// Returns: -/// - `Inactive`: feature not active, no validation performed +/// - `Inactive`: feature not active, or alpenglow is active, no validation performed /// - `Pass`: chained block ID matches parent's block ID (or parent has no -/// block ID yet), or the slot replays from an UpdateParent FEC set +/// block ID yet) /// - `Mismatch`: definitive mismatch between child's chained merkle root /// and parent's block ID /// - `Unavailable`: data shred 0 not received yet, cannot validate @@ -2737,26 +2739,14 @@ pub fn check_chained_block_id( bank: &Bank, migration_status: &MigrationStatus, ) -> ChainedBlockIdCheck { + let slot = bank.slot(); let feature_snapshot = bank.feature_set.snapshot(); if !(feature_snapshot.validate_chained_block_id || feature_snapshot.validate_chained_block_id_2) + || migration_status.should_use_double_merkle_block_id(slot) { return ChainedBlockIdCheck::Inactive; } - let slot = bank.slot(); - if migration_status.should_allow_fast_leader_handover(slot) - && leader_slot_index(slot) == 0 - && blockstore - .meta(slot) - .expect("Blockstore operations must succeed") - .is_some_and(|meta| meta.has_update_parent()) - { - // This block contains an `UpdateParent` and Alpenglow is active, so we - // rely on Double Merkle verification of parent chained block ID instead - // of CMR. - return ChainedBlockIdCheck::Pass; - } - let parent_slot = bank.parent_slot(); let Ok(expected_parent_block_id) = blockstore.get_parent_chained_block_id(slot) else { @@ -5200,11 +5190,7 @@ pub mod tests { fn test_replay_vote_sender() { let validator_keypairs: Vec<_> = (0..10).map(|_| ValidatorVoteKeypairs::new_rand()).collect(); - let GenesisConfigInfo { - genesis_config, - voting_keypair: _, - .. - } = create_genesis_config_with_vote_accounts( + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config_with_vote_accounts( 1_000_000_000, &validator_keypairs, vec![100; validator_keypairs.len()], @@ -6465,34 +6451,17 @@ pub mod tests { ChainedBlockIdCheck::Mismatch )); - // Case 5: Alpenglow UpdateParent slots skip shred-0 chained block ID - // validation because replay starts at the UpdateParent FEC set. + // Case 5: When alpenglow is active, SIMD-0340 is skipped assert!(matches!( check_chained_block_id( &blockstore, &child_bank, &MigrationStatus::post_migration_status() ), - ChainedBlockIdCheck::Pass - )); - - // Case 6: Non-first-window UpdateParent metadata does not bypass - // chained block ID validation. - insert_shreds_with_chained_merkle_root(13, 0, Hash::new_unique()); - let mut meta = blockstore.meta(13).unwrap().unwrap(); - meta.replay_fec_set_index = 32; - blockstore.put_meta(13, &meta).unwrap(); - let child_bank = Bank::new_from_parent(parent_bank.clone(), SlotLeader::default(), 13); - assert!(matches!( - check_chained_block_id( - &blockstore, - &child_bank, - &MigrationStatus::post_migration_status() - ), - ChainedBlockIdCheck::Mismatch + ChainedBlockIdCheck::Inactive )); - // Case 7: Parent has no shreds (get_block_merkle_root returns Err) — + // Case 6: Parent has no shreds (get_block_merkle_root returns Err) — // should return Pass regardless of chained merkle root. let no_shreds_parent_bank = Arc::new(Bank::new_from_parent( parent_bank, diff --git a/local-cluster/Cargo.toml b/local-cluster/Cargo.toml index d4d73e78ceb..50139d429ae 100644 --- a/local-cluster/Cargo.toml +++ b/local-cluster/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/logger/Cargo.toml b/logger/Cargo.toml index e929ccfa224..56c6b3df98c 100644 --- a/logger/Cargo.toml +++ b/logger/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "agave_logger" diff --git a/math-utils/Cargo.toml b/math-utils/Cargo.toml index a5666457841..e7575a60d9a 100644 --- a/math-utils/Cargo.toml +++ b/math-utils/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/measure/Cargo.toml b/measure/Cargo.toml index dcea4940cce..706ba201313 100644 --- a/measure/Cargo.toml +++ b/measure/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/merkle-tree/Cargo.toml b/merkle-tree/Cargo.toml index e96412c165d..e8cf2d39faa 100644 --- a/merkle-tree/Cargo.toml +++ b/merkle-tree/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/metrics/Cargo.toml b/metrics/Cargo.toml index 220496e2c09..f48a661c904 100644 --- a/metrics/Cargo.toml +++ b/metrics/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_metrics" diff --git a/net-utils/Cargo.toml b/net-utils/Cargo.toml index 1ca335d3dd6..ce5d90c2244 100644 --- a/net-utils/Cargo.toml +++ b/net-utils/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_net_utils" diff --git a/net-utils/src/tooling_for_tests.rs b/net-utils/src/tooling_for_tests.rs index 93f0b189bbb..a15cbf3e963 100644 --- a/net-utils/src/tooling_for_tests.rs +++ b/net-utils/src/tooling_for_tests.rs @@ -32,10 +32,7 @@ impl Iterator for PcapReader { fn next(&mut self) -> Option { loop { - let block = match self.reader.next_block() { - Some(block) => block.ok()?, - None => return None, - }; + let block = self.reader.next_block()?.ok()?; let data = match block { pcap_file::pcapng::Block::Packet(ref block) => { &block.data[0..block.original_len as usize] diff --git a/net/README.md b/net/README.md deleted file mode 120000 index 46a6d5baa9a..00000000000 --- a/net/README.md +++ /dev/null @@ -1 +0,0 @@ -../docs/src/clusters/testnet.md \ No newline at end of file diff --git a/net/README.md b/net/README.md new file mode 100644 index 00000000000..1c7419ee72b --- /dev/null +++ b/net/README.md @@ -0,0 +1,3 @@ +# Network Management + +See the [Test Network Management](https://docs.anza.xyz/clusters/testnet) guide. diff --git a/notifier/Cargo.toml b/notifier/Cargo.toml index 3837f078bc7..774c87b7d7c 100644 --- a/notifier/Cargo.toml +++ b/notifier/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_notifier" diff --git a/perf/Cargo.toml b/perf/Cargo.toml index a17062afc48..e1b3e5b1ca4 100644 --- a/perf/Cargo.toml +++ b/perf/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_perf" @@ -111,3 +113,18 @@ harness = false [lints.rust.unexpected_cfgs] level = "warn" check-cfg = ['cfg(build_target_feature_avx)', 'cfg(build_target_feature_avx2)'] + +# Duplicated from the root [workspace.lints.clippy]. This crate defines its own +# [lints.rust.unexpected_cfgs] above, and cargo does not support combining +# `lints.workspace = true` with crate-specific lints in the same manifest +# (see https://github.com/rust-lang/cargo/issues/13157). Keep in sync with the workspace. +[lints.clippy] +# Denied lints +arithmetic_side_effects = "deny" +default_trait_access = "deny" +manual_let_else = "deny" +uninlined_format_args = "deny" +used_underscore_binding = "deny" + +# Allowed lints +new_without_default = "allow" diff --git a/poh/Cargo.toml b/poh/Cargo.toml index d85b28b428e..38a73a10f2a 100644 --- a/poh/Cargo.toml +++ b/poh/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/precompiles/src/lib.rs b/precompiles/src/lib.rs index b3fef2b8c45..98f2dab8b0f 100644 --- a/precompiles/src/lib.rs +++ b/precompiles/src/lib.rs @@ -1,5 +1,5 @@ #![cfg(feature = "agave-unstable-api")] -#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use { agave_feature_set::FeatureSet, solana_message::compiled_instruction::CompiledInstruction, solana_precompile_error::PrecompileError, solana_pubkey::Pubkey, std::sync::LazyLock, diff --git a/program-binaries/Cargo.toml b/program-binaries/Cargo.toml index bb0fd5c5171..ffcbfe3fa8e 100644 --- a/program-binaries/Cargo.toml +++ b/program-binaries/Cargo.toml @@ -8,6 +8,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/program-runtime/Cargo.toml b/program-runtime/Cargo.toml index 5355cf3be4d..eaf76175e06 100644 --- a/program-runtime/Cargo.toml +++ b/program-runtime/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/program-runtime/src/cpi.rs b/program-runtime/src/cpi.rs index 5bfc8b18176..c661ec47d80 100644 --- a/program-runtime/src/cpi.rs +++ b/program-runtime/src/cpi.rs @@ -26,7 +26,7 @@ use { }; /// CPI-specific error types -#[derive(Debug, Error, PartialEq, Eq)] +#[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum CpiError { #[error("Invalid pointer")] InvalidPointer, diff --git a/program-test/Cargo.toml b/program-test/Cargo.toml index 1c0bcc37c46..35a3553cf44 100644 --- a/program-test/Cargo.toml +++ b/program-test/Cargo.toml @@ -8,6 +8,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/programs/bpf_loader/Cargo.toml b/programs/bpf_loader/Cargo.toml index 2dade0e942c..35cbe43c41c 100644 --- a/programs/bpf_loader/Cargo.toml +++ b/programs/bpf_loader/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/programs/compute-budget/Cargo.toml b/programs/compute-budget/Cargo.toml index dd11702619d..1213bdd4ba6 100644 --- a/programs/compute-budget/Cargo.toml +++ b/programs/compute-budget/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 32d2967b15f..d73f9e708ef 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -1034,19 +1034,20 @@ dependencies = [ [[package]] name = "aya" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d18bc4e506fbb85ab7392ed993a7db4d1a452c71b75a246af4a80ab8c9d2dd50" +checksum = "66e644424fada9fff4fdc63848db1732fb69b626e8328202ef55c03df1f4d939" dependencies = [ "assert_matches", "aya-obj", "bitflags 2.13.0", - "bytes", + "hashbrown 0.17.0", "libc", "log", "object", "once_cell", - "thiserror 1.0.69", + "scopeguard", + "thiserror 2.0.18", ] [[package]] @@ -1090,16 +1091,14 @@ dependencies = [ [[package]] name = "aya-obj" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51b96c5a8ed8705b40d655273bc4212cbbf38d4e3be2788f36306f154523ec7" +checksum = "8c76b9c75d9cdc155ff8f6a06d61e873f67bf47be8cfa92a3b5aaea43f4b4077" dependencies = [ "bytes", - "core-error", - "hashbrown 0.15.1", "log", "object", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1642,15 +1641,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "core-error" -version = "0.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efcdb2972eb64230b4c50646d8498ff73f5128d196a90c7236eec4cbe8619b8f" -dependencies = [ - "version_check", -] - [[package]] name = "core-foundation" version = "0.9.3" @@ -2791,7 +2781,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" dependencies = [ "allocator-api2", - "equivalent", "foldhash 0.1.5", ] @@ -4197,12 +4186,12 @@ dependencies = [ [[package]] name = "object" -version = "0.36.7" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "crc32fast", - "hashbrown 0.15.1", + "hashbrown 0.17.0", "indexmap", "memchr", ] @@ -4761,9 +4750,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -4781,9 +4770,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "fastbloom", @@ -6407,12 +6396,13 @@ dependencies = [ [[package]] name = "solana-clock" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea35d8f69b67daddb921a9da7f78ca591b533cf5e98833cd9ae62fdc2e4652c" +checksum = "f0acdace90d96e2c9e70d681465b4fe888b6bcf27c354ae9774e9f8a3b72923d" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -6818,12 +6808,13 @@ dependencies = [ [[package]] name = "solana-epoch-rewards" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cddf2388b28291210d9aa60690740733cab527531f06ed153c4d388951e407c" +checksum = "daf7eb4986b0b1d6f562b21f75a836f1a6df6e00c275efcef50aab5c144dc59e" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-hash 4.4.0", "solana-sdk-ids", "solana-sdk-macro", @@ -6843,12 +6834,14 @@ dependencies = [ [[package]] name = "solana-epoch-schedule" -version = "3.1.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad280b1ed803853f7b453cb3ea9a57e600ca5599a63e69f7be199b486c0ec93" +checksum = "8116e6ffa6002237d5ab5edcbda17f9ba66b6742c45a89c9fb40a94dbacd4c1d" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", + "solana-program-error", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -7010,6 +7003,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "solana-get-sysvar" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef3bc859fc036ed490146793557386cbfae614ebba4adc704c37d94350824ed4" +dependencies = [ + "solana-address 2.6.1", + "solana-define-syscall 5.1.0", + "solana-program-error", +] + [[package]] name = "solana-geyser-plugin-manager" version = "4.2.0-alpha.0" @@ -7052,6 +7056,7 @@ dependencies = [ "arc-swap", "assert_matches", "bv", + "bytes", "crossbeam-channel", "ed25519-dalek 2.2.0", "flate2", @@ -7170,9 +7175,9 @@ dependencies = [ [[package]] name = "solana-instruction-error" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b188842592fdf6cb96f55263ae1bf11713ab5114401d1d5a881ed7cc41bef6" +checksum = "3b7d34343838343a3755b7dfb1e438d94c6db2263b519cfe3c2257af932b6e93" dependencies = [ "num-traits", "serde", @@ -7413,9 +7418,9 @@ dependencies = [ [[package]] name = "solana-message" -version = "4.2.2" +version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee01edb797313c1c8e1961d8ac6befc7b7cd0f90d1e9cf8f784add2b08926a3" +checksum = "b94164f9740d40f41568f6f48140a0866251a79a7bce013eb4ffefe12d0e38cc" dependencies = [ "blake3", "serde", @@ -7982,12 +7987,13 @@ dependencies = [ [[package]] name = "solana-rent" -version = "4.2.1" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f02fbe2669ebe5d851dbf29a02e91ed6244b051bb64fcc57e6644aba636063" +checksum = "39f0d780bf8e8a1fe8b5b5fce1acad6b209485b86dec246e7523d5e4a8b7c7fc" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-sdk-ids", "solana-sdk-macro", "solana-sysvar-id", @@ -9058,9 +9064,9 @@ dependencies = [ [[package]] name = "solana-sbpf" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f84c593fa3d4131045b606dec5acf9d8eac73791bc786ca9911057aec8f43ec" +checksum = "d777d7a89267dd133e985113c7e7f820fb7cfd9123a4a350cf8b39ebae1920bc" dependencies = [ "byteorder", "combine 3.8.1", @@ -9285,12 +9291,13 @@ dependencies = [ [[package]] name = "solana-slot-hashes" -version = "3.0.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a57c158c35629f9e302ab385f16b15813f4927a31c27dda72f3df828bb08d93" +checksum = "5c7ce2b4b8911bf2db3de7b6266e67bfc21a6a9f8c566fb096d9782ca2ad16ee" dependencies = [ "serde", "serde_derive", + "solana-get-sysvar", "solana-hash 4.4.0", "solana-sdk-ids", "solana-sysvar-id", @@ -9422,6 +9429,7 @@ dependencies = [ "rustls", "smallvec", "solana-keypair", + "solana-measure", "solana-metrics", "solana-net-utils", "solana-packet 4.1.0", @@ -9824,9 +9832,9 @@ dependencies = [ [[package]] name = "solana-transaction" -version = "4.1.3" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d105ecce084206697230226c6b2230401c220feb4dc63e1274d58b38969292" +checksum = "2509e70bdce879db3e0f56cf97e40edd53742e8f0e6f34d64c46e7900071b53f" dependencies = [ "serde", "serde_derive", @@ -9862,9 +9870,9 @@ dependencies = [ [[package]] name = "solana-transaction-error" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441d6dcd51100e7d97c3fb3b723e08aa701066ff7afc00026fd8d8e222cb95b" +checksum = "757a648388ab1e7350a806ffceb31ce656dc5b5fe607b9f8209aa56f63040179" dependencies = [ "serde", "serde_derive", @@ -10088,9 +10096,9 @@ dependencies = [ [[package]] name = "solana-vote-interface" -version = "6.0.1" +version = "6.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab4307b353cbfab0ca1666c969f91fd7ca6f592abee2d03f5fa6a32c0e1a42b" +checksum = "61843d7be827cac5e025c3a16c1101a34fcdd8cf593f6a82eafdd253bd55a26b" dependencies = [ "bincode", "cfg_eval", diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index c6f309a072b..bd935c49fd2 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -88,6 +88,18 @@ check-cfg = [ 'cfg(target_feature, values("dynamic-frames"))', ] +# Keep in sync with the root [workspace.lints.clippy]. +[workspace.lints.clippy] +# Denied lints +arithmetic_side_effects = "deny" +default_trait_access = "deny" +manual_let_else = "deny" +uninlined_format_args = "deny" +used_underscore_binding = "deny" + +# Allowed lints +new_without_default = "allow" + [workspace.dependencies] agave-feature-set = { path = "../../feature-set", version = "=4.2.0-alpha.0" } agave-logger = { path = "../../logger", version = "=4.2.0-alpha.0" } @@ -108,7 +120,7 @@ solana-account-view = "=2.0.0" solana-address = "=2.6.1" solana-blake3-hasher = { version = "=3.1.0", features = ["blake3"] } solana-bn254 = "=3.2.1" -solana-clock = { version = "=3.1.0", features = ["serde", "sysvar"] } +solana-clock = { version = "=3.1.1", features = ["serde", "sysvar"] } solana-compute-budget = { path = "../../compute-budget", version = "=4.2.0-alpha.0" } solana-compute-budget-instruction = { path = "../../compute-budget-instruction", version = "=4.2.0-alpha.0" } solana-cpi = "=3.1.0" @@ -150,7 +162,7 @@ solana-system-interface = { version = "=3.2", features = ["bincode"] } solana-sysvar = "=4.0.0" solana-test-validator = { path = "../../test-validator", version = "=4.2.0-alpha.0" } solana-vote = { path = "../../vote", version = "=4.2.0-alpha.0" } -solana-vote-interface = "6.0.1" +solana-vote-interface = "6.0.2" solana-vote-program = { path = "../../programs/vote", version = "=4.2.0-alpha.0" } test-case = "3.3.1" thiserror = "2.0" @@ -211,7 +223,7 @@ solana-system-interface = { workspace = true } solana-sysvar = "4.0.0" solana-test-validator = { workspace = true, features = ["agave-unstable-api", "dev-context-only-utils"] } solana-transaction = "4.1.0" -solana-transaction-error = "3.2.0" +solana-transaction-error = "3.3.0" solana-vote = { workspace = true } solana-vote-interface = { workspace = true } solana-vote-program = { workspace = true } @@ -233,3 +245,6 @@ opt-level = 1 # The test programs are build in release mode # Minimize their file size so that they fit into the account size limit strip = true + +[lints] +workspace = true diff --git a/programs/sbf/rust/128bit_dep/Cargo.toml b/programs/sbf/rust/128bit_dep/Cargo.toml index 6015395f67b..e11624f48f0 100644 --- a/programs/sbf/rust/128bit_dep/Cargo.toml +++ b/programs/sbf/rust/128bit_dep/Cargo.toml @@ -9,3 +9,6 @@ license = { workspace = true } edition = { workspace = true } [dependencies] + +[lints] +workspace = true diff --git a/programs/sbf/rust/dep_crate/Cargo.toml b/programs/sbf/rust/dep_crate/Cargo.toml index 6902b2fd5ee..707a73025bd 100644 --- a/programs/sbf/rust/dep_crate/Cargo.toml +++ b/programs/sbf/rust/dep_crate/Cargo.toml @@ -14,3 +14,6 @@ crate-type = ["cdylib"] [dependencies] byteorder = { workspace = true } solana-program-entrypoint = { workspace = true } + +[lints] +workspace = true diff --git a/programs/sbf/rust/invoke_dep/Cargo.toml b/programs/sbf/rust/invoke_dep/Cargo.toml index 4b6a403c3fe..da6d28a3cc9 100644 --- a/programs/sbf/rust/invoke_dep/Cargo.toml +++ b/programs/sbf/rust/invoke_dep/Cargo.toml @@ -10,3 +10,6 @@ edition = { workspace = true } [lib] crate-type = ["lib"] + +[lints] +workspace = true diff --git a/programs/sbf/rust/invoked_dep/Cargo.toml b/programs/sbf/rust/invoked_dep/Cargo.toml index a70b7d87bfd..6b01f3768d3 100644 --- a/programs/sbf/rust/invoked_dep/Cargo.toml +++ b/programs/sbf/rust/invoked_dep/Cargo.toml @@ -14,3 +14,6 @@ solana-pubkey = { workspace = true } [lib] crate-type = ["lib"] + +[lints] +workspace = true diff --git a/programs/sbf/rust/many_args_dep/Cargo.toml b/programs/sbf/rust/many_args_dep/Cargo.toml index 0dde7fc44cc..f3e73a810d8 100644 --- a/programs/sbf/rust/many_args_dep/Cargo.toml +++ b/programs/sbf/rust/many_args_dep/Cargo.toml @@ -11,3 +11,6 @@ edition = { workspace = true } [dependencies] solana-msg = { workspace = true } solana-program = { workspace = true } + +[lints] +workspace = true diff --git a/programs/sbf/rust/mem_dep/Cargo.toml b/programs/sbf/rust/mem_dep/Cargo.toml index 8b0ef1caed9..1d82d46af19 100644 --- a/programs/sbf/rust/mem_dep/Cargo.toml +++ b/programs/sbf/rust/mem_dep/Cargo.toml @@ -12,3 +12,6 @@ edition = { workspace = true } crate-type = ["lib"] [dependencies] + +[lints] +workspace = true diff --git a/programs/sbf/rust/param_passing_dep/Cargo.toml b/programs/sbf/rust/param_passing_dep/Cargo.toml index 7c85e0cf5cf..3e831df1127 100644 --- a/programs/sbf/rust/param_passing_dep/Cargo.toml +++ b/programs/sbf/rust/param_passing_dep/Cargo.toml @@ -9,3 +9,6 @@ license = { workspace = true } edition = { workspace = true } [dependencies] + +[lints] +workspace = true diff --git a/programs/sbf/rust/realloc_dep/Cargo.toml b/programs/sbf/rust/realloc_dep/Cargo.toml index c2b90bb6f4a..40ff02c3316 100644 --- a/programs/sbf/rust/realloc_dep/Cargo.toml +++ b/programs/sbf/rust/realloc_dep/Cargo.toml @@ -14,3 +14,6 @@ solana-pubkey = { workspace = true } [lib] crate-type = ["lib"] + +[lints] +workspace = true diff --git a/programs/sbf/rust/realloc_invoke_dep/Cargo.toml b/programs/sbf/rust/realloc_invoke_dep/Cargo.toml index 85629d589d9..3b2a59ed237 100644 --- a/programs/sbf/rust/realloc_invoke_dep/Cargo.toml +++ b/programs/sbf/rust/realloc_invoke_dep/Cargo.toml @@ -10,3 +10,6 @@ edition = { workspace = true } [lib] crate-type = ["lib"] + +[lints] +workspace = true diff --git a/programs/system/Cargo.toml b/programs/system/Cargo.toml index a66dd851ed2..90206673be2 100644 --- a/programs/system/Cargo.toml +++ b/programs/system/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/programs/vote/Cargo.toml b/programs/vote/Cargo.toml index ca036c9e73e..c6691d0757a 100644 --- a/programs/vote/Cargo.toml +++ b/programs/vote/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/programs/zk-elgamal-proof/Cargo.toml b/programs/zk-elgamal-proof/Cargo.toml index 4943b8d65fe..f3ecfb4606b 100644 --- a/programs/zk-elgamal-proof/Cargo.toml +++ b/programs/zk-elgamal-proof/Cargo.toml @@ -8,6 +8,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/programs/zk-token-proof/Cargo.toml b/programs/zk-token-proof/Cargo.toml index ae27b00e111..778b9af02e0 100644 --- a/programs/zk-token-proof/Cargo.toml +++ b/programs/zk-token-proof/Cargo.toml @@ -8,6 +8,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/pubsub-client/Cargo.toml b/pubsub-client/Cargo.toml index e3e71855e58..59920d2319c 100644 --- a/pubsub-client/Cargo.toml +++ b/pubsub-client/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/quic-client/Cargo.toml b/quic-client/Cargo.toml index dd51e9b406d..a27ada1d2bf 100644 --- a/quic-client/Cargo.toml +++ b/quic-client/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/random/Cargo.toml b/random/Cargo.toml index 53b5402d6ef..fe6b5991ecb 100644 --- a/random/Cargo.toml +++ b/random/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/rayon-threadlimit/Cargo.toml b/rayon-threadlimit/Cargo.toml index 7a552c56eb4..205a4e1c552 100644 --- a/rayon-threadlimit/Cargo.toml +++ b/rayon-threadlimit/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/remote-wallet/Cargo.toml b/remote-wallet/Cargo.toml index d05c65d6de5..0ec898dcc80 100644 --- a/remote-wallet/Cargo.toml +++ b/remote-wallet/Cargo.toml @@ -11,10 +11,22 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +# `all-features` can't be used here: hidapi's linux backends +# (linux-{shared,static}-{hidraw,libusb}) are mutually exclusive. +# Document the default backend plus the unstable API instead. +features = ["agave-unstable-api"] +rustdoc-args = ["--cfg=docsrs"] [features] default = ["linux-static-hidraw"] agave-unstable-api = [] +keystone = [ + "dep:hex", + "dep:rusb", + "dep:serde_json", + "dep:ur-parse-lib", + "dep:ur-registry", +] linux-shared-hidraw = ["hidapi/linux-shared-hidraw"] linux-shared-libusb = ["hidapi/linux-shared-libusb"] linux-static-hidraw = ["hidapi/linux-static-hidraw"] @@ -23,12 +35,15 @@ linux-static-libusb = ["hidapi/linux-static-libusb"] [dependencies] console = { workspace = true } dialoguer = { workspace = true } +hex = { workspace = true, optional = true } hidapi = { workspace = true, optional = true } log = { workspace = true } num-derive = { workspace = true } num-traits = { workspace = true } parking_lot = { workspace = true } +rusb = { workspace = true, optional = true } semver = { workspace = true } +serde_json = { workspace = true, optional = true } solana-derivation-path = { workspace = true } solana-offchain-message = { workspace = true } solana-pubkey = { workspace = true, features = ["std"] } @@ -36,6 +51,8 @@ solana-signature = { workspace = true, features = ["std"] } solana-signer = { workspace = true } thiserror = { workspace = true } trezor-client = { workspace = true } +ur-parse-lib = { workspace = true, optional = true } +ur-registry = { workspace = true, optional = true } uriparse = { workspace = true } [dev-dependencies] diff --git a/remote-wallet/src/keystone.rs b/remote-wallet/src/keystone.rs new file mode 100644 index 00000000000..a04d68b42a1 --- /dev/null +++ b/remote-wallet/src/keystone.rs @@ -0,0 +1,806 @@ +use { + crate::{ + locator::Manufacturer, + remote_wallet::{RemoteWallet, RemoteWalletError, RemoteWalletInfo}, + }, + console::Emoji, + hex, + semver::Version as FirmwareVersion, + serde_json, + solana_derivation_path::DerivationPath, + solana_pubkey::Pubkey, + solana_signature::Signature, + std::{convert::TryFrom, fmt, time::Duration}, + ur_parse_lib::{keystone_ur_decoder::probe_decode, keystone_ur_encoder::probe_encode}, + ur_registry::{ + crypto_key_path::{CryptoKeyPath, PathComponent}, + extend::{ + crypto_multi_accounts::CryptoMultiAccounts, + key_derivation::KeyDerivationCall, + key_derivation_schema::{Curve, KeyDerivationSchema}, + qr_hardware_call::{CallParams, CallType, HardWareCallVersion, QRHardwareCall}, + }, + solana::{ + sol_sign_request::{SignType, SolSignRequest}, + sol_signature::SolSignature, + }, + traits::RegistryItem, + }, +}; + +static CHECK_MARK: Emoji = Emoji("✅ ", ""); + +const REQUEST_ID: u16 = 0x0000; + +/// Keystone vendor ID +const KEYSTONE_VID: u16 = 0x1209; +/// Keystone product ID +const KEYSTONE_PID: u16 = 0x3001; + +const HID_PACKET_SIZE: usize = 64; +const EAPDU_OFFSET_CLA: usize = 0; +const EAPDU_OFFSET_INS: usize = 1; +const EAPDU_OFFSET_P1: usize = 3; +const EAPDU_OFFSET_P2: usize = 5; +const EAPDU_OFFSET_LC: usize = 7; +const EAPDU_OFFSET_CDATA: usize = 9; +const EAPDU_RESPONSE_STATUS_LEN: usize = 2; +const EAPDU_MAX_REQ_DATA_PER_PACKET: usize = HID_PACKET_SIZE - EAPDU_OFFSET_CDATA; +const EAPDU_SUCCESS_STATUS: u16 = 0x0000; +const EAPDU_EXPORT_ADDRESS_PAGE_STATUS: u16 = 0x0006; +// USB operations are user-interactive and may wait on device screen approval. +// Keystone signing requires user approval on the device, so allow enough time +// for users to review and confirm requests before the USB read/write times out. +const USB_TIMEOUT: Duration = Duration::from_secs(60); +// Use a short timeout so stale packets are drained without delaying new requests. +const DRAIN_TIMEOUT: Duration = Duration::from_millis(20); +// Bound stale packet draining to at most one full-sized response window. +const MAX_DRAIN_PACKETS: usize = 64; +// Maximum UR fragment size; large enough to keep USB requests single-part. +const MAX_UR_FRAGMENT_LEN: usize = 0x0FFF_FFFF; + +// JSON response field names +const JSON_FIELD_PUBKEY: &str = "pubkey"; +const JSON_FIELD_PAYLOAD: &str = "payload"; +const JSON_FIELD_ERROR: &str = "error"; +const JSON_FIELD_FIRMWARE_VERSION: &str = "firmwareVersion"; +const JSON_FIELD_WALLET_MFP: &str = "walletMFP"; + +// Error messages +const ERROR_INVALID_JSON: &str = "Invalid JSON response"; +const ERROR_MISSING_FIELD: &str = "Missing required field"; +const ERROR_INVALID_HEX: &str = "Invalid hex data"; +const ERROR_SIGNATURE_SIZE: &str = "Signature packet size mismatch"; +const ERROR_KEY_SIZE: &str = "Key packet size mismatch"; +const ERROR_EXPORT_ADDRESS_PAGE: &str = "Export address is only allowed on specific pages"; + +// Keystone cached derivation-path ranges for USB pubkey export. +const CACHED_ACCOUNT_RANGE: u32 = 49; +const CACHED_FIXED_CHANGE: u32 = 0; +const SOLANA_COIN_TYPE: u32 = 501; + +#[derive(Debug, Clone, Copy, PartialEq)] +enum CommandType { + CmdEchoTest = 0x01, + CmdResolveUR = 0x02, + CmdCheckLockStatus = 0x03, + CmdExportAddress = 0x04, + CmdGetDeviceInfo = 0x05, + CmdGetDeviceUSBPubkey = 0x06, +} + +#[derive(Clone, Copy)] +struct UsbIo { + interface_number: u8, + setting_number: u8, + endpoint_out: u8, + endpoint_in: u8, + transfer_type: rusb::TransferType, +} + +struct EndpointPair { + output: u8, + input: u8, +} + +struct EapduHeader { + command: u16, + total_packets: u16, + packet_sequence: u16, + request_id: u16, +} + +impl EapduHeader { + fn parse(packet: &[u8]) -> Result { + if packet.len() < EAPDU_OFFSET_CDATA + EAPDU_RESPONSE_STATUS_LEN { + return Err(RemoteWalletError::Protocol("Invalid EAPDU packet size")); + } + + let command = u16::from_be_bytes([packet[EAPDU_OFFSET_INS], packet[EAPDU_OFFSET_INS + 1]]); + let total_packets = + u16::from_be_bytes([packet[EAPDU_OFFSET_P1], packet[EAPDU_OFFSET_P1 + 1]]); + let packet_sequence = + u16::from_be_bytes([packet[EAPDU_OFFSET_P2], packet[EAPDU_OFFSET_P2 + 1]]); + let request_id = u16::from_be_bytes([packet[EAPDU_OFFSET_LC], packet[EAPDU_OFFSET_LC + 1]]); + + if !is_valid_command(command) || total_packets == 0 || packet_sequence >= total_packets { + return Err(RemoteWalletError::Protocol("Unable to parse packet header")); + } + + Ok(Self { + command, + total_packets, + packet_sequence, + request_id, + }) + } +} + +/// Keystone hardware wallet device +pub struct KeystoneWallet { + pub device: rusb::Device, + pub handle: rusb::DeviceHandle, + usb_io: UsbIo, + pub pretty_path: String, + pub version: Option, + pub mfp: Option<[u8; 4]>, +} + +impl fmt::Debug for KeystoneWallet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "KeystoneWallet") + } +} + +impl KeystoneWallet { + pub fn new( + device: rusb::Device, + handle: rusb::DeviceHandle, + ) -> Result { + let usb_io = Self::discover_usb_io(&device)?; + + // Best effort: detach kernel driver where supported. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + if handle + .kernel_driver_active(usb_io.interface_number) + .unwrap_or(false) + { + let _ = handle.detach_kernel_driver(usb_io.interface_number); + } + } + + handle + .claim_interface(usb_io.interface_number) + .map_err(|e| { + RemoteWalletError::Hid(format!( + "Failed to claim USB interface {}: {e}", + usb_io.interface_number + )) + })?; + + Ok(Self { + device, + handle, + usb_io, + pretty_path: String::default(), + version: None, + mfp: None, + }) + } + + fn discover_usb_io(device: &rusb::Device) -> Result { + let config = device + .active_config_descriptor() + .or_else(|_| device.config_descriptor(0)) + .map_err(|e| { + RemoteWalletError::Hid(format!("Failed to read USB config descriptor: {e}")) + })?; + + // Match webusb-cli behavior first: interface 0, alternate setting 0, first IN/OUT endpoints. + for interface in config.interfaces() { + for descriptor in interface.descriptors() { + if descriptor.interface_number() != 0 || descriptor.setting_number() != 0 { + continue; + } + for transfer_type in [rusb::TransferType::Bulk, rusb::TransferType::Interrupt] { + if let Some(endpoints) = find_endpoint_pair(&descriptor, transfer_type) { + return Ok(UsbIo { + interface_number: descriptor.interface_number(), + setting_number: descriptor.setting_number(), + endpoint_out: endpoints.output, + endpoint_in: endpoints.input, + transfer_type, + }); + } + } + } + } + + // Fallback: scan all interfaces/settings, BULK first, then INTERRUPT. + for wanted_type in [rusb::TransferType::Bulk, rusb::TransferType::Interrupt] { + for interface in config.interfaces() { + for descriptor in interface.descriptors() { + if let Some(endpoints) = find_endpoint_pair(&descriptor, wanted_type) { + return Ok(UsbIo { + interface_number: descriptor.interface_number(), + setting_number: descriptor.setting_number(), + endpoint_out: endpoints.output, + endpoint_in: endpoints.input, + transfer_type: wanted_type, + }); + } + } + } + } + + Err(RemoteWalletError::Protocol( + "No suitable USB IN/OUT endpoints found", + )) + } + + /// Write data to device with Keystone USB transport framing + fn write(&self, command: CommandType, data: &[u8]) -> Result<(), RemoteWalletError> { + // Avoid carrying unread responses across requests; stale data can keep + // firmware IN endpoint busy and block subsequent sends. + self.drain_pending_input_packets(); + + let total_packets = std::cmp::max(1, data.len().div_ceil(EAPDU_MAX_REQ_DATA_PER_PACKET)); + + for packet_index in 0..total_packets { + let start = packet_index * EAPDU_MAX_REQ_DATA_PER_PACKET; + let end = std::cmp::min(start + EAPDU_MAX_REQ_DATA_PER_PACKET, data.len()); + let chunk = &data[start..end]; + + let mut eapdu_packet = [0u8; EAPDU_OFFSET_CDATA + EAPDU_MAX_REQ_DATA_PER_PACKET]; + eapdu_packet[EAPDU_OFFSET_CLA] = 0x00; + eapdu_packet[EAPDU_OFFSET_INS..EAPDU_OFFSET_INS + 2] + .copy_from_slice(&(command as u16).to_be_bytes()); + eapdu_packet[EAPDU_OFFSET_P1..EAPDU_OFFSET_P1 + 2] + .copy_from_slice(&(total_packets as u16).to_be_bytes()); + eapdu_packet[EAPDU_OFFSET_P2..EAPDU_OFFSET_P2 + 2] + .copy_from_slice(&(packet_index as u16).to_be_bytes()); + eapdu_packet[EAPDU_OFFSET_LC..EAPDU_OFFSET_LC + 2] + .copy_from_slice(&REQUEST_ID.to_be_bytes()); + eapdu_packet[EAPDU_OFFSET_CDATA..EAPDU_OFFSET_CDATA + chunk.len()] + .copy_from_slice(chunk); + + self.device_write(&eapdu_packet[..EAPDU_OFFSET_CDATA + chunk.len()])?; + } + + Ok(()) + } + + /// Read data from device with Keystone USB transport parsing + fn read(&self) -> Result, RemoteWalletError> { + let mut total_packets: Option = None; + let mut expected_req_id: Option = None; + let mut expected_command: Option = None; + let mut packet_chunks: Vec>> = Vec::new(); + let mut response_status: Option = None; + + loop { + let chunk = self.device_read()?; + if chunk.is_empty() || chunk.iter().all(|b| *b == 0) { + continue; + } + let packet = chunk.as_slice(); + let header = EapduHeader::parse(packet)?; + let packet_payload = &packet[EAPDU_OFFSET_CDATA..]; + if packet_payload.len() < EAPDU_RESPONSE_STATUS_LEN { + return Err(RemoteWalletError::Protocol("EAPDU payload too short")); + } + + let payload_len = packet_payload.len() - EAPDU_RESPONSE_STATUS_LEN; + let status = + u16::from_be_bytes([packet_payload[payload_len], packet_payload[payload_len + 1]]); + + if total_packets.is_none() { + total_packets = Some(header.total_packets); + expected_req_id = Some(header.request_id); + expected_command = Some(header.command); + packet_chunks = vec![None; header.total_packets as usize]; + } + + if total_packets != Some(header.total_packets) + || expected_req_id != Some(header.request_id) + || expected_command != Some(header.command) + { + return Err(RemoteWalletError::Protocol( + "Mismatched EAPDU packet header across fragments", + )); + } + + let idx = header.packet_sequence as usize; + if packet_chunks[idx].is_none() { + packet_chunks[idx] = Some(packet_payload[..payload_len].to_vec()); + } + + if let Some(prev_status) = response_status { + if prev_status != status { + return Err(RemoteWalletError::Protocol( + "Mismatched EAPDU status across fragments", + )); + } + } else { + response_status = Some(status); + } + + if packet_chunks.iter().all(|c| c.is_some()) { + break; + } + } + + let result_len = packet_chunks + .iter() + .map(|chunk| chunk.as_ref().map_or(0, Vec::len)) + .sum(); + let mut result_data = Vec::with_capacity(result_len); + for chunk in packet_chunks { + result_data.extend_from_slice(&chunk.unwrap()); + } + + match response_status { + Some(EAPDU_SUCCESS_STATUS) | None => {} + Some(_) => { + if let Some(payload) = keystone_response_payload(&result_data) { + return Err(RemoteWalletError::KeystoneError(payload)); + } + return Err(RemoteWalletError::Protocol( + "EAPDU returned non-success status", + )); + } + } + + Ok(result_data) + } + + /// Send APDU command and receive JSON response + fn send_apdu(&self, command: CommandType, data: &[u8]) -> Result { + self.write(command, data)?; + let message = self.read()?; + let message_str = String::from_utf8_lossy(&message); + + // Extract JSON from response + if let (Some(start), Some(end)) = (message_str.find('{'), message_str.rfind('}')) { + if start < end { + let json_str = &message_str[start..=end]; + return Ok(json_str.to_string()); + } + } + + Ok(message_str.to_string()) + } + + /// Get device firmware version and master fingerprint + fn get_device_info(&self) -> Result<(FirmwareVersion, Option<[u8; 4]>), RemoteWalletError> { + let json_str = self.send_apdu(CommandType::CmdGetDeviceInfo, &[])?; + let json = serde_json::from_str::(&json_str) + .map_err(|_| RemoteWalletError::Protocol(ERROR_INVALID_JSON))?; + + if let Some(device_error) = json.get(JSON_FIELD_ERROR).and_then(|v| v.as_str()) { + if !device_error.trim().is_empty() { + return Err(RemoteWalletError::KeystoneError(device_error.to_string())); + } + } + + // Parse firmware version + let version_str = json + .get(JSON_FIELD_FIRMWARE_VERSION) + .and_then(|v| v.as_str()) + .ok_or(RemoteWalletError::Protocol(ERROR_MISSING_FIELD))?; + + let version = FirmwareVersion::parse(version_str) + .map_err(|_| RemoteWalletError::Protocol("Invalid firmware version"))?; + + // Parse master fingerprint (MFP) + let mfp = json + .get(JSON_FIELD_WALLET_MFP) + .and_then(|v| v.as_str()) + .and_then(|hex_str| { + let bytes = hex::decode(hex_str).ok()?; + bytes.try_into().ok() + }); + + Ok((version, mfp)) + } + + /// Generate UR-encoded key derivation request for QR code display + fn generate_hardware_call( + &self, + derivation_path: &DerivationPath, + ) -> Result { + let key_path = parse_crypto_key_path(derivation_path, self.mfp); + let schema = KeyDerivationSchema::new(key_path, Some(Curve::Ed25519), None, None); + let schemas = vec![schema]; + let call = QRHardwareCall::new( + CallType::KeyDerivation, + CallParams::KeyDerivation(KeyDerivationCall::new(schemas)), + None, + HardWareCallVersion::V1, + ); + + let bytes: Vec = call + .try_into() + .map_err(|_| RemoteWalletError::Protocol("Failed to encode QR call"))?; + + let encoded = probe_encode( + &bytes, + MAX_UR_FRAGMENT_LEN, + QRHardwareCall::get_registry_type().get_type(), + ) + .map_err(|_| RemoteWalletError::Protocol("Failed to encode UR"))?; + + Ok(encoded.data) + } + + /// Generate UR-encoded sign request for transaction signing + fn generate_sol_sign_request( + &self, + derivation_path: &DerivationPath, + sign_data: &[u8], + ) -> Result { + let crypto_key_path = parse_crypto_key_path(derivation_path, self.mfp); + let request_id = [0u8; 16].to_vec(); + let sol_sign_request = SolSignRequest::new( + Some(request_id), + sign_data.to_vec(), + crypto_key_path, + None, + Some("solana cli".to_string()), + SignType::Transaction, + ); + + let bytes: Vec = sol_sign_request + .try_into() + .map_err(|_| RemoteWalletError::Protocol("Failed to encode sign request"))?; + + let encoded = probe_encode( + &bytes, + MAX_UR_FRAGMENT_LEN, + SolSignRequest::get_registry_type().get_type(), + ) + .map_err(|_| RemoteWalletError::Protocol("Failed to encode UR"))?; + + Ok(encoded.data) + } + + /// Low-level USB write to device + fn device_write(&self, data: &[u8]) -> Result<(), RemoteWalletError> { + match self.usb_io.transfer_type { + rusb::TransferType::Interrupt => self + .handle + .write_interrupt(self.usb_io.endpoint_out, data, USB_TIMEOUT) + .map_err(|e| RemoteWalletError::Hid(format!("USB write failed: {e}")))?, + rusb::TransferType::Bulk => self + .handle + .write_bulk(self.usb_io.endpoint_out, data, USB_TIMEOUT) + .map_err(|e| RemoteWalletError::Hid(format!("USB write failed: {e}")))?, + _ => { + return Err(RemoteWalletError::Protocol( + "Unsupported USB transfer type for write", + )); + } + }; + + Ok(()) + } + + fn drain_pending_input_packets(&self) { + for _ in 0..MAX_DRAIN_PACKETS { + match self.device_read_raw(DRAIN_TIMEOUT) { + Ok(chunk) => { + if chunk.is_empty() { + break; + } + } + Err(rusb::Error::Timeout) => break, + Err(rusb::Error::NotSupported) => break, + Err(_) => break, + } + } + } + + fn device_read_raw(&self, timeout: std::time::Duration) -> Result, rusb::Error> { + let mut buf = vec![0u8; HID_PACKET_SIZE]; + + let bytes_read = match self.usb_io.transfer_type { + rusb::TransferType::Interrupt => { + self.handle + .read_interrupt(self.usb_io.endpoint_in, &mut buf, timeout)? + } + rusb::TransferType::Bulk => { + self.handle + .read_bulk(self.usb_io.endpoint_in, &mut buf, timeout)? + } + _ => return Err(rusb::Error::NotSupported), + }; + + buf.truncate(bytes_read); + Ok(buf) + } + + /// Low-level USB read from device + fn device_read(&self) -> Result, RemoteWalletError> { + self.device_read_raw(USB_TIMEOUT).map_err(|e| match e { + rusb::Error::NotSupported => { + RemoteWalletError::Protocol("Unsupported USB transfer type for read") + } + _ => RemoteWalletError::Hid(format!("USB read failed: {e}")), + }) + } +} + +fn find_endpoint_pair( + descriptor: &rusb::InterfaceDescriptor<'_>, + transfer_type: rusb::TransferType, +) -> Option { + let mut endpoint_out = None; + let mut endpoint_in = None; + for ep in descriptor.endpoint_descriptors() { + if ep.transfer_type() != transfer_type { + continue; + } + match ep.direction() { + rusb::Direction::Out => { + endpoint_out.get_or_insert(ep.address()); + } + rusb::Direction::In => { + endpoint_in.get_or_insert(ep.address()); + } + } + } + match (endpoint_out, endpoint_in) { + (Some(output), Some(input)) => Some(EndpointPair { output, input }), + _ => None, + } +} + +fn is_valid_command(value: u16) -> bool { + matches!(value, 0x01..=0x06) +} + +fn keystone_response_payload(data: &[u8]) -> Option { + let message = String::from_utf8_lossy(data); + if let Ok(json) = serde_json::from_str::(&message) { + return json + .get(JSON_FIELD_PAYLOAD) + .and_then(|payload| payload.as_str()) + .map(str::to_string); + } + + (!message.trim().is_empty()).then(|| message.to_string()) +} + +fn parse_ur_pubkey(ur: &str) -> Result, RemoteWalletError> { + let result: ur_parse_lib::keystone_ur_decoder::URParseResult = + probe_decode(ur.to_lowercase()) + .map_err(|_| RemoteWalletError::Protocol("Failed to decode UR pubkey"))?; + + result + .data + .ok_or(RemoteWalletError::Protocol("No pubkey in response"))? + .get_keys() + .first() + .ok_or(RemoteWalletError::Protocol("Empty pubkey list")) + .map(|key| key.get_key()) +} + +fn parse_ur_signature(ur: &str) -> Result, RemoteWalletError> { + let result: ur_parse_lib::keystone_ur_decoder::URParseResult = + probe_decode(ur.to_lowercase()) + .map_err(|_| RemoteWalletError::Protocol("Failed to decode UR signature"))?; + + Ok(result + .data + .ok_or(RemoteWalletError::Protocol("No signature in response"))? + .get_signature() + .as_slice() + .to_vec()) +} + +/// Parse JSON field from response. +fn parse_json_field(json_str: &str, field_name: &str) -> Result { + let json = serde_json::from_str::(json_str) + .map_err(|_| RemoteWalletError::Protocol(ERROR_INVALID_JSON))?; + + if let Some(device_error) = json.get(JSON_FIELD_ERROR).and_then(|v| v.as_str()) { + if !device_error.trim().is_empty() { + return Err(RemoteWalletError::KeystoneError(device_error.to_string())); + } + } + + json.get(field_name) + .and_then(|v| v.as_str()) + .ok_or(RemoteWalletError::Protocol(ERROR_MISSING_FIELD)) + .map(String::from) +} + +impl RemoteWallet> for KeystoneWallet { + fn name(&self) -> &str { + "Keystone hardware wallet" + } + + fn read_device( + &mut self, + _dev_info: &rusb::Device, + ) -> Result { + // Get device info (firmware version and MFP) + let (version, mfp) = self.get_device_info()?; + self.version = Some(version); + self.mfp = mfp; + + // Get device descriptor for model and serial + let device_descriptor = self + .device + .device_descriptor() + .map_err(|e| RemoteWalletError::Hid(format!("Failed to get device descriptor: {e}")))?; + + let model = format!( + "Keystone {:04x}:{:04x}", + device_descriptor.vendor_id(), + device_descriptor.product_id() + ); + + let serial = self + .handle + .read_serial_number_string_ascii(&device_descriptor) + .unwrap_or_else(|_| "Unknown".to_string()); + + // Try to get default pubkey + let default_path = DerivationPath::new_bip44(Some(0), Some(0)); + let pubkey_result = self.get_pubkey(&default_path, false); + let (pubkey, error) = match pubkey_result { + Ok(pubkey) => (pubkey, None), + Err(err) => (Pubkey::default(), Some(err)), + }; + + let mut info = RemoteWalletInfo { + model, + manufacturer: Manufacturer::Keystone, + serial, + host_device_path: String::new(), + pubkey, + error, + }; + info.host_device_path = info.get_pretty_path(); + + Ok(info) + } + + fn get_pubkey( + &self, + derivation_path: &DerivationPath, + _confirm_key: bool, + ) -> Result { + let use_cached_usb_pubkey = is_path_in_cached_range(derivation_path); + + let pubkey_bytes = if use_cached_usb_pubkey { + let serialized_path = extend_and_serialize(derivation_path); + let json_response = + self.send_apdu(CommandType::CmdGetDeviceUSBPubkey, &serialized_path)?; + let pubkey_hex = parse_json_field(&json_response, JSON_FIELD_PUBKEY) + .or_else(|_| parse_json_field(&json_response, JSON_FIELD_PAYLOAD))?; + hex::decode(pubkey_hex).map_err(|_| RemoteWalletError::Protocol(ERROR_INVALID_HEX))? + } else { + let ur_request = self.generate_hardware_call(derivation_path)?; + let json_response = self.send_apdu(CommandType::CmdResolveUR, ur_request.as_bytes())?; + let pubkey_ur = parse_json_field(&json_response, JSON_FIELD_PAYLOAD)?; + if pubkey_ur.trim().is_empty() { + return Err(RemoteWalletError::Protocol( + "CmdResolveUR returned empty payload", + )); + } + parse_ur_pubkey(&pubkey_ur)? + }; + + Pubkey::try_from(pubkey_bytes).map_err(|_| RemoteWalletError::Protocol(ERROR_KEY_SIZE)) + } + + fn sign_message( + &self, + derivation_path: &DerivationPath, + data: &[u8], + ) -> Result { + let ur_request = self.generate_sol_sign_request(derivation_path, data)?; + + println!( + "Waiting for your approval on {} {}", + self.name(), + self.pretty_path + ); + + let json_response = self.send_apdu(CommandType::CmdResolveUR, ur_request.as_bytes())?; + + let signature_ur = parse_json_field(&json_response, JSON_FIELD_PAYLOAD)?; + let signature_bytes = parse_ur_signature(&signature_ur)?; + println!("{CHECK_MARK}Approved"); + + Signature::try_from(signature_bytes) + .map_err(|_| RemoteWalletError::Protocol(ERROR_SIGNATURE_SIZE)) + } + + fn sign_offchain_message( + &self, + derivation_path: &DerivationPath, + message: &[u8], + ) -> Result { + self.sign_message(derivation_path, message) + } +} + +/// Check if device is a Keystone +pub fn is_valid_keystone(vendor_id: u16, product_id: u16) -> bool { + vendor_id == KEYSTONE_VID && product_id == KEYSTONE_PID +} + +fn extend_and_serialize(derivation_path: &DerivationPath) -> Vec { + let path = derivation_path.path(); + // Firmware expects: [coin_type(4 bytes, BE)] [depth(1 byte)] [path components(4 bytes each, BE)] + let mut serialized = Vec::with_capacity(4 + 1 + path.len() * 4); + serialized.extend_from_slice(&SOLANA_COIN_TYPE.to_be_bytes()); + serialized.push(path.len() as u8); + for index in path { + serialized.extend_from_slice(&index.to_bits().to_be_bytes()); + } + serialized +} + +/// Check whether a derivation path can be served by Keystone's pre-cached usb pubkey range. +/// 44'/501' +/// 44'/501'/0' ... 44'/501'/49' +/// 44'/501'/0'/0 ... 44'/501'/49'/0 +fn is_path_in_cached_range(derivation_path: &DerivationPath) -> bool { + let path = derivation_path.path(); + if path.len() < 2 { + return false; + } + + let purpose = path[0].to_bits() & 0x7fff_ffff; + let coin = path[1].to_bits() & 0x7fff_ffff; + if purpose != 44 || coin != 501 { + return false; + } + + match path.len() { + 2 => true, // m/44'/501' + 3 => { + let account = path[2].to_bits() & 0x7fff_ffff; + account <= CACHED_ACCOUNT_RANGE + } + 4 => { + let account = path[2].to_bits() & 0x7fff_ffff; + let change = path[3].to_bits() & 0x7fff_ffff; + change == CACHED_FIXED_CHANGE && account <= CACHED_ACCOUNT_RANGE + } + _ => false, + } +} + +/// Parse derivation path into CryptoKeyPath for UR encoding +fn parse_crypto_key_path(derivation_path: &DerivationPath, mfp: Option<[u8; 4]>) -> CryptoKeyPath { + let mut path_components = Vec::new(); + + for index in derivation_path.path() { + let bits = index.to_bits(); + let hardened = (bits & 0x8000_0000) != 0; + let value = bits & 0x7fff_ffff; + if let Ok(component) = PathComponent::new(Some(value), hardened) { + path_components.push(component); + } + } + + if path_components.is_empty() { + if let Ok(component) = PathComponent::new(Some(44u32), true) { + path_components.push(component); + } + if let Ok(component) = PathComponent::new(Some(501u32), true) { + path_components.push(component); + } + if let Ok(component) = PathComponent::new(Some(0u32), true) { + path_components.push(component); + } + } + + CryptoKeyPath::new(path_components, mfp, None) +} diff --git a/remote-wallet/src/lib.rs b/remote-wallet/src/lib.rs index 300f619b57e..ae6088d5ec5 100644 --- a/remote-wallet/src/lib.rs +++ b/remote-wallet/src/lib.rs @@ -1,6 +1,8 @@ #![cfg(feature = "agave-unstable-api")] #![allow(clippy::arithmetic_side_effects)] #![allow(dead_code)] +#[cfg(feature = "keystone")] +pub mod keystone; pub mod ledger; pub mod ledger_error; pub mod locator; diff --git a/remote-wallet/src/locator.rs b/remote-wallet/src/locator.rs index aa299ea54ac..cfbb19eb0b0 100644 --- a/remote-wallet/src/locator.rs +++ b/remote-wallet/src/locator.rs @@ -14,11 +14,15 @@ pub enum Manufacturer { Unknown, Ledger, Trezor, + #[cfg(feature = "keystone")] + Keystone, } const MANUFACTURER_UNKNOWN: &str = "unknown"; const MANUFACTURER_LEDGER: &str = "ledger"; const MANUFACTURER_TREZOR: &str = "trezor"; +#[cfg(feature = "keystone")] +const MANUFACTURER_KEYSTONE: &str = "keystone"; #[derive(Clone, Debug, Error, PartialEq, Eq)] #[error("not a manufacturer")] @@ -37,6 +41,8 @@ impl FromStr for Manufacturer { match s.as_str() { MANUFACTURER_LEDGER => Ok(Self::Ledger), MANUFACTURER_TREZOR => Ok(Self::Trezor), + #[cfg(feature = "keystone")] + MANUFACTURER_KEYSTONE => Ok(Self::Keystone), _ => Err(ManufacturerError), } } @@ -55,6 +61,8 @@ impl AsRef for Manufacturer { Self::Unknown => MANUFACTURER_UNKNOWN, Self::Ledger => MANUFACTURER_LEDGER, Self::Trezor => MANUFACTURER_TREZOR, + #[cfg(feature = "keystone")] + Self::Keystone => MANUFACTURER_KEYSTONE, } } } diff --git a/remote-wallet/src/remote_keypair.rs b/remote-wallet/src/remote_keypair.rs index 5cce82980ff..9b8e1fb266c 100644 --- a/remote-wallet/src/remote_keypair.rs +++ b/remote-wallet/src/remote_keypair.rs @@ -3,8 +3,7 @@ use { ledger::get_wallet_from_info, locator::Locator, remote_wallet::{ - RemoteWallet, RemoteWalletError, RemoteWalletInfo, RemoteWalletManager, - RemoteWalletType, + RemoteWalletError, RemoteWalletInfo, RemoteWalletManager, RemoteWalletType, }, }, solana_derivation_path::DerivationPath, @@ -27,10 +26,7 @@ impl RemoteKeypair { confirm_key: bool, path: String, ) -> Result { - let pubkey = match &wallet_type { - RemoteWalletType::Ledger(wallet) => wallet.get_pubkey(&derivation_path, confirm_key)?, - RemoteWalletType::Trezor(wallet) => wallet.get_pubkey(&derivation_path, confirm_key)?, - }; + let pubkey = wallet_type.get_pubkey(&derivation_path, confirm_key)?; Ok(Self { wallet_type, @@ -47,14 +43,9 @@ impl Signer for RemoteKeypair { } fn try_sign_message(&self, message: &[u8]) -> Result { - match &self.wallet_type { - RemoteWalletType::Ledger(wallet) => wallet - .sign_message(&self.derivation_path, message) - .map_err(|e| e.into()), - RemoteWalletType::Trezor(wallet) => wallet - .sign_message(&self.derivation_path, message) - .map_err(|e| e.into()), - } + self.wallet_type + .sign_message(&self.derivation_path, message) + .map_err(|e| e.into()) } fn is_interactive(&self) -> bool { @@ -70,12 +61,16 @@ pub fn generate_remote_keypair( keypair_name: &str, ) -> Result { let remote_wallet_info = RemoteWalletInfo::parse_locator(locator); + let remote_wallet = get_wallet_from_info(remote_wallet_info, keypair_name, wallet_manager)?; let path = format!("{}{}", remote_wallet.path, derivation_path.get_query()); - RemoteKeypair::new( - remote_wallet.wallet_type, + let wallet_type = remote_wallet.wallet_type; + let pubkey = wallet_type.get_pubkey(&derivation_path, confirm_key)?; + + Ok(RemoteKeypair { + wallet_type, derivation_path, - confirm_key, + pubkey, path, - ) + }) } diff --git a/remote-wallet/src/remote_wallet.rs b/remote-wallet/src/remote_wallet.rs index 66f838134dc..6494bab72be 100644 --- a/remote-wallet/src/remote_wallet.rs +++ b/remote-wallet/src/remote_wallet.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "keystone")] +use crate::keystone::KeystoneWallet; +#[cfg(all(feature = "hidapi", feature = "keystone"))] +use rusb::UsbContext; #[cfg(feature = "hidapi")] use {crate::ledger::is_valid_ledger, parking_lot::Mutex, std::sync::Arc}; use { @@ -60,6 +64,10 @@ pub enum RemoteWalletError { #[error("trezor error: {0}")] TrezorError(String), + #[cfg(feature = "keystone")] + #[error("keystone error: {0}")] + KeystoneError(String), + #[error("remote wallet operation rejected by the user")] UserCancel, @@ -83,6 +91,8 @@ impl From for SignerError { RemoteWalletError::InvalidInput(input) => SignerError::InvalidInput(input), RemoteWalletError::LedgerError(e) => SignerError::Protocol(e.to_string()), RemoteWalletError::TrezorError(e) => SignerError::Protocol(e), + #[cfg(feature = "keystone")] + RemoteWalletError::KeystoneError(e) => SignerError::Protocol(e), RemoteWalletError::NoDeviceFound => SignerError::NoDeviceFound, RemoteWalletError::Protocol(e) => SignerError::Protocol(e.to_string()), RemoteWalletError::UserCancel => { @@ -122,12 +132,31 @@ impl RemoteWalletManager { pub fn update_devices(&self) -> Result { let mut usb = self.usb.lock(); usb.refresh_devices()?; - let devices = usb.device_list(); let num_prev_devices = self.devices.read().len(); let mut detected_devices = vec![]; let mut errors = vec![]; - for device_info in devices.filter(|&device_info| { + Self::scan_ledger_devices(&usb, &mut detected_devices, &mut errors); + Self::scan_trezor_devices(&mut detected_devices, &mut errors); + #[cfg(feature = "keystone")] + Self::scan_keystone_devices(&mut detected_devices, &mut errors); + let num_curr_devices = detected_devices.len(); + *self.devices.write() = detected_devices; + + if num_curr_devices == 0 && !errors.is_empty() { + return Err(errors[0].clone()); + } + + Ok(num_curr_devices - num_prev_devices) + } + + #[cfg(feature = "hidapi")] + fn scan_ledger_devices( + usb: &hidapi::HidApi, + detected_devices: &mut Vec, + errors: &mut Vec, + ) { + for device_info in usb.device_list().filter(|&device_info| { #[cfg(not(any(feature = "linux-static-libusb", feature = "linux-shared-libusb")))] let is_valid_hid_device = is_valid_hid_device(device_info.usage_page(), device_info.interface_number()); @@ -160,7 +189,13 @@ impl RemoteWalletManager { Err(err) => error!("Error connecting to ledger device to read info: {err}"), } } + } + #[cfg(feature = "hidapi")] + fn scan_trezor_devices( + detected_devices: &mut Vec, + errors: &mut Vec, + ) { for device in trezor_client::find_devices(false) { let mut trezor = match device.connect() { Ok(t) => t, @@ -189,14 +224,66 @@ impl RemoteWalletManager { wallet_type: RemoteWalletType::Trezor(Rc::new(wallet)), }); } - let num_curr_devices = detected_devices.len(); - *self.devices.write() = detected_devices; + } - if num_curr_devices == 0 && !errors.is_empty() { - return Err(errors[0].clone()); - } + #[cfg(all(feature = "hidapi", feature = "keystone"))] + fn scan_keystone_devices( + detected_devices: &mut Vec, + errors: &mut Vec, + ) { + let Ok(context) = rusb::Context::new() else { + return; + }; + let Ok(device_list) = context.devices() else { + return; + }; - Ok(num_curr_devices - num_prev_devices) + for device in device_list.iter() { + let Ok(desc) = device.device_descriptor() else { + continue; + }; + // Some firmware modes may expose a different PID; use VID-based prefilter, + // then still prefer known Keystone VID/PID path. + if !crate::keystone::is_valid_keystone(desc.vendor_id(), desc.product_id()) { + continue; + } + + let handle = match device.open() { + Ok(handle) => handle, + Err(err) => { + error!("Failed to open Keystone device: {err}"); + errors.push(RemoteWalletError::Hid(format!( + "Failed to open Keystone device {:04x}:{:04x}: {err}", + desc.vendor_id(), + desc.product_id() + ))); + continue; + } + }; + let mut keystone = match KeystoneWallet::new(device.clone(), handle) { + Ok(keystone) => keystone, + Err(err) => { + error!("Error initializing Keystone USB transport: {err}"); + errors.push(err); + continue; + } + }; + match keystone.read_device(&device) { + Ok(info) => { + keystone.pretty_path = info.get_pretty_path(); + trace!("Found Keystone device: {info:?}"); + detected_devices.push(Device { + path: info.host_device_path.clone(), + info, + wallet_type: RemoteWalletType::Keystone(Rc::new(keystone)), + }) + } + Err(err) => { + error!("Error connecting to Keystone device: {err}"); + errors.push(err); + } + } + } } #[cfg(not(feature = "hidapi"))] @@ -299,6 +386,36 @@ pub struct Device { pub enum RemoteWalletType { Ledger(Rc), Trezor(Rc), + #[cfg(feature = "keystone")] + Keystone(Rc), +} + +impl RemoteWalletType { + pub fn get_pubkey( + &self, + derivation_path: &DerivationPath, + confirm_key: bool, + ) -> Result { + match self { + Self::Ledger(wallet) => wallet.get_pubkey(derivation_path, confirm_key), + Self::Trezor(wallet) => wallet.get_pubkey(derivation_path, confirm_key), + #[cfg(feature = "keystone")] + Self::Keystone(wallet) => wallet.get_pubkey(derivation_path, confirm_key), + } + } + + pub fn sign_message( + &self, + derivation_path: &DerivationPath, + message: &[u8], + ) -> Result { + match self { + Self::Ledger(wallet) => wallet.sign_message(derivation_path, message), + Self::Trezor(wallet) => wallet.sign_message(derivation_path, message), + #[cfg(feature = "keystone")] + Self::Keystone(wallet) => wallet.sign_message(derivation_path, message), + } + } } /// Remote wallet information. diff --git a/reserved-account-keys/src/lib.rs b/reserved-account-keys/src/lib.rs index 49fdae78d6a..0f9cf44021b 100644 --- a/reserved-account-keys/src/lib.rs +++ b/reserved-account-keys/src/lib.rs @@ -3,7 +3,7 @@ //! New reserved account keys may be added as long as they specify a feature //! gate that transitions the key into read-only at an epoch boundary. #![cfg_attr(feature = "frozen-abi", feature(min_specialization))] -#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use { agave_feature_set::FeatureSet, solana_pubkey::Pubkey, diff --git a/rpc-client-api/Cargo.toml b/rpc-client-api/Cargo.toml index f6585059034..54fbe96d175 100644 --- a/rpc-client-api/Cargo.toml +++ b/rpc-client-api/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/rpc-client-nonce-utils/Cargo.toml b/rpc-client-nonce-utils/Cargo.toml index b7afd7b5a96..4b0a16a51c9 100644 --- a/rpc-client-nonce-utils/Cargo.toml +++ b/rpc-client-nonce-utils/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] default = [] diff --git a/rpc-client-types/Cargo.toml b/rpc-client-types/Cargo.toml index be050842515..a5f0d7813a3 100644 --- a/rpc-client-types/Cargo.toml +++ b/rpc-client-types/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/rpc-client/Cargo.toml b/rpc-client/Cargo.toml index c9e52e92c04..f228d96f0f6 100644 --- a/rpc-client/Cargo.toml +++ b/rpc-client/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] default = ["spinner"] diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 08f270ccdd0..dbc053bcf81 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/rpc/src/rpc_subscription_tracker.rs b/rpc/src/rpc_subscription_tracker.rs index 8c0aa8c1b7b..bae372d6c16 100644 --- a/rpc/src/rpc_subscription_tracker.rs +++ b/rpc/src/rpc_subscription_tracker.rs @@ -500,7 +500,6 @@ impl SubscriptionsTracker { } } - #[allow(clippy::collapsible_if)] pub fn unsubscribe(&mut self, params: SubscriptionParams, id: SubscriptionId) { match ¶ms { SubscriptionParams::Logs(params) => { @@ -520,20 +519,16 @@ impl SubscriptionsTracker { } _ => {} } - if params.is_commitment_watcher() { - if self.commitment_watchers.remove(&id).is_none() { - warn!("Subscriptions inconsistency (missing entry in commitment_watchers)"); - } + if params.is_commitment_watcher() && self.commitment_watchers.remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in commitment_watchers)"); } - if params.is_gossip_watcher() { - if self.gossip_watchers.remove(&id).is_none() { - warn!("Subscriptions inconsistency (missing entry in gossip_watchers)"); - } + if params.is_gossip_watcher() && self.gossip_watchers.remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in gossip_watchers)"); } - if params.is_node_progress_watcher() { - if self.node_progress_watchers.remove(¶ms).is_none() { - warn!("Subscriptions inconsistency (missing entry in node_progress_watchers)"); - } + if params.is_node_progress_watcher() + && self.node_progress_watchers.remove(¶ms).is_none() + { + warn!("Subscriptions inconsistency (missing entry in node_progress_watchers)"); } } @@ -571,7 +566,6 @@ impl fmt::Debug for SubscriptionTokenInner { } impl Drop for SubscriptionTokenInner { - #[allow(clippy::collapsible_if)] fn drop(&mut self) { match self.control.subscriptions.entry(self.params.clone()) { DashEntry::Vacant(_) => { diff --git a/runtime-transaction/Cargo.toml b/runtime-transaction/Cargo.toml index 44cc039a367..ade769c8f5f 100644 --- a/runtime-transaction/Cargo.toml +++ b/runtime-transaction/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 0527ce37996..2c2292f1d01 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index f963911b115..89a4de1f1a8 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -82,8 +82,10 @@ use { agave_reserved_account_keys::ReservedAccountKeys, agave_snapshots::snapshot_hash::SnapshotHash, agave_votor_messages::{ - certificate::Certificate, migration::GENESIS_CERTIFICATE_ACCOUNT, + certificate::{Certificate, CertificateType}, + migration::GENESIS_CERTIFICATE_ACCOUNT, unverified_vote_message::UnverifiedCertificate, + wire::{WireBlockCertMessage, WireCertSignature}, }, ahash::AHashSet, log::*, @@ -199,7 +201,6 @@ use { std::{ collections::{HashMap, HashSet}, fmt, - num::NonZero, ops::AddAssign, path::PathBuf, slice, @@ -229,11 +230,6 @@ use { solana_svm::program_loader::load_program_with_pubkey, }; -/// params to `verify_accounts_hash` -struct VerifyAccountsHashConfig { - require_rooted_bank: bool, -} - mod accounts_lt_hash; mod address_lookup_table; pub mod bank_hash_details; @@ -274,7 +270,7 @@ static NANOSECOND_CLOCK_ACCOUNT: LazyLock = LazyLock::new(|| { pub type BankStatusCache = StatusCache>; #[cfg_attr( feature = "frozen-abi", - frozen_abi(digest = "23uAyYmzMrmPvPDKf6SvF1YoojYstmEPmdkfAQDnpwsq") + frozen_abi(digest = "HvpA8mUc4TZAcDF3BpcynmYWYBK3scJRTem2qadCiF5Z") )] pub type BankSlotDelta = SlotDelta>; @@ -3345,14 +3341,28 @@ impl Bank { // The address is known in advance, so the account could already exist if it was prefunded. // However this account cannot be written to except by us in `set_alpenglow_genesis_certificate`, // so this deserialize is safe if the account is non-empty - wincode::deserialize(acct.data()) - .expect("Programmer error deserializing genesis certificate") + let cert: WireBlockCertMessage = wincode::deserialize(acct.data()) + .expect("Programmer error deserializing genesis certificate"); + Certificate { + cert_type: CertificateType::Genesis(cert.block), + signature: cert.signature.signature, + bitmap: cert.signature.bitmap, + } }) } /// For use in the first Alpenglow block, set the genesis certificate. pub fn set_alpenglow_genesis_certificate(&self, cert: &Certificate) { - let data = wincode::serialize(cert).unwrap(); + debug_assert!(cert.cert_type.is_genesis()); + let block = cert.cert_type.to_block().unwrap(); + let cert = WireBlockCertMessage { + block, + signature: WireCertSignature { + signature: cert.signature, + bitmap: cert.bitmap.clone(), + }, + }; + let data = wincode::serialize(&cert).unwrap(); let lamports = Rent::default().minimum_balance(data.len()); let mut cert_acct = AccountSharedData::new(lamports, data.len(), &system_program::ID); cert_acct.set_data_from_slice(&data); @@ -5321,12 +5331,7 @@ impl Bank { pub fn run_final_hash_calc(&self) { self.force_flush_accounts_cache(); // note that this slot may not be a root - _ = self.verify_accounts( - VerifyAccountsHashConfig { - require_rooted_bank: false, - }, - None, - ); + _ = self.verify_accounts(None); } /// Verify the account state as part of startup, typically from a snapshot. @@ -5342,31 +5347,9 @@ impl Bank { /// /// Only intended to be called at startup, or from tests/ledger-tool. #[must_use] - fn verify_accounts( - &self, - config: VerifyAccountsHashConfig, - calculated_accounts_lt_hash: Option<&AccountsLtHash>, - ) -> bool { + fn verify_accounts(&self, calculated_accounts_lt_hash: Option<&AccountsLtHash>) -> bool { let accounts_db = &self.rc.accounts.accounts_db; - let slot = self.slot(); - - if config.require_rooted_bank && !accounts_db.accounts_index.is_alive_root(slot) { - if let Some(parent) = self.parent() { - info!( - "slot {slot} is not a root, so verify accounts hash on parent bank at slot {}", - parent.slot(), - ); - // The calculated_accounts_lt_hash parameter is only valid for the current slot, so - // we must fall back to calculating the accounts lt hash with the index. - return parent.verify_accounts(config, None); - } else { - // this will result in mismatch errors - // accounts hash calc doesn't include unrooted slots - panic!("cannot verify accounts hash because slot {slot} is not a root"); - } - } - fn check_lt_hash( expected_accounts_lt_hash: &AccountsLtHash, calculated_accounts_lt_hash: &AccountsLtHash, @@ -5567,12 +5550,7 @@ impl Bank { let (verified_accounts, verify_accounts_time_us) = measure_us!({ let should_verify_accounts = !self.rc.accounts.accounts_db.skip_initial_hash_calc; if should_verify_accounts { - self.verify_accounts( - VerifyAccountsHashConfig { - require_rooted_bank: false, - }, - calculated_accounts_lt_hash, - ) + self.verify_accounts(calculated_accounts_lt_hash) } else { info!("Verifying accounts... Skipped."); true @@ -5768,8 +5746,7 @@ impl Bank { .epoch_stakes_from_slot(slot) .ok_or(CertVerifyError::MissingRankMap)?; let key_to_rank_map = epoch_stakes.bls_pubkey_to_rank_map(); - let total_stake = - NonZero::new(key_to_rank_map.total_stake()).expect("total stake cannot be 0"); + let total_stake = key_to_rank_map.total_stake(); let cert = cert_verify::verify_certificate(cert, key_to_rank_map.len(), total_stake, |rank| { diff --git a/runtime/src/bank/partitioned_epoch_rewards/calculation.rs b/runtime/src/bank/partitioned_epoch_rewards/calculation.rs index 0715eaa26e2..52da43dc76d 100644 --- a/runtime/src/bank/partitioned_epoch_rewards/calculation.rs +++ b/runtime/src/bank/partitioned_epoch_rewards/calculation.rs @@ -14,7 +14,7 @@ use { fee_distribution::ExternalCollectorType, null_tracer, }, inflation_rewards::{ - adjust_delegation_for_rent, + delegation_may_need_adjustment, points::{ CalculationEnvironment, DelegatedVoteState, PointValue, calculate_points_for_tower, }, @@ -205,6 +205,7 @@ impl Bank { distribution_starting_block_height, num_partitions, point_value, + 0, // block_rewards ); datapoint_info!( @@ -558,23 +559,23 @@ impl Bank { .rent_collector .rent .minimum_balance(stake_account.data_len()); - let mut stake = *stake_account.stake(); + let stake = *stake_account.stake(); let Some(vote_account) = distribution_epoch_vote_accounts.get(&vote_pubkey) else { debug!("could not find vote account {vote_pubkey} in cache"); // Even if the vote account doesn't exist, there might still be a // need to adjust the stake delegation if adjust_delegations_for_rent { - let delegation = stake.delegation.stake; - let stake_was_adjusted = adjust_delegation_for_rent( - &mut stake.delegation, - rewarded_epoch, - delegation, + if delegation_may_need_adjustment( + stake.delegation.stake, + stake.delegation.stake, current_lamports, minimum_lamports, - ); - if stake_was_adjusted { - debug!("delegation for stake {stake_pubkey} was adjusted"); + ) { + debug!( + "delegation for stake {stake_pubkey} may be adjusted at distribution, \ + unless lamports are transferred before distribution block" + ); let inflation = InflationReward { stake, stake_reward: 0, @@ -595,7 +596,7 @@ impl Bank { reward_commission, }); } else { - debug!("delegation for stake {stake_pubkey} was not adjusted"); + debug!("delegation for stake {stake_pubkey} will not be adjusted"); return None; } } else { diff --git a/runtime/src/bank/partitioned_epoch_rewards/distribution.rs b/runtime/src/bank/partitioned_epoch_rewards/distribution.rs index 076ef58939c..bca6d58c752 100644 --- a/runtime/src/bank/partitioned_epoch_rewards/distribution.rs +++ b/runtime/src/bank/partitioned_epoch_rewards/distribution.rs @@ -46,6 +46,35 @@ struct DistributionResults { updated_stake_rewards: StakeRewards, } +/// Adjusts stake delegation based on Rent sysvar parameters. +/// +/// As part of SIMD-0392, if Rent is ever increased, we need to make sure that +/// lamports are not double-counted for the rent-exempt minimum and the stake +/// delegation. This function adjusts the delegation in a Stake if needed, right +/// at distribution time. +fn adjust_delegation_for_rent( + delegation: &mut Delegation, + rewarded_epoch: Epoch, + new_delegation_with_rewards: u64, + lamports_with_rewards: u64, + minimum_lamports: u64, +) { + let new_delegation = std::cmp::min( + new_delegation_with_rewards, + lamports_with_rewards.saturating_sub(minimum_lamports), + ); + + if new_delegation != delegation.stake { + delegation.stake = new_delegation; + // Deactivate stake if needed. This deactivation is immediate, + // unlike a requested deactivation which happens at the next epoch + // boundary + if new_delegation == 0 { + delegation.deactivation_epoch = rewarded_epoch; + } + } +} + impl Bank { /// Process reward distribution for the block if it is inside reward interval. pub(in crate::bank) fn distribute_partitioned_epoch_rewards(&mut self) { @@ -165,6 +194,7 @@ impl Bank { // decrease distributed capital from epoch rewards sysvar self.update_epoch_rewards_sysvar( stake_reward_lamports_minted + stake_reward_lamports_burned, + 0, // debit_block_rewards ); // update reward history for this partitioned distribution @@ -224,13 +254,21 @@ impl Bank { account .checked_add_lamports(partitioned_stake_reward.inflation.stake_reward) .map_err(|_| DistributionError::ArithmeticOverflow)?; + + let mut new_stake = partitioned_stake_reward.inflation.stake; if adjust_delegations_for_rent { let minimum_balance = rent.minimum_balance(account.data().len()); - assert!( - partitioned_stake_reward.inflation.stake.delegation.stake - <= account.lamports().saturating_sub(minimum_balance), - "stake reward delegation must be consistent with the updated stake account \ - lamport balance" + // The rewarded epoch is right before the distribution epoch + let rewarded_epoch = distribution_epoch.saturating_sub(1); + // The entry in `partitioned_stake_reward` contains the rewards, + // calculated during the calculation phase + let delegation_with_rewards = new_stake.delegation.stake; + adjust_delegation_for_rent( + &mut new_stake.delegation, + rewarded_epoch, + delegation_with_rewards, + account.lamports(), + minimum_balance, ); } else { let expected_delegation = stake @@ -238,21 +276,17 @@ impl Bank { .stake .saturating_add(partitioned_stake_reward.inflation.stake_reward); assert_eq!( - expected_delegation, partitioned_stake_reward.inflation.stake.delegation.stake, + expected_delegation, new_stake.delegation.stake, "stake reward delegation must be consistent with the updated stake account \ lamport balance" ); } account - .set_state(&StakeStateV2::Stake( - meta, - partitioned_stake_reward.inflation.stake, - flags, - )) + .set_state(&StakeStateV2::Stake(meta, new_stake, flags)) .map_err(|_| DistributionError::UnableToSetState)?; let stake_at_distribution_epoch = delegation_effective_stake( - &partitioned_stake_reward.inflation.stake.delegation, + &new_stake.delegation, distribution_epoch, stake_history, new_warmup_cooldown_rate_epoch, @@ -367,6 +401,7 @@ mod tests { use { super::*, crate::{ + alpenglow_epoch_type::AlpenglowEpochType, bank::{ partitioned_epoch_rewards::{ InflationReward, PartitionedStakeRewards, REWARD_CALCULATION_NUM_BLOCKS, @@ -374,7 +409,10 @@ mod tests { }, tests::create_genesis_config, }, - inflation_rewards::points::PointValue, + inflation_rewards::{ + points::{CalculationEnvironment, DelegatedVoteState, PointValue, null_tracer}, + redeem_rewards, + }, reward_info::RewardInfo, stake_utils, }, @@ -392,7 +430,7 @@ mod tests { }, solana_sysvar as sysvar, solana_vote_interface::state::BLS_PUBLIC_KEY_COMPRESSED_SIZE, - solana_vote_program::vote_state, + solana_vote_program::vote_state::{self, VoteStateV4, handler::VoteStateHandler}, std::sync::Arc, test_case::test_case, }; @@ -511,6 +549,7 @@ mod tests { // Set up epoch_rewards sysvar with rewards with 1e9 lamports to distribute. let inflation_rewards = 1_000_000_000; + let block_rewards = 0; let num_partitions = 2; // num_partitions is arbitrary and unimportant for this test let total_points = (inflation_rewards * 42) as u128; // total_points is arbitrary for the purposes of this test bank.create_epoch_rewards_sysvar( @@ -521,6 +560,7 @@ mod tests { rewards: inflation_rewards, points: total_points, }, + block_rewards, ); let pre_epoch_rewards_account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); let expected_balance = @@ -1014,6 +1054,7 @@ mod tests { // Set up epoch_rewards sysvar with rewards with 10e9 lamports to distribute. let total_rewards = 10 * LAMPORTS_PER_SOL; + let block_rewards = 0; let num_partitions = 2; // num_partitions is arbitrary and unimportant for this test let total_points = (total_rewards * 42) as u128; // total_points is arbitrary for the purposes of this test bank.create_epoch_rewards_sysvar( @@ -1024,6 +1065,7 @@ mod tests { rewards: total_rewards, points: total_points, }, + block_rewards, ); let pre_epoch_rewards_account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); let expected_balance = @@ -1099,4 +1141,366 @@ mod tests { // distributed assert_eq!(pre_cap + rewards_to_distribute, post_cap); } + + #[test] + fn test_delegation_adjustment_at_distribution() { + let (mut genesis_config, _mint_keypair) = + create_genesis_config(1_000_000 * LAMPORTS_PER_SOL); + genesis_config.epoch_schedule = EpochSchedule::custom(432000, 432000, false); + let bank = Bank::new_for_tests(&genesis_config); + + // Set up epoch_rewards sysvar with rewards with 10e9 lamports to distribute. + let total_rewards = 10 * LAMPORTS_PER_SOL; + let block_rewards = 0; + let num_partitions = 2; // num_partitions is arbitrary and unimportant for this test + let total_points = (total_rewards * 42) as u128; // total_points is arbitrary for the purposes of this test + bank.create_epoch_rewards_sysvar( + 0, + 42, + num_partitions, + &PointValue { + rewards: total_rewards, + points: total_points, + }, + block_rewards, + ); + let pre_epoch_rewards_account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); + let expected_balance = + bank.get_minimum_balance_for_rent_exemption(pre_epoch_rewards_account.data().len()); + // Expected balance is the sysvar rent-exempt balance + assert_eq!(pre_epoch_rewards_account.lamports(), expected_balance); + + // Use lower lamports per byte for creating, bank has higher amount + let mut lower_rent = bank.rent_collector.rent.clone(); + lower_rent.lamports_per_byte /= 10; + + // Below new minimum, small reward, should normally be destaked + let reward_lamports = 1; + let stake_reward = StakeReward::new_with_pre_stake_account(reward_lamports, 1, &lower_rent); + bank.store_account(&stake_reward.1.stake_pubkey, &stake_reward.0); + + let stake_pubkey = stake_reward.1.stake_pubkey; + let mut stake_account = stake_reward.0; + + let expected_num = 1; + let rewards_to_distribute = stake_reward.1.stake_reward_info.lamports as u64; + let all_rewards = convert_rewards(vec![stake_reward.1]); + + let partitioned_rewards = StartBlockHeightAndPartitionedRewards { + distribution_starting_block_height: bank.block_height() + REWARD_CALCULATION_NUM_BLOCKS, + all_stake_rewards: Arc::new(all_rewards), + partition_indices: vec![(0..expected_num).collect::>()], + }; + + // But we transfer in more lamports before distribution time + stake_account.checked_add_lamports(1_000_000_000).unwrap(); + bank.store_account(&stake_pubkey, &stake_account); + + // Distribute rewards + let pre_cap = bank.capitalization(); + bank.distribute_epoch_rewards_in_partition(&partitioned_rewards, 0); + let post_cap = bank.capitalization(); + let post_epoch_rewards_account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); + + // Assert that epoch rewards sysvar lamports balance does not change + assert_eq!(post_epoch_rewards_account.lamports(), expected_balance); + + let epoch_rewards: sysvar::epoch_rewards::EpochRewards = + from_account(&post_epoch_rewards_account).unwrap(); + assert_eq!(epoch_rewards.total_rewards, total_rewards); + assert_eq!(epoch_rewards.distributed_rewards, rewards_to_distribute,); + + // Assert that the bank total capital changed by the amount of rewards + // distributed + assert_eq!(pre_cap + rewards_to_distribute, post_cap); + + // Check that delegation just gets rewards + let post_account = bank.get_account(&stake_pubkey).unwrap(); + let post_stake_state: StakeStateV2 = post_account.state().unwrap(); + let pre_stake_state: StakeStateV2 = stake_account.state().unwrap(); + assert_eq!( + post_stake_state.delegation().unwrap().stake, + pre_stake_state.delegation().unwrap().stake + reward_lamports as u64 + ); + } + + fn check_rent_adjusted_stake_delegation( + rewarded_epoch: u64, + pre_stake: Stake, + pre_lamports: u64, + new_minimum_balance: u64, + total_rewards: u64, + reward_info: Option<(u64, Stake)>, + ) { + let mut vote_state = VoteStateHandler::new_v4(VoteStateV4::default()); + // put 1 credit to create rewards + vote_state.increment_credits(rewarded_epoch, 1); + let stake_history: &StakeHistory = &StakeHistory::default(); + let new_rate_activation_epoch = None; + let commission_rate_in_basis_points = true; + let adjust_delegations_for_rent = true; + + let maybe_rewards = redeem_rewards( + pre_stake, + vote_state.as_ref_v4().inflation_rewards_commission_bps, + DelegatedVoteState::from(vote_state.as_ref_v4()), + CalculationEnvironment { + rewarded_epoch, + point_value: &PointValue { + rewards: total_rewards, + points: 1, + }, + stake_history, + new_rate_activation_epoch, + commission_rate_in_basis_points, + adjust_delegations_for_rent, + use_fixed_point_stake_math: true, + }, + null_tracer(), + &AlpenglowEpochType::Tower, + pre_lamports, + new_minimum_balance, + ); + + // fake the distribution portion which adjusts the delegation + let maybe_rewards = maybe_rewards + .map(|x| { + let stake_rewards = x.0; + let mut stake = x.2; + let new_delegation_with_rewards = stake.delegation.stake; + adjust_delegation_for_rent( + &mut stake.delegation, + rewarded_epoch, + new_delegation_with_rewards, + pre_lamports + stake_rewards, + new_minimum_balance, + ); + (stake_rewards, stake) + }) + .ok(); + assert_eq!(maybe_rewards, reward_info); + } + + #[test] + fn rent_adjusted_stake_delegation_calculations() { + let old_minimum_balance = 8; + let new_minimum_balance = 9; + let rewarded_epoch = 1; + + // No rewards at all -> updated (all stakes get driven forward if + // inflation is disabled) + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 1, + }, + new_minimum_balance + 1, + new_minimum_balance, + 0, + Some(( + 0, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 1, + }, + )), + ); + + // Stake receives no rewards or delegation adjustment -> no update + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 1, + }, + new_minimum_balance + 1, + new_minimum_balance, + 1, + None, + ); + + // Already destaked -> no update + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 0, + deactivation_epoch: 0, + ..Default::default() + }, + credits_observed: 0, + }, + old_minimum_balance - 1, + new_minimum_balance, + 1, + None, + ); + + // Staked, already below minimum, go further below minimum + // -> destaked, still one lamport of rewards though + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 0, + }, + old_minimum_balance - 1, + new_minimum_balance, + 1, + Some(( + 1, + Stake { + delegation: Delegation { + stake: 0, + deactivation_epoch: rewarded_epoch, + ..Default::default() + }, + credits_observed: 1, + }, + )), + ); + + // Delegation hits exactly 0 -> destaked, no rewards + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 0, + }, + new_minimum_balance, + new_minimum_balance, + 0, + Some(( + 0, + Stake { + delegation: Delegation { + stake: 0, + deactivation_epoch: rewarded_epoch, + ..Default::default() + }, + credits_observed: 1, + }, + )), + ); + + // Delegation decreases to 1 -> still staked + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 2, + ..Default::default() + }, + credits_observed: 0, + }, + new_minimum_balance + 1, + new_minimum_balance, + 0, + Some(( + 0, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 1, + }, + )), + ); + + // Rewards partially cover minimum balance change + // -> decrease stake + // This case is confusing because it pays out 2 lamports in rewards, + // so we adjust minimum up so that even with 2 lamports in rewards, the + // delegation goes down. + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 2, + ..Default::default() + }, + credits_observed: 0, + }, + new_minimum_balance, + new_minimum_balance + 1, + 1, + Some(( + 2, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 1, + }, + )), + ); + + // Rewards cover minimum balance change -> no change in stake + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 0, + }, + new_minimum_balance, + new_minimum_balance, + 1, + Some(( + 1, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 1, + }, + )), + ); + + // Well above new minimum balance -> delegation change capped to rewards + check_rent_adjusted_stake_delegation( + rewarded_epoch, + Stake { + delegation: Delegation { + stake: 1, + ..Default::default() + }, + credits_observed: 0, + }, + new_minimum_balance + 2, + new_minimum_balance, + 1, + Some(( + 1, + Stake { + delegation: Delegation { + stake: 2, + ..Default::default() + }, + credits_observed: 1, + }, + )), + ); + } } diff --git a/runtime/src/bank/partitioned_epoch_rewards/sysvar.rs b/runtime/src/bank/partitioned_epoch_rewards/sysvar.rs index 06ab9ec96ce..77b7477252a 100644 --- a/runtime/src/bank/partitioned_epoch_rewards/sysvar.rs +++ b/runtime/src/bank/partitioned_epoch_rewards/sysvar.rs @@ -2,16 +2,19 @@ use { super::Bank, crate::inflation_rewards::points::PointValue, log::info, - solana_account::{create_account_shared_data_with_fields as create_account, from_account}, - solana_sysvar as sysvar, + solana_account::{ + ReadableAccount, WritableAccount, create_account_shared_data_with_fields as create_account, + from_account, + }, + solana_clock::INITIAL_RENT_EPOCH, + solana_sysvar::{self as sysvar, epoch_rewards::EpochRewards}, }; impl Bank { /// Helper fn to log epoch_rewards sysvar fn log_epoch_rewards_sysvar(&self, prefix: &str) { if let Some(account) = self.get_account(&sysvar::epoch_rewards::id()) { - let epoch_rewards: sysvar::epoch_rewards::EpochRewards = - from_account(&account).unwrap(); + let epoch_rewards: EpochRewards = from_account(&account).unwrap(); info!("{prefix} epoch_rewards sysvar: {epoch_rewards:?}"); } else { info!("{prefix} epoch_rewards sysvar: none"); @@ -27,12 +30,13 @@ impl Bank { distribution_starting_block_height: u64, num_partitions: u64, point_value: &PointValue, + block_rewards: u64, ) { assert!(point_value.rewards >= distributed_rewards); let parent_blockhash = self.last_blockhash(); - let epoch_rewards = sysvar::epoch_rewards::EpochRewards { + let epoch_rewards = EpochRewards { distribution_starting_block_height, num_partitions, parent_blockhash, @@ -42,6 +46,8 @@ impl Bank { active: true, }; + // Do the first store to create the account from scratch, update + // capitalization if needed, etc self.update_sysvar_account(&sysvar::epoch_rewards::id(), |account| { create_account( &epoch_rewards, @@ -49,6 +55,19 @@ impl Bank { ) }); + // Now add the lamports separately without updating capitalization, + // since block reward lamports already existed + let mut account = self + .get_account_with_fixed_root(&sysvar::epoch_rewards::id()) + .expect("created sysvar account exists"); + + // SAFETY: block rewards come from existing lamports, which cannot + // overflow + account + .checked_add_lamports(block_rewards) + .expect("block rewards and sysvar account rent exemption must fit in a u64"); + self.store_account(&sysvar::epoch_rewards::id(), &account); + self.log_epoch_rewards_sysvar("create"); } @@ -56,6 +75,7 @@ impl Bank { pub(in crate::bank::partitioned_epoch_rewards) fn update_epoch_rewards_sysvar( &self, distributed: u64, + debit_block_reward_lamports: u64, ) { let mut epoch_rewards = self.get_epoch_rewards_sysvar(); assert!(epoch_rewards.active); @@ -69,19 +89,48 @@ impl Bank { ) }); + // Debit the lamports separately without updating capitalization, + // since block reward lamports already existed + let mut account = self + .get_account_with_fixed_root(&sysvar::epoch_rewards::id()) + .expect("created sysvar account exists"); + + // SAFETY: programmer error if we debit too many block rewards + account + .checked_sub_lamports(debit_block_reward_lamports) + .expect("epoch reward sysvar has enough lamports for distribution"); + assert!( + account.lamports() >= self.get_minimum_balance_for_rent_exemption(account.data().len()), + "Sysvar account must have enough for rent exemption after debiting block rewards" + ); + self.store_account(&sysvar::epoch_rewards::id(), &account); + self.log_epoch_rewards_sysvar("update"); } - /// Update EpochRewards sysvar with distributed rewards + /// Update EpochRewards sysvar with distributed rewards and burn any + /// remaining lamports over the rent-exempt reserve pub(in crate::bank::partitioned_epoch_rewards) fn set_epoch_rewards_sysvar_to_inactive(&self) { + const RENT_UNADJUSTED_INITIAL_BALANCE: u64 = 1; + let mut epoch_rewards = self.get_epoch_rewards_sysvar(); assert!(epoch_rewards.total_rewards >= epoch_rewards.distributed_rewards); epoch_rewards.active = false; self.update_sysvar_account(&sysvar::epoch_rewards::id(), |account| { + // Don't use `inherit_specially_retained_account_fields()` to + // ensure that any remaining lamports get burned, lamports are + // set to the rent-exempt minimum during `update_sysvar_account`, + // and capitalization is updated create_account( &epoch_rewards, - self.inherit_specially_retained_account_fields(account), + ( + RENT_UNADJUSTED_INITIAL_BALANCE, + account + .as_ref() + .map(|a| a.rent_epoch()) + .unwrap_or(INITIAL_RENT_EPOCH), + ), ) }); @@ -92,7 +141,7 @@ impl Bank { /// account cannot be found or cannot be deserialized. pub(in crate::bank::partitioned_epoch_rewards) fn get_epoch_rewards_sysvar( &self, - ) -> sysvar::epoch_rewards::EpochRewards { + ) -> EpochRewards { from_account( &self .get_account(&sysvar::epoch_rewards::id()) @@ -107,7 +156,6 @@ mod tests { use { super::*, crate::bank::{SlotLeader, tests::create_genesis_config}, - solana_account::ReadableAccount, solana_epoch_schedule::EpochSchedule, solana_native_token::LAMPORTS_PER_SOL, }; @@ -131,8 +179,10 @@ mod tests { points: total_points, }; + let first_block_rewards = 5_000_000_000; + // create epoch rewards sysvar - let expected_epoch_rewards = sysvar::epoch_rewards::EpochRewards { + let expected_epoch_rewards = EpochRewards { distribution_starting_block_height: 42, num_partitions, parent_blockhash: bank.last_blockhash(), @@ -143,19 +193,34 @@ mod tests { }; let epoch_rewards = bank.get_epoch_rewards_sysvar(); - assert_eq!( - epoch_rewards, - sysvar::epoch_rewards::EpochRewards::default() - ); + assert_eq!(epoch_rewards, EpochRewards::default()); + + let pre_capitalization = bank.capitalization(); + bank.create_epoch_rewards_sysvar(10, 42, num_partitions, &point_value, first_block_rewards); + let post_capitalization = bank.capitalization(); - bank.create_epoch_rewards_sysvar(10, 42, num_partitions, &point_value); let account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); - let expected_balance = bank.get_minimum_balance_for_rent_exemption(account.data().len()); - // Expected balance is the sysvar rent-exempt balance + let rent_exempt_reserve = bank.get_minimum_balance_for_rent_exemption(account.data().len()); + let expected_balance = rent_exempt_reserve + first_block_rewards; + // Expected balance is the sysvar rent-exempt balance plus block rewards assert_eq!(account.lamports(), expected_balance); - let epoch_rewards: sysvar::epoch_rewards::EpochRewards = from_account(&account).unwrap(); + + // Expect capitalization to only change by rent exempt minimum + assert_eq!( + post_capitalization, + pre_capitalization + rent_exempt_reserve + ); + + let epoch_rewards: EpochRewards = from_account(&account).unwrap(); assert_eq!(epoch_rewards, expected_epoch_rewards); + // Unsetting should burn all block rewards + bank.set_epoch_rewards_sysvar_to_inactive(); + assert_eq!( + post_capitalization - first_block_rewards, + bank.capitalization() + ); + // Create a child bank to test parent_blockhash let parent_blockhash = bank.last_blockhash(); let parent_slot = bank.slot(); @@ -165,11 +230,26 @@ mod tests { SlotLeader::default(), parent_slot + 1, ); + let second_block_rewards = 500_000_000; + // Also note that running `create_epoch_rewards_sysvar()` against a bank // with an existing EpochRewards sysvar clobbers the previous values - bank.create_epoch_rewards_sysvar(10, 42, num_partitions, &point_value); + let pre_capitalization = bank.capitalization(); + bank.create_epoch_rewards_sysvar( + 10, + 42, + num_partitions, + &point_value, + second_block_rewards, + ); + let post_capitalization = bank.capitalization(); + + // Capitalization shouldn't change this time, no new lamports minted + // since account already existed, but different amount of lamports on + // account + assert_eq!(post_capitalization, pre_capitalization); - let expected_epoch_rewards = sysvar::epoch_rewards::EpochRewards { + let expected_epoch_rewards = EpochRewards { distribution_starting_block_height: 42, num_partitions, parent_blockhash, @@ -179,16 +259,31 @@ mod tests { active: true, }; + let account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); + let expected_balance = bank.get_minimum_balance_for_rent_exemption(account.data().len()) + + second_block_rewards; + // Expected balance is the sysvar rent-exempt balance with new block rewards + assert_eq!(account.lamports(), expected_balance); + let epoch_rewards = bank.get_epoch_rewards_sysvar(); assert_eq!(epoch_rewards, expected_epoch_rewards); // make a distribution from epoch rewards sysvar - bank.update_epoch_rewards_sysvar(10); + let block_reward_distribution = 1_000_000; + bank.update_epoch_rewards_sysvar(10, block_reward_distribution); let account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); - // Balance should not change - assert_eq!(account.lamports(), expected_balance); - let epoch_rewards: sysvar::epoch_rewards::EpochRewards = from_account(&account).unwrap(); - let expected_epoch_rewards = sysvar::epoch_rewards::EpochRewards { + + // Balance should change + assert_eq!( + account.lamports(), + expected_balance - block_reward_distribution + ); + + // Capitalization should not + assert_eq!(post_capitalization, bank.capitalization()); + + let epoch_rewards: EpochRewards = from_account(&account).unwrap(); + let expected_epoch_rewards = EpochRewards { distribution_starting_block_height: 42, num_partitions, parent_blockhash, @@ -198,5 +293,26 @@ mod tests { active: true, }; assert_eq!(epoch_rewards, expected_epoch_rewards); + + // Unsetting should burn the rest + bank.set_epoch_rewards_sysvar_to_inactive(); + assert_eq!( + bank.capitalization(), + post_capitalization + block_reward_distribution - second_block_rewards + ); + + let account = bank.get_account(&sysvar::epoch_rewards::id()).unwrap(); + let epoch_rewards: EpochRewards = from_account(&account).unwrap(); + let expected_epoch_rewards = EpochRewards { + distribution_starting_block_height: 42, + num_partitions, + parent_blockhash, + total_points, + total_rewards, + distributed_rewards: 20, + active: false, + }; + assert_eq!(epoch_rewards, expected_epoch_rewards); + assert_eq!(account.lamports(), rent_exempt_reserve); } } diff --git a/runtime/src/bank/sysvar_cache.rs b/runtime/src/bank/sysvar_cache.rs index 3840bd5505e..2dc5fcb53a6 100644 --- a/runtime/src/bank/sysvar_cache.rs +++ b/runtime/src/bank/sysvar_cache.rs @@ -125,6 +125,7 @@ mod tests { // inject a reward sysvar for test let num_partitions = 2; // num_partitions is arbitrary and unimportant for this test let total_points = 42_000; // total_points is arbitrary for the purposes of this test + let block_rewards = 42_000_000; // block_rewards are arbitrary for this test let expected_epoch_rewards = EpochRewards { distribution_starting_block_height: 42, num_partitions, @@ -142,6 +143,7 @@ mod tests { rewards: 100, points: total_points, }, + block_rewards, ); bank1 diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 217369efbbf..0e553825597 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -1006,14 +1006,6 @@ fn test_bank_update_rewards_determinism() { } } -impl VerifyAccountsHashConfig { - fn default_for_test() -> Self { - Self { - require_rooted_bank: false, - } - } -} - // Test that purging 0 lamports accounts works. #[test] fn test_purge_empty_accounts() { @@ -1083,7 +1075,7 @@ fn test_purge_empty_accounts() { if pass == 0 { add_root_and_flush_write_cache(&bank0); - assert!(bank0.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank0.verify_accounts(None)); continue; } @@ -1092,14 +1084,14 @@ fn test_purge_empty_accounts() { bank0.squash(); add_root_and_flush_write_cache(&bank0); if pass == 1 { - assert!(bank0.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank0.verify_accounts(None)); continue; } bank1.freeze(); bank1.squash(); add_root_and_flush_write_cache(&bank1); - assert!(bank1.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank1.verify_accounts(None)); // keypair should have 0 tokens on both forks assert_eq!(bank0.get_account(&keypair.pubkey()), None); @@ -1107,7 +1099,7 @@ fn test_purge_empty_accounts() { bank1.clean_accounts_for_tests(); - assert!(bank1.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank1.verify_accounts(None)); } } @@ -2286,7 +2278,7 @@ fn test_bank_hash_internal_state() { bank2.transfer(amount, &mint_keypair, &pubkey2).unwrap(); bank2.squash(); bank2.force_flush_accounts_cache(); - assert!(bank2.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank2.verify_accounts(None)); } #[test] @@ -2320,7 +2312,7 @@ fn test_bank_hash_internal_state_verify() { // we later modify bank 2, so this flush is destructive to the test bank2.freeze(); add_root_and_flush_write_cache(&bank2); - assert!(bank2.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank2.verify_accounts(None)); } let bank3 = Bank::new_from_parent_with_bank_forks( &bank_forks, @@ -2331,7 +2323,7 @@ fn test_bank_hash_internal_state_verify() { assert_eq!(bank0_state, bank0.hash_internal_state()); if pass == 0 { // this relies on us having set bank2's accounts hash in the pass==0 if above - assert!(bank2.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank2.verify_accounts(None)); continue; } if pass == 1 { @@ -2340,7 +2332,7 @@ fn test_bank_hash_internal_state_verify() { // Doing so throws an assert. So, we can't flush 3 until 2 is flushed. bank3.freeze(); add_root_and_flush_write_cache(&bank3); - assert!(bank3.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank3.verify_accounts(None)); continue; } @@ -2349,7 +2341,7 @@ fn test_bank_hash_internal_state_verify() { bank2.freeze(); // <-- keep freeze() *outside* `if pass == 2 {}` if pass == 2 { add_root_and_flush_write_cache(&bank2); - assert!(bank2.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank2.verify_accounts(None)); // Verifying the accounts lt hash is only intended to be called at startup, and // normally in the background. Since here we're *not* at startup, and doing it @@ -2364,7 +2356,7 @@ fn test_bank_hash_internal_state_verify() { bank3.freeze(); add_root_and_flush_write_cache(&bank3); - assert!(bank3.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank3.verify_accounts(None)); } } @@ -10997,7 +10989,7 @@ fn test_verify_accounts() { bank.force_flush_accounts_cache(); // ensure the accounts verify successfully - assert!(bank.verify_accounts(VerifyAccountsHashConfig::default_for_test(), None)); + assert!(bank.verify_accounts(None)); } #[test] diff --git a/runtime/src/bank_forks.rs b/runtime/src/bank_forks.rs index 395f3006ab0..c4ae16421eb 100644 --- a/runtime/src/bank_forks.rs +++ b/runtime/src/bank_forks.rs @@ -789,6 +789,7 @@ mod tests { certificate::{Certificate, CertificateType}, consensus_message::Block, migration::{GENESIS_CERTIFICATE_ACCOUNT, MIGRATION_SLOT_OFFSET}, + wire::{WireBlockCertMessage, WireCertSignature}, }, assert_matches::assert_matches, crossbeam_channel::{Receiver, Sender, bounded}, @@ -992,8 +993,15 @@ mod tests { } = create_genesis_config(10_000); genesis_config.epoch_schedule = EpochSchedule::new(32); - if let Some(genesis_cert) = genesis_cert.as_ref() { - let cert_data = wincode::serialize(genesis_cert).unwrap(); + if let Some(genesis_cert) = genesis_cert { + let cert = WireBlockCertMessage { + block: genesis_cert.cert_type.to_block().unwrap(), + signature: WireCertSignature { + signature: genesis_cert.signature, + bitmap: genesis_cert.bitmap, + }, + }; + let cert_data = wincode::serialize(&cert).unwrap(); let lamports = Rent::default().minimum_balance(cert_data.len()); let mut cert_account = Account::new(lamports, cert_data.len(), &system_program::ID); cert_account.data = cert_data; @@ -1112,7 +1120,10 @@ mod tests { // Migration can still succeed let mut bank = Bank::new_from_parent(root_bank, SlotLeader::default(), 10); let genesis_cert = Certificate { - cert_type: CertificateType::Finalize(1), + cert_type: CertificateType::Genesis(Block { + slot: 1, + block_id: Hash::new_unique(), + }), signature: BLSSignature([0; BLS_SIGNATURE_AFFINE_SIZE]), bitmap: vec![], }; diff --git a/runtime/src/block_component_processor.rs b/runtime/src/block_component_processor.rs index 3ee043db2ca..6722357970c 100644 --- a/runtime/src/block_component_processor.rs +++ b/runtime/src/block_component_processor.rs @@ -130,12 +130,6 @@ impl BlockComponentProcessor { return Ok(()); } - // If we encounter an UpdateParent when fast leader handover is disabled, error. - if !migration_status.should_allow_fast_leader_handover(slot) && self.update_parent.is_some() - { - return Err(BlockComponentProcessorError::SpuriousUpdateParent); - } - // Post-migration: both header and footer are required if !self.has_footer { return Err(BlockComponentProcessorError::MissingBlockFooter); @@ -190,6 +184,8 @@ impl BlockComponentProcessor { let markers_fully_enabled = migration_status.should_allow_block_markers(slot); let in_migration = migration_status.is_in_migration(); + let fast_leader_handover_active = + bank.feature_set.snapshot().alpenglow_fast_leader_handover; match marker { // Header and genesis cert can be processed either: @@ -219,7 +215,11 @@ impl BlockComponentProcessor { ), BlockMarkerV1::UpdateParent(update_parent) if markers_fully_enabled => { - self.on_update_parent(slot, update_parent.inner(), allow_initial_update_parent) + if fast_leader_handover_active { + self.on_update_parent(slot, update_parent.inner(), allow_initial_update_parent) + } else { + Err(BlockComponentProcessorError::SpuriousUpdateParent) + } } // Any other combination means we saw a marker too early @@ -227,12 +227,40 @@ impl BlockComponentProcessor { } } + /// Processes the genesis block marker with full verification pub fn on_genesis_cert_block_marker( &self, bank: Arc, shred_version: u16, genesis_block_marker: GenesisCertBlockMarker, migration_status: &MigrationStatus, + ) -> Result<(), BlockComponentProcessorError> { + self.process_genesis_cert_block_marker( + bank, + genesis_block_marker, + migration_status, + Some(shred_version), + ) + } + + /// Processes a locally produced genesis certificate marker without + /// re-verifying the certificate signature. + pub fn on_genesis_cert_block_marker_leader( + &self, + bank: Arc, + genesis_block_marker: GenesisCertBlockMarker, + migration_status: &MigrationStatus, + ) -> Result<(), BlockComponentProcessorError> { + self.process_genesis_cert_block_marker(bank, genesis_block_marker, migration_status, None) + } + + /// Performs verification if `shred_version` is specified + fn process_genesis_cert_block_marker( + &self, + bank: Arc, + genesis_block_marker: GenesisCertBlockMarker, + migration_status: &MigrationStatus, + shred_version: Option, ) -> Result<(), BlockComponentProcessorError> { // Genesis Certificate is only allowed for direct child of genesis if bank.parent_slot() == 0 { @@ -252,16 +280,26 @@ impl BlockComponentProcessor { return Err(BlockComponentProcessorError::GenesisCertificateAlreadyPopulated); } - let unverified_genesis_cert = UnverifiedCertificate { - cert_type: CertificateType::Genesis(Block { - slot: genesis_block_marker.slot, - block_id: genesis_block_marker.block_id, - }), - signature: genesis_block_marker.bls_signature, - bitmap: genesis_block_marker.bitmap, - shred_version, + let genesis_cert_type = CertificateType::Genesis(Block { + slot: genesis_block_marker.slot, + block_id: genesis_block_marker.block_id, + }); + let genesis_cert = match shred_version { + Some(shred_version) => { + let unverified_genesis_cert = UnverifiedCertificate { + cert_type: genesis_cert_type, + signature: genesis_block_marker.bls_signature, + bitmap: genesis_block_marker.bitmap, + shred_version, + }; + Self::verify_genesis_certificate(&bank, unverified_genesis_cert)? + } + None => Certificate { + cert_type: genesis_cert_type, + signature: genesis_block_marker.bls_signature, + bitmap: genesis_block_marker.bitmap, + }, }; - let genesis_cert = Self::verify_genesis_certificate(&bank, unverified_genesis_cert)?; bank.set_alpenglow_genesis_certificate(&genesis_cert); bank.set_hashes_per_tick(None); @@ -1521,7 +1559,6 @@ mod tests { } #[test] - #[ignore] // TODO(ksn): Enable when fast leader handover is enabled in MigrationPhase::should_allow_fast_leader_handover fn test_workflow_with_update_parent() { let migration_status = MigrationStatus::post_migration_status(); let mut processor = BlockComponentProcessor::default(); diff --git a/runtime/src/block_component_processor/vote_reward.rs b/runtime/src/block_component_processor/vote_reward.rs index 02f535a6295..e8b2b5d73ac 100644 --- a/runtime/src/block_component_processor/vote_reward.rs +++ b/runtime/src/block_component_processor/vote_reward.rs @@ -608,7 +608,7 @@ mod tests { create_genesis_config_with_alpenglow_vote_accounts, create_genesis_config_with_leader_ex, create_validator, }, - inflation_rewards::commission_split, + inflation_rewards::commission_split_preserve_lamports, stake_utils, validated_block_finalization::ValidatedBlockFinalizationCert, }, @@ -1272,7 +1272,7 @@ mod tests { let stake = initial_lamports - rent_exempt_reserve; let stake_weighted_reward = validator_reward * stake / validator_stake; let (voter_reward, staker_reward, is_split) = - commission_split(self.commission_bps, stake_weighted_reward); + commission_split_preserve_lamports(self.commission_bps, stake_weighted_reward); assert!(is_split); assert_eq!( staker_reward, diff --git a/runtime/src/block_component_processor/vote_reward/migration_test.rs b/runtime/src/block_component_processor/vote_reward/migration_test.rs index 9a07c87f81f..a28b6170a99 100644 --- a/runtime/src/block_component_processor/vote_reward/migration_test.rs +++ b/runtime/src/block_component_processor/vote_reward/migration_test.rs @@ -12,7 +12,7 @@ mod tests { ValidatorVoteKeypairs, activate_all_features, create_genesis_config_with_leader_ex, create_validator, }, - inflation_rewards::commission_split, + inflation_rewards::commission_split_preserve_lamports, stake_utils, }, agave_feature_set::FeatureSet, @@ -356,7 +356,7 @@ mod tests { self.pay_type.ag().map(NonZero::get).unwrap_or(0) * stake / validator_stake; let stake_weighted_reward = stake_weighted_tower + stake_weighted_ag; let (voter_reward, staker_reward, is_split) = - commission_split(self.commission_bps, stake_weighted_reward); + commission_split_preserve_lamports(self.commission_bps, stake_weighted_reward); assert!(is_split); assert_eq!( staker_reward, diff --git a/runtime/src/epoch_stakes.rs b/runtime/src/epoch_stakes.rs index 6dbb2c87970..a1d182884fd 100644 --- a/runtime/src/epoch_stakes.rs +++ b/runtime/src/epoch_stakes.rs @@ -17,6 +17,7 @@ use { std::{ collections::HashMap, fmt, + num::NonZero, sync::{Arc, OnceLock}, }, }; @@ -36,7 +37,7 @@ pub struct BLSPubkeyStakeEntry { /// The bls pubkey of the validator specified in the vote account pub bls_pubkey: PopVerified, /// The stake of the validator - pub stake: u64, + pub stake: NonZero, } /// Container to store a mapping from validator [`BLSPubkeyAffine`] to rank. @@ -49,7 +50,7 @@ pub struct BLSPubkeyToRankMap { rank_map: HashMap, vote_pubkey_to_rank: HashMap, sorted_pubkeys: Vec, - total_stake: u64, + total_stake: NonZero, } // We cannot auto derive `AbiExample` for `BLSPubkeyToRankMap` because @@ -61,7 +62,7 @@ impl solana_frozen_abi::abi_example::AbiExample for BLSPubkeyToRankMap { rank_map: HashMap::new(), vote_pubkey_to_rank: HashMap::new(), sorted_pubkeys: Vec::new(), - total_stake: 0, + total_stake: NonZero::new(1).unwrap(), } } } @@ -84,9 +85,9 @@ impl BLSPubkeyToRankMap { let mut bls_pubkey_counts = HashMap::new(); let mut node_pubkey_counts = HashMap::new(); for (&vote_account_pubkey, (stake, account)) in epoch_vote_accounts_hash_map { - if *stake == 0 { + let Some(stake) = NonZero::new(*stake) else { continue; - } + }; let node_pubkey = *account.vote_state_view().node_pubkey(); let Some((bls_pubkey_compressed, bls_pubkey)) = account .vote_state_view() @@ -99,7 +100,7 @@ impl BLSPubkeyToRankMap { vote_account_pubkey, node_pubkey, bls_pubkey, - stake: *stake, + stake, }; *bls_pubkey_counts.entry(bls_pubkey_compressed).or_insert(0) += 1; *node_pubkey_counts.entry(node_pubkey).or_insert(0) += 1; @@ -116,7 +117,10 @@ impl BLSPubkeyToRankMap { .collect(); let total_stake = keys_stake_entry_with_compressed .iter() - .fold(0u64, |stake, (entry, _)| stake.saturating_add(entry.stake)); + .fold(0u64, |stake, (entry, _)| { + stake.saturating_add(entry.stake.get()) + }); + let total_stake = NonZero::new(total_stake).expect("total stakes should not be 0"); keys_stake_entry_with_compressed.sort_by( |(a_entry, a_pubkey_compressed), (b_entry, b_pubkey_compressed)| { b_entry @@ -153,7 +157,7 @@ impl BLSPubkeyToRankMap { self.sorted_pubkeys.len() } - pub fn total_stake(&self) -> u64 { + pub fn total_stake(&self) -> NonZero { self.total_stake } @@ -662,14 +666,13 @@ pub(crate) mod tests { } } - #[test_case(1; "single_vote_account")] - #[test_case(2; "multiple_vote_accounts")] - fn test_bls_pubkey_rank_map(num_vote_accounts_per_node: usize) { + #[test] + fn test_bls_pubkey_rank_map() { agave_logger::setup(); let num_nodes = 10; - let num_vote_accounts = num_nodes * num_vote_accounts_per_node; + let num_vote_accounts = num_nodes; - let vote_accounts_map = new_vote_accounts(num_nodes, num_vote_accounts_per_node, true); + let vote_accounts_map = new_vote_accounts(num_nodes, 1, true); let node_id_to_stake_map = vote_accounts_map .keys() .enumerate() @@ -680,18 +683,13 @@ pub(crate) mod tests { }); let epoch_stakes = VersionedEpochStakes::new_for_tests(epoch_vote_accounts.clone(), 0); let bls_pubkey_to_rank_map = epoch_stakes.bls_pubkey_to_rank_map(); - let expected_num_vote_accounts = if num_vote_accounts_per_node == 1 { - num_vote_accounts - } else { - 0 - }; + let expected_num_vote_accounts = num_vote_accounts; assert_eq!(bls_pubkey_to_rank_map.len(), expected_num_vote_accounts); - let expected_total_stake = if num_vote_accounts_per_node == 1 { - epoch_stakes.total_stake() - } else { - 0 - }; - assert_eq!(bls_pubkey_to_rank_map.total_stake(), expected_total_stake); + let expected_total_stake = epoch_stakes.total_stake(); + assert_eq!( + bls_pubkey_to_rank_map.total_stake().get(), + expected_total_stake + ); for (vote_account_pubkey, (stake, vote_account)) in epoch_vote_accounts { let vote_state_view = vote_account.vote_state_view(); let (_comp, bls_pubkey) = bls_pubkey_compressed_bytes_to_bls_pubkey( @@ -699,15 +697,6 @@ pub(crate) mod tests { ) .unwrap(); let node_pubkey = *vote_state_view.node_pubkey(); - if num_vote_accounts_per_node > 1 { - assert!(bls_pubkey_to_rank_map.get_rank(&bls_pubkey).is_none()); - assert!( - bls_pubkey_to_rank_map - .get_rank_for_vote_pubkey(&vote_account_pubkey) - .is_none() - ); - continue; - } let index = bls_pubkey_to_rank_map.get_rank(&bls_pubkey).unwrap(); assert!(index >= &0 && index < &(expected_num_vote_accounts as u16)); assert_eq!( @@ -716,7 +705,7 @@ pub(crate) mod tests { vote_account_pubkey, node_pubkey, bls_pubkey, - stake, + stake: NonZero::new(stake).unwrap(), }) ); } @@ -731,6 +720,25 @@ pub(crate) mod tests { assert_eq!(bls_pubkey_to_rank_map2, bls_pubkey_to_rank_map); } + #[test] + #[should_panic(expected = "total stakes should not be 0")] + fn test_multiple_vote_accounts_panics() { + agave_logger::setup(); + let num_nodes = 10; + + let vote_accounts_map = new_vote_accounts(num_nodes, 2, true); + let node_id_to_stake_map = vote_accounts_map + .keys() + .enumerate() + .map(|(index, node_id)| (*node_id, ((index + 1) * 100) as u64)) + .collect::>(); + let epoch_vote_accounts = new_epoch_vote_accounts(&vote_accounts_map, |node_id| { + *node_id_to_stake_map.get(node_id).unwrap() + }); + let epoch_stakes = VersionedEpochStakes::new_for_tests(epoch_vote_accounts.clone(), 0); + epoch_stakes.bls_pubkey_to_rank_map(); + } + #[test] fn test_bls_pubkey_rank_map_excludes_duplicate_bls_and_identity() { let new_bls_pubkey = || { @@ -865,7 +873,7 @@ pub(crate) mod tests { let rank_map = BLSPubkeyToRankMap::new(&epoch_vote_accounts); assert_eq!(rank_map.len(), 3); - assert_eq!(rank_map.total_stake(), 250); + assert_eq!(rank_map.total_stake().get(), 250); for bls_pubkey in [ duplicate_bls_pubkey, duplicate_node_bls_pubkey, @@ -905,7 +913,7 @@ pub(crate) mod tests { vote_account_pubkey, node_pubkey, bls_pubkey, - stake: 100, + stake: NonZero::new(100).unwrap(), }) ); } @@ -921,7 +929,7 @@ pub(crate) mod tests { vote_account_pubkey: unique_vote_pubkey, node_pubkey: unique_node_pubkey, bls_pubkey: unique_bls_pubkey, - stake: 50, + stake: NonZero::new(50).unwrap(), }) ); assert!(rank_map.get_pubkey_stake_entry(rank_map.len()).is_none()); diff --git a/runtime/src/genesis_utils.rs b/runtime/src/genesis_utils.rs index cb8c7980201..2d01fbfe8f0 100644 --- a/runtime/src/genesis_utils.rs +++ b/runtime/src/genesis_utils.rs @@ -9,9 +9,9 @@ use { agave_feature_set::{FEATURE_NAMES, FeatureSet}, agave_votor_messages::{ self, - certificate::{Certificate, CertificateType}, consensus_message::{BLS_KEYPAIR_DERIVE_SEED, Block}, migration::GENESIS_CERTIFICATE_ACCOUNT, + wire::{WireBlockCertMessage, WireCertSignature}, }, bincode::serialize, bitvec::vec::BitVec, @@ -336,13 +336,15 @@ fn configure_alpenglow_at_genesis(genesis_config: &mut GenesisConfig) { // This is a dev cluster with alpenglow enabled at genesis. We don't want to test the migration pathway // so we add a fake genesis certificate. - let cert = Certificate { - cert_type: CertificateType::Genesis(Block { + let cert = WireBlockCertMessage { + block: Block { slot: 0, block_id: Hash::default(), - }), - signature: BLSSignature([0; BLS_SIGNATURE_AFFINE_SIZE]), - bitmap: encode_base2(&BitVec::new()).unwrap(), + }, + signature: WireCertSignature { + signature: BLSSignature([0; BLS_SIGNATURE_AFFINE_SIZE]), + bitmap: encode_base2(&BitVec::new()).unwrap(), + }, }; let cert_size = bincode::serialized_size(&cert).unwrap(); let lamports = Rent::default().minimum_balance(cert_size as usize); diff --git a/runtime/src/inflation_rewards/mod.rs b/runtime/src/inflation_rewards/mod.rs index 1b137b2fb78..4f1996e6426 100644 --- a/runtime/src/inflation_rewards/mod.rs +++ b/runtime/src/inflation_rewards/mod.rs @@ -7,12 +7,8 @@ use { InflationPointCalculationEvent, SkippedReason, calculate_stake_points_and_credits, }, crate::{alpenglow_epoch_type::AlpenglowEpochType, stake_delegation::effective_stake}, - solana_clock::Epoch, solana_instruction::error::InstructionError, - solana_stake_interface::{ - error::StakeError, - state::{Delegation, Stake}, - }, + solana_stake_interface::{error::StakeError, state::Stake}, }; pub mod points; @@ -109,7 +105,6 @@ fn redeem_stake_rewards<'a>( )); } - let rewarded_epoch = calculation_environment.rewarded_epoch; let adjust_delegations_for_rent = calculation_environment.adjust_delegations_for_rent; let maybe_rewards = calculate_stake_rewards( stake, @@ -136,16 +131,16 @@ fn redeem_stake_rewards<'a>( let staker_rewards = maybe_rewards.map(|x| x.0).unwrap_or(0); if adjust_delegations_for_rent { let new_delegation_with_rewards = stake.delegation.stake.saturating_add(staker_rewards); - let stake_was_adjusted = adjust_delegation_for_rent( - &mut stake.delegation, - rewarded_epoch, + let needs_adjustment = delegation_may_need_adjustment( + stake.delegation.stake, new_delegation_with_rewards, current_lamports.saturating_add(staker_rewards), minimum_lamports, ); // If `maybe_rewards.is_some()`, need to drive forward credits, even // if rewards are zero - if stake_was_adjusted || maybe_rewards.is_some() { + if needs_adjustment || maybe_rewards.is_some() { + stake.delegation.stake = new_delegation_with_rewards; let voter_rewards = maybe_rewards.map(|x| x.1).unwrap_or(0); Some((staker_rewards, voter_rewards)) } else { @@ -157,14 +152,14 @@ fn redeem_stake_rewards<'a>( } } -/// Adjusts stake delegation based on Rent sysvar parameters at epoch boundary +/// Returns `true` if stake delegation needs to be adjusted during distribution +/// based on Rent sysvar parameters at epoch boundary /// -/// As part of SIMD-0392, if Rent is ever increased, we need to make sure that -/// lamports are not double-counted for the rent-exempt minimum and the stake -/// delegation. This function adjusts the delegation in a Stake if needed. -pub(crate) fn adjust_delegation_for_rent( - delegation: &mut Delegation, - rewarded_epoch: Epoch, +/// The actual adjustment happens at distribution, to account for any lamports +/// credited to the account during partitioned epoch rewards, before the +/// distribution has occurred. +pub(crate) fn delegation_may_need_adjustment( + current_delegation: u64, new_delegation_with_rewards: u64, lamports_with_rewards: u64, minimum_lamports: u64, @@ -174,18 +169,7 @@ pub(crate) fn adjust_delegation_for_rent( lamports_with_rewards.saturating_sub(minimum_lamports), ); - if new_delegation != delegation.stake { - delegation.stake = new_delegation; - // Deactivate stake if needed. This deactivation is immediate, - // unlike a requested deactivation which happens at the next epoch - // boundary - if new_delegation == 0 { - delegation.deactivation_epoch = rewarded_epoch; - } - true - } else { - false - } + new_delegation != current_delegation } /// for a given stake and vote_state, calculate what distributions and what updates should be made @@ -331,7 +315,11 @@ fn calculate_stake_rewards<'a>( if rewards == 0 { return skip_reward(SkippedReason::ZeroReward); } - let (voter_rewards, staker_rewards, is_split) = commission_split(voter_commission_bps, rewards); + let (voter_rewards, staker_rewards, is_split) = if is_tower_epoch { + commission_split(voter_commission_bps, rewards) + } else { + commission_split_preserve_lamports(voter_commission_bps, rewards) + }; if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer.as_ref() { inflation_point_calc_tracer(&InflationPointCalculationEvent::SplitRewards( rewards, @@ -396,6 +384,35 @@ fn commission_split(commission_bps: u16, on: u64) -> (u64, u64, bool) { } } +/// returns commission split as (voter_portion, staker_portion, was_split) tuple, +/// assigning any fractional-lamport remainder to the voter so no lamports are lost. +/// +/// This is used only for non-Tower epochs, where small unfair splits no longer defer redemption. +#[cfg_attr(any(test, feature = "dev-context-only-utils"), qualifiers(pub(crate)))] +fn commission_split_preserve_lamports(commission_bps: u16, on: u64) -> (u64, u64, bool) { + const MAX_BPS: u16 = 10_000; + const MAX_BPS_U128: u128 = MAX_BPS as u128; + match commission_bps.min(MAX_BPS) { + 0 => (0, on, false), + MAX_BPS => (on, 0, false), + split => { + let staker_bps = MAX_BPS + .checked_sub(split) + .expect("commission cannot be greater than MAX_BPS"); + let staker_rewards = u128::from(on) + .checked_mul(u128::from(staker_bps)) + .expect("multiplication of a u64 and u16 should not overflow") + / MAX_BPS_U128; + let staker_rewards = staker_rewards as u64; + let voter_rewards = on + .checked_sub(staker_rewards) + .expect("staker rewards cannot exceed total rewards"); + + (voter_rewards, staker_rewards, true) + } + } +} + #[cfg(test)] mod tests { use { @@ -827,14 +844,14 @@ mod tests { let small_redemption_result = || { ag_enabled.then_some(CalculatedStakeRewards { staker_rewards: 0, - voter_rewards: 0, + voter_rewards: 1, new_credits_observed: 4 * ag_total_stake_multiplier, }) }; // same as above, but is a small enough reward that both sides round to - // zero after the commission split. Tower defers; AG records the - // zero-lamport payout and advances credits. + // zero after the Tower commission split. Tower defers; AG assigns the + // remainder to the voter and advances credits. vote_state.set_inflation_rewards_commission_bps(100); assert_eq!( small_redemption_result(), @@ -1100,272 +1117,6 @@ mod tests { ); } - fn check_rent_adjusted_stake_delegation( - rewarded_epoch: u64, - mut pre_stake: Stake, - pre_lamports: u64, - new_minimum_balance: u64, - total_rewards: u64, - post_stake: Stake, - staker_rewards: Option, - ) { - let mut vote_state = VoteStateHandler::new_v4(VoteStateV4::default()); - // put 1 credit to create rewards - vote_state.increment_credits(rewarded_epoch, 1); - let stake_history: &StakeHistory = &StakeHistory::default(); - let new_rate_activation_epoch = None; - let commission_rate_in_basis_points = true; - let adjust_delegations_for_rent = true; - - let maybe_rewards = redeem_stake_rewards( - &mut pre_stake, - vote_state.as_ref_v4().inflation_rewards_commission_bps, - DelegatedVoteState::from(vote_state.as_ref_v4()), - CalculationEnvironment { - rewarded_epoch, - point_value: &PointValue { - rewards: total_rewards, - points: 1, - }, - stake_history, - new_rate_activation_epoch, - commission_rate_in_basis_points, - adjust_delegations_for_rent, - use_fixed_point_stake_math: true, - }, - null_tracer(), - &AlpenglowEpochType::Tower, - pre_lamports, - new_minimum_balance, - ); - assert_eq!(pre_stake, post_stake); - assert_eq!(maybe_rewards.map(|x| x.0), staker_rewards) - } - - #[test] - fn rent_adjusted_stake_delegation_calculations() { - let old_minimum_balance = 8; - let new_minimum_balance = 9; - let rewarded_epoch = 1; - - // No rewards at all -> updated (all stakes get driven forward if - // inflation is disabled) - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 1, - }, - new_minimum_balance + 1, - new_minimum_balance, - 0, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 1, - }, - Some(0), - ); - - // Stake receives no rewards or delegation adjustment -> no update - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 1, - }, - new_minimum_balance + 1, - new_minimum_balance, - 1, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 1, - }, - None, - ); - - // Already destaked -> no update - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 0, - deactivation_epoch: 0, - ..Default::default() - }, - credits_observed: 0, - }, - old_minimum_balance - 1, - new_minimum_balance, - 1, - Stake { - delegation: Delegation { - stake: 0, - deactivation_epoch: 0, - ..Default::default() - }, - credits_observed: 0, - }, - None, - ); - - // Staked, already below minimum, go further below minimum - // -> destaked, still one lamport of rewards though - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 0, - }, - old_minimum_balance - 1, - new_minimum_balance, - 1, - Stake { - delegation: Delegation { - stake: 0, - deactivation_epoch: rewarded_epoch, - ..Default::default() - }, - credits_observed: 1, - }, - Some(1), - ); - - // Delegation hits exactly 0 -> destaked, no rewards - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 0, - }, - new_minimum_balance, - new_minimum_balance, - 0, - Stake { - delegation: Delegation { - stake: 0, - deactivation_epoch: rewarded_epoch, - ..Default::default() - }, - credits_observed: 1, - }, - Some(0), - ); - - // Delegation decreases to 1 -> still staked - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 2, - ..Default::default() - }, - credits_observed: 0, - }, - new_minimum_balance + 1, - new_minimum_balance, - 0, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 1, - }, - Some(0), - ); - - // Rewards partially cover minimum balance change - // -> decrease stake - // This case is confusing because it pays out 2 lamports in rewards, - // so we adjust minimum up so that even with 2 lamports in rewards, the - // delegation goes down. - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 2, - ..Default::default() - }, - credits_observed: 0, - }, - new_minimum_balance, - new_minimum_balance + 1, - 1, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 1, - }, - Some(2), - ); - - // Rewards cover minimum balance change -> no change in stake - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 0, - }, - new_minimum_balance, - new_minimum_balance, - 1, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 1, - }, - Some(1), - ); - - // Well above new minimum balance -> delegation change capped to rewards - check_rent_adjusted_stake_delegation( - rewarded_epoch, - Stake { - delegation: Delegation { - stake: 1, - ..Default::default() - }, - credits_observed: 0, - }, - new_minimum_balance + 2, - new_minimum_balance, - 1, - Stake { - delegation: Delegation { - stake: 2, - ..Default::default() - }, - credits_observed: 1, - }, - Some(1), - ); - } - #[test_case(u64::MAX, 1_000, u64::MAX => panics "Rewards intermediate calculation should fit within u128")] #[test_case(1, u64::MAX, u64::MAX => panics "Rewards should fit within u64")] fn calculate_rewards_tests(stake: u64, rewards: u64, credits: u64) { @@ -1491,7 +1242,7 @@ mod tests { assert_eq!( Some(CalculatedStakeRewards { staker_rewards: 3, - voter_rewards: 0, + voter_rewards: 1, new_credits_observed: 4 * ag_total_stake_multiplier, }), calculate_stake_rewards( @@ -1508,7 +1259,7 @@ mod tests { assert_eq!( Some(CalculatedStakeRewards { staker_rewards: 0, - voter_rewards: 3, + voter_rewards: 4, new_credits_observed: 4 * ag_total_stake_multiplier, }), calculate_stake_rewards( @@ -1630,6 +1381,92 @@ mod tests { ); // 1-lamport truncation } + #[test] + fn test_commission_split_preserve_lamports_bps() { + // 0% commission + assert_eq!(commission_split_preserve_lamports(0, 1), (0, 1, false)); + assert_eq!(commission_split_preserve_lamports(0, 10), (0, 10, false)); + assert_eq!(commission_split_preserve_lamports(0, 100), (0, 100, false)); + assert_eq!( + commission_split_preserve_lamports(0, u64::MAX), + (0, u64::MAX, false) + ); + + // 100% commission (10,000 bps) + assert_eq!(commission_split_preserve_lamports(10_000, 1), (1, 0, false)); + assert_eq!( + commission_split_preserve_lamports(10_000, 10), + (10, 0, false) + ); + assert_eq!( + commission_split_preserve_lamports(10_000, 100), + (100, 0, false) + ); + assert_eq!( + commission_split_preserve_lamports(10_000, u64::MAX), + (u64::MAX, 0, false) + ); + + // Values > 10,000 bps are capped at 100% + assert_eq!( + commission_split_preserve_lamports(u16::MAX, 1), + (1, 0, false) + ); + assert_eq!( + commission_split_preserve_lamports(u16::MAX, u64::MAX), + (u64::MAX, 0, false) + ); + + // Remainder lamports go to the voter. + assert_eq!(commission_split_preserve_lamports(9_900, 1), (1, 0, true)); + assert_eq!(commission_split_preserve_lamports(9_900, 10), (10, 0, true)); + assert_eq!( + commission_split_preserve_lamports(9_900, 100), + (99, 1, true) + ); + assert_eq!( + commission_split_preserve_lamports(9_900, 1_000), + (990, 10, true) + ); + + assert_eq!(commission_split_preserve_lamports(100, 1), (1, 0, true)); + assert_eq!(commission_split_preserve_lamports(100, 10), (1, 9, true)); + assert_eq!(commission_split_preserve_lamports(100, 100), (1, 99, true)); + assert_eq!( + commission_split_preserve_lamports(100, 1_000), + (10, 990, true) + ); + + assert_eq!(commission_split_preserve_lamports(5_000, 1), (1, 0, true)); + assert_eq!(commission_split_preserve_lamports(5_000, 10), (5, 5, true)); + assert_eq!( + commission_split_preserve_lamports(5_000, 100), + (50, 50, true) + ); + + assert_eq!(commission_split_preserve_lamports(1_234, 1), (1, 0, true)); + assert_eq!(commission_split_preserve_lamports(1_234, 10), (2, 8, true)); + assert_eq!( + commission_split_preserve_lamports(1_234, 1_000), + (124, 876, true) + ); + assert_eq!( + commission_split_preserve_lamports(1_234, 10_000), + (1_234, 8_766, true) + ); + + assert_eq!(commission_split_preserve_lamports(3_333, 1), (1, 0, true)); + assert_eq!(commission_split_preserve_lamports(3_333, 10), (4, 6, true)); + assert_eq!( + commission_split_preserve_lamports(3_333, 1_000), + (334, 666, true) + ); + assert_eq!( + commission_split_preserve_lamports(3_333, 10_000), + (3_333, 6_667, true) + ); + } + proptest! { #[test] fn test_commission_split_properties( @@ -1690,5 +1527,61 @@ mod tests { prop_assert_eq!(voter, expected_voter); prop_assert_eq!(staker, expected_staker); } + + #[test] + fn test_commission_split_preserve_lamports_properties( + commission_bps in 0..=u16::MAX, + rewards in 0..=u64::MAX, + ) { + let (voter, staker, was_split) = + commission_split_preserve_lamports(commission_bps, rewards); + + // Invariant 1: The full reward amount is assigned. + prop_assert_eq!(voter + staker, rewards); + + // Invariant 2: was_split is false only at the 0% and 100% boundaries. + let effective_bps = commission_bps.min(10_000); + if effective_bps == 0 || effective_bps == 10_000 { + prop_assert!(!was_split); + } else { + prop_assert!(was_split); + } + + // Invariant 3: Boundary - 0% commission gives everything to staker. + if effective_bps == 0 { + prop_assert_eq!(voter, 0); + prop_assert_eq!(staker, rewards); + } + + // Invariant 4: Boundary - 100% commission gives everything to voter. + if effective_bps == 10_000 { + prop_assert_eq!(voter, rewards); + prop_assert_eq!(staker, 0); + } + + // Invariant 5: Clamping - values above 10,000 bps behave as 10,000. + if commission_bps > 10_000 { + let (clamped_voter, clamped_staker, clamped_ws) = + commission_split_preserve_lamports(10_000, rewards); + prop_assert_eq!(voter, clamped_voter); + prop_assert_eq!(staker, clamped_staker); + prop_assert_eq!(was_split, clamped_ws); + } + + // Invariant 6: Higher commission does not decrease the voter amount. + if commission_bps > 0 { + let lower_bps = commission_bps - 1; + let (lower_voter, _, _) = commission_split_preserve_lamports(lower_bps, rewards); + prop_assert!(voter >= lower_voter); + } + + // Invariant 7: The staker side is floored, and the voter gets the remainder. + let staker_bps = 10_000 - effective_bps; + let expected_staker = + (u128::from(rewards) * u128::from(staker_bps) / 10_000) as u64; + let expected_voter = rewards - expected_staker; + prop_assert_eq!(voter, expected_voter); + prop_assert_eq!(staker, expected_staker); + } } } diff --git a/runtime/src/serde_snapshot/status_cache.rs b/runtime/src/serde_snapshot/status_cache.rs index f84da15d3a6..a7ba005c910 100644 --- a/runtime/src/serde_snapshot/status_cache.rs +++ b/runtime/src/serde_snapshot/status_cache.rs @@ -18,7 +18,7 @@ use { #[cfg_attr( feature = "frozen-abi", - frozen_abi(digest = "AardUUq1At4qq6oNNp9V2JZFsMR5k54RZmBmZkxUfk7m") + frozen_abi(digest = "DM9FgEZxfdt43ZgxNAtU2YoGV2P1NgiABgJJunURuV2p") )] type SerdeBankSlotDelta = SerdeSlotDelta>; type SerdeSlotDelta = (Slot, bool, SerdeStatus); @@ -112,7 +112,7 @@ pub fn deserialize_status_cache( /// contain a string in the BorshIoError variant. #[cfg_attr( feature = "frozen-abi", - frozen_abi(digest = "5pMgydVNgsYbg64Trhjxbftsug5La7fRDmooyrsHd4wy"), + frozen_abi(digest = "H4jrGnmko28mgcxgsVyC49ihwiZmBJbSFAnYGHYtNpS"), derive(AbiExample, AbiEnumVisitor) )] #[derive(Debug, PartialEq, Eq, Clone, Serialize, SchemaRead, SchemaWrite)] @@ -156,6 +156,7 @@ enum SerdeTransactionError { UnbalancedTransaction, ProgramCacheHitMaxLimit, CommitCancelled, + InstructionsSysvarOverflow, } impl From<&TransactionError> for SerdeTransactionError { @@ -224,6 +225,7 @@ impl From<&TransactionError> for SerdeTransactionError { TransactionError::UnbalancedTransaction => Self::UnbalancedTransaction, TransactionError::ProgramCacheHitMaxLimit => Self::ProgramCacheHitMaxLimit, TransactionError::CommitCancelled => Self::CommitCancelled, + TransactionError::InstructionsSysvarOverflow => Self::InstructionsSysvarOverflow, } } } @@ -294,6 +296,7 @@ impl From for TransactionError { SerdeTransactionError::UnbalancedTransaction => Self::UnbalancedTransaction, SerdeTransactionError::ProgramCacheHitMaxLimit => Self::ProgramCacheHitMaxLimit, SerdeTransactionError::CommitCancelled => Self::CommitCancelled, + SerdeTransactionError::InstructionsSysvarOverflow => Self::InstructionsSysvarOverflow, } } } diff --git a/runtime/src/validated_block_finalization.rs b/runtime/src/validated_block_finalization.rs index 3c255eda018..f28d8dd2cda 100644 --- a/runtime/src/validated_block_finalization.rs +++ b/runtime/src/validated_block_finalization.rs @@ -406,7 +406,7 @@ mod tests { signing_ranks: &[usize], validator_keypairs: &[ValidatorVoteKeypairs], ) -> Certificate { - let serialized_vote = get_vote_payload_to_sign(&vote, shred_version); + let serialized_vote = get_vote_payload_to_sign(vote, shred_version); // Aggregate signatures let mut signature = SignatureProjective::identity(); diff --git a/runtime/src/validated_reward_certificate.rs b/runtime/src/validated_reward_certificate.rs index 610588e05d3..947c5fda955 100644 --- a/runtime/src/validated_reward_certificate.rs +++ b/runtime/src/validated_reward_certificate.rs @@ -104,7 +104,7 @@ impl ValidatedRewardCert { if let Some(skip) = skip { let vote = Vote::new_skip_vote(skip.slot); - let payload = get_vote_payload_to_sign(&vote, shred_version); + let payload = get_vote_payload_to_sign(vote, shred_version); verify_base2( &payload, &skip.signature, @@ -118,7 +118,7 @@ impl ValidatedRewardCert { slot: notar.slot, block_id: notar.block_id, }); - let payload = get_vote_payload_to_sign(&vote, shred_version); + let payload = get_vote_payload_to_sign(vote, shred_version); verify_base2( &payload, ¬ar.signature, @@ -136,6 +136,31 @@ impl ValidatedRewardCert { })) } + /// Constructs a [`ValidatedRewardCert`] for a block produced locally. + /// + /// The leader-side reward certificate builder receives verified votes and + /// tracks the validator set while aggregating them, so block production + /// only needs the reward slot and validator set for bank reward + /// calculation. + pub fn try_new_for_leader( + current_slot: Slot, + skip: &Option, + notar: &Option, + validators: impl IntoIterator, + ) -> Result, Error> { + let Some(reward_slot) = extract_slot(current_slot, skip, notar)? else { + return Ok(None); + }; + let validators: HashSet<_> = validators.into_iter().collect(); + if validators.is_empty() { + return Ok(None); + } + Ok(Some(Self { + validators, + reward_slot, + })) + } + pub(crate) fn slot(&self) -> Slot { self.reward_slot } @@ -176,7 +201,7 @@ mod tests { }; fn new_vote(vote: Vote, rank: usize, keypair: &BlsKeypair, shred_version: u16) -> VoteMessage { - let payload = get_vote_payload_to_sign(&vote, shred_version); + let payload = get_vote_payload_to_sign(vote, shred_version); let signature = keypair.sign(&payload).into(); VoteMessage { vote, diff --git a/scheduler-bindings/Cargo.toml b/scheduler-bindings/Cargo.toml index 05b0235b018..ae966018f5a 100644 --- a/scheduler-bindings/Cargo.toml +++ b/scheduler-bindings/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/scheduler-bindings/src/lib.rs b/scheduler-bindings/src/lib.rs index c3b78acc54d..c641fa50abf 100644 --- a/scheduler-bindings/src/lib.rs +++ b/scheduler-bindings/src/lib.rs @@ -439,11 +439,13 @@ pub mod worker_message_types { pub const UNBALANCED_TRANSACTION: u8 = 100; /// Program cache hit max limit. pub const PROGRAM_CACHE_HIT_MAX_LIMIT: u8 = 101; + /// Instructions sysvar overflowed format. + pub const INSTRUCTIONS_SYSVAR_OVERFLOW: u8 = 102; // This error in agave is only internal, and to avoid updating the sdk // it is reused for mapping into `ALL_OR_NOTHING_BATCH_FAILURE`. // /// Commit cancelled internally. - // pub const COMMIT_CANCELLED: u8 = 102; + // pub const COMMIT_CANCELLED: u8 = 103; } /// Tag indicating [`CheckResponse`] inner message. diff --git a/scheduling-utils/Cargo.toml b/scheduling-utils/Cargo.toml index 08749ea9d4c..232a2d863ed 100644 --- a/scheduling-utils/Cargo.toml +++ b/scheduling-utils/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = ["dep:wincode", "dep:solana-transaction"] diff --git a/scheduling-utils/src/error.rs b/scheduling-utils/src/error.rs index ef2902b7550..41cd80fafa3 100644 --- a/scheduling-utils/src/error.rs +++ b/scheduling-utils/src/error.rs @@ -87,6 +87,9 @@ pub fn transaction_error_to_not_included_reason(error: &TransactionError) -> u8 TransactionError::ProgramCacheHitMaxLimit => { not_included_reasons::PROGRAM_CACHE_HIT_MAX_LIMIT } + TransactionError::InstructionsSysvarOverflow => { + not_included_reasons::INSTRUCTIONS_SYSVAR_OVERFLOW + } // SPECIAL CASE - CommitCancelled is an internal error reused to avoid breaking sdk TransactionError::CommitCancelled => not_included_reasons::ALL_OR_NOTHING_BATCH_FAILURE, diff --git a/scripts/cargo-clippy-nightly.sh b/scripts/cargo-clippy-nightly.sh index 7a63529cb92..195935c41f0 100755 --- a/scripts/cargo-clippy-nightly.sh +++ b/scripts/cargo-clippy-nightly.sh @@ -25,9 +25,4 @@ source "$here/../ci/rust-version.sh" nightly "$here/cargo-for-all-lock-files.sh" -- \ "+${rust_nightly}" clippy \ --workspace --all-targets --features dummy-for-ci-check,frozen-abi -- \ - --deny=warnings \ - --deny=clippy::default_trait_access \ - --deny=clippy::arithmetic_side_effects \ - --deny=clippy::manual_let_else \ - --deny=clippy::uninlined-format-args \ - --deny=clippy::used_underscore_binding + --deny=warnings diff --git a/send-transaction-service/Cargo.toml b/send-transaction-service/Cargo.toml index 31f330d9752..990f3711569 100644 --- a/send-transaction-service/Cargo.toml +++ b/send-transaction-service/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/snapshots/Cargo.toml b/snapshots/Cargo.toml index 1181b418328..048b89797e9 100644 --- a/snapshots/Cargo.toml +++ b/snapshots/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/stake-accounts/Cargo.toml b/stake-accounts/Cargo.toml index 947fa69bab8..2bea8eebd91 100644 --- a/stake-accounts/Cargo.toml +++ b/stake-accounts/Cargo.toml @@ -11,6 +11,11 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +# `all-features` can't be used: remote-wallet-hidraw and remote-wallet-libusb +# forward to hidapi's mutually-exclusive linux backends. Document the default +# (hidraw) backend plus the unstable API. +features = ["agave-unstable-api"] +rustdoc-args = ["--cfg=docsrs"] [features] default = ["remote-wallet-hidraw"] diff --git a/storage-bigtable/Cargo.toml b/storage-bigtable/Cargo.toml index 429b9d5e8b1..1294a38014f 100644 --- a/storage-bigtable/Cargo.toml +++ b/storage-bigtable/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/storage-proto/Cargo.toml b/storage-proto/Cargo.toml index 916e94ac721..147f4ae2323 100644 --- a/storage-proto/Cargo.toml +++ b/storage-proto/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/storage-proto/proto/transaction_by_addr.proto b/storage-proto/proto/transaction_by_addr.proto index 5748b05655e..ace8657f617 100644 --- a/storage-proto/proto/transaction_by_addr.proto +++ b/storage-proto/proto/transaction_by_addr.proto @@ -64,6 +64,7 @@ enum TransactionErrorType { UNBALANCED_TRANSACTION = 36; PROGRAM_CACHE_HIT_MAX_LIMIT = 37; COMMIT_CANCELLED = 38; + INSTRUCTIONS_SYSVAR_OVERFLOW = 39; } message InstructionError { diff --git a/storage-proto/src/convert.rs b/storage-proto/src/convert.rs index ba69e521b84..a34fa2fd21b 100644 --- a/storage-proto/src/convert.rs +++ b/storage-proto/src/convert.rs @@ -932,6 +932,7 @@ impl TryFrom for TransactionError { 36 => TransactionError::UnbalancedTransaction, 37 => TransactionError::ProgramCacheHitMaxLimit, 38 => TransactionError::CommitCancelled, + 39 => TransactionError::InstructionsSysvarOverflow, _ => return Err("Invalid TransactionError"), }) } @@ -1056,6 +1057,9 @@ impl From for tx_by_addr::TransactionError { TransactionError::CommitCancelled => { tx_by_addr::TransactionErrorType::CommitCancelled } + TransactionError::InstructionsSysvarOverflow => { + tx_by_addr::TransactionErrorType::InstructionsSysvarOverflow + } } as i32, instruction_error: match transaction_error { TransactionError::InstructionError(index, ref instruction_error) => { diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml index e33dda5a727..647af100c00 100644 --- a/streamer/Cargo.toml +++ b/streamer/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] @@ -38,6 +40,7 @@ rand = { workspace = true } rustls = { workspace = true } smallvec = { workspace = true } solana-keypair = { workspace = true } +solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-net-utils = { workspace = true } solana-packet = { workspace = true } diff --git a/streamer/src/quic_socket.rs b/streamer/src/quic_socket.rs index 61f351138c8..d61367f16e8 100644 --- a/streamer/src/quic_socket.rs +++ b/streamer/src/quic_socket.rs @@ -7,7 +7,6 @@ use { }, bytes::Bytes, crossbeam_channel::TrySendError, - nix::ifaddrs::getifaddrs, quinn::{ AsyncUdpSocket, Runtime, TokioRuntime, UdpPoller, udp::{EcnCodepoint as QuinnEcnCodepoint, RecvMeta, Transmit}, @@ -273,7 +272,10 @@ impl QuicXdpSender { } /// Collects IPv4 addresses assigned to local network interfaces. +#[cfg(target_os = "linux")] fn collect_local_ipv4_ips() -> io::Result> { + use nix::ifaddrs::getifaddrs; + let mut ips = Vec::new(); for ifa in getifaddrs().map_err(io::Error::other)? { let Some(addr) = ifa.address else { continue }; @@ -287,6 +289,11 @@ fn collect_local_ipv4_ips() -> io::Result> { Ok(ips) } +#[cfg(not(target_os = "linux"))] +fn collect_local_ipv4_ips() -> io::Result> { + Ok(Vec::new()) +} + #[inline] const fn quinn_ecn_to_xdp(ecn: QuinnEcnCodepoint) -> XdpEcnCodepoint { match ecn { diff --git a/streamer/src/streamer.rs b/streamer/src/streamer.rs index dccd4a0d772..cbe1d39c444 100644 --- a/streamer/src/streamer.rs +++ b/streamer/src/streamer.rs @@ -11,6 +11,7 @@ use { }, crossbeam_channel::{Receiver, RecvTimeoutError, SendError, Sender, TrySendError}, histogram::Histogram, + solana_measure::measure::Measure, solana_net_utils::{ SocketAddrSpace, multihomed_sockets::{ @@ -19,11 +20,10 @@ use { }, }, solana_pubkey::Pubkey, - solana_time_utils::timestamp, std::{ cmp::Reverse, collections::HashMap, - net::{IpAddr, UdpSocket}, + net::{IpAddr, SocketAddr, UdpSocket}, sync::{ Arc, atomic::{AtomicBool, AtomicUsize, Ordering}, @@ -452,26 +452,6 @@ impl StakedNodes { } } -fn recv_send( - sock: &UdpSocket, - r: &PacketBatchReceiver, - socket_addr_space: &SocketAddrSpace, - stats: &mut Option, -) -> Result<()> { - let timer = Duration::new(1, 0); - let packet_batch = r.recv_timeout(timer)?; - if let Some(stats) = stats { - packet_batch.iter().for_each(|p| stats.record(p)); - } - let packets = packet_batch.iter().filter_map(|pkt| { - let addr = pkt.meta().socket_addr(); - let data = pkt.data(..)?; - socket_addr_space.check(&addr).then_some((data, addr)) - }); - batch_send(sock, packets.collect::>())?; - Ok(()) -} - pub fn recv_packet_batches( recvr: &PacketBatchReceiver, soft_receive_limit: usize, @@ -500,26 +480,27 @@ pub fn recv_packet_batches( Ok((packet_batches, num_packets, recv_duration)) } -pub fn responder_atomic( - name: &'static str, - sockets: Arc<[UdpSocket]>, - bind_ip_addrs: Arc, - r: PacketBatchReceiver, +struct ServeRepairSocketProvider { + socket: Arc, socket_addr_space: SocketAddrSpace, - stats_reporter_sender: Option>>, -) -> JoinHandle<()> { - Builder::new() - .name(format!("solRspndr{name}")) - .spawn(move || { - responder_loop( - MultihomedSocketProvider::new(sockets, bind_ip_addrs), - name, - r, - socket_addr_space, - stats_reporter_sender, - ); - }) - .unwrap() +} + +impl ResponseSender for ServeRepairSocketProvider { + fn send_batch(&self, batch: PacketBatch) -> std::result::Result<(), SendPktsError> { + let packets = filter_packets_by_socket_addr_space(batch.iter(), &self.socket_addr_space); + batch_send(self.socket.as_ref(), packets.collect::>()) + } +} + +pub fn filter_packets_by_socket_addr_space<'a>( + packets: impl Iterator> + 'a, + socket_addr_space: &'a SocketAddrSpace, +) -> impl Iterator + 'a { + packets.filter_map(move |pkt| { + let addr = pkt.meta().socket_addr(); + let data = pkt.data(..)?; + socket_addr_space.check(&addr).then_some((data, addr)) + }) } pub fn responder( @@ -533,26 +514,40 @@ pub fn responder( .name(format!("solRspndr{name}")) .spawn(move || { responder_loop( - FixedSocketProvider::new(sock), name, r, - socket_addr_space, + ServeRepairSocketProvider { + socket: sock, + socket_addr_space, + }, stats_reporter_sender, ); }) .unwrap() } -fn responder_loop( - provider: P, +pub trait ResponseSender { + /// Send a batch of packets. + /// + /// Returns Ok if all the packets with valid destination within batch were sent successfully, + /// and returns an error if any packet within the batch failed to send with number of failed + /// packets. + fn send_batch(&self, batch: PacketBatch) -> std::result::Result<(), SendPktsError>; +} + +pub fn responder_loop( name: &'static str, r: PacketBatchReceiver, - socket_addr_space: SocketAddrSpace, + sender: G, stats_reporter_sender: Option>>, ) { + const SEND_REPORTING_INTERVAL: Duration = Duration::from_secs(1); let mut errors = 0; let mut last_error = None; - let mut last_print = 0; + let mut send_elapsed_us: u64 = 0; + let mut send_batch_count: u64 = 0; + + let mut now = Instant::now(); let mut stats = None; if stats_reporter_sender.is_some() { @@ -560,24 +555,55 @@ fn responder_loop( } loop { - let sock = provider.current_socket_ref(); - if let Err(e) = recv_send(sock, &r, &socket_addr_space, &mut stats) { - match e { - StreamerError::RecvTimeout(RecvTimeoutError::Disconnected) => break, - StreamerError::RecvTimeout(RecvTimeoutError::Timeout) => (), - _ => { - errors += 1; - last_error = Some(e); - } + let timer = Duration::new(1, 0); + let packet_batch = match r.recv_timeout(timer) { + Ok(batch) => Some(batch), + Err(RecvTimeoutError::Disconnected) => break, + Err(RecvTimeoutError::Timeout) => None, + }; + if let Some(packet_batch) = packet_batch { + if let Some(stats) = stats.as_mut() { + packet_batch.iter().for_each(|p| stats.record(p)); + } + let mut measure_send = Measure::start("send batch"); + if let Err(e) = sender.send_batch(packet_batch) { + errors += 1; + last_error = Some(StreamerError::SendPktsError(e)); } + measure_send.stop(); + send_elapsed_us = send_elapsed_us.saturating_add(measure_send.as_us()); + send_batch_count = send_batch_count.saturating_add(1); } - let now = timestamp(); - if now - last_print > 1000 && errors != 0 { - datapoint_info!(name, ("errors", errors, i64),); - info!("{name} last-error: {last_error:?} count: {errors}"); - last_print = now; - errors = 0; + + // Metrics reporting + let sample_duration = now.elapsed(); + if sample_duration > SEND_REPORTING_INTERVAL { + datapoint_info!( + name, + // how long it took to send batches of packets during this interval + ("streamer-send-egress_time_us", send_elapsed_us as i64, i64), + ( + "streamer-send-egress_batch_count", + send_batch_count as i64, + i64 + ), + ( + "streamer-send-egress_sample_duration_ms", + sample_duration.as_millis() as i64, + i64 + ), + ); + send_elapsed_us = 0; + send_batch_count = 0; + if errors != 0 { + datapoint_info!(name, ("errors", errors, i64),); + info!("{name} last-error: {last_error:?} count: {errors}"); + errors = 0; + last_error = None; + } + now = Instant::now(); } + if let Some(ref stats_reporter_sender) = stats_reporter_sender { if let Some(ref mut stats) = stats { stats.maybe_submit(name, stats_reporter_sender); diff --git a/svm-callback/Cargo.toml b/svm-callback/Cargo.toml index 37772d763c8..46382a93d72 100644 --- a/svm-callback/Cargo.toml +++ b/svm-callback/Cargo.toml @@ -9,6 +9,11 @@ license = { workspace = true } edition = { workspace = true } readme = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/svm-feature-set/Cargo.toml b/svm-feature-set/Cargo.toml index 6c4b24d9022..b6758caa35a 100644 --- a/svm-feature-set/Cargo.toml +++ b/svm-feature-set/Cargo.toml @@ -9,6 +9,11 @@ license = { workspace = true } edition = { workspace = true } readme = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/svm-log-collector/Cargo.toml b/svm-log-collector/Cargo.toml index ad678fe7bdc..3691684e904 100644 --- a/svm-log-collector/Cargo.toml +++ b/svm-log-collector/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/svm-measure/Cargo.toml b/svm-measure/Cargo.toml index dd29a885ce1..e5f9bc77b02 100644 --- a/svm-measure/Cargo.toml +++ b/svm-measure/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/svm-timings/Cargo.toml b/svm-timings/Cargo.toml index 69aef7670ab..71753c6db3e 100644 --- a/svm-timings/Cargo.toml +++ b/svm-timings/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/svm-type-overrides/Cargo.toml b/svm-type-overrides/Cargo.toml index eca951aa7b6..2a89fa4fd73 100644 --- a/svm-type-overrides/Cargo.toml +++ b/svm-type-overrides/Cargo.toml @@ -8,6 +8,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] shuttle-test = ["dep:shuttle"] diff --git a/svm/Cargo.toml b/svm/Cargo.toml index 68ca5b74aa5..926586e0d49 100644 --- a/svm/Cargo.toml +++ b/svm/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] @@ -25,6 +27,7 @@ conformance = [ "dep:bincode", "dep:prost", "dep:protosol", + "dep:solana-poseidon", "dep:solana-vote-program", "dep:solana-zk-elgamal-proof-program", "dep:xxhash-rust", @@ -92,6 +95,7 @@ solana-loader-v4-interface = { workspace = true } solana-message = { workspace = true } solana-nonce = { workspace = true } solana-nonce-account = { workspace = true, features = ["wincode"] } +solana-poseidon = { workspace = true, optional = true } solana-precompile-error = { workspace = true, optional = true } solana-program-entrypoint = { workspace = true } solana-program-pack = { workspace = true } diff --git a/svm/doc/spec.md b/svm/doc/spec.md index 16c311bb9b4..74333b443e0 100644 --- a/svm/doc/spec.md +++ b/svm/doc/spec.md @@ -28,7 +28,7 @@ We envision the following applications for SVM The SVM is currently viewed as realizing two stages of the Transaction Engine Execution pipeline as described in Solana Architecture documentation - [https://docs.solana.com/validator/runtime#execution](https://docs.solana.com/validator/runtime#execution), + [https://docs.anza.xyz/validator/runtime#execution](https://docs.anza.xyz/validator/runtime#execution), namely ‘load accounts’ and ‘execute’ stages. - **SVM Rollups** diff --git a/svm/src/conformance/elf_loader.rs b/svm/src/conformance/elf_loader.rs index 501c73512c1..adedefe42a3 100644 --- a/svm/src/conformance/elf_loader.rs +++ b/svm/src/conformance/elf_loader.rs @@ -2,6 +2,7 @@ use { crate::conformance::{ + err::elf_error_code, fd_hash::{fd_hash_u64_without_seed, fd_hash_without_seed}, feature_set::feature_set_from_proto, }, @@ -10,10 +11,7 @@ use { ElfLoaderCtx as ProtoElfLoaderCtx, ElfLoaderEffects as ProtoElfLoaderEffects, }, solana_compute_budget::compute_budget::ComputeBudget, - solana_program_runtime::solana_sbpf::{ - ebpf, - elf::{ElfError, Executable}, - }, + solana_program_runtime::solana_sbpf::{ebpf, elf::Executable}, solana_syscalls::create_program_runtime_environment, std::{collections::BTreeSet, ffi::c_int}, }; @@ -41,7 +39,7 @@ pub fn execute_elf_loader(input: &ProtoElfLoaderCtx) -> ProtoElfLoaderEffects { Ok(executable) => executable, Err(err) => { return ProtoElfLoaderEffects { - err_code: elf_err_to_num(&err) as u32, + err_code: elf_error_code(&err), ..Default::default() }; } @@ -66,34 +64,6 @@ pub fn execute_elf_loader(input: &ProtoElfLoaderCtx) -> ProtoElfLoaderEffects { } } -fn elf_err_to_num(error: &ElfError) -> u8 { - match error { - ElfError::FailedToParse(_) => 1, - ElfError::EntrypointOutOfBounds => 2, - ElfError::InvalidEntrypoint => 3, - ElfError::FailedToGetSection(_) => 4, - ElfError::UnresolvedSymbol(_, _, _) => 5, - ElfError::SectionNotFound(_) => 6, - ElfError::RelativeJumpOutOfBounds(_) => 7, - ElfError::SymbolHashCollision(_) => 8, - ElfError::WrongEndianess => 9, - ElfError::WrongAbi => 10, - ElfError::WrongMachine => 11, - ElfError::WrongClass => 12, - ElfError::NotOneTextSection => 13, - ElfError::WritableSectionNotSupported(_) => 14, - ElfError::AddressOutsideLoadableSection(_) => 15, - ElfError::InvalidVirtualAddress(_) => 16, - ElfError::UnknownRelocation(_) => 17, - ElfError::FailedToReadRelocationInfo => 18, - ElfError::WrongType => 19, - ElfError::UnknownSymbol(_) => 20, - ElfError::ValueOutOfBounds => 21, - ElfError::UnsupportedSBPFVersion => 22, - ElfError::InvalidProgramHeader => 23, - } -} - /// # Safety /// /// `in_ptr` must point to `in_sz` initialized bytes. `out_ptr` must point diff --git a/svm/src/conformance/err.rs b/svm/src/conformance/err.rs new file mode 100644 index 00000000000..9f331aab337 --- /dev/null +++ b/svm/src/conformance/err.rs @@ -0,0 +1,105 @@ +//! Error-code mapping for VM execution results. + +use { + solana_instruction::error::InstructionError, + solana_poseidon::PoseidonSyscallError, + solana_program_runtime::{ + cpi::CpiError, + memory::MemoryTranslationError, + solana_sbpf::{ + elf::ElfError, + error::{EbpfError, StableResult}, + }, + }, + solana_syscalls::SyscallError, +}; + +pub(crate) fn elf_error_code(error: &ElfError) -> u32 { + (error.discriminant() as u32).saturating_add(1) +} + +fn ebpf_error_code(error: &EbpfError) -> i64 { + (error.discriminant() as i64).saturating_add(1) +} + +fn syscall_error_code(error: &SyscallError) -> i64 { + (error.discriminant() as i64).saturating_add(1) +} + +pub(crate) fn instruction_error_code(error: &InstructionError) -> i32 { + let serialized = bincode::serialize(error).unwrap(); + i32::from_le_bytes(serialized[0..4].try_into().unwrap()).saturating_add(1) +} + +/// A VM `program_result` mapped into the fields a conformance fixture compares. +pub(crate) struct UnpackedResult { + /// Error number, or `0` on success. + pub error: i64, + /// Which error taxonomy `error` belongs to (`ERR_KIND_*`). + pub error_kind: i32, + /// The program return value (`r0`), only meaningful on success. + pub r0: u64, +} + +impl UnpackedResult { + const ERR_KIND_UNSPECIFIED: i32 = 0; + const ERR_KIND_EBPF: i32 = 1; + const ERR_KIND_SYSCALL: i32 = 2; + const ERR_KIND_INSTRUCTION: i32 = 3; + + fn ok(r0: u64) -> Self { + Self { + error: 0, + error_kind: Self::ERR_KIND_UNSPECIFIED, + r0, + } + } + + fn err(error: i64, error_kind: i32) -> Self { + Self { + error, + error_kind, + r0: 0, + } + } + + fn from_ebpf_err(ebpf_err: EbpfError) -> Self { + // Agave wraps syscall-side failures in `EbpfError::SyscallError`; recover + // the concrete error by downcasting so we report the matching code and + // kind. Anything else is a plain VM error. + match &ebpf_err { + EbpfError::SyscallError(boxed) => { + if let Some(e) = boxed.downcast_ref::() { + Self::err(instruction_error_code(e) as i64, Self::ERR_KIND_INSTRUCTION) + } else if let Some(e) = boxed.downcast_ref::() { + Self::err(syscall_error_code(e), Self::ERR_KIND_SYSCALL) + } else if let Some(e) = boxed.downcast_ref::() { + Self::err( + syscall_error_code(&e.clone().into()), + Self::ERR_KIND_SYSCALL, + ) + } else if let Some(e) = boxed.downcast_ref::() { + Self::err( + syscall_error_code(&e.clone().into()), + Self::ERR_KIND_SYSCALL, + ) + } else if let Some(e) = boxed.downcast_ref::() { + Self::err(ebpf_error_code(e), Self::ERR_KIND_EBPF) + } else if boxed.downcast_ref::().is_some() { + Self::err(-1, Self::ERR_KIND_SYSCALL) + } else { + Self::err(-1, Self::ERR_KIND_UNSPECIFIED) + } + } + _ => Self::err(ebpf_error_code(&ebpf_err), Self::ERR_KIND_EBPF), + } + } +} + +/// Map a VM `program_result` to its [`UnpackedResult`]. +pub(crate) fn unpack_stable_result(program_result: StableResult) -> UnpackedResult { + match program_result { + StableResult::Ok(r0) => UnpackedResult::ok(r0), + StableResult::Err(ebpf_err) => UnpackedResult::from_ebpf_err(ebpf_err), + } +} diff --git a/svm/src/conformance/instr/context.rs b/svm/src/conformance/instr/context.rs index bbc6a6bf846..8f014f6e49a 100644 --- a/svm/src/conformance/instr/context.rs +++ b/svm/src/conformance/instr/context.rs @@ -68,8 +68,17 @@ impl From for InstrContext { }) .collect::>(); + // Match Firedancer harness limit (FD_INSTR_ACCT_MAX = 1094) + // which is derived from the MTU + // (see FD_BPF_INSTR_ACCT_MAX comment in Firedancer) + // + // TODO: This limit exceeds 255 because native programs can currently + // be invoked with more than 255 instruction accounts. Once the + // feature gate restricting instruction accounts to 255 is activated + // (https://github.com/anza-xyz/feature-gate-tracker/issues/115), + // this limit should be tightened to 255, eliminating any ambiguity. assert!( - instruction_accounts.len() <= 128, + instruction_accounts.len() <= 1094, "too many instruction accounts", ); diff --git a/svm/src/conformance/instr/effects.rs b/svm/src/conformance/instr/effects.rs index 0d7b64e9e86..5ca95b429e1 100644 --- a/svm/src/conformance/instr/effects.rs +++ b/svm/src/conformance/instr/effects.rs @@ -2,7 +2,7 @@ #[cfg(feature = "conformance")] use { - crate::conformance::account_state::account_to_proto, + crate::conformance::{account_state::account_to_proto, err::instruction_error_code}, protosol::protos::InstrEffects as ProtoInstrEffects, }; use {solana_account::Account, solana_instruction::error::InstructionError, solana_pubkey::Pubkey}; @@ -42,11 +42,7 @@ impl From for ProtoInstrEffects { Self { result: result .as_ref() - .map(|error| { - let serialized_err = bincode::serialize(error).unwrap(); - i32::from_le_bytes((&serialized_err[0..4]).try_into().unwrap()) - .saturating_add(1) - }) + .map(instruction_error_code) .unwrap_or_default(), custom_err: custom_err.unwrap_or_default(), modified_accounts: resulting_accounts diff --git a/svm/src/conformance/instr/harness.rs b/svm/src/conformance/instr/harness.rs index a929756b62f..b1a3123856b 100644 --- a/svm/src/conformance/instr/harness.rs +++ b/svm/src/conformance/instr/harness.rs @@ -5,20 +5,20 @@ use { crate::{ conformance::{ callback::DefaultCallback, - setup::{compile_transaction_context, program_runtime_environments, recent_blockhash}, + setup::{ + InvokeContextFields, compute_budget, prepare_invoke_context_fields, + program_runtime_environments, + }, }, message_processor::process_message, }, - solana_compute_budget::compute_budget::ComputeBudget, solana_instruction::error::InstructionError, solana_program_runtime::{ - invoke_context::{EnvironmentConfig, InvokeContext}, - loaded_programs::ProgramCacheForTxBatch, + invoke_context::InvokeContext, loaded_programs::ProgramCacheForTxBatch, sysvar_cache::SysvarCache, }, solana_pubkey::Pubkey, solana_svm_callback::InvokeContextCallback, - solana_svm_log_collector::LogCollector, solana_svm_timings::ExecuteTimings, solana_transaction_error::TransactionError, std::rc::Rc, @@ -30,6 +30,7 @@ use { programs::{fill_program_cache_from_accounts, new_program_cache_with_builtins}, setup::sysvar_cache_from_accounts, }, + agave_precompiles::is_precompile, prost::Message, protosol::protos::{InstrContext as ProtoInstrContext, InstrEffects as ProtoInstrEffects}, std::ffi::c_int, @@ -55,52 +56,43 @@ pub fn execute_instr_with_callback( let mut compute_units_consumed = 0; let mut timings = ExecuteTimings::default(); - let log_collector = LogCollector::new_ref(); - let feature_set = input.feature_set; - let simd_0268_active = feature_set.raise_cpi_nesting_limit_to_8; - - let mut compute_budget = ComputeBudget::new_with_defaults(simd_0268_active); + let mut compute_budget = compute_budget(&input.feature_set); compute_budget.compute_unit_limit = input.cu_avail; // Clamp budget for execution by cu_avail - let rent = sysvar_cache.get_rent().unwrap(); - let program_id = &input.instruction.program_id; let loader_key = program_cache - .find(program_id) + .find(&input.instruction.program_id) .expect("program not loaded in cache") .account_owner(); - let (sanitized_message, mut transaction_context) = compile_transaction_context( - &input.instruction, - &input.accounts, - program_id, + let program_runtime_environments = + program_runtime_environments(&input.feature_set, &compute_budget); + + let InvokeContextFields { + sanitized_message, + mut transaction_context, + environment_config, + log_collector, + execution_budget, + execution_cost, + } = prepare_invoke_context_fields( + input, + callback, &loader_key, + sysvar_cache, &compute_budget, - (*rent).clone(), + &program_runtime_environments, ); - let runtime_environments = program_runtime_environments(&input.feature_set, &compute_budget); - let result = { - let (blockhash, blockhash_lamports_per_signature) = recent_blockhash(sysvar_cache); - - let environment_config = EnvironmentConfig::new( - blockhash, - blockhash_lamports_per_signature, - false, - callback, - &feature_set, - &runtime_environments, - sysvar_cache, - ); - let mut invoke_context = InvokeContext::new( &mut transaction_context, program_cache, environment_config, Some(log_collector.clone()), - compute_budget.to_budget(), - compute_budget.to_cost(), + execution_budget, + execution_cost, ); + match process_message( &sanitized_message, &mut invoke_context, @@ -165,8 +157,7 @@ pub fn execute_instr_proto(input: ProtoInstrContext) -> ProtoInstrEffects { let mut program_cache = { let slot = sysvar_cache.get_clock().unwrap().slot; let feature_set = &instr_context.feature_set; - let simd_0268_active = feature_set.raise_cpi_nesting_limit_to_8; - let compute_budget = ComputeBudget::new_with_defaults(simd_0268_active); + let compute_budget = compute_budget(feature_set); let environments = program_runtime_environments(feature_set, &compute_budget); let mut cache = new_program_cache_with_builtins(slot); @@ -181,13 +172,57 @@ pub fn execute_instr_proto(input: ProtoInstrContext) -> ProtoInstrEffects { cache }; - execute_instr_with_callback( + let mut effects = execute_instr_with_callback( &instr_context, &ConformanceCallback, &mut program_cache, &sysvar_cache, - ) - .into() + ); + + // Precompile verification failures surface as `Custom`, but Firedancer + // reports a custom error code of 0 for precompiles. + if effects.custom_err.is_some() + && is_precompile(&instr_context.instruction.program_id, |_| true) + { + effects.custom_err = Some(0); + } + + // TODO: Firedancer's tooling compares resulting account contents even + // when execution fails, so the harness must report them. Account + // contents are not meaningful on error (partial writes can diverge based + // on timing, e.g. with direct mapping or builtins), so once the tooling + // supports it, the harness should skip the account comparison on error + // entirely, which would also make the CU-exhaustion workaround below + // unnecessary. + direct_mapping_handle_cu_exhaustion( + instr_context.feature_set.virtual_address_space_adjustments, + effects.cu_avail, + effects.result.is_some(), + effects + .resulting_accounts + .iter_mut() + .map(|(_, account)| &mut account.data), + ); + + effects.into() +} + +/// Due to how Firedancer's VM CU accounting works, when +/// virtual_address_space_adjustments is enabled and execution fails with the +/// CU meter exhausted, we cannot compare the data region of the accounts with +/// Agave. Clears each supplied data buffer in that case. +#[cfg(feature = "conformance")] +fn direct_mapping_handle_cu_exhaustion<'a>( + virtual_address_space_adjustments_active: bool, + cu_avail: u64, + has_err: bool, + account_data: impl IntoIterator>, +) { + if virtual_address_space_adjustments_active && cu_avail == 0 && has_err { + for data in account_data { + data.clear(); + } + } } /// # Safety diff --git a/svm/src/conformance/mod.rs b/svm/src/conformance/mod.rs index 74022a04263..70cd70a22c6 100644 --- a/svm/src/conformance/mod.rs +++ b/svm/src/conformance/mod.rs @@ -6,6 +6,8 @@ pub mod callback; #[cfg(feature = "conformance")] pub mod elf_loader; #[cfg(feature = "conformance")] +mod err; +#[cfg(feature = "conformance")] pub mod fd_hash; #[cfg(feature = "conformance")] pub mod feature_set; @@ -14,3 +16,5 @@ pub mod programs; #[cfg(feature = "conformance")] pub mod serialization; mod setup; +#[cfg(feature = "conformance")] +pub mod syscall; diff --git a/svm/src/conformance/serialization.rs b/svm/src/conformance/serialization.rs index f24f9292102..bfeb15900e8 100644 --- a/svm/src/conformance/serialization.rs +++ b/svm/src/conformance/serialization.rs @@ -6,8 +6,8 @@ use { fd_hash::fd_hash, instr::context::InstrContext, setup::{ - compile_transaction_context, program_runtime_environments, recent_blockhash, - sysvar_cache_from_accounts, + InvokeContextFields, compute_budget, prepare_invoke_context_fields, program_loader_key, + program_runtime_environments, sysvar_cache_from_accounts, }, }, prost::Message, @@ -16,62 +16,52 @@ use { VmSerializationEffects as ProtoVmSerializationEffects, VmSerializedAccountMetadata as ProtoVmSerializedAccountMetadata, }, - solana_compute_budget::compute_budget::ComputeBudget, + solana_instruction::error::InstructionError, + solana_message::SanitizedMessage, solana_program_runtime::{ - invoke_context::{EnvironmentConfig, InvokeContext}, + invoke_context::InvokeContext, loaded_programs::ProgramCacheForTxBatch, memory_context::SerializedAccountMetadata, serialization::serialize_parameters, - solana_sbpf::memory_region::MemoryRegion, + solana_sbpf::{ + aligned_memory::AlignedMemory, ebpf::HOST_ALIGN, memory_region::MemoryRegion, + }, }, - solana_svm_log_collector::LogCollector, + solana_svm_feature_set::SVMFeatureSet, std::ffi::c_int, }; pub fn execute_vm_serialize(input: ProtoInstrContext) -> ProtoVmSerializationEffects { let instr_context = InstrContext::from(input); - let log_collector = LogCollector::new_ref(); let feature_set = instr_context.feature_set; - let virtual_address_space_adjustments = feature_set.virtual_address_space_adjustments; - let direct_mapping = feature_set.account_data_direct_mapping; - let direct_account_pointers = feature_set.direct_account_pointers_in_program_input; - let compute_budget = ComputeBudget::new_with_defaults(feature_set.raise_cpi_nesting_limit_to_8); + let compute_budget = compute_budget(&feature_set); // No CU limit for this harness. let sysvar_cache = sysvar_cache_from_accounts(&instr_context.accounts); - let rent = sysvar_cache.get_rent().unwrap(); let program_id = instr_context.instruction.program_id; - let loader_key = instr_context - .accounts - .iter() - .find(|(key, _)| *key == program_id) - .map(|(_, account)| account.owner) - .expect("program not found in accounts"); + let loader_key = program_loader_key(&instr_context.accounts, &program_id); - let (sanitized_message, mut transaction_context) = compile_transaction_context( - &instr_context.instruction, - &instr_context.accounts, - &program_id, - &loader_key, - &compute_budget, - (*rent).clone(), - ); + let program_runtime_environments = program_runtime_environments(&feature_set, &compute_budget); // We're only testing the parameter serialization, so use an empty cache. let mut program_cache = ProgramCacheForTxBatch::default(); - let runtime_environments = program_runtime_environments(&feature_set, &compute_budget); - let (blockhash, lamports_per_signature) = recent_blockhash(&sysvar_cache); - let environment_config = EnvironmentConfig::new( - blockhash, - lamports_per_signature, - false, + let InvokeContextFields { + sanitized_message, + mut transaction_context, + environment_config, + log_collector, + execution_budget, + execution_cost, + } = prepare_invoke_context_fields( + &instr_context, &DefaultCallback, - &feature_set, - &runtime_environments, + &loader_key, &sysvar_cache, + &compute_budget, + &program_runtime_environments, ); let mut invoke_context = InvokeContext::new( @@ -79,33 +69,22 @@ pub fn execute_vm_serialize(input: ProtoInstrContext) -> ProtoVmSerializationEff &mut program_cache, environment_config, Some(log_collector.clone()), - compute_budget.to_budget(), - compute_budget.to_cost(), + execution_budget, + execution_cost, ); - invoke_context - .prepare_top_level_instructions(&sanitized_message) - .unwrap(); - invoke_context.push().unwrap(); - - let instruction_context = invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(); - - match serialize_parameters( - &instruction_context, - virtual_address_space_adjustments, - direct_mapping, - direct_account_pointers, - ) { - Ok((aligned_memory, input_memory_regions, account_metadatas, _instruction_data_offset)) => { + match push_and_serialize_parameters(&mut invoke_context, &sanitized_message, &feature_set) { + Ok(SerializedParameters { + aligned_memory, + input_memory_regions, + account_metadata, + }) => { let serialized_memory_hash = fd_hash(0, aligned_memory.as_slice()); let vm_input_memory_regions = input_memory_regions .iter() .map(memory_region_to_proto) .collect(); - let serialized_account_metadata = account_metadatas + let serialized_account_metadata = account_metadata .iter() .map(serialized_acct_meta_to_proto) .collect(); @@ -123,6 +102,49 @@ pub fn execute_vm_serialize(input: ProtoInstrContext) -> ProtoVmSerializationEff } } +/// The product of serializing a program's input parameters into VM memory: the +/// serialized region itself plus the metadata needed to map accounts back out. +pub(crate) struct SerializedParameters { + pub(crate) aligned_memory: AlignedMemory, + pub(crate) input_memory_regions: Vec, + pub(crate) account_metadata: Vec, +} + +/// Push the message's single top-level instruction onto `invoke_context`, then +/// serialize that instruction's program input parameters into VM memory. +pub(crate) fn push_and_serialize_parameters<'ix_data>( + invoke_context: &mut InvokeContext<'_, 'ix_data>, + sanitized_message: &'ix_data SanitizedMessage, + feature_set: &SVMFeatureSet, +) -> Result { + invoke_context + .prepare_top_level_instructions(sanitized_message) + .expect("failed to prepare top-level instructions"); + invoke_context + .push() + .expect("failed to push instruction context"); + + let instruction_context = invoke_context + .transaction_context + .get_current_instruction_context() + .unwrap(); + serialize_parameters( + &instruction_context, + feature_set.virtual_address_space_adjustments, + feature_set.account_data_direct_mapping, + feature_set.direct_account_pointers_in_program_input, + ) + .map( + |(aligned_memory, input_memory_regions, account_metadata, _instruction_data_offset)| { + SerializedParameters { + aligned_memory, + input_memory_regions, + account_metadata, + } + }, + ) +} + fn memory_region_to_proto(region: &MemoryRegion) -> ProtoVmInputMemoryRegion { ProtoVmInputMemoryRegion { vm_address: region.vm_addr, diff --git a/svm/src/conformance/setup.rs b/svm/src/conformance/setup.rs index 697ca973930..a65689194fe 100644 --- a/svm/src/conformance/setup.rs +++ b/svm/src/conformance/setup.rs @@ -1,33 +1,105 @@ //! Shared setup helpers for the execution harnesses. -//! -//! Each helper builds one owned prerequisite for an `InvokeContext` (the -//! transaction context, the runtime environments, the blockhash). The harness -//! still assembles its own `EnvironmentConfig`/`InvokeContext`, since those -//! borrow these pieces — so rather than one big `create_invoke_context_fields` -//! returning a tuple of everything, this is a handful of small, composable -//! pieces the harnesses pick from. #[cfg(feature = "conformance")] use solana_account::ReadableAccount; use { + crate::conformance::instr::context::InstrContext, solana_account::Account, solana_compute_budget::compute_budget::ComputeBudget, solana_hash::Hash, solana_instruction::Instruction, solana_message::SanitizedMessage, solana_program_runtime::{ - invoke_context::mock_compile_message, + execution_budget::{SVMTransactionExecutionBudget, SVMTransactionExecutionCost}, + invoke_context::{EnvironmentConfig, mock_compile_message}, loaded_programs::{ProgramRuntimeEnvironment, ProgramRuntimeEnvironments}, sysvar_cache::SysvarCache, }, solana_pubkey::Pubkey, solana_rent::Rent, + solana_svm_callback::InvokeContextCallback, solana_svm_feature_set::SVMFeatureSet, + solana_svm_log_collector::LogCollector, solana_svm_transaction::svm_message::SVMStaticMessage, solana_syscalls::create_program_runtime_environment, solana_transaction_context::transaction::TransactionContext, + std::{cell::RefCell, rc::Rc}, }; +/// Fields required by `InvokeContext::new`. +pub(crate) struct InvokeContextFields<'a, 'ix_data> { + pub(crate) sanitized_message: SanitizedMessage, + pub(crate) transaction_context: TransactionContext<'ix_data>, + pub(crate) environment_config: EnvironmentConfig<'a>, + pub(crate) log_collector: Rc>, + pub(crate) execution_budget: SVMTransactionExecutionBudget, + pub(crate) execution_cost: SVMTransactionExecutionCost, +} + +/// Compile a sanitized transaction message then instantiate a transaction +/// context as well as the remaining fields required by `InvokeContext::new`. +pub(crate) fn prepare_invoke_context_fields<'a, C: InvokeContextCallback>( + instr_context: &'a InstrContext, + callback: &'a C, + loader_key: &Pubkey, + sysvar_cache: &'a SysvarCache, + compute_budget: &ComputeBudget, + program_runtime_environments: &'a ProgramRuntimeEnvironments, +) -> InvokeContextFields<'a, 'a> { + let rent = sysvar_cache.get_rent().unwrap(); + + let (sanitized_message, transaction_context) = compile_transaction_context( + &instr_context.instruction, + &instr_context.accounts, + &instr_context.instruction.program_id, + loader_key, + compute_budget, + (*rent).clone(), + ); + + let (blockhash, blockhash_lamports_per_signature) = recent_blockhash(sysvar_cache); + let environment_config = EnvironmentConfig::new( + blockhash, + blockhash_lamports_per_signature, + false, + callback, + &instr_context.feature_set, + program_runtime_environments, + sysvar_cache, + ); + + let log_collector = LogCollector::new_ref(); + let execution_budget = compute_budget.to_budget(); + let execution_cost = compute_budget.to_cost(); + + InvokeContextFields { + sanitized_message, + transaction_context, + environment_config, + log_collector, + execution_budget, + execution_cost, + } +} + +// Create a compute budget from the given feature set. +pub(crate) fn compute_budget(feature_set: &SVMFeatureSet) -> ComputeBudget { + let simd_0268_active = feature_set.raise_cpi_nesting_limit_to_8; + ComputeBudget::new_with_defaults(simd_0268_active) +} + +/// The loader that owns the program account in `accounts`, used as the program +/// account's owner when compiling the transaction. `None` if the program +/// account isn't present. +#[cfg(feature = "conformance")] +pub(crate) fn program_loader_key(accounts: &[(Pubkey, Account)], program_id: &Pubkey) -> Pubkey { + accounts + .iter() + .find(|(key, _)| key == program_id) + .map(|(_, account)| account.owner) + .expect("program not found in accounts") +} + /// Compile `instruction` into a sanitized message and a fresh transaction /// context sized for a single top-level instruction. pub(crate) fn compile_transaction_context( diff --git a/svm/src/conformance/syscall.rs b/svm/src/conformance/syscall.rs new file mode 100644 index 00000000000..d90eaa3030e --- /dev/null +++ b/svm/src/conformance/syscall.rs @@ -0,0 +1,472 @@ +//! VM syscall conformance harness. + +use { + crate::conformance::{ + callback::DefaultCallback, + err::{UnpackedResult, unpack_stable_result}, + instr::context::InstrContext, + programs::{fill_program_cache_from_accounts, new_program_cache_with_builtins}, + serialization::{SerializedParameters, push_and_serialize_parameters}, + setup::{ + InvokeContextFields, compute_budget, prepare_invoke_context_fields, program_loader_key, + program_runtime_environments, sysvar_cache_from_accounts, + }, + }, + prost::Message, + protosol::protos::{ + InputDataRegion as ProtoInputDataRegion, SyscallContext as ProtoSyscallContext, + SyscallEffects as ProtoSyscallEffects, SyscallInvocation as ProtoSyscallInvocation, + VmContext as ProtoVmContext, + }, + solana_program_runtime::{ + invoke_context::{BpfAllocator, InvokeContext}, + loaded_programs::ProgramCacheForTxBatch, + memory_context::MemoryContext, + solana_sbpf::{ + aligned_memory::AlignedMemory, + ebpf::{HOST_ALIGN, MM_BYTECODE_START, MM_HEAP_START, MM_INPUT_START, MM_STACK_START}, + error::{ProgramResult, StableResult}, + memory_region::{AccessViolationHandler, MemoryMapping, MemoryRegion}, + program::{BuiltinProgram, SBPFVersion}, + vm::{Config, ContextObject, EbpfVm}, + }, + }, + solana_pubkey::Pubkey, + std::{ffi::c_int, sync::Arc}, +}; + +const STACK_GAP_SIZE: u64 = 4_096; +const STACK_SIZE: usize = 64 * STACK_GAP_SIZE as usize; +/// Upper bound on `vm_context.heap_max` — matches Firedancer's cap so the same +/// fuzzer inputs run on either implementation. +const HEAP_MAX: usize = 256 * 1024; +const SBPF_VERSION: SBPFVersion = SBPFVersion::V0; + +pub fn execute_vm_syscall(input: ProtoSyscallContext) -> ProtoSyscallEffects { + let instr_context = InstrContext::from(input.instr_ctx.expect("missing instr context")); + let mut vm_context = input.vm_ctx.expect("missing vm context"); + let syscall_invocation = input.syscall_invocation.unwrap_or_default(); + let registers = get_registers(&vm_context); + + let feature_set = instr_context.feature_set; + let virtual_address_space_adjustments = feature_set.virtual_address_space_adjustments; + let account_data_direct_mapping = feature_set.account_data_direct_mapping; + + let mut compute_budget = compute_budget(&feature_set); + compute_budget.compute_unit_limit = instr_context.cu_avail; // Clamp budget for execution by cu_avail + + let sysvar_cache = sysvar_cache_from_accounts(&instr_context.accounts); + + let program_id = instr_context.instruction.program_id; + let loader_key = program_loader_key(&instr_context.accounts, &program_id); + + let program_runtime_environments = program_runtime_environments(&feature_set, &compute_budget); + let deployment_environment = program_runtime_environments.get_env_for_deployment(); + let execution_environment = program_runtime_environments.get_env_for_execution(); + let config = execution_environment.get_config().clone(); + + // Only build out the program cache if the syscall is CPI. + let mut program_cache = if contains_cpi(&syscall_invocation) { + let slot = sysvar_cache + .get_clock() + .map(|clock| clock.slot) + .unwrap_or_default(); + let mut cache = new_program_cache_with_builtins(slot); + fill_program_cache_from_accounts( + &mut cache, + deployment_environment, + &instr_context.accounts, + slot, + ) + .expect("failed to fill program cache from accounts"); + cache + } else { + ProgramCacheForTxBatch::default() + }; + + let InvokeContextFields { + sanitized_message, + mut transaction_context, + environment_config, + execution_budget, + execution_cost, + .. + } = prepare_invoke_context_fields( + &instr_context, + &DefaultCallback, + &loader_key, + &sysvar_cache, + &compute_budget, + &program_runtime_environments, + ); + + // Replay any prior return data the fuzzer wants in scope before the syscall. + if let Some(return_data) = vm_context.return_data.take() { + let return_program_id = + Pubkey::try_from(return_data.program_id).expect("invalid return data program id"); + transaction_context + .set_return_data(return_program_id, return_data.data) + .expect("failed to set return data"); + } + + let access_violation_handler = transaction_context.access_violation_handler( + virtual_address_space_adjustments, + account_data_direct_mapping, + ); + + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache, + environment_config, + None, + execution_budget, + execution_cost, + ); + + let SerializedParameters { + aligned_memory: _input_memory, // <-- Keep bound + input_memory_regions, + account_metadata, + } = push_and_serialize_parameters(&mut invoke_context, &sanitized_message, &feature_set) + .expect("failed to serialize parameters"); + + let [rodata, mut stack, mut heap] = allocate_memory(&vm_context, &syscall_invocation); + let memory_mapping = unsafe { + create_memory_mapping( + &rodata, + &mut stack, + &mut heap, + input_memory_regions, + &config, + access_violation_handler, + ) + }; + let memory_context = MemoryContext::new( + BpfAllocator::new(vm_context.heap_max), + account_metadata, + memory_mapping, + ); + invoke_context + .memory_contexts + .set_memory_context_abi_v1(memory_context) + .expect("failed to set memory context"); + + let syscall_function = execution_environment + .get_function_registry() + .lookup_by_name(&syscall_invocation.function_name) + .expect("syscall function not registered") + .1 + .0; + + let (program_result, call_depth) = { + // Invoke the syscall with a `&'static` InvokeContext, then take the + // result, dropping the VM. Avoids dangling memory. + let loader = Arc::new(BuiltinProgram::new_loader(config)); + let invoke_context_static: &mut InvokeContext<'static, 'static> = + unsafe { std::mem::transmute(&mut invoke_context) }; + + let mut vm = EbpfVm::new(loader, SBPF_VERSION, invoke_context_static, STACK_SIZE); + vm.registers = registers; + + vm.invoke_function(syscall_function); + + let program_result = std::mem::replace(&mut vm.program_result, ProgramResult::Ok(0)); + let call_depth = vm.call_depth; + (program_result, call_depth) + }; + + let input_data_regions = extract_input_data_regions( + &invoke_context, + &program_result, + virtual_address_space_adjustments, + ); + + let UnpackedResult { + error, + error_kind, + r0, + } = unpack_stable_result(program_result); + let cu_avail = invoke_context.get_remaining(); + invoke_context + .pop() + .expect("failed to pop instruction context"); + + ProtoSyscallEffects { + error, + error_kind, + r0, + cu_avail, + heap: heap.as_slice().to_vec(), + stack: stack.as_slice().to_vec(), + input_data_regions, + frame_count: call_depth, + rodata: rodata.as_slice().to_vec(), + pc: 0, + ..Default::default() + } +} + +fn get_registers(vm_context: &ProtoVmContext) -> [u64; 12] { + [ + vm_context.r0, + vm_context.r1, + vm_context.r2, + vm_context.r3, + vm_context.r4, + vm_context.r5, + vm_context.r6, + vm_context.r7, + vm_context.r8, + vm_context.r9, + vm_context.r10, + vm_context.r11, + ] +} + +fn contains_cpi(syscall_invocation: &ProtoSyscallInvocation) -> bool { + syscall_invocation.function_name == b"sol_invoke_signed_c" + || syscall_invocation.function_name == b"sol_invoke_signed_rust" +} + +fn allocate_memory( + vm_context: &ProtoVmContext, + syscall_invocation: &ProtoSyscallInvocation, +) -> [AlignedMemory; 3] { + assert!( + vm_context.heap_max as usize <= HEAP_MAX, + "vm_context.heap_max ({}) exceeds HEAP_MAX ({HEAP_MAX})", + vm_context.heap_max, + ); + let rodata = AlignedMemory::from(&vm_context.rodata); + let mut stack = AlignedMemory::from(&vec![0; STACK_SIZE]); + let mut heap = AlignedMemory::from(&vec![0; vm_context.heap_max as usize]); + + copy_memory_prefix(heap.as_slice_mut(), &syscall_invocation.heap_prefix); + copy_memory_prefix(stack.as_slice_mut(), &syscall_invocation.stack_prefix); + + [rodata, stack, heap] +} + +fn copy_memory_prefix(dst: &mut [u8], src: &[u8]) { + let size = dst.len().min(src.len()); + dst[..size].copy_from_slice(&src[..size]); +} + +// SAFETY: The backing memory for rodata/stack/heap should live at least as +// long as this function's returned `MemoryMapping`. +unsafe fn create_memory_mapping( + rodata: &AlignedMemory, + stack: &mut AlignedMemory, + heap: &mut AlignedMemory, + input_memory_regions: Vec, + config: &Config, + acces_violation_handler: AccessViolationHandler, +) -> MemoryMapping { + let stack_frame_gap = if SBPF_VERSION.stack_frame_gaps() && config.enable_stack_frame_gaps { + config.stack_frame_size as u64 + } else { + 0 + }; + let regions = [ + MemoryRegion::new(rodata.as_slice() as *const [u8], MM_BYTECODE_START), + MemoryRegion::new_gapped( + stack.as_slice_mut() as *mut [u8], + MM_STACK_START, + stack_frame_gap, + ), + MemoryRegion::new(heap.as_slice_mut() as *mut [u8], MM_HEAP_START), + ] + .into_iter() + .chain(input_memory_regions) + .collect(); + unsafe { + MemoryMapping::new_with_access_violation_handler( + regions, + config, + SBPF_VERSION, + acces_violation_handler, + ) + .expect("failed to create memory mapping") + } +} + +fn extract_input_data_regions( + invoke_context: &InvokeContext, + program_result: &ProgramResult, + virtual_address_space_adjustments: bool, +) -> Vec { + // When virtual_address_space_adjustments is enabled, Agave calls + // update_caller_account_region only after a _successful_ CPI execution, so + // on failure the input regions can hold stale data — return empty instead. + if virtual_address_space_adjustments && matches!(program_result, StableResult::Err(_)) { + return Vec::new(); + } + invoke_context + .memory_contexts + .memory_mapping() + .ok() + .map(|mapping| { + let mut regions: Vec = mapping + .get_regions() + .iter() + .filter(|region| region.vm_addr >= MM_INPUT_START) + .map(mem_region_to_input_data_region) + .collect(); + regions.sort_by_key(|region| region.offset); + regions + }) + .unwrap_or_default() +} + +fn mem_region_to_input_data_region(region: &MemoryRegion) -> ProtoInputDataRegion { + ProtoInputDataRegion { + content: unsafe { + std::slice::from_raw_parts(region.host_addr as *const u8, region.len as usize).to_vec() + }, + offset: region.vm_addr.saturating_sub(MM_INPUT_START), + is_writable: region.writable, + } +} + +/// # Safety +/// +/// `in_ptr` must point to `in_sz` initialized bytes. `out_ptr` must point +/// to a writable buffer of at least `*out_psz` bytes. On return, `*out_psz` +/// is updated to the number of bytes written. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sol_compat_vm_syscall_execute_v1( + out_ptr: *mut u8, + out_psz: *mut u64, + in_ptr: *mut u8, + in_sz: u64, +) -> c_int { + let in_slice = unsafe { std::slice::from_raw_parts(in_ptr, in_sz as usize) }; + let Ok(syscall_context) = ProtoSyscallContext::decode(in_slice) else { + return 0; + }; + + let syscall_effects = execute_vm_syscall(syscall_context); + let out_slice = unsafe { std::slice::from_raw_parts_mut(out_ptr, (*out_psz) as usize) }; + let out_vec = syscall_effects.encode_to_vec(); + if out_vec.len() > out_slice.len() { + return 0; + } + out_slice[..out_vec.len()].copy_from_slice(&out_vec); + unsafe { *out_psz = out_vec.len() as u64 }; + + 1 +} + +#[cfg(test)] +mod tests { + use { + super::*, + protosol::protos::{ + AcctState as ProtoAcctState, InstrContext as ProtoInstrContext, + SyscallInvocation as ProtoSyscallInvocation, VmContext as ProtoVmContext, + }, + solana_rent::Rent, + solana_sdk_ids::sysvar, + }; + + const PROGRAM_ID: [u8; 32] = [7; 32]; + + fn syscall_context( + function_name: &[u8], + r1: u64, + r2: u64, + r3: u64, + r4: u64, + heap_prefix: Vec, + ) -> ProtoSyscallContext { + let program_account = ProtoAcctState { + address: PROGRAM_ID.to_vec(), + lamports: 0, + data: vec![], + executable: true, + owner: Pubkey::default().to_bytes().to_vec(), + }; + let rent_sysvar = ProtoAcctState { + address: sysvar::rent::id().to_bytes().to_vec(), + lamports: 1, + data: bincode::serialize(&Rent::default()).unwrap(), + executable: false, + owner: sysvar::id().to_bytes().to_vec(), + }; + ProtoSyscallContext { + instr_ctx: Some(ProtoInstrContext { + program_id: PROGRAM_ID.to_vec(), + accounts: vec![program_account, rent_sysvar], + instr_accounts: vec![], + data: vec![], + cu_avail: 200_000, + features: None, + }), + vm_ctx: Some(ProtoVmContext { + heap_max: 1024, + r1, + r2, + r3, + r4, + ..Default::default() + }), + syscall_invocation: Some(ProtoSyscallInvocation { + function_name: function_name.to_vec(), + heap_prefix, + stack_prefix: vec![], + }), + } + } + + #[test] + fn test_sol_log() { + let msg = b"hello"; + let effects = execute_vm_syscall(syscall_context( + b"sol_log_", + MM_HEAP_START, // r1: msg address in heap + msg.len() as u64, // r2: length + 0, + 0, + msg.to_vec(), + )); + + assert_eq!(effects.error, 0); + // Logs are no longer collected (the harness runs without a log + // collector), so the syscall succeeding is all we assert here. + assert!(effects.cu_avail < 200_000, "syscall should consume compute"); + } + + #[test] + fn test_sol_memset() { + let effects = execute_vm_syscall(syscall_context( + b"sol_memset_", + MM_HEAP_START, // r1: dst + 0x42, // r2: byte + 8, // r3: count + 0, + vec![0u8; 16], + )); + + assert_eq!(effects.error, 0); + assert_eq!(&effects.heap[..8], &[0x42; 8]); + assert_eq!( + &effects.heap[8..16], + &[0u8; 8], + "must not write past length" + ); + } + + #[test] + fn test_sol_panic_surfaces_error() { + let effects = execute_vm_syscall(syscall_context( + b"sol_panic_", + MM_HEAP_START, // r1: file address + 1, // r2: file length + 10, // r3: line + 5, // r4: column + b"x".to_vec(), + )); + + assert_ne!(effects.error, 0); + } +} diff --git a/svm/tests/concurrent_tests.rs b/svm/tests/concurrent_tests.rs index be8701d3493..cab75d9a462 100644 --- a/svm/tests/concurrent_tests.rs +++ b/svm/tests/concurrent_tests.rs @@ -18,6 +18,7 @@ use { ProgramToLoad, }, program_cache_entry::{ProgramCacheEntryOwner, ProgramCacheEntryType}, + program_metrics::ProgramStatistics, }, solana_pubkey::Pubkey, solana_svm::{ @@ -96,6 +97,33 @@ fn program_cache_execution(threads: usize) { } }) }) + .chain(programs.iter().map(|program| { + let program = *program; + let local_bank = mock_bank.clone(); + let processor = TransactionBatchProcessor::new_from( + &batch_processor, + batch_processor.slot, + batch_processor.epoch, + ); + thread::spawn(move || { + let feature_set = SVMFeatureSet::all_enabled(); + let account_loader = AccountLoader::new_with_loaded_accounts_capacity( + None, + &local_bank, + &feature_set, + 0, + ); + let upcoming_environment = + processor.program_runtime_environment_for_epoch(processor.epoch + 1); + processor.prepare_one_program_for_upcoming_feature_set( + &account_loader, + false, + &upcoming_environment, + &program, + &ProgramStatistics::default(), + ); + }) + })) .collect(); for th in ths { diff --git a/syscalls/Cargo.toml b/syscalls/Cargo.toml index d8cad2fbf8d..6ab736d24e4 100644 --- a/syscalls/Cargo.toml +++ b/syscalls/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] default = ["metrics"] diff --git a/syscalls/src/lib.rs b/syscalls/src/lib.rs index 22df6344d10..2c81cc0102c 100644 --- a/syscalls/src/lib.rs +++ b/syscalls/src/lib.rs @@ -59,7 +59,10 @@ mod mem_ops; mod sysvar; /// Error definitions +// Note: `#[repr(u64)]` is used for `Self::discriminant`, but the actual +// memory layout of this enum's variants is not depended on by the VM. #[derive(Debug, ThisError, PartialEq, Eq)] +#[repr(u64)] pub enum SyscallError { #[error("{0}: {1:?}")] InvalidString(Utf8Error, Vec), @@ -114,6 +117,15 @@ pub enum SyscallError { ArithmeticOverflow, } +impl SyscallError { + /// Returns the enum discriminant as a `u64`. + /// + /// This is sound only because of the `#[repr(u64)]` attribute on the enum. + pub fn discriminant(&self) -> u64 { + unsafe { *std::ptr::addr_of!(*self).cast::() } + } +} + impl From for SyscallError { fn from(error: MemoryTranslationError) -> Self { match error { diff --git a/test-validator/Cargo.toml b/test-validator/Cargo.toml index 6be86b55a32..cfcd28cf525 100644 --- a/test-validator/Cargo.toml +++ b/test-validator/Cargo.toml @@ -10,6 +10,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/tls-utils/Cargo.toml b/tls-utils/Cargo.toml index d0a76f2d775..6754782b669 100644 --- a/tls-utils/Cargo.toml +++ b/tls-utils/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/tokens/Cargo.toml b/tokens/Cargo.toml index 6318551d4ba..0d416d43651 100644 --- a/tokens/Cargo.toml +++ b/tokens/Cargo.toml @@ -9,6 +9,14 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +# `all-features` can't be used: remote-wallet-hidraw and remote-wallet-libusb +# forward to hidapi's mutually-exclusive linux backends. Document the default +# (hidraw) backend plus the unstable API. +features = ["agave-unstable-api"] +rustdoc-args = ["--cfg=docsrs"] + [features] default = ["remote-wallet-hidraw"] agave-unstable-api = [] diff --git a/tpu-client-next/Cargo.toml b/tpu-client-next/Cargo.toml index 084ed02608b..d0ce3d31249 100644 --- a/tpu-client-next/Cargo.toml +++ b/tpu-client-next/Cargo.toml @@ -10,6 +10,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/tpu-client/Cargo.toml b/tpu-client/Cargo.toml index 70cb0de90df..9362a68c540 100644 --- a/tpu-client/Cargo.toml +++ b/tpu-client/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] default = ["spinner"] diff --git a/transaction-context/src/lib.rs b/transaction-context/src/lib.rs index 537d4a56f89..7a0e3a37ecd 100644 --- a/transaction-context/src/lib.rs +++ b/transaction-context/src/lib.rs @@ -1,7 +1,7 @@ #![cfg(feature = "agave-unstable-api")] //! Data shared between program runtime and built-in programs as well as SBF programs. #![deny(clippy::indexing_slicing)] -#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod instruction; pub mod instruction_accounts; diff --git a/transaction-status-client-types/Cargo.toml b/transaction-status-client-types/Cargo.toml index ed511cf4ca5..4bff6a5edb7 100644 --- a/transaction-status-client-types/Cargo.toml +++ b/transaction-status-client-types/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/transaction-status/Cargo.toml b/transaction-status/Cargo.toml index d5413c0c551..dde180bbede 100644 --- a/transaction-status/Cargo.toml +++ b/transaction-status/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/transaction-view/Cargo.toml b/transaction-view/Cargo.toml index 499801466ed..0c278922281 100644 --- a/transaction-view/Cargo.toml +++ b/transaction-view/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [] diff --git a/transaction-view/src/sanitize.rs b/transaction-view/src/sanitize.rs index fbd17419121..88d4eef8978 100644 --- a/transaction-view/src/sanitize.rs +++ b/transaction-view/src/sanitize.rs @@ -85,17 +85,14 @@ fn sanitize_config( view: &UnsanitizedTransactionView, config: &SanitizeConfig, ) -> Result<()> { - #[allow(clippy::collapsible_if)] if let Some(requested_heap_bytes) = view .transaction_config() .and_then(|config| config.requested_heap_size()) - { - if !(config.min_requested_heap_size..=config.max_requested_heap_size) + && (!(config.min_requested_heap_size..=config.max_requested_heap_size) .contains(&requested_heap_bytes) - || !requested_heap_bytes.is_multiple_of(1024) - { - return Err(TransactionViewError::SanitizeError); - } + || !requested_heap_bytes.is_multiple_of(1024)) + { + return Err(TransactionViewError::SanitizeError); } Ok(()) @@ -181,10 +178,10 @@ fn sanitize_instructions( } } - if let Some(max_accounts_per_instruction) = config.max_accounts_per_instruction { - if instruction.accounts.len() > max_accounts_per_instruction { - return Err(TransactionViewError::SanitizeError); - } + if let Some(max_accounts_per_instruction) = config.max_accounts_per_instruction + && instruction.accounts.len() > max_accounts_per_instruction + { + return Err(TransactionViewError::SanitizeError); } } diff --git a/turbine/Cargo.toml b/turbine/Cargo.toml index 9a7f462de87..58500b78282 100644 --- a/turbine/Cargo.toml +++ b/turbine/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/udp-client/Cargo.toml b/udp-client/Cargo.toml index ea710e31ab1..18b323f0a63 100644 --- a/udp-client/Cargo.toml +++ b/udp-client/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [] diff --git a/unified-scheduler-logic/Cargo.toml b/unified-scheduler-logic/Cargo.toml index 5784d7add2c..8da344e14fa 100644 --- a/unified-scheduler-logic/Cargo.toml +++ b/unified-scheduler-logic/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] diff --git a/unified-scheduler-pool/Cargo.toml b/unified-scheduler-pool/Cargo.toml index d821b010894..c81163fcddc 100644 --- a/unified-scheduler-pool/Cargo.toml +++ b/unified-scheduler-pool/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [] diff --git a/validator/Cargo.toml b/validator/Cargo.toml index ea6b8f476d0..2b8895239f4 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -13,6 +13,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index 544d87b2a71..9bb25827b42 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -166,6 +166,7 @@ fn start_gossip_node( &cluster_info, None, gossip_sockets, + None, gossip_validators, should_check_duplicate_instance, None, diff --git a/validator/src/commands/run/args.rs b/validator/src/commands/run/args.rs index dedab1df151..15df3194b63 100644 --- a/validator/src/commands/run/args.rs +++ b/validator/src/commands/run/args.rs @@ -1104,6 +1104,7 @@ pub fn add_args<'a>(app: App<'a, 'a>, default_args: &'a DefaultArgs) -> App<'a, Arg::with_name("allow_private_addr") .long("allow-private-addr") .takes_value(false) + .requires("no_xdp") .help("Allow contacting private ip addresses") .hidden(hidden_unless_forced()), ) @@ -1849,7 +1850,7 @@ mod tests { }; verify_args_struct_by_command_run_with_identity_setup( default_run_args, - vec!["--allow-private-addr"], + vec!["--allow-private-addr", "--no-xdp"], expected_args, ); } diff --git a/validator/src/commands/run/execute.rs b/validator/src/commands/run/execute.rs index 472ac6e34c6..bc78f2ae351 100644 --- a/validator/src/commands/run/execute.rs +++ b/validator/src/commands/run/execute.rs @@ -84,7 +84,8 @@ use { }; #[cfg(target_os = "linux")] use { - agave_cpu_utils::cpu_affinity, agave_xdp::transmitter::XdpConfig, + agave_cpu_utils::cpu_affinity, + agave_xdp::transmitter::{QueueCpuBinding, XdpConfig}, solana_clap_utils::input_parsers::parse_cpu_ranges, }; @@ -1451,7 +1452,16 @@ fn build_xdp_config( }; Ok(cpus.map(|cpus| { info!("XDP enabled on CPU cores: {cpus:?}"); - XdpConfig::new(xdp_interface, cpus, xdp_zero_copy) + // Map the CPU list onto hardware queues sequentially (queue i -> cpus[i]). + let queues = cpus + .into_iter() + .enumerate() + .map(|(queue, cpu)| QueueCpuBinding { + queue: queue as u32, + cpu, + }) + .collect(); + XdpConfig::new(xdp_interface, queues, xdp_zero_copy) })) } diff --git a/version/Cargo.toml b/version/Cargo.toml index 48466aeb6f0..fbf410a9280 100644 --- a/version/Cargo.toml +++ b/version/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] name = "solana_version" diff --git a/vote/Cargo.toml b/vote/Cargo.toml index 2b71a638192..4fb37196868 100644 --- a/vote/Cargo.toml +++ b/vote/Cargo.toml @@ -11,6 +11,8 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [lib] crate-type = ["lib"] diff --git a/votor-messages/Cargo.toml b/votor-messages/Cargo.toml index aaaefd2b38c..308afcacd26 100644 --- a/votor-messages/Cargo.toml +++ b/votor-messages/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [] diff --git a/votor-messages/src/certificate.rs b/votor-messages/src/certificate.rs index 411531a7249..284ce8a434e 100644 --- a/votor-messages/src/certificate.rs +++ b/votor-messages/src/certificate.rs @@ -82,33 +82,30 @@ impl CertificateType { match self { Self::Notarize(block) | Self::FinalizeFast(block) => { let vote = Vote::new_notarization_vote(*block); - (get_vote_payload_to_sign(&vote, shred_version), None) + (get_vote_payload_to_sign(vote, shred_version), None) } Self::Genesis(block) => { let vote = Vote::new_genesis_vote(*block); - (get_vote_payload_to_sign(&vote, shred_version), None) + (get_vote_payload_to_sign(vote, shred_version), None) } Self::Finalize(slot) => { let vote = Vote::new_finalization_vote(*slot); - (get_vote_payload_to_sign(&vote, shred_version), None) + (get_vote_payload_to_sign(vote, shred_version), None) } Self::Skip(slot) => { let skip_vote = Vote::new_skip_vote(*slot); let skip_fallback_vote = Vote::new_skip_fallback_vote(*slot); ( - get_vote_payload_to_sign(&skip_vote, shred_version), - Some(get_vote_payload_to_sign(&skip_fallback_vote, shred_version)), + get_vote_payload_to_sign(skip_vote, shred_version), + Some(get_vote_payload_to_sign(skip_fallback_vote, shred_version)), ) } Self::NotarizeFallback(block) => { let notar_vote = Vote::new_notarization_vote(*block); let notar_fallback_vote = Vote::new_notarization_fallback_vote(*block); ( - get_vote_payload_to_sign(¬ar_vote, shred_version), - Some(get_vote_payload_to_sign( - ¬ar_fallback_vote, - shred_version, - )), + get_vote_payload_to_sign(notar_vote, shred_version), + Some(get_vote_payload_to_sign(notar_fallback_vote, shred_version)), ) } } diff --git a/votor-messages/src/consensus_message.rs b/votor-messages/src/consensus_message.rs index e3060e39237..4c0487cd259 100644 --- a/votor-messages/src/consensus_message.rs +++ b/votor-messages/src/consensus_message.rs @@ -52,19 +52,18 @@ pub struct Block { } /// A consensus vote. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, SchemaWrite, SchemaRead)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct VoteMessage { /// The type of the vote. pub vote: Vote, /// The signature. - #[wincode(with = "PodBLSSignature")] pub signature: BLSSignature, /// The rank of the validator. pub rank: u16, } /// A consensus message sent between validators. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, SchemaWrite, SchemaRead)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] #[allow(clippy::large_enum_variant)] pub enum ConsensusMessage { /// A vote from a single party. diff --git a/votor-messages/src/migration.rs b/votor-messages/src/migration.rs index c8916945d95..d2abd403829 100644 --- a/votor-messages/src/migration.rs +++ b/votor-messages/src/migration.rs @@ -288,11 +288,6 @@ impl MigrationPhase { fn should_use_double_merkle_block_id(&self, slot: Slot) -> bool { self.is_alpenglow_block(slot) } - - /// Should this block allow the UpdateParent marker, i.e., support fast leader handover? - fn should_allow_fast_leader_handover(&self, slot: Slot) -> bool { - self.is_alpenglow_block(slot) - } } /// Keeps track of the current migration status @@ -459,7 +454,6 @@ impl MigrationStatus { dispatch!(pub fn should_respond_to_ancestor_hashes_requests(&self, slot: Slot) -> bool); dispatch!(pub fn should_have_alpenglow_ticks(&self, slot: Slot) -> bool); dispatch!(pub fn should_allow_block_markers(&self, slot: Slot) -> bool); - dispatch!(pub fn should_allow_fast_leader_handover(&self, slot: Slot) -> bool); dispatch!(pub fn should_use_double_merkle_block_id(&self, slot: Slot) -> bool); /// The alpenglow feature flag has been activated in slot `slot`. diff --git a/votor-messages/src/vote.rs b/votor-messages/src/vote.rs index 9610de3b40b..91a83ada86e 100644 --- a/votor-messages/src/vote.rs +++ b/votor-messages/src/vote.rs @@ -1,22 +1,9 @@ //! Vote data types for use by clients -use { - crate::consensus_message::Block, - serde::{Deserialize, Serialize}, - solana_clock::Slot, - solana_hash::Hash, - wincode::{SchemaRead, SchemaWrite}, -}; +use {crate::consensus_message::Block, solana_clock::Slot, solana_hash::Hash}; /// Enum that clients can use to parse and create the vote /// structures expected by the program -#[cfg_attr( - feature = "frozen-abi", - derive(AbiExample, AbiEnumVisitor), - frozen_abi(digest = "Fd13KXQMkc1mCJEoHwyXWkcewqBCdRcAiMhS7Aqe4sm1") -)] -#[derive( - Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, SchemaWrite, SchemaRead, -)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Vote { /// A notarization vote Notarize(NotarizationVote), @@ -182,48 +169,14 @@ impl From for Vote { } /// A notarization vote -#[cfg_attr( - feature = "frozen-abi", - derive(AbiExample), - frozen_abi(digest = "F9veHPmSwMyrYNSVuBLcvLGYSLgc7voTD3kUhxUHUTRU") -)] -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Default, - Serialize, - Deserialize, - SchemaWrite, - SchemaRead, -)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] pub struct NotarizationVote { /// The block this vote is cast for pub block: Block, } /// A finalization vote -#[cfg_attr( - feature = "frozen-abi", - derive(AbiExample), - frozen_abi(digest = "2XQ5N6YLJjF28w7cMFFUQ9SDgKuf9JpJNtAiXSPA8vR2") -)] -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Default, - Serialize, - Deserialize, - SchemaWrite, - SchemaRead, -)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] pub struct FinalizationVote { /// The slot this vote is cast for. pub slot: Slot, @@ -232,96 +185,28 @@ pub struct FinalizationVote { /// A skip vote /// Represents a range of slots to skip /// inclusive on both ends -#[cfg_attr( - feature = "frozen-abi", - derive(AbiExample), - frozen_abi(digest = "G8Nrx3sMYdnLpHsCNark3BGA58BmW2sqNnqjkYhQHtN") -)] -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Default, - Serialize, - Deserialize, - SchemaWrite, - SchemaRead, -)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] pub struct SkipVote { /// The slot this vote is cast for. pub slot: Slot, } /// A notarization fallback vote -#[cfg_attr( - feature = "frozen-abi", - derive(AbiExample), - frozen_abi(digest = "6UW4zutbRvyri4z8WAyKx8aUZkJrZX4XoiqC4XMUnUZk") -)] -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Default, - Serialize, - Deserialize, - SchemaWrite, - SchemaRead, -)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] pub struct NotarizationFallbackVote { /// The block this vote is cast for pub block: Block, } /// A skip fallback vote -#[cfg_attr( - feature = "frozen-abi", - derive(AbiExample), - frozen_abi(digest = "WsUNum8V62gjRU1yAnPuBMAQui4YvMwD1RwrzHeYkeF") -)] -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Default, - Serialize, - Deserialize, - SchemaWrite, - SchemaRead, -)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] pub struct SkipFallbackVote { /// The slot this vote is cast for. pub slot: Slot, } /// A genesis vote. Only used during the migration from TowerBFT -#[cfg_attr( - feature = "frozen-abi", - derive(AbiExample), - frozen_abi(digest = "8ty2gETfpyVGPNMYrEFS1YXeDRprfZaisSAmJwoAYusb") -)] -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Default, - Serialize, - Deserialize, - SchemaWrite, - SchemaRead, -)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] pub struct GenesisVote { /// The block this vote is cast for pub block: Block, diff --git a/votor-messages/src/wire.rs b/votor-messages/src/wire.rs index 595c525da1c..2006ebea3e9 100644 --- a/votor-messages/src/wire.rs +++ b/votor-messages/src/wire.rs @@ -103,14 +103,17 @@ pub(crate) struct WireSlotVoteMessage { #[cfg_attr(feature = "frozen-abi", derive(AbiExample, StableAbi, StableAbiSample))] #[derive(Clone, Debug, Hash, PartialEq, Eq, SchemaRead, SchemaWrite, Serialize)] -pub(crate) struct WireCertSignature { +/// Signature on a wire cert message +pub struct WireCertSignature { #[cfg_attr( feature = "frozen-abi", stable_abi_sample(with = "sample_bls_signature(rng)") )] #[wincode(with = "PodBLSSignature")] - pub(crate) signature: BLSSignature, - pub(crate) bitmap: Vec, + /// the aggregate signature + pub signature: BLSSignature, + /// bitmap of ranks of validators included in the aggregate. + pub bitmap: Vec, } impl From for WireCertSignature { @@ -131,9 +134,12 @@ pub(crate) struct WireSlotCertMessage { #[cfg_attr(feature = "frozen-abi", derive(AbiExample, StableAbi, StableAbiSample))] #[derive(Debug, Clone, Hash, PartialEq, Eq, SchemaRead, SchemaWrite, Serialize)] -pub(crate) struct WireBlockCertMessage { - pub(crate) block: Block, - pub(crate) signature: WireCertSignature, +/// A wire cert message that holds a block. +pub struct WireBlockCertMessage { + /// the block the cert is certifying. + pub block: Block, + /// the signature of the cert message. + pub signature: WireCertSignature, } #[cfg_attr( @@ -340,48 +346,119 @@ impl VersionedWireConsensusMessage { #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, SchemaWrite, SchemaRead)] #[wincode(tag_encoding = "u8")] /// Vote payload that must be signed -enum VotePayloadToSign { +pub enum VotePayloadToSign { #[wincode(tag = 1)] - Notar { block: Block, shred_version: u16 }, + /// notar vote + Notar { + /// block + block: Block, + /// shred version + shred_version: u16, + }, #[wincode(tag = 2)] - Finalize { slot: Slot, shred_version: u16 }, + /// finalize vote + Finalize { + /// slot + slot: Slot, + /// shred version + shred_version: u16, + }, #[wincode(tag = 3)] - Skip { slot: Slot, shred_version: u16 }, + /// skip vote + Skip { + /// slot + slot: Slot, + /// shred version + shred_version: u16, + }, #[wincode(tag = 4)] - NotarFallback { block: Block, shred_version: u16 }, + /// notar fallback vote + NotarFallback { + /// block + block: Block, + /// shred version + shred_version: u16, + }, #[wincode(tag = 5)] - SkipFallback { slot: Slot, shred_version: u16 }, + /// skip fallback vote + SkipFallback { + /// slot + slot: Slot, + /// shred version + shred_version: u16, + }, #[wincode(tag = 6)] - Genesis { block: Block, shred_version: u16 }, + /// genesis vote + Genesis { + /// block + block: Block, + /// shred version + shred_version: u16, + }, +} + +impl VotePayloadToSign { + /// Converts a `Vote` into a `VotePayloadToSign` + pub fn new_from_vote(vote: Vote, shred_version: u16) -> Self { + match vote { + Vote::Notarize(v) => Self::Notar { + block: v.block, + shred_version, + }, + Vote::NotarizeFallback(v) => Self::NotarFallback { + block: v.block, + shred_version, + }, + Vote::Genesis(v) => Self::Genesis { + block: v.block, + shred_version, + }, + Vote::Finalize(v) => Self::Finalize { + slot: v.slot, + shred_version, + }, + Vote::Skip(v) => Self::Skip { + slot: v.slot, + shred_version, + }, + Vote::SkipFallback(v) => Self::SkipFallback { + slot: v.slot, + shred_version, + }, + } + } + + /// Returns the slot the vote is for. + pub fn slot(&self) -> Slot { + match self { + Self::Notar { block, .. } + | Self::NotarFallback { block, .. } + | Self::Genesis { block, .. } => block.slot, + Self::Finalize { slot, .. } + | Self::Skip { slot, .. } + | Self::SkipFallback { slot, .. } => *slot, + } + } +} + +impl From for Vote { + /// Converts a `VotePayloadToSign` back into a `Vote`, dropping the shred version. + fn from(vote_payload: VotePayloadToSign) -> Self { + match vote_payload { + VotePayloadToSign::Notar { block, .. } => Self::new_notarization_vote(block), + VotePayloadToSign::Finalize { slot, .. } => Self::new_finalization_vote(slot), + VotePayloadToSign::Skip { slot, .. } => Self::new_skip_vote(slot), + VotePayloadToSign::NotarFallback { block, .. } => { + Self::new_notarization_fallback_vote(block) + } + VotePayloadToSign::SkipFallback { slot, .. } => Self::new_skip_fallback_vote(slot), + VotePayloadToSign::Genesis { block, .. } => Self::new_genesis_vote(block), + } + } } /// Returns the appropriate vote payload to sign. -pub fn get_vote_payload_to_sign(vote: &Vote, shred_version: u16) -> Vec { - let vote_to_sign = match vote { - Vote::Notarize(v) => VotePayloadToSign::Notar { - block: v.block, - shred_version, - }, - Vote::NotarizeFallback(v) => VotePayloadToSign::NotarFallback { - block: v.block, - shred_version, - }, - Vote::Genesis(v) => VotePayloadToSign::Genesis { - block: v.block, - shred_version, - }, - Vote::Finalize(v) => VotePayloadToSign::Finalize { - slot: v.slot, - shred_version, - }, - Vote::Skip(v) => VotePayloadToSign::Skip { - slot: v.slot, - shred_version, - }, - Vote::SkipFallback(v) => VotePayloadToSign::SkipFallback { - slot: v.slot, - shred_version, - }, - }; +pub fn get_vote_payload_to_sign(vote: Vote, shred_version: u16) -> Vec { + let vote_to_sign = VotePayloadToSign::new_from_vote(vote, shred_version); wincode::serialize(&vote_to_sign).unwrap() } diff --git a/votor/Cargo.toml b/votor/Cargo.toml index dbb47a201b1..e204884d182 100644 --- a/votor/Cargo.toml +++ b/votor/Cargo.toml @@ -9,6 +9,11 @@ homepage = { workspace = true } license = { workspace = true } edition = { workspace = true } +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] dev-context-only-utils = [ diff --git a/votor/src/common.rs b/votor/src/common.rs index c5620b23b31..0ad4d3e90cc 100644 --- a/votor/src/common.rs +++ b/votor/src/common.rs @@ -10,21 +10,27 @@ use { // Core consensus types and constants pub type Stake = u64; -pub const fn conflicting_types(vote_type: VoteType) -> &'static [VoteType] { +pub(crate) const fn conflicting_types(vote_type: VoteType) -> &'static [VoteType] { match vote_type { VoteType::Finalize => &[ VoteType::NotarizeFallback, VoteType::Skip, VoteType::SkipFallback, + VoteType::Genesis, ], - VoteType::Notarize => &[VoteType::Skip, VoteType::NotarizeFallback], - VoteType::NotarizeFallback => &[VoteType::Finalize, VoteType::Notarize], + VoteType::Notarize => &[ + VoteType::Skip, + VoteType::NotarizeFallback, + VoteType::Genesis, + ], + VoteType::NotarizeFallback => &[VoteType::Finalize, VoteType::Notarize, VoteType::Genesis], VoteType::Skip => &[ VoteType::Finalize, VoteType::Notarize, VoteType::SkipFallback, + VoteType::Genesis, ], - VoteType::SkipFallback => &[VoteType::Skip, VoteType::Finalize], + VoteType::SkipFallback => &[VoteType::Skip, VoteType::Finalize, VoteType::Genesis], VoteType::Genesis => &[ VoteType::Finalize, VoteType::Notarize, diff --git a/votor/src/consensus_pool.rs b/votor/src/consensus_pool.rs index 6e56f945b39..8138bf7ded1 100644 --- a/votor/src/consensus_pool.rs +++ b/votor/src/consensus_pool.rs @@ -74,13 +74,8 @@ fn get_key_and_stakes( }; Ok(( entry.vote_account_pubkey, - NonZero::new(entry.stake).unwrap_or_else(|| { - panic!( - "Validator stake is zero for pubkey: {}", - entry.vote_account_pubkey, - ) - }), - NonZero::new(rank_map.total_stake()).expect("expect rank-map total stake to not be 0"), + entry.stake, + rank_map.total_stake(), )) } @@ -782,7 +777,7 @@ mod tests { let bls_keypair = BLSKeypair::derive_from_signer(&keypairs[rank].vote_keypair, BLS_KEYPAIR_DERIVE_SEED) .unwrap(); - let payload = get_vote_payload_to_sign(vote, shred_version); + let payload = get_vote_payload_to_sign(*vote, shred_version); let signature: BLSSignature = bls_keypair.sign(&payload).into(); ConsensusMessage::new_vote(*vote, signature, rank as u16) } @@ -2236,7 +2231,7 @@ mod tests { BLSKeypair::derive_from_signer(validator_vote_keypair, BLS_KEYPAIR_DERIVE_SEED) .unwrap(); - let payload = get_vote_payload_to_sign(&vote, ctx.pool.cluster_info.my_shred_version()); + let payload = get_vote_payload_to_sign(vote, ctx.pool.cluster_info.my_shred_version()); vote_message .signature .verify(&bls_keypair.public, &payload) diff --git a/votor/src/consensus_pool/certificate_builder.rs b/votor/src/consensus_pool/certificate_builder.rs index 381303793d3..af06c45646a 100644 --- a/votor/src/consensus_pool/certificate_builder.rs +++ b/votor/src/consensus_pool/certificate_builder.rs @@ -414,7 +414,7 @@ mod tests { let mut keypairs = Vec::new(); let mut vote_messages = Vec::new(); let vote = Vote::new_notarization_vote(block); - let serialized_vote = get_vote_payload_to_sign(&vote, shred_version); + let serialized_vote = get_vote_payload_to_sign(vote, shred_version); for i in 0..num_validators { let keypair = BLSKeypair::new(); @@ -459,7 +459,7 @@ mod tests { let mut all_pubkeys = Vec::new(); // Group 1: Signs a Notarize vote. let notarize_vote = Vote::new_notarization_vote(block); - let serialized_notarize_vote = get_vote_payload_to_sign(¬arize_vote, shred_version); + let serialized_notarize_vote = get_vote_payload_to_sign(notarize_vote, shred_version); for i in 0..3 { let keypair = BLSKeypair::new(); let signature = keypair.sign(&serialized_notarize_vote); @@ -474,7 +474,7 @@ mod tests { // Group 2: Signs a NotarizeFallback vote. let notarize_fallback_vote = Vote::new_notarization_fallback_vote(block); let serialized_fallback_vote = - get_vote_payload_to_sign(¬arize_fallback_vote, shred_version); + get_vote_payload_to_sign(notarize_fallback_vote, shred_version); for i in 3..6 { let keypair = BLSKeypair::new(); let signature = keypair.sign(&serialized_fallback_vote); diff --git a/votor/src/consensus_pool/slot_stake_counters.rs b/votor/src/consensus_pool/slot_stake_counters.rs index 9685a5756cd..67331e107a0 100644 --- a/votor/src/consensus_pool/slot_stake_counters.rs +++ b/votor/src/consensus_pool/slot_stake_counters.rs @@ -115,10 +115,10 @@ impl SlotStakeCounters { // White paper v1.1 page 22: The event is only issued if the node voted in slot s already, // but not to notarize b. Moreover: // notar(b) >= 40% or (skip(s) + notar(b) >= 60% and notar(b) >= 20%) - if let Some(Vote::Notarize(my_vote)) = self.my_first_vote.as_ref() { - if &my_vote.block.block_id == block_id { - return false; // I voted for the same block, no need to send NotarizeFallback - } + if let Some(Vote::Notarize(my_vote)) = self.my_first_vote.as_ref() + && &my_vote.block.block_id == block_id + { + return false; // I voted for the same block, no need to send NotarizeFallback } trace!( "safe_to_notar {block_id:?} skip_ratio={} notarized_ratio={}", diff --git a/votor/src/consensus_pool_service.rs b/votor/src/consensus_pool_service.rs index 86c6842a72c..58748f52621 100644 --- a/votor/src/consensus_pool_service.rs +++ b/votor/src/consensus_pool_service.rs @@ -814,7 +814,7 @@ mod tests { let bls_keypair = BLSKeypair::derive_from_signer(vote_keypair, BLS_KEYPAIR_DERIVE_SEED).unwrap(); let vote_serialized = - get_vote_payload_to_sign(¬arize_vote, ctx.ctx.cluster_info.my_shred_version()); + get_vote_payload_to_sign(notarize_vote, ctx.ctx.cluster_info.my_shred_version()); let message = ConsensusMessage::Vote(VoteMessage { vote: notarize_vote, signature: bls_keypair.sign(&vote_serialized).into(), diff --git a/votor/src/event_handler.rs b/votor/src/event_handler.rs index d59ab8397a9..10fa753f84e 100644 --- a/votor/src/event_handler.rs +++ b/votor/src/event_handler.rs @@ -450,15 +450,15 @@ impl EventHandler { stats, ); - if let Some(slot) = *standstill_slot { - if block.slot > slot { - *standstill_slot = None; - info!( - "{my_pubkey}: Standstill initially detected at slot={slot} has ended \ - at slot={}. Ending timeout extension", - block.slot - ); - } + if let Some(slot) = *standstill_slot + && block.slot > slot + { + *standstill_slot = None; + info!( + "{my_pubkey}: Standstill initially detected at slot={slot} has ended at \ + slot={}. Ending timeout extension", + block.slot + ); } if let Some(parent_block) = @@ -876,8 +876,10 @@ impl EventHandler { panic!( "{my_pubkey}: Block {block:?} has been finalized, however we have a bank \ hash mismatch. The cluster bank hash is {expected_hash} however we \ - computed {}. At this point we will be unable to recover. Please save a \ - copy of your ledger to share on discord and restart from a snapshot > {}.", + computed {}. At this point we will be unable to recover. Ensure that you \ + are running a supported Agave version for this cluster. If this is not \ + operator error,please save a copy of your ledger to share on discord and \ + restart from a snapshot > {}.", bank.hash(), block.slot ); @@ -1398,7 +1400,7 @@ mod tests { fn expected_vote_message(&self, expected_vote: &Vote) -> VoteMessage { let payload = - get_vote_payload_to_sign(expected_vote, self.cluster_info.my_shred_version()); + get_vote_payload_to_sign(*expected_vote, self.cluster_info.my_shred_version()); let signature: BLSSignature = self.my_bls_keypair.sign(&payload).into(); VoteMessage { vote: *expected_vote, diff --git a/votor/src/event_handler/stats.rs b/votor/src/event_handler/stats.rs index c1d6bdee9bf..d6e12eedc81 100644 --- a/votor/src/event_handler/stats.rs +++ b/votor/src/event_handler/stats.rs @@ -185,19 +185,23 @@ impl EventHandlerStats { } pub fn incr_vote(&mut self, bls_op: &BLSOp) { - if let BLSOp::PushVote { vote, .. } = bls_op { - let vote_type = vote.vote.get_type(); - let entry = self.sent_votes.entry(vote_type).or_insert(0); - *entry = entry.saturating_add(1); - if vote_type == VoteType::Notarize { - let entry = self.slot_tracking_map.entry(vote.vote.slot()).or_default(); - entry.vote_notarize = Some(Instant::now()); - } else if vote_type == VoteType::Skip { - let entry = self.slot_tracking_map.entry(vote.vote.slot()).or_default(); - entry.vote_skip = Some(Instant::now()); + match bls_op { + BLSOp::PushVote { vote, .. } => { + let vote_type = vote.vote.get_type(); + let entry = self.sent_votes.entry(vote_type).or_insert(0); + *entry = entry.saturating_add(1); + if vote_type == VoteType::Notarize { + let entry = self.slot_tracking_map.entry(vote.vote.slot()).or_default(); + entry.vote_notarize = Some(Instant::now()); + } else if vote_type == VoteType::Skip { + let entry = self.slot_tracking_map.entry(vote.vote.slot()).or_default(); + entry.vote_skip = Some(Instant::now()); + } + } + BLSOp::RefreshVotes { .. } => (), + _ => { + warn!("Unexpected BLS operation: {bls_op:?}"); } - } else { - warn!("Unexpected BLS operation: {bls_op:?}"); } } diff --git a/votor/src/staked_validators_cache.rs b/votor/src/staked_validators_cache.rs index 94fec023736..a0eb00e3c5c 100644 --- a/votor/src/staked_validators_cache.rs +++ b/votor/src/staked_validators_cache.rs @@ -156,18 +156,16 @@ impl StakedValidatorsCache { ) -> (&[SocketAddr], bool) { // Check if self.alpenglow_port_override has a different last_modified. // Immediately refresh the cache if it does. - if let Some(alpenglow_port_override) = &self.alpenglow_port_override { - if alpenglow_port_override.has_new_override(self.alpenglow_port_override_last_modified) - { - self.alpenglow_port_override_last_modified = - alpenglow_port_override.last_modified(); - trace!( - "refreshing cache entry for epoch {} due to alpenglow port override \ - last_modified change", - self.cur_epoch(slot) - ); - self.refresh_cache_entry(self.cur_epoch(slot), cluster_info, access_time); - } + if let Some(alpenglow_port_override) = &self.alpenglow_port_override + && alpenglow_port_override.has_new_override(self.alpenglow_port_override_last_modified) + { + self.alpenglow_port_override_last_modified = alpenglow_port_override.last_modified(); + trace!( + "refreshing cache entry for epoch {} due to alpenglow port override last_modified \ + change", + self.cur_epoch(slot) + ); + self.refresh_cache_entry(self.cur_epoch(slot), cluster_info, access_time); } self.get_staked_validators_by_epoch(self.cur_epoch(slot), cluster_info, access_time) diff --git a/votor/src/vote_history.rs b/votor/src/vote_history.rs index 31fa9d278e5..bf6199291fd 100644 --- a/votor/src/vote_history.rs +++ b/votor/src/vote_history.rs @@ -2,8 +2,7 @@ use { super::vote_history_storage::{ Result, SavedVoteHistory, SavedVoteHistoryVersions, VoteHistoryStorage, }, - agave_votor_messages::{consensus_message::Block, vote::Vote}, - serde::{Deserialize, Serialize}, + agave_votor_messages::{consensus_message::Block, vote::Vote, wire::VotePayloadToSign}, solana_clock::Slot, solana_hash::Hash, solana_keypair::Keypair, @@ -13,7 +12,11 @@ use { wincode::{SchemaRead, SchemaWrite}, }; -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +// Dummy shred version used for persisted votes - will be re-signed with the +// correct shred version when performing the refresh +const VOTE_HISTORY_SHRED_VERSION: u16 = 0; + +#[derive(Debug, SchemaRead, SchemaWrite, PartialEq, Clone)] pub enum VoteHistoryVersions { Current(VoteHistory), } @@ -31,10 +34,13 @@ impl VoteHistoryVersions { #[cfg_attr( feature = "frozen-abi", - derive(AbiExample), - frozen_abi(digest = "BdNXvHhpPUTEba3DfDCLqirLoLXAmja2jyhroicM63bT") + derive(AbiExample, StableAbi, StableAbiSample), + frozen_abi( + abi_digest = "MYpTecggZfULsn6SC1bojefNFK1R5kjBZg7wE8H8dHF", + abi_serializer = "wincode" + ) )] -#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Default, SchemaWrite, SchemaRead)] +#[derive(Clone, Debug, PartialEq, Default, SchemaWrite, SchemaRead)] pub struct VoteHistory { /// The validator identity that cast votes pub node_pubkey: Pubkey, @@ -64,7 +70,7 @@ pub struct VoteHistory { its_over: HashSet, /// All votes cast for a `slot`, for use in refresh - votes_cast: HashMap>, + votes_cast: HashMap>, /// Blocks which have a notarization certificate via the certificate pool notarized_blocks: HashSet, @@ -131,7 +137,8 @@ impl VoteHistory { .iter() .filter(|&(&s, _)| s > slot) .flat_map(|(_, votes)| votes.iter()) - .cloned() + .copied() + .map(Vote::from) .collect() } @@ -203,7 +210,13 @@ impl VoteHistory { // votor, we do not need to insert anything here. } } - self.votes_cast.entry(vote.slot()).or_default().push(vote); + self.votes_cast + .entry(vote.slot()) + .or_default() + .push(VotePayloadToSign::new_from_vote( + vote, + VOTE_HISTORY_SHRED_VERSION, + )); } /// Add a new notarized block diff --git a/votor/src/voting_utils.rs b/votor/src/voting_utils.rs index 73d3ecc0963..986e5332929 100644 --- a/votor/src/voting_utils.rs +++ b/votor/src/voting_utils.rs @@ -162,10 +162,10 @@ pub fn generate_vote_tx( if bank.get_vote_account(&vote_account_pubkey).is_none() { return GenerateVoteTxResult::VoteAccountNotFound(vote_account_pubkey); } - if let Some(slot) = wait_to_vote_slot { - if vote.slot() < slot { - return GenerateVoteTxResult::WaitToVoteSlot(slot); - } + if let Some(slot) = wait_to_vote_slot + && vote.slot() < slot + { + return GenerateVoteTxResult::WaitToVoteSlot(slot); } let rank_map = bank @@ -219,7 +219,7 @@ pub fn generate_vote_tx( return GenerateVoteTxResult::NonVoting; }; - let vote_payload_to_sign = get_vote_payload_to_sign(&vote, shred_version); + let vote_payload_to_sign = get_vote_payload_to_sign(vote, shred_version); GenerateVoteTxResult::Vote(VoteMessage { vote, signature: bls_keypair.sign(&vote_payload_to_sign).into(), @@ -363,7 +363,7 @@ mod tests { vote: Vote, my_bls_keypair: &BLSKeypair, ) -> VoteMessage { - let payload = get_vote_payload_to_sign(&vote, ctx.cluster_info.my_shred_version()); + let payload = get_vote_payload_to_sign(vote, ctx.cluster_info.my_shred_version()); let signature = my_bls_keypair.sign(&payload); VoteMessage { vote, diff --git a/watchtower/Cargo.toml b/watchtower/Cargo.toml index 055510618be..6a6fbc76aff 100644 --- a/watchtower/Cargo.toml +++ b/watchtower/Cargo.toml @@ -11,11 +11,14 @@ edition = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] [features] agave-unstable-api = [] [dependencies] +agave-feature-set = { workspace = true } agave-logger = { workspace = true } clap = { workspace = true } humantime = { workspace = true } @@ -31,6 +34,7 @@ solana-pubkey = { version = "=4.2.0", default-features = false } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-version = { workspace = true } +solana-vote-interface = { workspace = true } [lints] workspace = true diff --git a/watchtower/README.md b/watchtower/README.md index c94fdf52d91..e6efdbd5895 100644 --- a/watchtower/README.md +++ b/watchtower/README.md @@ -8,6 +8,13 @@ If you only care about the health of several specific validators, the `--validator-identity` command-line argument can be used to restrict failure notifications to issues only affecting that set of validators. +When the Validator Admission Ticket feature is active, watchtower also alerts +when a monitored validator's vote account balance is below the bank-side VAT +balance threshold computed from RPC-visible state. This check only verifies the +vote account balance over public RPC. It does not verify full VAT eligibility, +including BLS pubkey presence, stake, top validator set inclusion, or tie cutoff +behavior. + User can provide either 1 or 3 RPC URLs for the cluster via the `--url` or `--urls` command-line arguments respectively. 2 URLs are not accepted because it's not enough to have redundnacy, and more than 3 URLs are not accepted because there's little diff --git a/watchtower/src/main.rs b/watchtower/src/main.rs index 0a838196be1..48ce73eb820 100644 --- a/watchtower/src/main.rs +++ b/watchtower/src/main.rs @@ -17,6 +17,7 @@ use { solana_pubkey::Pubkey, solana_rpc_client::rpc_client::RpcClient, solana_rpc_client_api::{client_error, response::RpcVoteAccountStatus}, + solana_vote_interface::state::VoteStateV4, std::{ collections::HashMap, error, @@ -25,6 +26,8 @@ use { }, }; +const LEGACY_VAT_TO_BURN_PER_EPOCH: u64 = 1_600_000_000; + struct Config { address_labels: HashMap, ignore_http_bad_gateway: bool, @@ -284,6 +287,88 @@ struct EndpointData { last_recent_blockhash: Hash, } +fn slot_time_reduction_vat_burns() -> [(Pubkey, u64); 4] { + [ + ( + agave_feature_set::reduce_slot_time_to_200ms::id(), + 800_000_000, + ), + ( + agave_feature_set::reduce_slot_time_to_250ms::id(), + 1_000_000_000, + ), + ( + agave_feature_set::reduce_slot_time_to_300ms::id(), + 1_200_000_000, + ), + ( + agave_feature_set::reduce_slot_time_to_350ms::id(), + 1_400_000_000, + ), + ] +} + +fn is_feature_active_at_slot( + rpc_client: &RpcClient, + feature_id: &Pubkey, + slot: u64, +) -> client_error::Result { + Ok(matches!( + rpc_client.get_feature_activation_slot(feature_id)?, + Some(activation_slot) if activation_slot <= slot + )) +} + +fn vat_to_burn_per_epoch(rpc_client: &RpcClient, slot: u64) -> client_error::Result { + let epoch_schedule = rpc_client.get_epoch_schedule()?; + + // Keep this table in sync with runtime/src/slot_params.rs. Slot-time + // feature gates take effect at the first slot of the epoch after activation. + for (feature_id, vat_to_burn_per_epoch) in slot_time_reduction_vat_burns() { + let Some(activation_slot) = rpc_client.get_feature_activation_slot(&feature_id)? else { + continue; + }; + if activation_slot > slot { + continue; + } + + let activation_epoch = epoch_schedule.get_epoch(activation_slot); + let effective_slot = + epoch_schedule.get_first_slot_in_epoch(activation_epoch.saturating_add(1)); + if effective_slot <= slot { + return Ok(vat_to_burn_per_epoch); + } + } + + Ok(LEGACY_VAT_TO_BURN_PER_EPOCH) +} + +fn get_minimum_vat_vote_account_balance( + rpc_client: &RpcClient, +) -> client_error::Result> { + let slot = rpc_client.get_slot()?; + if !is_feature_active_at_slot( + rpc_client, + &agave_feature_set::validator_admission_ticket::id(), + slot, + )? { + return Ok(None); + } + + let vote_account_rent_exempt_minimum = + rpc_client.get_minimum_balance_for_rent_exemption(VoteStateV4::size_of())?; + let vat_to_burn_per_epoch = + if is_feature_active_at_slot(rpc_client, &agave_feature_set::alpenglow::id(), slot)? { + vat_to_burn_per_epoch(rpc_client, slot)? + } else { + 0 + }; + + Ok(Some( + vote_account_rent_exempt_minimum + vat_to_burn_per_epoch, + )) +} + fn query_endpoint( config: &Config, endpoint: &mut EndpointData, @@ -354,19 +439,25 @@ fn query_endpoint( } let mut validator_errors = vec![]; + let minimum_vat_vote_account_balance = if config.validator_identity_pubkeys.is_empty() { + None + } else { + get_minimum_vat_vote_account_balance(&endpoint.rpc_client)? + }; for validator_identity in config.validator_identity_pubkeys.iter() { + let validator_identity_string = validator_identity.to_string(); let formatted_validator_identity = - format_labeled_address(&validator_identity.to_string(), &config.address_labels); + format_labeled_address(&validator_identity_string, &config.address_labels); if vote_accounts .delinquent .iter() - .any(|vai| vai.node_pubkey == *validator_identity.to_string()) + .any(|vai| vai.node_pubkey == validator_identity_string.as_str()) { validator_errors.push(format!("{formatted_validator_identity} delinquent")); } else if !vote_accounts .current .iter() - .any(|vai| vai.node_pubkey == *validator_identity.to_string()) + .any(|vai| vai.node_pubkey == validator_identity_string.as_str()) { validator_errors.push(format!("{formatted_validator_identity} missing")); } @@ -379,6 +470,41 @@ fn query_endpoint( )); } } + + if let Some(minimum_vat_vote_account_balance) = minimum_vat_vote_account_balance { + for vote_account in vote_accounts + .current + .iter() + .chain(vote_accounts.delinquent.iter()) + .filter(|vai| vai.node_pubkey == validator_identity_string.as_str()) + { + let Ok(vote_pubkey) = vote_account.vote_pubkey.parse::() else { + failures.push(( + "vat-vote-account-balance", + format!( + "{} vote account {} is not a valid pubkey", + formatted_validator_identity, vote_account.vote_pubkey + ), + )); + continue; + }; + + let balance = endpoint.rpc_client.get_balance(&vote_pubkey)?; + if balance < minimum_vat_vote_account_balance { + failures.push(( + "vat-vote-account-balance", + format!( + "{} vote account {} has {}, below required VAT balance \ + threshold of {}", + formatted_validator_identity, + vote_pubkey, + Sol(balance), + Sol(minimum_vat_vote_account_balance) + ), + )); + } + } + } } if !validator_errors.is_empty() { diff --git a/xdp-ebpf/Cargo.toml b/xdp-ebpf/Cargo.toml index 5c751eebfc2..c27982b5e66 100644 --- a/xdp-ebpf/Cargo.toml +++ b/xdp-ebpf/Cargo.toml @@ -9,6 +9,11 @@ license = { workspace = true } edition = { workspace = true } include = ["Cargo.toml", "src/**", "README", "agave-xdp-prog"] +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [[bin]] name = "agave-xdp-prog" path = "src/bin/agave-xdp-prog.rs" diff --git a/xdp/Cargo.toml b/xdp/Cargo.toml index cd916284dbc..7ec0c768fd8 100644 --- a/xdp/Cargo.toml +++ b/xdp/Cargo.toml @@ -8,6 +8,11 @@ license = { workspace = true } edition = { workspace = true } publish = true +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true +rustdoc-args = ["--cfg=docsrs"] + [features] agave-unstable-api = [] @@ -26,5 +31,29 @@ arrayvec = { workspace = true } aya = { workspace = true } caps = { workspace = true } +[target.'cfg(target_os = "linux")'.dev-dependencies] +aya = { workspace = true, features = ["test-helpers"] } +nix = { workspace = true, features = ["net", "poll", "socket"] } + +[[test]] +name = "netlink_snapshot" +path = "tests/netlink_snapshot.rs" +required-features = ["agave-unstable-api"] + +[[test]] +name = "route_monitor" +path = "tests/route_monitor.rs" +required-features = ["agave-unstable-api"] + +[[test]] +name = "router_snapshot" +path = "tests/router_snapshot.rs" +required-features = ["agave-unstable-api"] + +[[test]] +name = "transmitter_smoke" +path = "tests/transmitter_smoke.rs" +required-features = ["agave-unstable-api"] + [lints] workspace = true diff --git a/xdp/src/netlink.rs b/xdp/src/netlink.rs index e2d4b576df5..991eece9213 100644 --- a/xdp/src/netlink.rs +++ b/xdp/src/netlink.rs @@ -632,10 +632,10 @@ pub fn parse_rtm_newneigh(msg: &NetlinkMessage, if_index: Option) -> Option return None; } let nd_msg = unsafe { ptr::read_unaligned(msg.data.as_ptr() as *const ndmsg) }; - if let Some(idx) = if_index { - if nd_msg.ndm_ifindex != idx as i32 { - return None; - } + if let Some(idx) = if_index + && nd_msg.ndm_ifindex != idx as i32 + { + return None; } let Ok(attrs) = parse_attrs(&msg.data[mem::size_of::()..]) else { return None; @@ -649,12 +649,12 @@ pub fn parse_rtm_newneigh(msg: &NetlinkMessage, if_index: Option) -> Option if let Some(dst_attr) = attrs.get(&NDA_DST) { neighbor.destination = parse_ip_address(dst_attr.data, nd_msg.ndm_family); } - if let Some(lladdr_attr) = attrs.get(&NDA_LLADDR) { - if lladdr_attr.data.len() >= 6 { - let mut mac = [0u8; 6]; - mac.copy_from_slice(&lladdr_attr.data[0..6]); - neighbor.lladdr = Some(MacAddress(mac)); - } + if let Some(lladdr_attr) = attrs.get(&NDA_LLADDR) + && lladdr_attr.data.len() >= 6 + { + let mut mac = [0u8; 6]; + mac.copy_from_slice(&lladdr_attr.data[0..6]); + neighbor.lladdr = Some(MacAddress(mac)); } Some(neighbor) } diff --git a/xdp/src/program.rs b/xdp/src/program.rs index a9ce2e34b98..3775a42751b 100644 --- a/xdp/src/program.rs +++ b/xdp/src/program.rs @@ -2,7 +2,10 @@ use { crate::device::NetworkDevice, - aya::{Ebpf, EbpfLoader, programs::Xdp}, + aya::{ + Ebpf, EbpfLoader, + programs::{Xdp, xdp::XdpMode}, + }, std::io::{Cursor, Write}, }; @@ -44,7 +47,7 @@ pub fn load_xdp_program(dev: &NetworkDevice) -> Result Result, - pub cpus: Vec, + /// NIC-queue -> CPU-core bindings. One TX worker is created per entry, in + /// order. The queue id is taken explicitly from the binding rather than + /// inferred from position, so callers can target arbitrary hardware queues. + pub queues: Vec, pub zero_copy: bool, // The capacity of the channel that sits between senders and each XDP thread that enqueues // packets to the NIC. @@ -54,7 +69,7 @@ impl Default for XdpConfig { fn default() -> Self { Self { interface: None, - cpus: vec![], + queues: vec![], zero_copy: false, tx_channel_cap: Self::DEFAULT_TX_CHANNEL_CAP, } @@ -62,10 +77,14 @@ impl Default for XdpConfig { } impl XdpConfig { - pub fn new(interface: Option>, cpus: Vec, zero_copy: bool) -> Self { + pub fn new( + interface: Option>, + queues: Vec, + zero_copy: bool, + ) -> Self { Self { interface: interface.map(|s| s.into()), - cpus, + queues, zero_copy, tx_channel_cap: XdpConfig::DEFAULT_TX_CHANNEL_CAP, } @@ -236,7 +255,7 @@ impl TransmitterBuilder { }; let XdpConfig { interface: maybe_interface, - cpus, + queues, zero_copy, tx_channel_cap, } = config; @@ -251,10 +270,9 @@ impl TransmitterBuilder { tx_loop_config_builder.zero_copy(zero_copy); let tx_loop_config = tx_loop_config_builder.build_with_src_device(&dev); - let reserved_cores = cpus + let reserved_cores = queues .iter() - .copied() - .map(CpuId::new) + .map(|binding| CpuId::new(binding.cpu)) .collect::>>()?; let unreserved_cores = cpu_affinity(None)? .into_iter() @@ -265,15 +283,19 @@ impl TransmitterBuilder { return Err("all CPUs are reserved; no CPU available for the main thread".into()); } - let mut tx_loop_builders = Vec::with_capacity(cpus.len()); - for (i, cpu_id) in cpus.into_iter().enumerate() { + let mut tx_loop_builders = Vec::with_capacity(queues.len()); + for binding in queues { // since we aren't necessarily allocating from the thread that we intend to run on, // temporarily switch to the target cpu for each TxLoop to ensure that the Umem region // is allocated to the correct numa node - let cpu = CpuId::new(cpu_id)?; + let cpu = CpuId::new(binding.cpu)?; set_cpu_affinity(None, [cpu])?; - let tx_loop_builder = - TxLoopBuilder::new(cpu_id, QueueId(i as u64), tx_loop_config.clone(), &dev); + let tx_loop_builder = TxLoopBuilder::new( + binding.cpu, + QueueId(binding.queue as u64), + tx_loop_config.clone(), + &dev, + ); // migrate main thread back off of the last xdp reserved cpu set_cpu_affinity(None, unreserved_cores.iter().copied())?; tx_loop_builders.push(tx_loop_builder); diff --git a/xdp/tests/README.md b/xdp/tests/README.md new file mode 100644 index 00000000000..c46e9f3f9da --- /dev/null +++ b/xdp/tests/README.md @@ -0,0 +1,51 @@ +# XDP Integration Tests + +These tests are marked as ignored so ordinary workspace test jobs do not run them without privileges. They are run through `cargo xtask xdp-test`. + +The tests run directly on the host and require root or equivalent network admin privileges because the harness creates a temporary network namespace, `veth` interfaces, routes, and neighbors. `xtask` builds the test binaries as the current user, then applies `--runner` only when running the compiled test executables: + +```bash +cargo xtask xdp-test --runner "sudo -n -E" +``` + +To run a single test binary locally, use this form: + +```bash +cargo xtask xdp-test --runner "sudo -n -E" --test -- --exact --nocapture +``` + +## Test Topology + +Each integration test runs in a fresh temporary network namespace created with `unshare(CLONE_NEWNET)`. The tests bring `lo` up, create the interfaces needed by that test, and restore the original namespace when the test exits. + +The primary topology is one veth pair inside that namespace: + +```text +temporary test network namespace + + route and neighbor state under test + | + v + axdp0 10.0.0.1/24 02:aa:bb:cc:dd:01 + | + | veth peer + | + axdp1 10.0.0.2/24 02:aa:bb:cc:dd:02 + + neighbor: 10.0.0.2 -> 02:aa:bb:cc:dd:02 dev axdp0 + route example: 203.0.113.0/24 via 10.0.0.2 dev axdp0 +``` + +GRE coverage adds `gxdp0` on top of the veth pair and routes the overlay prefix through that tunnel: + +```text +GRE overlay route: + 192.0.2.0/24 dev gxdp0 src 192.0.2.1 + +GRE tunnel: + gxdp0 + local underlay: 10.0.0.1 (axdp0) + remote underlay: 10.0.0.2 (axdp1) + overlay source: 192.0.2.1/32 + ttl: 64 +``` diff --git a/xdp/tests/common/mod.rs b/xdp/tests/common/mod.rs new file mode 100644 index 00000000000..b91c2429b88 --- /dev/null +++ b/xdp/tests/common/mod.rs @@ -0,0 +1,221 @@ +#![cfg(target_os = "linux")] +#![allow(dead_code)] + +pub use aya::test_helpers::NetNsGuard; +use { + agave_xdp::netlink::MacAddress, + std::{ + ffi::{CString, OsString}, + os::unix::ffi::OsStringExt, + path::{Path, PathBuf}, + process::{Command, Output}, + sync::OnceLock, + thread, + time::{Duration, Instant}, + }, +}; + +pub const LEFT_IFACE: &str = "axdp0"; +pub const RIGHT_IFACE: &str = "axdp1"; +pub const GRE_IFACE: &str = "gxdp0"; + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct TestLinks { + pub left_if_index: u32, + pub right_if_index: u32, + pub left_ip: std::net::Ipv4Addr, + pub right_ip: std::net::Ipv4Addr, + pub left_mac: MacAddress, + pub right_mac: MacAddress, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct TestGreTunnel { + pub if_index: u32, + pub local_ip: std::net::Ipv4Addr, + pub remote_ip: std::net::Ipv4Addr, + pub overlay_ip: std::net::Ipv4Addr, +} + +pub fn setup_veth_pair() -> TestLinks { + let left_ip = std::net::Ipv4Addr::new(10, 0, 0, 1); + let right_ip = std::net::Ipv4Addr::new(10, 0, 0, 2); + let left_mac = MacAddress([0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0x01]); + let right_mac = MacAddress([0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0x02]); + + run_ip(&[ + "link", + "add", + LEFT_IFACE, + "type", + "veth", + "peer", + "name", + RIGHT_IFACE, + ]); + set_link_mac(LEFT_IFACE, &left_mac.to_string()); + set_link_mac(RIGHT_IFACE, &right_mac.to_string()); + add_ipv4_addr(&format!("{left_ip}/24"), LEFT_IFACE); + add_ipv4_addr(&format!("{right_ip}/24"), RIGHT_IFACE); + set_link_up(LEFT_IFACE); + set_link_up(RIGHT_IFACE); + + TestLinks { + left_if_index: if_index(LEFT_IFACE), + right_if_index: if_index(RIGHT_IFACE), + left_ip, + right_ip, + left_mac, + right_mac, + } +} + +pub fn setup_gre_tunnel(underlay: &TestLinks) -> TestGreTunnel { + let local = underlay.left_ip.to_string(); + let remote = underlay.right_ip.to_string(); + let overlay_ip = std::net::Ipv4Addr::new(192, 0, 2, 1); + + run_ip(&[ + "tunnel", "add", GRE_IFACE, "mode", "gre", "local", &local, "remote", &remote, "ttl", "64", + ]); + add_ipv4_addr(&format!("{overlay_ip}/32"), GRE_IFACE); + set_link_up(GRE_IFACE); + + TestGreTunnel { + if_index: if_index(GRE_IFACE), + local_ip: underlay.left_ip, + remote_ip: underlay.right_ip, + overlay_ip, + } +} + +pub fn add_route(destination: &str, via: std::net::Ipv4Addr, dev: &str) { + let via = via.to_string(); + run_ip(&["route", "replace", destination, "via", &via, "dev", dev]); +} + +pub fn add_route_to_dev(destination: &str, dev: &str) { + run_ip(&["route", "replace", destination, "dev", dev]); +} + +pub fn add_route_to_dev_with_src(destination: &str, dev: &str, src: std::net::Ipv4Addr) { + let src = src.to_string(); + run_ip(&["route", "replace", destination, "dev", dev, "src", &src]); +} + +pub fn delete_link(dev: &str) { + run_ip(&["link", "del", dev]); +} + +#[allow(dead_code)] +pub fn delete_route(destination: &str) { + run_ip(&["route", "del", destination]); +} + +pub fn replace_neighbor(ip: std::net::Ipv4Addr, mac: MacAddress, dev: &str) { + let ip = ip.to_string(); + let mac = mac.to_string(); + run_ip(&[ + "neigh", + "replace", + &ip, + "lladdr", + &mac, + "dev", + dev, + "nud", + "permanent", + ]); +} + +pub fn delete_neighbor(ip: std::net::Ipv4Addr, dev: &str) { + let ip = ip.to_string(); + run_ip(&["neigh", "del", &ip, "dev", dev]); +} + +#[allow(dead_code)] +pub fn wait_until(description: &str, timeout: Duration, mut predicate: F) -> T +where + F: FnMut() -> Option, +{ + let start = Instant::now(); + loop { + if let Some(value) = predicate() { + return value; + } + + if start.elapsed() >= timeout { + panic!("timed out waiting for {description}"); + } + + thread::sleep(Duration::from_millis(10)); + } +} + +fn set_link_mac(dev: &str, mac: &str) { + run_ip(&["link", "set", "dev", dev, "address", mac]); +} + +fn set_link_up(dev: &str) { + run_ip(&["link", "set", "dev", dev, "up"]); +} + +fn add_ipv4_addr(addr: &str, dev: &str) { + run_ip(&["addr", "add", addr, "dev", dev]); +} + +pub fn if_index(dev: &str) -> u32 { + let dev = CString::new(dev).expect("interface name must not contain NUL"); + let index = unsafe { libc::if_nametoindex(dev.as_ptr()) }; + assert_ne!(index, 0, "failed to resolve ifindex for interface"); + index +} + +fn run_ip(args: &[&str]) { + run_command(ip_command(), args); +} + +fn run_command(program: &Path, args: &[&str]) { + let Output { + status, + stdout, + stderr, + } = Command::new(program) + .args(args) + .output() + .unwrap_or_else(|err| panic!("failed to run {program:?} {args:?}: {err}")); + if status.success() { + return; + } + + panic!( + "{program:?} {args:?} failed: {} +stdout: +{} +stderr: +{}", + status, + OsString::from_vec(stdout).display(), + OsString::from_vec(stderr).display(), + ); +} + +fn ip_command() -> &'static PathBuf { + static IP_COMMAND: OnceLock = OnceLock::new(); + IP_COMMAND.get_or_init(|| { + let mut candidates = std::env::var_os("IP") + .into_iter() + .map(PathBuf::from) + .chain([ + PathBuf::from("/usr/sbin/ip"), + PathBuf::from("/sbin/ip"), + PathBuf::from("ip"), + ]); + + candidates + .find(|path| path == Path::new("ip") || path.exists()) + .unwrap_or_else(|| PathBuf::from("ip")) + }) +} diff --git a/xdp/tests/netlink_snapshot.rs b/xdp/tests/netlink_snapshot.rs new file mode 100644 index 00000000000..0aa1d6840da --- /dev/null +++ b/xdp/tests/netlink_snapshot.rs @@ -0,0 +1,73 @@ +#![cfg(target_os = "linux")] + +mod common; + +use { + agave_xdp::{ + netlink::{netlink_get_interfaces, netlink_get_neighbors, netlink_get_routes}, + route::RouteTable, + }, + libc::{AF_INET, NUD_PERMANENT}, + std::net::{IpAddr, Ipv4Addr}, +}; + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn netlink_snapshot_reads_the_prepared_namespace() { + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + + let routed_prefix = "203.0.113.0/24"; + common::replace_neighbor(links.right_ip, links.right_mac, common::LEFT_IFACE); + common::add_route(routed_prefix, links.right_ip, common::LEFT_IFACE); + + let interfaces = netlink_get_interfaces(AF_INET as u8).expect("read interfaces from netlink"); + assert!( + interfaces + .iter() + .any(|interface| interface.if_index == links.left_if_index) + ); + assert!( + interfaces + .iter() + .any(|interface| interface.if_index == links.right_if_index) + ); + + let routes = + netlink_get_routes(AF_INET as u8, u32::from(RouteTable::Main)).expect("read routes"); + assert!(routes.iter().any(|route| { + route.destination == Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 0))) + && route.gateway == Some(IpAddr::V4(links.right_ip)) + && route.out_if_index == Some(links.left_if_index as i32) + && route.dst_len == 24 + })); + + let neighbors = + netlink_get_neighbors(None, AF_INET as u8).expect("read neighbor table from netlink"); + assert!(neighbors.iter().any(|neighbor| { + neighbor.destination == Some(IpAddr::V4(links.right_ip)) + && neighbor.lladdr == Some(links.right_mac) + && neighbor.ifindex == links.left_if_index as i32 + && neighbor.state == NUD_PERMANENT + })); +} + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn netlink_snapshot_reads_gre_tunnel_metadata() { + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + let gre = common::setup_gre_tunnel(&links); + + let interfaces = netlink_get_interfaces(AF_INET as u8).expect("read interfaces from netlink"); + let tunnel = interfaces + .iter() + .find(|interface| interface.if_index == gre.if_index) + .and_then(|interface| interface.gre_tunnel.as_ref()) + .expect("read GRE tunnel metadata from netlink"); + + assert_eq!(tunnel.local, IpAddr::V4(gre.local_ip)); + assert_eq!(tunnel.remote, IpAddr::V4(gre.remote_ip)); + assert_eq!(tunnel.ttl, 64); + assert_eq!(tunnel.tos, 0); +} diff --git a/xdp/tests/route_monitor.rs b/xdp/tests/route_monitor.rs new file mode 100644 index 00000000000..fa6fb7a01d6 --- /dev/null +++ b/xdp/tests/route_monitor.rs @@ -0,0 +1,275 @@ +#![cfg(target_os = "linux")] + +mod common; + +use { + agave_xdp::{ + netlink::MacAddress, + route::{RouteError, RouteTable, Router}, + route_monitor::RouteMonitor, + }, + arc_swap::ArcSwap, + std::{ + net::{IpAddr, Ipv4Addr}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + thread::JoinHandle, + time::Duration, + }, +}; + +struct RouteMonitorGuard { + router: Arc>, + exit: Arc, + handle: Option>, +} + +impl Drop for RouteMonitorGuard { + fn drop(&mut self) { + self.exit.store(true, Ordering::Relaxed); + let Some(handle) = self.handle.take() else { + return; + }; + if let Err(err) = handle.join() { + if std::thread::panicking() { + eprintln!("route monitor thread panicked: {err:?}"); + } else { + std::panic::resume_unwind(err); + } + } + } +} + +fn start_route_monitor() -> RouteMonitorGuard { + let router = Router::new().expect("build initial router"); + let router = Arc::new(ArcSwap::from_pointee(router)); + let exit = Arc::new(AtomicBool::new(false)); + let handle = RouteMonitor::start( + Arc::clone(&router), + RouteTable::Main, + Arc::clone(&exit), + Duration::ZERO, + || {}, + ); + RouteMonitorGuard { + router, + exit, + handle: Some(handle), + } +} + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn route_monitor_publishes_live_route_updates() { + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + + let monitor = start_route_monitor(); + + let routed_destination = Ipv4Addr::new(203, 0, 113, 7); + assert!(matches!( + monitor.router.load().route_v4(routed_destination), + Err(RouteError::NoRouteFound(_)) + )); + + common::replace_neighbor(links.right_ip, links.right_mac, common::LEFT_IFACE); + common::add_route("203.0.113.0/24", links.right_ip, common::LEFT_IFACE); + + common::wait_until( + "the route monitor to publish a newly added route", + Duration::from_secs(2), + || { + let router = monitor.router.load(); + match router.route_v4(routed_destination) { + Ok(next_hop) + if next_hop.if_index == links.left_if_index + && next_hop.ip_addr == IpAddr::V4(links.right_ip) + && next_hop.mac_addr == Some(links.right_mac) => + { + Some(()) + } + _ => None, + } + }, + ); + + common::delete_route("203.0.113.0/24"); + common::wait_until( + "the route monitor to publish a removed route", + Duration::from_secs(2), + || { + matches!( + monitor.router.load().route_v4(routed_destination), + Err(RouteError::NoRouteFound(_)) + ) + .then_some(()) + }, + ); +} + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn route_monitor_publishes_live_neighbor_updates() { + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + + let monitor = start_route_monitor(); + let routed_destination = Ipv4Addr::new(203, 0, 113, 7); + + common::add_route("203.0.113.0/24", links.right_ip, common::LEFT_IFACE); + let initial_mac = links.right_mac; + common::replace_neighbor(links.right_ip, initial_mac, common::LEFT_IFACE); + + common::wait_until( + "the route monitor to publish the initial neighbor", + Duration::from_secs(2), + || { + let router = monitor.router.load(); + match router.route_v4(routed_destination) { + Ok(next_hop) + if next_hop.if_index == links.left_if_index + && next_hop.ip_addr == IpAddr::V4(links.right_ip) + && next_hop.mac_addr == Some(initial_mac) => + { + Some(()) + } + _ => None, + } + }, + ); + + let updated_mac = MacAddress([0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0x44]); + common::replace_neighbor(links.right_ip, updated_mac, common::LEFT_IFACE); + + common::wait_until( + "the route monitor to publish a replaced neighbor", + Duration::from_secs(2), + || { + let router = monitor.router.load(); + match router.route_v4(routed_destination) { + Ok(next_hop) if next_hop.mac_addr == Some(updated_mac) => Some(()), + _ => None, + } + }, + ); + + common::delete_neighbor(links.right_ip, common::LEFT_IFACE); + common::wait_until( + "the route monitor to publish a removed neighbor", + Duration::from_secs(2), + || { + let router = monitor.router.load(); + match router.route_v4(routed_destination) { + Ok(next_hop) + if next_hop.if_index == links.left_if_index + && next_hop.ip_addr == IpAddr::V4(links.right_ip) + && next_hop.mac_addr.is_none() => + { + Some(()) + } + _ => None, + } + }, + ); +} + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn route_monitor_publishes_link_removals() { + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + + common::replace_neighbor(links.right_ip, links.right_mac, common::LEFT_IFACE); + common::add_route("203.0.113.0/24", links.right_ip, common::LEFT_IFACE); + + let monitor = start_route_monitor(); + let routed_destination = Ipv4Addr::new(203, 0, 113, 7); + common::wait_until( + "the route monitor to publish the initial link-backed route", + Duration::from_secs(2), + || { + let router = monitor.router.load(); + match router.route_v4(routed_destination) { + Ok(next_hop) + if next_hop.if_index == links.left_if_index + && next_hop.ip_addr == IpAddr::V4(links.right_ip) => + { + Some(()) + } + _ => None, + } + }, + ); + + common::delete_link(common::LEFT_IFACE); + common::wait_until( + "the route monitor to publish a removed link", + Duration::from_secs(2), + || { + matches!( + monitor.router.load().route_v4(routed_destination), + Err(RouteError::NoRouteFound(_)) + ) + .then_some(()) + }, + ); +} + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn route_monitor_publishes_live_gre_route_updates() { + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + + let monitor = start_route_monitor(); + let overlay_destination = Ipv4Addr::new(192, 0, 2, 99); + assert!(matches!( + monitor.router.load().route_v4(overlay_destination), + Err(RouteError::NoRouteFound(_)) + )); + + common::replace_neighbor(links.right_ip, links.right_mac, common::LEFT_IFACE); + common::add_route_to_dev(&format!("{}/32", links.right_ip), common::LEFT_IFACE); + let gre = common::setup_gre_tunnel(&links); + common::add_route_to_dev_with_src("192.0.2.0/24", common::GRE_IFACE, gre.overlay_ip); + + common::wait_until( + "the route monitor to publish a GRE overlay route", + Duration::from_secs(2), + || { + let router = monitor.router.load(); + match router.route_v4(overlay_destination) { + Ok(next_hop) + if next_hop.if_index == gre.if_index + && next_hop.ip_addr == IpAddr::V4(overlay_destination) + && next_hop.mac_addr == Some(links.right_mac) + && next_hop.preferred_src_ip == Some(gre.overlay_ip) + && next_hop.gre.as_ref().is_some_and(|gre_route| { + gre_route.if_index == gre.if_index + && gre_route.mac_addr == links.right_mac + && gre_route.tunnel_info.local == IpAddr::V4(gre.local_ip) + && gre_route.tunnel_info.remote == IpAddr::V4(gre.remote_ip) + }) => + { + Some(()) + } + _ => None, + } + }, + ); + + common::delete_link(common::GRE_IFACE); + common::wait_until( + "the route monitor to publish a removed GRE link", + Duration::from_secs(2), + || { + matches!( + monitor.router.load().route_v4(overlay_destination), + Err(RouteError::NoRouteFound(_)) + ) + .then_some(()) + }, + ); +} diff --git a/xdp/tests/router_snapshot.rs b/xdp/tests/router_snapshot.rs new file mode 100644 index 00000000000..0392e0837f3 --- /dev/null +++ b/xdp/tests/router_snapshot.rs @@ -0,0 +1,44 @@ +#![cfg(target_os = "linux")] + +mod common; + +use { + agave_xdp::route::{RouteTable, Router, RoutingTables}, + std::net::{IpAddr, Ipv4Addr}, +}; + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn router_snapshot_resolves_gre_routes_from_netlink() { + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + + common::replace_neighbor(links.right_ip, links.right_mac, common::LEFT_IFACE); + common::add_route_to_dev(&format!("{}/32", links.right_ip), common::LEFT_IFACE); + let gre = common::setup_gre_tunnel(&links); + common::add_route_to_dev_with_src("192.0.2.0/24", common::GRE_IFACE, gre.overlay_ip); + + let router_from_tables = + Router::from_tables(RoutingTables::from_netlink(RouteTable::Main).expect("read tables")) + .expect("build router from snapshot tables"); + let router_from_netlink = Router::new().expect("build router directly from netlink"); + let overlay_destination = Ipv4Addr::new(192, 0, 2, 99); + + for router in [&router_from_tables, &router_from_netlink] { + let next_hop = router + .route_v4(overlay_destination) + .expect("resolve GRE overlay route"); + assert_eq!(next_hop.if_index, gre.if_index); + assert_eq!(next_hop.ip_addr, IpAddr::V4(overlay_destination)); + assert_eq!(next_hop.mac_addr, Some(links.right_mac)); + assert_eq!(next_hop.preferred_src_ip, Some(gre.overlay_ip)); + + let gre_route = next_hop.gre.as_ref().expect("route should use GRE"); + assert_eq!(gre_route.if_index, gre.if_index); + assert_eq!(gre_route.mac_addr, links.right_mac); + assert_eq!(gre_route.tunnel_info.local, IpAddr::V4(gre.local_ip)); + assert_eq!(gre_route.tunnel_info.remote, IpAddr::V4(gre.remote_ip)); + assert_eq!(gre_route.tunnel_info.ttl, 64); + assert_eq!(gre_route.tunnel_info.tos, 0); + } +} diff --git a/xdp/tests/transmitter_smoke.rs b/xdp/tests/transmitter_smoke.rs new file mode 100644 index 00000000000..df933c598b7 --- /dev/null +++ b/xdp/tests/transmitter_smoke.rs @@ -0,0 +1,447 @@ +#![cfg(target_os = "linux")] + +mod common; + +use { + agave_cpu_utils::cpu_affinity, + agave_xdp::{ + gre::packet::GRE_HEADER_BASE_SIZE, + netlink::MacAddress, + packet::{ETH_HEADER_SIZE, IP_HEADER_SIZE, UDP_HEADER_SIZE}, + transmitter::{ + BytesTxPacket, QueueCpuBinding, Transmitter, TransmitterBuilder, XdpConfig, XdpSender, + }, + }, + bytes::Bytes, + nix::{ + errno::Errno, + poll::{PollFd, PollFlags, PollTimeout, poll}, + sys::socket::{ + AddressFamily, MsgFlags, SockFlag, SockProtocol, SockType, SockaddrLike, + SockaddrStorage, bind, recv, socket, + }, + }, + std::{ + io, mem, + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + os::fd::{AsFd, AsRawFd, OwnedFd}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, + }, +}; + +fn transmitter_cpu() -> usize { + let cores = cpu_affinity(None).expect("linux provides affine cores"); + assert!( + cores.len() >= 2, + "transmitter smoke test requires at least 2 affine CPU cores, found {}", + cores.len(), + ); + **cores.first().expect("at least two affine cores") +} + +struct PacketSocket { + fd: OwnedFd, +} + +struct TransmitterGuard { + transmitter: Option, + sender: Option, + exit: Arc, +} + +impl TransmitterGuard { + fn new(transmitter: Transmitter, sender: XdpSender, exit: Arc) -> Self { + Self { + transmitter: Some(transmitter), + sender: Some(sender), + exit, + } + } + + fn sender(&self) -> &XdpSender { + self.sender.as_ref().expect("sender is live") + } +} + +impl Drop for TransmitterGuard { + fn drop(&mut self) { + self.exit.store(true, Ordering::Relaxed); + drop(self.sender.take()); + let Some(transmitter) = self.transmitter.take() else { + return; + }; + if let Err(err) = transmitter.join() { + if std::thread::panicking() { + eprintln!("transmitter thread panicked: {err:?}"); + } else { + std::panic::resume_unwind(err); + } + } + } +} + +impl PacketSocket { + fn bind(if_index: u32) -> io::Result { + let fd = socket( + AddressFamily::Packet, + SockType::Raw, + SockFlag::SOCK_CLOEXEC, + SockProtocol::EthAll, + ) + .map_err(io::Error::from)?; + let addr = libc::sockaddr_ll { + sll_family: libc::AF_PACKET as u16, + sll_protocol: (libc::ETH_P_ALL as u16).to_be(), + sll_ifindex: if_index as i32, + sll_hatype: 0, + sll_pkttype: 0, + sll_halen: 0, + sll_addr: [0; 8], + }; + let addr = unsafe { + SockaddrStorage::from_raw( + (&addr as *const libc::sockaddr_ll).cast(), + Some(mem::size_of::() as libc::socklen_t), + ) + } + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid packet address"))?; + bind(fd.as_raw_fd(), &addr).map_err(io::Error::from)?; + Ok(Self { fd }) + } + + fn recv_matching_udp( + &self, + expected: &ExpectedUdpPacket<'_>, + timeout: Duration, + ) -> io::Result> { + self.recv_matching_payload("matching UDP frame", timeout, |frame| { + matching_udp_payload(frame, expected) + }) + } + + fn recv_matching_gre_udp( + &self, + expected: &ExpectedGreUdpPacket<'_>, + timeout: Duration, + ) -> io::Result> { + self.recv_matching_payload("matching GRE UDP frame", timeout, |frame| { + matching_gre_udp_payload(frame, expected) + }) + } + + fn recv_matching_payload( + &self, + description: &str, + timeout: Duration, + mut matcher: F, + ) -> io::Result> + where + F: for<'a> FnMut(&'a [u8]) -> Option<&'a [u8]>, + { + let deadline = Instant::now().checked_add(timeout).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "timeout overflows instant") + })?; + let mut frame = [0u8; 2048]; + loop { + let now = Instant::now(); + if now >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!("timed out waiting for {description}"), + )); + } + let remaining = deadline.saturating_duration_since(now); + let mut pfd = [PollFd::new(self.fd.as_fd(), PollFlags::POLLIN)]; + let timeout = PollTimeout::try_from(remaining).unwrap_or(PollTimeout::MAX); + match poll(&mut pfd, timeout) { + Ok(0) => continue, + Ok(_) => {} + Err(Errno::EINTR) => continue, + Err(err) => return Err(io::Error::from(err)), + } + + let len = match recv(self.fd.as_raw_fd(), &mut frame, MsgFlags::empty()) { + Ok(len) => len, + Err(Errno::EINTR) => continue, + Err(err) => return Err(io::Error::from(err)), + }; + let frame = &frame[..len]; + if let Some(payload) = matcher(frame) { + return Ok(payload.to_vec()); + } + } + } +} + +struct ExpectedUdpPacket<'a> { + src_mac: MacAddress, + dst_mac: MacAddress, + src_ip: Ipv4Addr, + dst_ip: Ipv4Addr, + src_port: u16, + dst_port: u16, + payload: &'a [u8], +} + +struct ExpectedGreUdpPacket<'a> { + outer_src_mac: MacAddress, + outer_dst_mac: MacAddress, + outer_src_ip: Ipv4Addr, + outer_dst_ip: Ipv4Addr, + inner_src_ip: Ipv4Addr, + inner_dst_ip: Ipv4Addr, + src_port: u16, + dst_port: u16, + payload: &'a [u8], +} + +struct ExpectedUdpDatagram<'a> { + src_ip: Ipv4Addr, + dst_ip: Ipv4Addr, + src_port: u16, + dst_port: u16, + payload: &'a [u8], +} + +fn matching_udp_payload<'a>(frame: &'a [u8], expected: &ExpectedUdpPacket<'_>) -> Option<&'a [u8]> { + if frame.len() < ETH_HEADER_SIZE { + return None; + } + if frame[0..6] != expected.dst_mac.0 || frame[6..12] != expected.src_mac.0 { + return None; + } + if u16::from_be_bytes([frame[12], frame[13]]) != libc::ETH_P_IP as u16 { + return None; + } + + matching_ipv4_udp_payload( + &frame[ETH_HEADER_SIZE..], + &ExpectedUdpDatagram { + src_ip: expected.src_ip, + dst_ip: expected.dst_ip, + src_port: expected.src_port, + dst_port: expected.dst_port, + payload: expected.payload, + }, + ) +} + +fn matching_gre_udp_payload<'a>( + frame: &'a [u8], + expected: &ExpectedGreUdpPacket<'_>, +) -> Option<&'a [u8]> { + const GRE_FLAGS_VERSION_BASIC: u16 = 0x0000; + + if frame.len() < ETH_HEADER_SIZE.checked_add(IP_HEADER_SIZE)? { + return None; + } + if frame[0..6] != expected.outer_dst_mac.0 || frame[6..12] != expected.outer_src_mac.0 { + return None; + } + if u16::from_be_bytes([frame[12], frame[13]]) != libc::ETH_P_IP as u16 { + return None; + } + + let outer_ip = &frame[ETH_HEADER_SIZE..]; + let outer_ihl = usize::from(outer_ip[0] & 0x0f).checked_mul(4)?; + let gre_offset = ETH_HEADER_SIZE.checked_add(outer_ihl)?; + let min_frame_len = gre_offset + .checked_add(GRE_HEADER_BASE_SIZE)? + .checked_add(IP_HEADER_SIZE)?; + if outer_ihl < IP_HEADER_SIZE || frame.len() < min_frame_len { + return None; + } + if outer_ip[9] != libc::IPPROTO_GRE as u8 { + return None; + } + if outer_ip[12..16] != expected.outer_src_ip.octets() + || outer_ip[16..20] != expected.outer_dst_ip.octets() + { + return None; + } + + let gre = &frame[gre_offset..]; + if u16::from_be_bytes([gre[0], gre[1]]) != GRE_FLAGS_VERSION_BASIC { + return None; + } + if u16::from_be_bytes([gre[2], gre[3]]) != libc::ETH_P_IP as u16 { + return None; + } + + let inner_offset = gre_offset.checked_add(GRE_HEADER_BASE_SIZE)?; + matching_ipv4_udp_payload( + frame.get(inner_offset..)?, + &ExpectedUdpDatagram { + src_ip: expected.inner_src_ip, + dst_ip: expected.inner_dst_ip, + src_port: expected.src_port, + dst_port: expected.dst_port, + payload: expected.payload, + }, + ) +} + +fn matching_ipv4_udp_payload<'a>( + ip: &'a [u8], + expected: &ExpectedUdpDatagram<'_>, +) -> Option<&'a [u8]> { + let min_udp_len = IP_HEADER_SIZE.checked_add(UDP_HEADER_SIZE)?; + if ip.len() < min_udp_len { + return None; + } + + let ihl = usize::from(ip[0] & 0x0f).checked_mul(4)?; + let min_packet_len = ihl.checked_add(UDP_HEADER_SIZE)?; + if ihl < IP_HEADER_SIZE || ip.len() < min_packet_len { + return None; + } + if ip[9] != libc::IPPROTO_UDP as u8 { + return None; + } + if ip[12..16] != expected.src_ip.octets() || ip[16..20] != expected.dst_ip.octets() { + return None; + } + + let udp = &ip[ihl..]; + if u16::from_be_bytes([udp[0], udp[1]]) != expected.src_port + || u16::from_be_bytes([udp[2], udp[3]]) != expected.dst_port + { + return None; + } + let udp_len = usize::from(u16::from_be_bytes([udp[4], udp[5]])); + if udp_len < UDP_HEADER_SIZE || udp.len() < udp_len { + return None; + } + let payload = &udp[UDP_HEADER_SIZE..udp_len]; + (payload == expected.payload).then_some(payload) +} + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn transmitter_sends_udp_payload_over_veth_in_copy_mode() { + let cpu_id = transmitter_cpu(); + + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + common::replace_neighbor(links.right_ip, links.right_mac, common::LEFT_IFACE); + + let receiver = PacketSocket::bind(links.right_if_index).expect("bind raw packet receiver"); + let dst_port = 45_678; + let src_port = 12_345; + let destination = SocketAddr::V4(SocketAddrV4::new(links.right_ip, dst_port)); + let payload = Bytes::from_static(b"agave-xdp-transmitter-smoke"); + + let exit = Arc::new(AtomicBool::new(false)); + let mut config = XdpConfig::new( + Some(common::LEFT_IFACE.to_string()), + vec![QueueCpuBinding { + queue: 0, + cpu: cpu_id, + }], + false, + ); + config.tx_channel_cap = 16; + + let (transmitter, sender) = TransmitterBuilder::new(config, Arc::clone(&exit)) + .expect("build copy-mode transmitter") + .build(); + let transmitter = TransmitterGuard::new(transmitter, sender, exit); + + let packet = BytesTxPacket::new( + SocketAddrV4::new(links.left_ip, src_port), + destination, + None, + payload.clone(), + ); + transmitter + .sender() + .try_send(0, packet) + .expect("queue packet through XdpSender::try_send"); + + let received = receiver + .recv_matching_udp( + &ExpectedUdpPacket { + src_mac: links.left_mac, + dst_mac: links.right_mac, + src_ip: links.left_ip, + dst_ip: links.right_ip, + src_port, + dst_port, + payload: payload.as_ref(), + }, + Duration::from_secs(3), + ) + .expect("receive UDP frame from AF_XDP transmitter"); + assert_eq!(received, payload.as_ref()); +} + +#[test] +#[ignore = "requires root and network namespace privileges"] +fn transmitter_sends_udp_payload_over_gre_tunnel_in_copy_mode() { + let cpu_id = transmitter_cpu(); + + let _netns = common::NetNsGuard::new().expect("create network namespace"); + let links = common::setup_veth_pair(); + common::replace_neighbor(links.right_ip, links.right_mac, common::LEFT_IFACE); + common::add_route_to_dev(&format!("{}/32", links.right_ip), common::LEFT_IFACE); + let gre = common::setup_gre_tunnel(&links); + common::add_route_to_dev_with_src("192.0.2.0/24", common::GRE_IFACE, gre.overlay_ip); + + // Sending to the overlay destination exercises route lookup plus GRE encapsulation. + // The raw receiver observes the outer packet on the underlay veth peer. + let receiver = PacketSocket::bind(links.right_if_index).expect("bind raw packet receiver"); + let dst_port = 45_679; + let src_port = 12_346; + let overlay_destination = Ipv4Addr::new(192, 0, 2, 99); + let destination = SocketAddr::V4(SocketAddrV4::new(overlay_destination, dst_port)); + let payload = Bytes::from_static(b"agave-xdp-transmitter-gre-smoke"); + + let exit = Arc::new(AtomicBool::new(false)); + let mut config = XdpConfig::new( + Some(common::LEFT_IFACE.to_string()), + vec![QueueCpuBinding { + queue: 0, + cpu: cpu_id, + }], + false, + ); + config.tx_channel_cap = 16; + + let (transmitter, sender) = TransmitterBuilder::new(config, Arc::clone(&exit)) + .expect("build copy-mode transmitter") + .build(); + let transmitter = TransmitterGuard::new(transmitter, sender, exit); + + let packet = BytesTxPacket::new( + SocketAddrV4::new(links.left_ip, src_port), + destination, + None, + payload.clone(), + ); + transmitter + .sender() + .try_send(0, packet) + .expect("queue packet through XdpSender::try_send"); + + let received = receiver + .recv_matching_gre_udp( + &ExpectedGreUdpPacket { + outer_src_mac: links.left_mac, + outer_dst_mac: links.right_mac, + outer_src_ip: gre.local_ip, + outer_dst_ip: gre.remote_ip, + inner_src_ip: gre.overlay_ip, + inner_dst_ip: overlay_destination, + src_port, + dst_port, + payload: payload.as_ref(), + }, + Duration::from_secs(3), + ) + .expect("receive GRE-encapsulated UDP frame from AF_XDP transmitter"); + assert_eq!(received, payload.as_ref()); +}