diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index db3a66881..5986ce37d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -2,8 +2,6 @@ name: electrs on: push: - branches: - - master pull_request: schedule: - cron: "0 0 * * *" # once a day @@ -28,15 +26,16 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@stable + - name: Checkout bitcoin-capnp-types (experimental chain-interface branch) + run: | + git clone --depth 1 --branch 2026/04/chain-interface \ + https://github.com/Sjors/bitcoin-capnp-types.git ../capnp-types + - uses: actions-rust-lang/setup-rust-toolchain@v1 with: components: rustfmt, clippy - - name: Install Rust - run: rustup component add rustfmt clippy - - name: Install other dependencies - run: sudo apt install build-essential libclang-dev + run: sudo apt install build-essential libclang-dev capnproto libcapnp-dev - name: Format run: cargo fmt --all -- --check @@ -60,3 +59,77 @@ jobs: run: docker build -f Dockerfile.ci . --rm -t electrs:tests - name: Test run: docker run -v $PWD/contrib/:/contrib -v $PWD/tests/:/tests --rm electrs:tests bash /tests/run.sh + + integration-ipc: + name: Integration (IPC, experimental, PR 29409) + runs-on: ubuntu-latest + steps: + - name: Checkout electrs + uses: actions/checkout@v6 + + - name: Checkout bitcoin-capnp-types (experimental chain-interface branch) + run: | + git clone --depth 1 --branch 2026/04/chain-interface \ + https://github.com/Sjors/bitcoin-capnp-types.git ../capnp-types + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Install build & test dependencies + run: | + sudo apt-get update -qqy + sudo apt-get install -qqy \ + build-essential cmake pkgconf python3 \ + libevent-dev libboost-dev capnproto libcapnp-dev \ + librocksdb-dev libclang-dev ccache \ + python3-venv python3-pip git jq curl libsecp256k1-dev + + - name: ccache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-bitcoin-29409-${{ runner.os }}-${{ github.sha }} + restore-keys: ccache-bitcoin-29409-${{ runner.os }}- + + - name: Checkout Bitcoin Core PR #29409 + run: | + git clone --depth 1 https://github.com/bitcoin/bitcoin.git + cd bitcoin + git fetch --depth 1 origin pull/29409/head:pr-29409 + git checkout pr-29409 + + - name: Build Bitcoin Core (multiprocess) + run: | + cd bitcoin + cmake -B build \ + -DENABLE_WALLET=ON \ + -DBUILD_TESTS=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + cmake --build build -j $(nproc) --target bitcoin bitcoin-node bitcoin-cli + + - name: Install Electrum wallet (4.7.2) + run: | + git clone --depth 1 --branch 4.7.2 --recurse-submodules https://github.com/spesmilo/electrum/ + python3 -m venv --system-site-packages venv + ELECTRUM_ECC_DONT_COMPILE=1 venv/bin/pip install -e electrum/ + sudo ln -s "$PWD/venv/bin/electrum" /usr/local/bin/electrum + electrum version --offline + + - name: Build electrs + run: cargo build --locked --bin electrs + + - name: Run IPC integration test + env: + BITCOIN_MULTIPROCESS_BIN: ${{ github.workspace }}/bitcoin/build/bin/bitcoin + run: | + export PATH="$PWD/target/debug:$PATH" + ./tests/run-ipc.sh + + - name: Upload electrs debug log on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: electrs-ipc-debug-log + path: | + data/electrs/regtest-debug.log + data/electrum/regtest-debug.log diff --git a/Cargo.lock b/Cargo.lock index 7f3557643..5464c1076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,6 +95,14 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcoin-capnp-types" +version = "0.2.0" +dependencies = [ + "capnp", + "capnpc", +] + [[package]] name = "bitcoin-internals" version = "0.3.0" @@ -196,6 +204,12 @@ dependencies = [ "objc2", ] +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -206,6 +220,46 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "capnp" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63da65e5e9ffc3b8f993d4ad222a548152549351a643f6b850a7773cb6ff2809" +dependencies = [ + "embedded-io", +] + +[[package]] +name = "capnp-futures" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b69dfddccc57844f9a90f9d72b44b97c326914851ea94fb7da40ef9cad6e8d" +dependencies = [ + "capnp", + "futures-channel", + "futures-util", +] + +[[package]] +name = "capnp-rpc" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c74c337e87f75f3174ffb3513738ab0acc631c04f64cc33b867c60f84771da" +dependencies = [ + "capnp", + "capnp-futures", + "futures", +] + +[[package]] +name = "capnpc" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca02be865c8c5a78bfc24b9819006ab6b59bef238467203928e26459557af93" +dependencies = [ + "capnp", +] + [[package]] name = "cargo_toml" version = "0.14.1" @@ -410,15 +464,19 @@ version = "0.11.1" dependencies = [ "anyhow", "bitcoin", + "bitcoin-capnp-types", "bitcoin-test-data", "bitcoin_slices", "bitcoincore-rpc", + "capnp", + "capnp-rpc", "configure_me", "configure_me_codegen", "crossbeam-channel", "ctrlc", "dirs-next", "env_logger", + "futures", "hex_lit", "log", "parking_lot", @@ -431,8 +489,16 @@ dependencies = [ "signal-hook", "tempfile", "tiny_http", + "tokio", + "tokio-util", ] +[[package]] +name = "embedded-io" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb1aa714776b75c7e67e1da744b81a129b3ff919c8712b5e1b32252c1f07cc7" + [[package]] name = "env_logger" version = "0.10.2" @@ -474,6 +540,94 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -690,6 +844,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.61.0", +] + [[package]] name = "nix" version = "0.31.1" @@ -771,6 +936,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bddc33f680b79eaf1e2e56da792c3c2236f86985bbc3a886e8ddee17ae4d3a4" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.32" @@ -1131,12 +1302,28 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.0", +] + [[package]] name = "syn" version = "2.0.101" @@ -1222,6 +1409,46 @@ dependencies = [ "log", ] +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.0", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.5.11" diff --git a/Cargo.toml b/Cargo.toml index e18346bcc..5e6e03da4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,11 +25,15 @@ spec = "internal/config_specification.toml" anyhow = "1.0" bitcoin = { version = "0.32.8", features = ["serde", "rand-std"] } bitcoin_slices = { version = "0.11.0", features = ["bitcoin", "sha2"] } +bitcoin-capnp-types = { path = "../capnp-types" } bitcoincore-rpc = { version = "0.19.0" } +capnp = "0.25.0" +capnp-rpc = "0.25.0" configure_me = "0.4" crossbeam-channel = "0.5" dirs-next = "2.0" env_logger = "0.10" +futures = "0.3" log = "0.4" parking_lot = "0.12" prometheus = { version = "0.14", optional = true } @@ -38,6 +42,8 @@ serde = "1.0.184" serde_derive = "1.0.184" serde_json = "1.0" tiny_http = { version = "0.12", optional = true } +tokio = { version = "1", features = ["rt", "net", "macros", "io-util", "sync", "time"] } +tokio-util = { version = "0.7", features = ["compat"] } [target.'cfg(windows)'.dependencies] ctrlc = "=3.5.2" diff --git a/Dockerfile b/Dockerfile index 055398535..987aaede9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,13 @@ RUN apt-get install -qqy librocksdb-dev curl ### Electrum Rust Server ### FROM base AS electrs-build -RUN apt-get install -qqy cargo build-essential libclang-dev +RUN apt-get install -qqy cargo build-essential libclang-dev git capnproto libcapnp-dev -# Install electrs +# Install electrs (with sibling bitcoin-capnp-types checkout for the +# experimental chain-interface branch -- see Cargo.toml path dep). +WORKDIR /build/capnp-types +RUN git clone --depth 1 --branch 2026/04/chain-interface \ + https://github.com/Sjors/bitcoin-capnp-types.git . WORKDIR /build/electrs COPY . . ENV ROCKSDB_INCLUDE_DIR=/usr/include diff --git a/Dockerfile.ci b/Dockerfile.ci index 9106ac10b..6008f30ba 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -8,9 +8,13 @@ RUN apt-get install -qqy librocksdb-dev wget ### Electrum Rust Server ### FROM base AS electrs-build -RUN apt-get install -qqy cargo build-essential libclang-dev +RUN apt-get install -qqy cargo build-essential libclang-dev git capnproto libcapnp-dev -# Install electrs +# Install electrs (with sibling bitcoin-capnp-types checkout for the +# experimental chain-interface branch -- see Cargo.toml path dep). +WORKDIR /build/capnp-types +RUN git clone --depth 1 --branch 2026/04/chain-interface \ + https://github.com/Sjors/bitcoin-capnp-types.git . WORKDIR /build/electrs COPY . . ENV ROCKSDB_INCLUDE_DIR=/usr/include diff --git a/internal/config_specification.toml b/internal/config_specification.toml index f4ae65e36..cad145e7a 100644 --- a/internal/config_specification.toml +++ b/internal/config_specification.toml @@ -81,6 +81,12 @@ name = "daemon_p2p_addr" type = "crate::config::ResolvAddr" doc = "Bitcoin daemon p2p 'addr:port' to connect (default: 127.0.0.1:8333 for mainnet, 127.0.0.1:18333 for testnet, 127.0.0.1:18444 for regtest and 127.0.0.1:38333 for signet)" +[[param]] +name = "daemon_ipc_socket" +type = "std::path::PathBuf" +doc = "EXPERIMENTAL: path to a Bitcoin Core multiprocess (`bitcoin-node -ipcbind=unix`) socket. When set, electrs uses Cap'n Proto IPC (Chain interface from PR #29409) for block fetching and tip queries instead of the P2P protocol." +optional = true + [[param]] name = "monitoring_addr" type = "crate::config::ResolvAddr" diff --git a/src/config.rs b/src/config.rs index b844dda06..81ca7210f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -132,6 +132,7 @@ pub struct Config { pub daemon_auth: SensitiveAuth, pub daemon_rpc_addr: SocketAddr, pub daemon_p2p_addr: SocketAddr, + pub daemon_ipc_socket: Option, pub electrum_rpc_addr: SocketAddr, pub monitoring_addr: SocketAddr, pub wait_duration: Duration, @@ -345,6 +346,7 @@ impl Config { daemon_auth, daemon_rpc_addr, daemon_p2p_addr, + daemon_ipc_socket: config.daemon_ipc_socket, electrum_rpc_addr, monitoring_addr, wait_duration: Duration::from_secs(config.wait_duration_secs), diff --git a/src/daemon.rs b/src/daemon.rs index 40df169ba..9c7d313cf 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use bitcoin::consensus::encode::serialize_hex; use bitcoin::{consensus::deserialize, hashes::hex::FromHex}; -use bitcoin::{Amount, BlockHash, Transaction, Txid}; +use bitcoin::{Amount, BlockHash, OutPoint, Transaction, Txid}; use bitcoincore_rpc::{json, jsonrpc, Auth, Client, RpcApi}; use crossbeam_channel::Receiver; use parking_lot::Mutex; @@ -16,6 +16,8 @@ use std::path::Path; use crate::{ chain::{Chain, NewHeader}, config::Config, + ipc::{IpcChain, IpcCoin}, + mempool::MempoolEvent, metrics::Metrics, p2p::Connection, signals::ExitFlag, @@ -100,8 +102,18 @@ fn rpc_connect(config: &Config) -> Result { } pub struct Daemon { - p2p: Mutex, + /// P2P connection to bitcoind. Only constructed when IPC is *not* + /// configured; with IPC, header sync, block fetch, and new-block + /// notifications all go through the Chain interface and the P2P + /// connection is unnecessary. + p2p: Option>, rpc: Client, + ipc: Option, + /// When IPC is configured, a long-running ChainNotifications handler on + /// the IPC worker signals on this channel each time the tip changes. Used + /// in place of the P2P `inv`-watching path for `new_block_notification`. + ipc_block_notifier: Option>, + ipc_mempool_events: Option>, } impl Daemon { @@ -131,23 +143,81 @@ impl Daemon { if network_info.version < 21_00_00 { bail!("electrs requires bitcoind 0.21+"); } - if !network_info.network_active { - bail!("electrs requires active bitcoind p2p network"); - } - let info = rpc.get_blockchain_info()?; - if info.pruned { + + let ipc = match config.daemon_ipc_socket.as_deref() { + Some(path) => { + info!( + "connecting to bitcoin-node IPC socket: {} (experimental)", + path.display() + ); + Some(IpcChain::connect(path).context("IPC chain connection failed")?) + } + None => None, + }; + + // Skip the P2P connection entirely when IPC is available: header + // sync, block fetch, and new-block notifications all go through the + // Chain interface in that case. Without IPC we still need P2P, so + // also enforce that bitcoind has its P2P network enabled. + let p2p = if ipc.is_some() { + None + } else { + if !network_info.network_active { + bail!("electrs requires active bitcoind p2p network"); + } + Some(Mutex::new(Connection::connect( + config.daemon_p2p_addr, + metrics, + config.magic, + )?)) + }; + + // Check the node is non-pruned, preferring the IPC chain interface + // over the JSON-RPC `getblockchaininfo` call when it is available. + let pruned = match &ipc { + Some(ipc) => ipc.have_pruned().context("Chain.havePruned failed")?, + None => rpc.get_blockchain_info()?.pruned, + }; + if pruned { bail!("electrs requires non-pruned bitcoind node"); } - let p2p = Mutex::new(Connection::connect( - config.daemon_p2p_addr, - metrics, - config.magic, - )?); - Ok(Self { p2p, rpc }) + // Register one IPC ChainNotifications handler for both mempool events + // and tip changes. This avoids the P2P `inv`-watching path and lets + // the mempool tracker consume node-pushed updates instead of polling + // `getrawmempool`. + let (ipc_mempool_events, ipc_block_notifier) = match &ipc { + Some(ipc) => { + let (mempool_events, block_notifier) = ipc + .start_notifications() + .context("failed to start IPC chain notifications")?; + (Some(mempool_events), Some(block_notifier)) + } + None => (None, None), + }; + + Ok(Self { + p2p, + rpc, + ipc, + ipc_block_notifier, + ipc_mempool_events, + }) } pub(crate) fn estimate_fee(&self, nblocks: u16) -> Result> { + if let Some(ipc) = &self.ipc { + // Chain.estimateSmartFee returns CFeeRate{} (size==0) when no + // estimate is available, which mirrors the JSON-RPC -32603 + // ("Insufficient data or no feerate found") behaviour we map to + // None below. + let sat_per_kvb = ipc + .estimate_smart_fee_sat_per_kvb(nblocks.into()) + .context("failed to estimate fee via IPC")?; + return Ok(sat_per_kvb + .filter(|n| *n > 0) + .map(|n| Amount::from_sat(n as u64))); + } let res = self.rpc.estimate_smart_fee(nblocks, None); if let Err(bitcoincore_rpc::Error::JsonRpc(jsonrpc::Error::Rpc(RpcError { code: -32603, @@ -160,6 +230,16 @@ impl Daemon { } pub(crate) fn get_relay_fee(&self) -> Result { + if let Some(ipc) = &self.ipc { + let sat_per_kvb = ipc + .relay_min_fee_sat_per_kvb() + .context("failed to fetch relay min fee via IPC")?; + // sat_per_kvb is non-negative in practice (the node clamps the + // configured -minrelaytxfee at 0); guard against a corrupt or + // hostile reply by saturating at 0 rather than panicking on the + // i64 -> u64 conversion. + return Ok(Amount::from_sat(sat_per_kvb.max(0) as u64)); + } Ok(self .rpc .get_network_info() @@ -168,6 +248,12 @@ impl Daemon { } pub(crate) fn broadcast(&self, tx: &Transaction) -> Result { + if let Some(ipc) = &self.ipc { + let bytes = bitcoin::consensus::serialize(tx); + ipc.broadcast_transaction(bytes) + .context("failed to broadcast transaction via IPC")?; + return Ok(tx.compute_txid()); + } self.rpc .send_raw_transaction(tx) .context("failed to broadcast transaction") @@ -219,6 +305,15 @@ impl Daemon { } pub(crate) fn get_block_txids(&self, blockhash: BlockHash) -> Result> { + if let Some(ipc) = &self.ipc { + let bytes = ipc + .get_block(blockhash) + .with_context(|| format!("IPC findBlock failed for {blockhash}"))? + .ok_or_else(|| anyhow!("IPC findBlock: block {blockhash} not found"))?; + let block = crate::ipc::decode_block(&bytes) + .with_context(|| format!("failed to decode block {blockhash}"))?; + return Ok(block.txdata.iter().map(|tx| tx.compute_txid()).collect()); + } Ok(self .rpc .get_block_info(&blockhash) @@ -288,20 +383,64 @@ impl Daemon { .collect()) } + pub(crate) fn find_coins(&self, outpoints: Vec) -> Result> { + self.ipc + .as_ref() + .context("Chain.findCoins requested without IPC connection")? + .find_coins(outpoints) + .context("Chain.findCoins failed") + } + pub(crate) fn get_new_headers(&self, chain: &Chain) -> Result> { - self.p2p.lock().get_new_headers(chain) + if let Some(ipc) = &self.ipc { + return ipc + .get_new_headers(chain) + .context("IPC get_new_headers failed"); + } + self.p2p().lock().get_new_headers(chain) } - pub(crate) fn for_blocks(&self, blockhashes: B, func: F) -> Result<()> + pub(crate) fn for_blocks(&self, blockhashes: B, mut func: F) -> Result<()> where B: IntoIterator, F: FnMut(BlockHash, SerBlock), { - self.p2p.lock().for_blocks(blockhashes, func) + if let Some(ipc) = &self.ipc { + // Fetch each block over IPC instead of P2P. The Chain interface's + // `findBlock(wantData=true)` returns the same serialized form + // electrs's P2P path delivers, so the downstream pipeline is + // unchanged. + for hash in blockhashes { + let block = ipc + .get_block(hash) + .with_context(|| format!("IPC findBlock failed for {hash}"))? + .ok_or_else(|| anyhow!("IPC findBlock: block {hash} not found"))?; + func(hash, block); + } + Ok(()) + } else { + self.p2p().lock().for_blocks(blockhashes, func) + } } pub(crate) fn new_block_notification(&self) -> Receiver<()> { - self.p2p.lock().new_block_notification() + if let Some(rx) = &self.ipc_block_notifier { + return rx.clone(); + } + self.p2p().lock().new_block_notification() + } + + pub(crate) fn mempool_events(&self) -> Option<&Receiver> { + self.ipc_mempool_events.as_ref() + } + + /// Accessor for the P2P connection. Panics if called when IPC is + /// configured (which is a programmer error: the IPC fast paths above + /// must be checked first). + fn p2p(&self) -> &Mutex { + self.p2p + .as_ref() + .expect("P2P connection accessed but only IPC is configured") } } diff --git a/src/ipc.rs b/src/ipc.rs new file mode 100644 index 000000000..dd477f604 --- /dev/null +++ b/src/ipc.rs @@ -0,0 +1,983 @@ +//! Cap'n Proto IPC client for Bitcoin Core's `interfaces::Chain`. +//! +//! This is an experimental backend that talks to a multiprocess `bitcoin-node` +//! (Bitcoin Core PR #29409) over a unix socket using the libmultiprocess +//! Cap'n Proto framing. When configured, electrs uses this in place of the P2P +//! protocol for fetching raw blocks and tip information. +//! +//! The Cap'n Proto stack is asynchronous and not `Send` (the generated client +//! types contain `Rc>`). To bridge it to electrs's synchronous, +//! threaded code we run a dedicated worker thread that owns a current-thread +//! tokio runtime + `LocalSet`, and communicate via an mpsc channel of +//! type-erased async closures returning their results through a oneshot. +#![allow(dead_code)] + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::rc::Rc; +use std::sync::Arc; +use std::thread as stdthread; +use std::time::Duration; + +use anyhow::{anyhow, bail, Context, Result}; +use bitcoin::blockdata::block::Header as BlockHeader; +use bitcoin::{ + consensus::deserialize, consensus::Decodable, hashes::Hash, Amount, BlockHash, OutPoint, +}; +use bitcoin_capnp_types::{ + chain_capnp::{chain, chain_notifications}, + init_capnp::init, + proxy_capnp::{thread as proxy_thread, thread_map}, +}; +use capnp_rpc::{rpc_twoparty_capnp::Side, twoparty::VatNetwork, RpcSystem}; +use crossbeam_channel::{bounded, unbounded, Receiver, Sender}; +use futures::{future::LocalBoxFuture, io::BufReader, FutureExt}; +use parking_lot::Mutex; +use tokio::net::UnixStream; +use tokio::sync::{mpsc, oneshot}; +use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt}; + +use crate::chain::{Chain, NewHeader}; +use crate::mempool::MempoolEvent; +use crate::types::SerBlock; + +/// Boxed (non-Send) future returned by a job. The closure itself must be +/// `Send` so it can be shipped from caller threads to the IPC worker, but +/// once on the worker it runs on a single-threaded LocalSet, so the future +/// it produces does not need to be `Send`. Each job receives an owned `Ctx` +/// (a cheap pair of `Rc`-backed capnp clients), letting the worker spawn +/// jobs concurrently on the LocalSet without lifetime gymnastics. +type Job = Box LocalBoxFuture<'static, ()> + Send + 'static>; + +/// IPC context held by the worker thread: the chain client and the proxy +/// thread handle that every call must reference. Cloning is cheap (both +/// fields are `Rc`-backed capnp clients). +#[derive(Clone)] +struct Ctx { + chain: chain::Client, + thread: proxy_thread::Client, +} + +/// Sync handle to the IPC worker thread. Cloneable; clones share the same +/// connection and the same header-fetch block cache. +#[derive(Clone)] +pub(crate) struct IpcChain { + tx: mpsc::UnboundedSender, + /// Blocks that were downloaded by [`IpcChain::get_new_headers`] (which + /// has to fetch the full block payload over IPC just to extract the 80 + /// header bytes, since the Chain interface exposes no header-only + /// fetch). The very next thing electrs's indexing pipeline does is call + /// [`Daemon::for_blocks`] over the same hashes, so handing those + /// payloads back from the cache avoids a duplicate IPC round-trip per + /// block. Bounded by total bytes; on overflow, inserts are skipped + /// (best-effort) so the cache never grows without bound during initial + /// sync. + header_fetch_cache: Arc>, +} + +pub(crate) struct IpcCoin { + pub(crate) value: Amount, + pub(crate) height: u32, +} + +/// In-memory cache of recently fetched blocks, drained by +/// [`IpcChain::get_block`] on hit. Keyed by block hash, bounded by total +/// payload bytes. +#[derive(Default)] +struct BlockCache { + map: HashMap, + bytes: usize, +} + +/// Soft upper bound on the bytes held in the header-fetch cache. Sized to +/// comfortably hold one [`IpcChain::get_new_headers`] batch (capped at +/// `MAX_HEADERS` blocks) under typical mainnet block sizes (~2 MiB), with +/// headroom for a few outliers; oversized batches simply skip insertion +/// and refetch on demand. +const HEADER_FETCH_CACHE_MAX_BYTES: usize = 256 * 1024 * 1024; + +impl BlockCache { + /// Best-effort insert. Skips the block if the cache is already full or + /// if the block alone exceeds the budget; the caller's get_block path + /// will simply refetch on the (rare) miss that follows. + fn try_insert(&mut self, hash: BlockHash, block: SerBlock) { + let block_bytes = block.len(); + if block_bytes > HEADER_FETCH_CACHE_MAX_BYTES { + return; + } + if self.map.contains_key(&hash) { + return; + } + if self.bytes + block_bytes > HEADER_FETCH_CACHE_MAX_BYTES { + return; + } + self.bytes += block_bytes; + self.map.insert(hash, block); + } + + /// Remove and return a cached block, decreasing the byte count. + fn take(&mut self, hash: &BlockHash) -> Option { + let block = self.map.remove(hash)?; + self.bytes = self.bytes.saturating_sub(block.len()); + Some(block) + } +} + +impl IpcChain { + /// Spawn the worker thread, connect to the unix socket, and bootstrap an + /// `Init` client + `Chain` client. Blocks until the connection is ready. + pub(crate) fn connect(socket: &Path) -> Result { + let socket = socket.to_path_buf(); + let (tx, rx) = mpsc::unbounded_channel::(); + let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); + stdthread::Builder::new() + .name("electrs-ipc".into()) + .spawn(move || worker(socket, rx, ready_tx)) + .context("failed to spawn IPC worker thread")?; + ready_rx + .recv() + .context("IPC worker thread exited before signalling readiness")??; + Ok(Self { + tx, + header_fetch_cache: Arc::new(Mutex::new(BlockCache::default())), + }) + } + + /// Submit a job to the worker and block on its result. + fn call(&self, f: F) -> Result + where + R: Send + 'static, + F: FnOnce(Ctx) -> LocalBoxFuture<'static, Result> + Send + 'static, + { + let (otx, orx) = oneshot::channel::>(); + let job: Job = Box::new(move |ctx| { + async move { + let res = f(ctx).await; + let _ = otx.send(res); + } + .boxed_local() + }); + self.tx + .send(job) + .map_err(|_| anyhow!("IPC worker thread is gone"))?; + orx.blocking_recv() + .map_err(|_| anyhow!("IPC worker dropped the response"))? + } + + pub(crate) fn get_height(&self) -> Result> { + self.call(|ctx| { + async move { + let mut req = ctx.chain.get_height_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + let resp = req.send().promise.await?; + let r = resp.get()?; + Ok(if r.get_has_result() { + Some(r.get_result()) + } else { + None + }) + } + .boxed_local() + }) + } + + pub(crate) fn is_initial_block_download(&self) -> Result { + self.call(|ctx| { + async move { + let mut req = ctx.chain.is_initial_block_download_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + let resp = req.send().promise.await?; + Ok(resp.get()?.get_result()) + } + .boxed_local() + }) + } + + /// Register a `ChainNotifications` handler and request a one-shot replay of + /// the node's current mempool. The returned receivers are fed by the IPC + /// worker thread for as long as the chain notification handler is alive. + pub(crate) fn start_notifications(&self) -> Result<(Receiver, Receiver<()>)> { + let (mempool_tx, mempool_rx) = unbounded::(); + let (block_tx, block_rx) = bounded::<()>(1); + let (ready_tx, ready_rx) = oneshot::channel::>(); + let job: Job = Box::new(move |ctx| { + async move { + register_notifications(ctx, mempool_tx, block_tx, ready_tx).await; + } + .boxed_local() + }); + self.tx + .send(job) + .map_err(|_| anyhow!("IPC worker thread is gone"))?; + ready_rx + .blocking_recv() + .map_err(|_| anyhow!("IPC worker dropped notification setup response"))??; + Ok((mempool_rx, block_rx)) + } + + pub(crate) fn have_pruned(&self) -> Result { + self.call(|ctx| { + async move { + let mut req = ctx.chain.have_pruned_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + let resp = req.send().promise.await?; + Ok(resp.get()?.get_result()) + } + .boxed_local() + }) + } + + pub(crate) fn get_block_hash(&self, height: i32) -> Result { + self.call(move |ctx| { + async move { + let mut req = ctx.chain.get_block_hash_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + req.get().set_height(height); + let resp = req.send().promise.await?; + let bytes = resp.get()?.get_result()?; + if bytes.len() != 32 { + bail!("unexpected block hash length {}", bytes.len()); + } + let mut buf = [0u8; 32]; + buf.copy_from_slice(bytes); + Ok(BlockHash::from_byte_array(buf)) + } + .boxed_local() + }) + } + + /// Fetch a serialized block by hash via `Chain.findBlock(wantData=true)`. + /// Returns `Ok(None)` if the block is not known to the node. + /// + /// Consults the header-fetch cache first: blocks downloaded earlier by + /// [`Self::get_new_headers`] are handed back without an IPC round-trip, + /// and removed from the cache (consume-once semantics). + pub(crate) fn get_block(&self, hash: BlockHash) -> Result> { + if let Some(block) = self.header_fetch_cache.lock().take(&hash) { + return Ok(Some(block)); + } + self.fetch_block(hash) + } + + /// Unconditional `findBlock(wantData=true)` fetch with no cache + /// interaction; used internally by [`Self::get_block`] (on cache miss) + /// and by [`Self::get_new_headers`] (which inserts the result into the + /// cache for the immediately-following `for_blocks` pass). + fn fetch_block(&self, hash: BlockHash) -> Result> { + let raw: [u8; 32] = hash.to_byte_array(); + self.call(move |ctx| { + async move { + let mut req = ctx.chain.find_block_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + req.get().set_hash(&raw); + { + let mut params = req.get().init_block(); + params.set_want_data(true); + } + let resp = req.send().promise.await?; + let r = resp.get()?; + if !r.get_result() { + return Ok(None); + } + let block = r.get_block()?; + let data = block.get_data()?.to_vec(); + Ok(Some(data)) + } + .boxed_local() + }) + } + + /// Broadcast a serialized transaction via `Chain.broadcastTransaction`, + /// using the same `MEMPOOL_AND_BROADCAST_TO_ALL` semantics as the + /// JSON-RPC `sendrawtransaction` call. `max_tx_fee` matches the RPC + /// default (`DEFAULT_MAX_RAW_TX_FEE = 0.10 BTC`). On failure the node's + /// error string is propagated. + pub(crate) fn broadcast_transaction(&self, tx_bytes: Vec) -> Result<()> { + // node::TxBroadcast::MEMPOOL_AND_BROADCAST_TO_ALL = 0 + const BROADCAST_METHOD_DEFAULT: i32 = 0; + // src/policy/policy.h: DEFAULT_MAX_RAW_TX_FEE = 0.10 * COIN + const MAX_TX_FEE_DEFAULT: i64 = 10_000_000; + self.call(move |ctx| { + async move { + let mut req = ctx.chain.broadcast_transaction_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + { + let mut params = req.get(); + params.set_tx(&tx_bytes); + params.set_max_tx_fee(MAX_TX_FEE_DEFAULT); + params.set_broadcast_method(BROADCAST_METHOD_DEFAULT); + } + let resp = req.send().promise.await?; + let r = resp.get()?; + if r.get_result() { + Ok(()) + } else { + let err = r.get_error()?.to_str()?; + if err.is_empty() { + bail!("Chain.broadcastTransaction failed"); + } + bail!("Chain.broadcastTransaction failed: {err}"); + } + } + .boxed_local() + }) + } + + /// Get the node's minimum relay feerate via `Chain.relayMinFee`. The + /// reply is a serialized `CFeeRate` blob (see [`decode_fee_rate_kvb`]); + /// this method returns the rate as satoshis per kilo-vbyte. + /// + /// Replaces the JSON-RPC `getnetworkinfo.relayfee` lookup used by + /// [`Daemon::get_relay_fee`] when IPC is configured. + pub(crate) fn relay_min_fee_sat_per_kvb(&self) -> Result { + self.call(|ctx| { + async move { + let mut req = ctx.chain.relay_min_fee_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + let resp = req.send().promise.await?; + let blob = resp.get()?.get_result()?; + Ok(decode_fee_rate_kvb(blob)?.unwrap_or(0)) + } + .boxed_local() + }) + } + + /// Smart fee estimate for confirmation within `nblocks`, via + /// `Chain.estimateSmartFee`. Returns `Ok(None)` if the node has no + /// estimate available (a `CFeeRate{}` with `size == 0`), otherwise the + /// rate as satoshis per kilo-vbyte. + /// + /// Replaces the JSON-RPC `estimatesmartfee` call used by + /// [`Daemon::estimate_fee`] when IPC is configured. + pub(crate) fn estimate_smart_fee_sat_per_kvb(&self, nblocks: i32) -> Result> { + self.call(move |ctx| { + async move { + let mut req = ctx.chain.estimate_smart_fee_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + { + let mut params = req.get(); + params.set_num_blocks(nblocks); + // Match the JSON-RPC default for `estimatesmartfee`: + // economical (non-conservative) mode. + params.set_conservative(false); + // We don't surface FeeCalculation diagnostics to clients. + params.set_want_calc(false); + } + let resp = req.send().promise.await?; + let blob = resp.get()?.get_result()?; + decode_fee_rate_kvb(blob) + } + .boxed_local() + }) + } + + /// Look up prevout coins through `Chain.findCoins`. The interface takes a + /// `std::map&`, encoded as key/value byte pairs. The + /// incoming `Coin` values are placeholders; the node overwrites them with + /// coins from the active UTXO set or mempool. + pub(crate) fn find_coins(&self, outpoints: Vec) -> Result> { + self.call(move |ctx| { + async move { + let mut req = ctx.chain.find_coins_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + { + let mut coins = req.get().init_coins(outpoints.len() as u32); + for (i, outpoint) in outpoints.iter().enumerate() { + let mut pair = coins.reborrow().get(i as u32); + pair.set_key(&serialize_outpoint(outpoint)[..])?; + pair.set_value(DUMMY_COIN)?; + } + } + let resp = req.send().promise.await?; + let coins = resp.get()?.get_coins()?; + if coins.len() != outpoints.len() as u32 { + bail!( + "Chain.findCoins returned {} coins, expected {}", + coins.len(), + outpoints.len() + ); + } + + let mut result = Vec::with_capacity(outpoints.len()); + for i in 0..coins.len() { + let pair = coins.get(i); + result.push(decode_coin(pair.get_value()?)?); + } + Ok(result) + } + .boxed_local() + }) + } + + /// Find the height of the highest block in `locator` that is part of the + /// node's active chain, via `Chain.findLocatorFork`. Returns `None` if no + /// hash in the locator is on the active chain (e.g. completely diverged + /// chain), in which case the caller should fall back to walking from + /// genesis. + /// + /// `locator` is the list of block hashes returned by [`Chain::locator`], + /// ordered tip→genesis with exponentially increasing gaps. + pub(crate) fn find_locator_fork(&self, locator: &[BlockHash]) -> Result> { + // Bitcoin Core CBlockLocator wire format: int32 LE version + // (DUMMY_VERSION = 70016) + CompactSize count + 32*count hashes. + let mut buf = Vec::with_capacity(4 + 9 + locator.len() * 32); + buf.extend_from_slice(&70016i32.to_le_bytes()); + write_compact_size(&mut buf, locator.len() as u64); + for hash in locator { + buf.extend_from_slice(hash.as_byte_array()); + } + self.call(move |ctx| { + async move { + let mut req = ctx.chain.find_locator_fork_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + req.get().set_locator(&buf); + let resp = req.send().promise.await?; + let r = resp.get()?; + Ok(if r.get_has_result() { + Some(r.get_result()) + } else { + None + }) + } + .boxed_local() + }) + } + + /// Replacement for the P2P `getheaders` exchange in [`Daemon::get_new_headers`]. + /// + /// Walks the IPC chain forward from the fork point between the locator + /// (taken from `chain`) and the node's active chain, returning every new + /// header up to the node's current tip. Each header is fetched by way of + /// `findBlock(wantData=true)` followed by parsing the first 80 bytes as a + /// `BlockHeader`; the chain interface does not expose a header-only + /// fetch, so the full block payload is downloaded. To avoid an immediate + /// duplicate fetch, the payload is stashed in the header-fetch cache and + /// handed back to the subsequent `for_blocks` indexing pass. + /// + /// Caps the response at [`MAX_HEADERS`] per call so the caller can drive + /// multiple iterations during initial sync. The cap is small (vs. + /// the 2000-header P2P `getheaders` batch) for two reasons: (1) each + /// header costs a full block over IPC until upstream grows a header-only + /// fetch, so larger batches blow past the in-process block cache budget + /// on mainnet; (2) over a local unix socket, round-trip cost is dominated + /// by per-block work, not per-batch overhead, so the historical P2P + /// argument for huge batches doesn't carry over. + pub(crate) fn get_new_headers(&self, chain: &Chain) -> Result> { + /// Headers per `get_new_headers` call. See the function-level + /// comment above for why this is much smaller than the P2P value. + const MAX_HEADERS: usize = 100; + + let tip_height = match self.get_height()? { + Some(h) if h >= 0 => h as usize, + _ => return Ok(vec![]), + }; + + // Find the fork height between our chain's locator and the node's + // active chain. + let locator = chain.locator(); + let fork_height = self + .find_locator_fork(&locator)? + .ok_or_else(|| anyhow!("Chain.findLocatorFork: no common ancestor"))?; + if fork_height < 0 { + bail!("Chain.findLocatorFork returned negative fork height {fork_height}"); + } + let fork_height = fork_height as usize; + + if tip_height <= fork_height { + return Ok(vec![]); + } + + let new_first = fork_height + 1; + let new_last = tip_height.min(fork_height + MAX_HEADERS); + let mut headers = Vec::with_capacity(new_last - new_first + 1); + for h in new_first..=new_last { + let hash = self.get_block_hash(h as i32)?; + let block = self + .fetch_block(hash)? + .ok_or_else(|| anyhow!("findBlock: missing block {hash} at height {h}"))?; + if block.len() < 80 { + bail!( + "findBlock returned {} bytes for {hash}; need >=80", + block.len() + ); + } + let header = BlockHeader::consensus_decode(&mut &block[..80]) + .with_context(|| format!("parsing header at height {h}"))?; + headers.push(NewHeader::from((header, h))); + // Stash the payload for the for_blocks pass that will follow + // immediately. Best-effort: when the cache is full, additional + // blocks will simply be re-fetched on demand. + self.header_fetch_cache.lock().try_insert(hash, block); + } + Ok(headers) + } + + /// Spawn a long-running task on the worker that watches for tip changes + /// via `Chain.waitForNotificationsIfTipChanged`, and returns a receiver + /// that fires once per detected tip change. + /// + /// This replaces the P2P `inv`-watching path in `src/p2p.rs` for IPC + /// deployments. The receiver is bounded to capacity 1 with `try_send` + /// semantics: if the consumer is busy, additional notifications are + /// coalesced into the pending one. + pub(crate) fn start_block_notifier(&self) -> Result> { + let (notif_tx, notif_rx) = bounded::<()>(1); + let job: Job = Box::new(move |ctx| { + async move { + loop { + // Snapshot the current tip hash so we can ask the node + // to wake us when it changes. + let tip_hash = match snapshot_tip_hash(&ctx).await { + Ok(h) => h, + Err(e) => { + warn!("IPC notifier: tip snapshot failed: {e:#}"); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + }; + + // Block until the tip differs from `tip_hash`. Returns + // immediately if it already does. + let mut req = ctx.chain.wait_for_notifications_if_tip_changed_request(); + match req.get().get_context() { + Ok(mut c) => c.set_thread(ctx.thread.clone()), + Err(e) => { + warn!("IPC notifier: failed to set thread: {e:#}"); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + } + req.get().set_old_tip(&tip_hash); + if let Err(e) = req.send().promise.await { + warn!("IPC notifier: waitForNotifications failed: {e:#}"); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + + // Coalesce: if the receiver is already pending, drop this + // signal (the indexer will see all changes on its next + // sync iteration regardless). + let _ = notif_tx.try_send(()); + } + } + .boxed_local() + }); + self.tx + .send(job) + .map_err(|_| anyhow!("IPC worker thread is gone"))?; + Ok(notif_rx) + } +} + +struct ChainNotificationHandler { + mempool_tx: Sender, + block_tx: Sender<()>, +} + +impl ChainNotificationHandler { + fn send_mempool_event(&self, event: MempoolEvent) { + if let Err(e) = self.mempool_tx.try_send(event) { + warn!("IPC notification: dropping mempool event: {e}"); + } + } + + fn send_block_event(&self) { + let _ = self.block_tx.try_send(()); + } +} + +impl chain_notifications::Server for ChainNotificationHandler { + fn destroy( + self: Rc, + _: chain_notifications::DestroyParams, + _: chain_notifications::DestroyResults, + ) -> impl std::future::Future> + 'static { + std::future::ready(Ok(())) + } + + async fn transaction_added_to_mempool( + self: Rc, + params: chain_notifications::TransactionAddedToMempoolParams, + _: chain_notifications::TransactionAddedToMempoolResults, + ) -> std::result::Result<(), capnp::Error> { + let p = params.get()?; + match deserialize(p.get_tx()?) { + Ok(tx) => self.send_mempool_event(MempoolEvent::Added(tx)), + Err(e) => warn!("IPC notification: invalid mempool transaction: {e}"), + } + Ok(()) + } + + async fn transaction_removed_from_mempool( + self: Rc, + params: chain_notifications::TransactionRemovedFromMempoolParams, + _: chain_notifications::TransactionRemovedFromMempoolResults, + ) -> std::result::Result<(), capnp::Error> { + let p = params.get()?; + match deserialize::(p.get_tx()?) { + Ok(tx) => self.send_mempool_event(MempoolEvent::Removed(tx.compute_txid())), + Err(e) => warn!("IPC notification: invalid removed mempool transaction: {e}"), + } + Ok(()) + } + + async fn block_connected( + self: Rc, + params: chain_notifications::BlockConnectedParams, + _: chain_notifications::BlockConnectedResults, + ) -> std::result::Result<(), capnp::Error> { + let p = params.get()?; + let data = p.get_block()?.get_data()?; + if data.is_empty() { + warn!("IPC notification: connected block without block data"); + return Ok(()); + } + match deserialize::(data) { + Ok(block) => { + for tx in block.txdata { + self.send_mempool_event(MempoolEvent::Removed(tx.compute_txid())); + } + } + Err(e) => warn!("IPC notification: invalid connected block: {e}"), + } + self.send_block_event(); + Ok(()) + } + + fn block_disconnected( + self: Rc, + _: chain_notifications::BlockDisconnectedParams, + _: chain_notifications::BlockDisconnectedResults, + ) -> impl std::future::Future> + 'static { + std::future::ready(Ok(())) + } + + fn updated_block_tip( + self: Rc, + _: chain_notifications::UpdatedBlockTipParams, + _: chain_notifications::UpdatedBlockTipResults, + ) -> impl std::future::Future> + 'static { + self.send_block_event(); + std::future::ready(Ok(())) + } + + fn chain_state_flushed( + self: Rc, + _: chain_notifications::ChainStateFlushedParams, + _: chain_notifications::ChainStateFlushedResults, + ) -> impl std::future::Future> + 'static { + std::future::ready(Ok(())) + } +} + +async fn register_notifications( + ctx: Ctx, + mempool_tx: Sender, + block_tx: Sender<()>, + ready_tx: oneshot::Sender>, +) { + let setup = async { + let handler = Rc::new(ChainNotificationHandler { + mempool_tx, + block_tx, + }); + let notifications: chain_notifications::Client = capnp_rpc::new_client_from_rc(handler); + + let mut req = ctx.chain.handle_notifications_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + req.get().set_notifications(notifications.clone()); + let resp = req.send().promise.await?; + let _handler = resp.get()?.get_result()?; + + let mut req = ctx.chain.request_mempool_transactions_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + req.get().set_notifications(notifications); + req.send().promise.await?; + + Ok::<_, anyhow::Error>(_handler) + } + .await; + + match setup { + Ok(_handler) => { + let _ = ready_tx.send(Ok(())); + futures::future::pending::<()>().await; + } + Err(e) => { + let _ = ready_tx.send(Err(e)); + } + } +} + +/// Helper for the notifier loop: read the current tip hash via +/// `getHeight` + `getBlockHash`. Returns the all-zeros hash when the node +/// reports no tip yet (genesis only), which still works for +/// `waitForNotificationsIfTipChanged` (it'll wake us as soon as a real +/// tip exists). +async fn snapshot_tip_hash(ctx: &Ctx) -> Result<[u8; 32]> { + let mut req = ctx.chain.get_height_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + let resp = req.send().promise.await?; + let r = resp.get()?; + if !r.get_has_result() { + return Ok([0u8; 32]); + } + let height = r.get_result(); + let mut req = ctx.chain.get_block_hash_request(); + req.get().get_context()?.set_thread(ctx.thread.clone()); + req.get().set_height(height); + let resp = req.send().promise.await?; + let bytes = resp.get()?.get_result()?; + if bytes.len() != 32 { + bail!("unexpected block hash length {}", bytes.len()); + } + let mut buf = [0u8; 32]; + buf.copy_from_slice(bytes); + Ok(buf) +} + +/// Decode a serialized Bitcoin Core `CFeeRate` blob into satoshis per +/// kilo-vbyte (matching `CFeeRate::GetFeePerK()`). Wire format is +/// `FeeFrac { int64_t fee; int32_t size; }` (12 bytes LE), per +/// `SERIALIZE_METHODS(CFeeRate, ...)`. Returns `Ok(None)` for the empty +/// `CFeeRate{}` case (`size == 0`), which Bitcoin Core uses to mean "no +/// estimate available". +fn decode_fee_rate_kvb(blob: &[u8]) -> Result> { + if blob.len() != 12 { + bail!( + "expected serialized CFeeRate to be 12 bytes (int64 fee + int32 size), got {}", + blob.len() + ); + } + let fee = i64::from_le_bytes(blob[0..8].try_into().expect("len 8")); + let size = i32::from_le_bytes(blob[8..12].try_into().expect("len 4")); + if size == 0 { + return Ok(None); + } + Ok(Some(fee.saturating_mul(1000) / size as i64)) +} + +/// Serialized `Coin{CTxOut{0, empty script}, height=0, coinbase=false}`. +/// `findCoins` treats request coin values as placeholders, but the map +/// decoder still needs each value to deserialize as a valid `Coin`. +const DUMMY_COIN: &[u8] = &[0x00, 0x00, 0x06]; + +fn serialize_outpoint(outpoint: &OutPoint) -> [u8; 36] { + let mut buf = [0u8; 36]; + buf[..32].copy_from_slice(outpoint.txid.as_byte_array()); + buf[32..].copy_from_slice(&outpoint.vout.to_le_bytes()); + buf +} + +fn decode_coin(blob: &[u8]) -> Result { + let mut pos = 0; + let code = read_core_varint(blob, &mut pos)?; + let compressed_amount = read_core_varint(blob, &mut pos)?; + let script_size = read_core_varint(blob, &mut pos)?; + let script_bytes = match script_size { + 0 | 1 => 20, + 2..=5 => 32, + n => n + .checked_sub(6) + .ok_or_else(|| anyhow!("invalid compressed script size code {n}"))?, + } as usize; + if blob.len().saturating_sub(pos) != script_bytes { + bail!( + "serialized Coin script has {} bytes, expected {}", + blob.len().saturating_sub(pos), + script_bytes + ); + } + Ok(IpcCoin { + value: Amount::from_sat(decompress_amount(compressed_amount)), + height: (code >> 1) + .try_into() + .context("serialized Coin height overflow")?, + }) +} + +/// Bitcoin Core VARINT encoding (base-128 with subtract-one continuation). +fn read_core_varint(blob: &[u8], pos: &mut usize) -> Result { + let mut n = 0u64; + loop { + let Some(&ch) = blob.get(*pos) else { + bail!("unexpected end of Bitcoin Core VARINT"); + }; + *pos += 1; + n = n + .checked_shl(7) + .and_then(|n| n.checked_add((ch & 0x7f) as u64)) + .ok_or_else(|| anyhow!("Bitcoin Core VARINT overflow"))?; + if ch & 0x80 == 0 { + return Ok(n); + } + n = n + .checked_add(1) + .ok_or_else(|| anyhow!("Bitcoin Core VARINT overflow"))?; + } +} + +/// Inverse of Bitcoin Core's `CompressAmount`. +fn decompress_amount(mut x: u64) -> u64 { + if x == 0 { + return 0; + } + x -= 1; + let e = x % 10; + x /= 10; + let mut n = if e < 9 { + let d = (x % 9) + 1; + x /= 9; + x * 10 + d + } else { + x + 1 + }; + for _ in 0..e { + n *= 10; + } + n +} + +/// Bitcoin Core CompactSize encoding (src/serialize.h). +fn write_compact_size(buf: &mut Vec, n: u64) { + if n < 253 { + buf.push(n as u8); + } else if n <= u16::MAX as u64 { + buf.push(253); + buf.extend_from_slice(&(n as u16).to_le_bytes()); + } else if n <= u32::MAX as u64 { + buf.push(254); + buf.extend_from_slice(&(n as u32).to_le_bytes()); + } else { + buf.push(255); + buf.extend_from_slice(&n.to_le_bytes()); + } +} + +/// Worker thread entry point. Owns the tokio runtime, the capnp RPC system, +/// and the dispatch loop. Never returns under normal operation. +fn worker( + socket: PathBuf, + mut rx: mpsc::UnboundedReceiver, + ready: std::sync::mpsc::Sender>, +) { + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + let _ = ready.send(Err(anyhow!("tokio runtime build: {e}"))); + return; + } + }; + let local = tokio::task::LocalSet::new(); + rt.block_on(local.run_until(async move { + let ctx = match bootstrap(&socket).await { + Ok(ctx) => ctx, + Err(e) => { + let _ = ready.send(Err(e)); + return; + } + }; + let _ = ready.send(Ok(())); + while let Some(job) = rx.recv().await { + // Spawn each job onto the LocalSet so long-running jobs (e.g. the + // block-tip notifier loop) do not block other in-flight calls. + tokio::task::spawn_local(job(ctx.clone())); + } + })); +} + +async fn bootstrap(socket: &Path) -> Result { + let stream = UnixStream::connect(socket) + .await + .with_context(|| format!("connecting to bitcoin-node IPC socket {}", socket.display()))?; + let (reader, writer) = stream.into_split(); + let buf_reader = BufReader::new(reader.compat()); + let buf_writer = futures::io::BufWriter::new(writer.compat_write()); + let network = VatNetwork::new(buf_reader, buf_writer, Side::Client, Default::default()); + let mut rpc_system = RpcSystem::new(Box::new(network), None); + let init: init::Client = rpc_system.bootstrap(Side::Server); + tokio::task::spawn_local(rpc_system); + + // Construct + obtain a thread handle. + let resp = init + .construct_request() + .send() + .promise + .await + .context("Init.construct failed")?; + let thread_map: thread_map::Client = resp + .get() + .context("Init.construct response")? + .get_thread_map() + .context("get_thread_map")?; + let resp = thread_map + .make_thread_request() + .send() + .promise + .await + .context("ThreadMap.makeThread failed")?; + let thread: proxy_thread::Client = resp + .get() + .context("makeThread response")? + .get_result() + .context("makeThread result")?; + + // Make a Chain client. + let mut req = init.make_chain_request(); + req.get() + .get_context() + .context("makeChain context")? + .set_thread(thread.clone()); + let resp = req.send().promise.await.context("Init.makeChain failed")?; + let chain: chain::Client = resp + .get() + .context("makeChain response")? + .get_result() + .context("makeChain result")?; + + Ok(Ctx { chain, thread }) +} + +/// Decode a serialized block. Helper that keeps callers free of the +/// `bitcoin::consensus` import. +pub(crate) fn decode_block(buf: &[u8]) -> Result { + let mut cursor = std::io::Cursor::new(buf); + bitcoin::Block::consensus_decode(&mut cursor) + .map_err(|e| anyhow!("failed to decode block: {e}")) +} + +#[cfg(test)] +mod tests { + use super::{decompress_amount, read_core_varint}; + + #[test] + fn decodes_core_varint_examples() { + for (bytes, expected) in [ + (&[0x00][..], 0), + (&[0x7f][..], 0x7f), + (&[0x80, 0x00][..], 0x80), + (&[0xa3, 0x34][..], 0x1234), + (&[0x82, 0xfe, 0x7f][..], 0xffff), + ] { + let mut pos = 0; + assert_eq!(read_core_varint(bytes, &mut pos).unwrap(), expected); + assert_eq!(pos, bytes.len()); + } + } + + #[test] + fn decompresses_bitcoin_amounts() { + for (compressed, sats) in [ + (0, 0), + (1, 1), + (7, 1000000), + (9, 100000000), + (50, 5000000000), + ] { + assert_eq!(decompress_amount(compressed), sats); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 1458f06ff..4dc159bf7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ mod daemon; mod db; mod electrum; mod index; +mod ipc; mod mempool; mod merkle; mod metrics; diff --git a/src/mempool.rs b/src/mempool.rs index 04457c95c..7e8d7c4ea 100644 --- a/src/mempool.rs +++ b/src/mempool.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::convert::TryFrom; @@ -7,6 +7,7 @@ use std::ops::Bound; use bitcoin::hashes::Hash; use bitcoin::{Amount, OutPoint, Transaction, Txid}; +use crossbeam_channel::Receiver; use serde::ser::{Serialize, SerializeSeq, Serializer}; use crate::{ @@ -24,6 +25,14 @@ pub(crate) struct Entry { pub has_unconfirmed_inputs: bool, } +pub(crate) enum MempoolEvent { + Added(Transaction), + Removed(Txid), +} + +/// Bitcoin Core marks mempool prevout coins with this synthetic height. +const MEMPOOL_HEIGHT: u32 = 0x7fffffff; + /// Mempool current state pub(crate) struct Mempool { entries: HashMap, @@ -183,10 +192,15 @@ impl Mempool { let added = update.new_entries.len(); for txid_to_remove in update.removed_entries { - self.remove_entry(txid_to_remove); + if self.entries.contains_key(&txid_to_remove) { + self.remove_entry(txid_to_remove); + } } for entry in update.new_entries { + if self.entries.contains_key(&entry.txid) { + self.remove_entry(entry.txid); + } self.add_entry(entry); } @@ -210,7 +224,48 @@ impl Mempool { } } + fn sync_notifications(&mut self, daemon: &Daemon, events: &Receiver) { + let mut added_txs = HashMap::::new(); + let mut removed_entries = HashSet::::new(); + + for event in events.try_iter() { + match event { + MempoolEvent::Added(tx) => { + let txid = tx.compute_txid(); + removed_entries.remove(&txid); + added_txs.insert(txid, tx); + } + MempoolEvent::Removed(txid) => { + added_txs.remove(&txid); + removed_entries.insert(txid); + } + } + } + + let mut new_entries = Vec::with_capacity(added_txs.len()); + for (txid, tx) in added_txs { + match entry_from_notification_tx(daemon, txid, tx) { + Ok(entry) => new_entries.push(entry), + Err(e) => warn!("mempool notification metadata sync failed for {txid}: {e}"), + } + } + + if new_entries.is_empty() && removed_entries.is_empty() { + return; + } + + self.apply_sync_update(MempoolSyncUpdate { + new_entries, + removed_entries, + }); + } + pub fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) { + if let Some(events) = daemon.mempool_events() { + self.sync_notifications(daemon, events); + return; + } + let loaded = match daemon.get_mempool_info() { Ok(info) => info.loaded.unwrap_or(true), Err(e) => { @@ -285,6 +340,48 @@ impl Mempool { } } +fn entry_from_notification_tx(daemon: &Daemon, txid: Txid, tx: Transaction) -> Result { + let prevouts: Vec = tx.input.iter().map(|txin| txin.previous_output).collect(); + let coins = daemon + .find_coins(prevouts) + .with_context(|| format!("looking up prevout coins for {txid}"))?; + ensure!( + coins.len() == tx.input.len(), + "got {} prevout coins, expected {}", + coins.len(), + tx.input.len() + ); + + let mut input_value = Amount::ZERO; + let mut has_unconfirmed_inputs = false; + for coin in coins { + input_value = input_value + .checked_add(coin.value) + .ok_or_else(|| anyhow!("input value overflow for {txid}"))?; + has_unconfirmed_inputs |= coin.height == MEMPOOL_HEIGHT; + } + + let mut output_value = Amount::ZERO; + for txout in &tx.output { + output_value = output_value + .checked_add(txout.value) + .ok_or_else(|| anyhow!("output value overflow for {txid}"))?; + } + + let fee = input_value + .checked_sub(output_value) + .ok_or_else(|| anyhow!("negative fee for {txid}"))?; + let vsize = tx.vsize() as u64; + + Ok(Entry { + txid, + tx, + fee, + vsize, + has_unconfirmed_inputs, + }) +} + pub(crate) struct FeeHistogram { /// bins[64-i] contains transactions' statistics inside the fee band of [2**(i-1), 2**i). /// bins[64] = [0, 1) diff --git a/tests/run-ipc.sh b/tests/run-ipc.sh new file mode 100755 index 000000000..89a7b17f1 --- /dev/null +++ b/tests/run-ipc.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# Integration test variant that exercises the experimental Cap'n Proto IPC +# backend introduced for Bitcoin Core PR #29409. Requires a multiprocess +# `bitcoin` multi-call binary (resolved via $BITCOIN_MULTIPROCESS_BIN or +# `bitcoin` on PATH) and a freshly built electrs. +set -euo pipefail + +BITCOIN_MULTIPROCESS_BIN="${BITCOIN_MULTIPROCESS_BIN:-$(command -v bitcoin || true)}" + +if [ -z "$BITCOIN_MULTIPROCESS_BIN" ] || [ ! -x "$BITCOIN_MULTIPROCESS_BIN" ]; then + echo "ERROR: multiprocess 'bitcoin' multi-call binary not found." >&2 + echo "Set BITCOIN_MULTIPROCESS_BIN to the binary built from bitcoin/bitcoin#29409," >&2 + echo "or place it on PATH as 'bitcoin'." >&2 + exit 1 +fi + +rm -rf data/ +mkdir -p data/{bitcoin,electrum,electrs} +touch data/electrs/regtest-debug.log data/electrum/regtest-debug.log + +cleanup() { + trap - SIGTERM SIGINT + set +e +o pipefail + jobs + for j in `jobs -rp` + do + kill $j + wait $j + done +} +trap cleanup SIGINT SIGTERM EXIT + +wait_for() { + CMD=$1 + shift + TEST_ARGS=$* + for _ in `seq 0 9`; do + if test "$($CMD | jq -c .)" $TEST_ARGS; then + return 0 + fi + sleep 2 + done + + echo "Timed out waiting for condition: $CMD $TEST_ARGS" >&2 + return 1 +} + +wait_for_log() { + LOG_FILE=$1 + PATTERN=$2 + DESCRIPTION=$3 + for _ in `seq 0 99`; do + if grep -Fq "$PATTERN" "$LOG_FILE"; then + return 0 + fi + sleep 0.2 + done + + echo "Timed out waiting for $DESCRIPTION in $LOG_FILE" >&2 + tail -n +1 "$LOG_FILE" >&2 || true + return 1 +} + +wait_for_http() { + URL=$1 + OUTPUT=$2 + for _ in `seq 0 99`; do + if curl --silent --show-error --fail "$URL" -o "$OUTPUT"; then + return 0 + fi + sleep 0.2 + done + + echo "Timed out waiting for HTTP endpoint $URL" >&2 + return 1 +} + +BTC="$BITCOIN_MULTIPROCESS_BIN rpc -chain=regtest -datadir=$PWD/data/bitcoin" +ELECTRUM="electrum --regtest" +EL="$ELECTRUM --wallet=data/electrum/wallet" +SOCK="$PWD/data/bitcoin/regtest/node.sock" + +echo "Starting $($BITCOIN_MULTIPROCESS_BIN node -version | head -n1) (multiprocess, IPC enabled)..." +$BITCOIN_MULTIPROCESS_BIN node \ + -regtest -datadir=$PWD/data/bitcoin \ + -ipcbind=unix \ + -listen=0 \ + -printtoconsole=0 \ + -fallbackfee=0.0001 & +BITCOIND_PID=$! + +# Wait for both RPC and the IPC socket. +$BTC -rpcwait getblockcount > /dev/null +for _ in `seq 0 50`; do + test -S "$SOCK" && break || sleep 0.2 +done +test -S "$SOCK" || { echo "IPC socket $SOCK never appeared" >&2; exit 1; } +echo "IPC socket ready: $SOCK" + +echo "Creating Electrum `electrum version --offline` wallet..." +WALLET=`$EL --offline create --seed_type=segwit` +MINING_ADDR=`$EL --offline getunusedaddress` + +$BTC generatetoaddress 110 $MINING_ADDR > /dev/null +echo `$BTC getblockchaininfo | jq -r '"Generated \(.blocks) regtest blocks (\(.size_on_disk/1e3) kB)"'` to $MINING_ADDR + +TIP=`$BTC getbestblockhash` + +export RUST_LOG=electrs=debug +electrs \ + --db-dir=data/electrs \ + --daemon-dir=data/bitcoin \ + --network=regtest \ + --daemon-ipc-socket="$SOCK" \ + 2> data/electrs/regtest-debug.log & +ELECTRS_PID=$! +wait_for_log data/electrs/regtest-debug.log "serving Electrum RPC" "electrs Electrum RPC startup" +wait_for_log data/electrs/regtest-debug.log "connecting to bitcoin-node IPC socket" "electrs IPC backend connection" + +wait_for_http http://localhost:24224 metrics.txt + +$ELECTRUM daemon --server localhost:60401:t -1 -vDEBUG 2> data/electrum/regtest-debug.log & +ELECTRUM_PID=$! +wait_for_log data/electrum/regtest-debug.log "connection established" "Electrum daemon connection" +$EL getinfo | jq . + +echo "Loading Electrum wallet..." +$EL load_wallet + +echo "Running integration tests (IPC backend):" + +echo " * getbalance" +wait_for "$EL getbalance" == '{"confirmed":"550","unmatured":"4950"}' + +echo " * getunusedaddress" +NEW_ADDR=`$EL getunusedaddress` + +echo " * payto & broadcast" +TXID=$($EL broadcast $($EL payto $NEW_ADDR 123 --fee 0.001 --password='')) + +echo " * get_tx_status" +wait_for "$EL get_tx_status $TXID" == '{"confirmations":0}' + +echo " * getaddresshistory" +test "`$EL getaddresshistory $NEW_ADDR | jq -c .`" == "[{\"fee\":100000,\"height\":0,\"tx_hash\":\"$TXID\"}]" + +echo " * getbalance" +test "`$EL getbalance | jq -c .`" == '{"confirmed":"549.999","unmatured":"4950"}' + +echo "Generating bitcoin block..." +$BTC generatetoaddress 1 $MINING_ADDR > /dev/null +$BTC getblockcount > /dev/null + +echo " * wait for new block" +kill -USR1 $ELECTRS_PID # notify server to index new block +wait_for_log data/electrum/regtest-debug.log "verified $TXID" "Electrum wallet verification of mined transaction" + +echo " * get_tx_status" +test "`$EL get_tx_status $TXID | jq -c .`" == '{"confirmations":1}' + +echo " * getaddresshistory" +test "`$EL getaddresshistory $NEW_ADDR | jq -c .`" == "[{\"height\":111,\"tx_hash\":\"$TXID\"}]" + +echo " * getbalance" +test "`$EL getbalance | jq -c .`" == '{"confirmed":"599.999","unmatured":"4950.001"}' + +echo "Electrum `$EL stop`" # disconnect wallet +wait $ELECTRUM_PID + +kill -INT $ELECTRS_PID # close server +wait_for_log data/electrs/regtest-debug.log "electrs stopped" "electrs shutdown" +wait $ELECTRS_PID + +# When the multiprocess node has IPC clients still attached at shutdown, the +# RPC `stop` request hangs indefinitely waiting for them to disconnect. The +# IPC client lives in electrs, which has already stopped above but whose +# capnp connection drop is not flushed back to the node. Skip the polite +# stop and just kill the node — this is a regtest fixture. +kill $BITCOIND_PID 2>/dev/null || true +wait $BITCOIND_PID 2>/dev/null || true + +echo "=== PASSED (IPC) ===" diff --git a/tests/run.sh b/tests/run.sh index 1e2c03c1f..5e11588e3 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -3,6 +3,7 @@ set -euo pipefail rm -rf data/ mkdir -p data/{bitcoin,electrum,electrs} +touch data/electrs/regtest-debug.log data/electrum/regtest-debug.log cleanup() { trap - SIGTERM SIGINT @@ -21,18 +22,50 @@ wait_for() { shift TEST_ARGS=$* for _ in `seq 0 9`; do - test "$($CMD | jq -c .)" $TEST_ARGS && break || sleep 2 + if test "$($CMD | jq -c .)" $TEST_ARGS; then + return 0 + fi + sleep 2 done + + echo "Timed out waiting for condition: $CMD $TEST_ARGS" >&2 + return 1 +} + +wait_for_log() { + LOG_FILE=$1 + PATTERN=$2 + DESCRIPTION=$3 + for _ in `seq 0 99`; do + if grep -Fq "$PATTERN" "$LOG_FILE"; then + return 0 + fi + sleep 0.2 + done + + echo "Timed out waiting for $DESCRIPTION in $LOG_FILE" >&2 + tail -n +1 "$LOG_FILE" >&2 || true + return 1 +} + +wait_for_http() { + URL=$1 + OUTPUT=$2 + for _ in `seq 0 99`; do + if curl --silent --show-error --fail "$URL" -o "$OUTPUT"; then + return 0 + fi + sleep 0.2 + done + + echo "Timed out waiting for HTTP endpoint $URL" >&2 + return 1 } BTC="bitcoin-cli -regtest -datadir=data/bitcoin" ELECTRUM="electrum --regtest" EL="$ELECTRUM --wallet=data/electrum/wallet" -tail_log() { - tail -n +0 -F $1 || true -} - echo "Starting $(bitcoind -version | head -n1)..." bitcoind -regtest -datadir=data/bitcoin -printtoconsole=0 & BITCOIND_PID=$! @@ -55,12 +88,12 @@ electrs \ --network=regtest \ 2> data/electrs/regtest-debug.log & ELECTRS_PID=$! -tail_log data/electrs/regtest-debug.log | grep -m1 "serving Electrum RPC" -curl localhost:24224 -o metrics.txt +wait_for_log data/electrs/regtest-debug.log "serving Electrum RPC" "electrs Electrum RPC startup" +wait_for_http http://localhost:24224 metrics.txt $ELECTRUM daemon --server localhost:60401:t -1 -vDEBUG 2> data/electrum/regtest-debug.log & ELECTRUM_PID=$! -tail_log data/electrum/regtest-debug.log | grep -m1 "connection established" +wait_for_log data/electrum/regtest-debug.log "connection established" "Electrum daemon connection" $EL getinfo | jq . echo "Loading Electrum wallet..." @@ -92,7 +125,7 @@ $BTC getblockcount > /dev/null echo " * wait for new block" kill -USR1 $ELECTRS_PID # notify server to index new block -tail_log data/electrum/regtest-debug.log | grep -m1 "verified $TXID" > /dev/null +wait_for_log data/electrum/regtest-debug.log "verified $TXID" "Electrum wallet verification of mined transaction" echo " * get_tx_status" test "`$EL get_tx_status $TXID | jq -c .`" == '{"confirmations":1}' @@ -107,10 +140,12 @@ echo "Electrum `$EL stop`" # disconnect wallet wait $ELECTRUM_PID kill -INT $ELECTRS_PID # close server -tail_log data/electrs/regtest-debug.log | grep -m1 "electrs stopped" +wait_for_log data/electrs/regtest-debug.log "electrs stopped" "electrs shutdown" wait $ELECTRS_PID -$BTC stop # stop bitcoind -wait $BITCOIND_PID +# Try a graceful stop; if the node has already exited the RPC call will fail, +# which is fine. +$BTC stop 2>/dev/null || kill $BITCOIND_PID 2>/dev/null || true +wait $BITCOIND_PID 2>/dev/null || true echo "=== PASSED ==="