From 3c0711f1609a6d698cf389554011956570909edd Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 6 Jul 2026 23:57:20 -0700 Subject: [PATCH] fix(zaino-state): replace sleep-ordered mines with handshakes in the mempool-stream tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both mempool-stream mockchain tests spawn a collector task whose stream must open before the main task mines the block that ends the test, and both enforced that ordering with a 500 ms sleep. Under full-suite parallel load the collector can lose the race: the expected-tip variant then arms its stream against the post-mine tip and hangs forever awaiting a second mine, and the no-expected-tip variant collects the drained post-mine mempool and fails its assertion on an empty vector. Both failure modes were observed on loaded machines and neither reproduces solo. Each test now passes a oneshot handshake: the collector signals after get_mempool_stream returns — the point where the stream's termination condition is locked to the pre-mine tip — and the main task awaits that signal before mining. The ordering becomes deterministic, and each test runs about 1.5 s faster by shedding the dead sleep. No fails-before sibling test accompanies the fix: the race window only opens under machine load and cannot be made deterministic in a demonstration. The hangdebug nextest profile that caught the no-expected-tip flake is committed alongside the fix. The default profile deliberately runs the production unit tests with no slow-timeout, so a hung test pends silently and anonymously; a parallel run under this profile names laggards at 60 seconds and terminates a stuck test at 300, dumping its captured output. Co-Authored-By: Claude Fable 5 --- .config/nextest.toml | 16 ++++++++++ .../src/chain_index/tests/mockchain_tests.rs | 31 +++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 79608e2ff..c02287239 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -54,3 +54,19 @@ default-filter = """ not test(testnet) & not test(client_rpcs) """ + +# Debug profile for hunting hangs and load-dependent flakes in the production +# unit tests, which the default profile deliberately runs without any +# slow-timeout. A parallel run under this profile names laggards with a SLOW +# line at 60s and terminates a genuinely stuck test at 300s, dumping its +# captured output — run with RUST_LOG=trace so that capture holds the full +# instrumentation: +# +# NEXTEST_PROFILE=hangdebug RUST_LOG=trace cargo nextest run +# +[profile.hangdebug] +fail-fast = false + +[[profile.hangdebug.overrides]] +filter = 'package(zaino-state)' +slow-timeout = { period = "60s", terminate-after = "5" } diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 2ee2de790..86ffba684 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -11,7 +11,7 @@ use crate::{ }, BlockchainSource as _, Outpoint, }; -use tokio::time::{sleep, Duration}; +use tokio::time::Duration; use tokio_stream::StreamExt as _; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, @@ -423,10 +423,20 @@ async fn get_mempool_stream_no_expected_chain_tip_snapshot() { .unwrap_or_default(); mempool_transactions.sort_by_key(|transaction| transaction.hash()); + // Same ordering constraint as the expected-tip variant below: the stream + // must open before the mine, or it observes the drained post-mine + // mempool and collects nothing. Without an expected tip there is no + // guard at open, so the lost race presents as an assertion failure + // rather than a hang. The handshake makes the ordering deterministic. + let (stream_opened_tx, stream_opened_rx) = tokio::sync::oneshot::channel(); + let mempool_stream_task = tokio::spawn(async move { let mut mempool_stream = index_reader .get_mempool_stream(None) .expect("failed to create mempool stream"); + stream_opened_tx + .send(()) + .expect("the main task awaits the handshake"); let mut indexer_mempool_transactions: Vec = Vec::new(); @@ -442,7 +452,9 @@ async fn get_mempool_stream_no_expected_chain_tip_snapshot() { indexer_mempool_transactions }); - sleep(Duration::from_millis(500)).await; + stream_opened_rx + .await + .expect("the collector task opens the stream"); mockchain.mine_blocks(1); @@ -484,11 +496,22 @@ async fn get_mempool_stream_correct_expected_chain_tip_snapshot() { .unwrap_or_default(); mempool_transactions.sort_by_key(|transaction| transaction.hash()); + // The stream closes only when the chain tip moves away from the tip its + // snapshot recorded, and the mine below is that one move: the snapshot + // and stream-open must therefore happen strictly before the mine, or the + // stream arms itself against the post-mine tip and waits forever for a + // second mine that never comes. A handshake makes the ordering + // deterministic where a sleep only made it likely on an idle machine. + let (stream_opened_tx, stream_opened_rx) = tokio::sync::oneshot::channel(); + let mempool_stream_task = tokio::spawn(async move { let nonfinalized_snapshot = index_reader.snapshot_nonfinalized_state().await.unwrap(); let mut mempool_stream = index_reader .get_mempool_stream(Some(&nonfinalized_snapshot)) .expect("failed to create mempool stream"); + stream_opened_tx + .send(()) + .expect("the main task awaits the handshake"); let mut indexer_mempool_transactions: Vec = Vec::new(); @@ -504,7 +527,9 @@ async fn get_mempool_stream_correct_expected_chain_tip_snapshot() { indexer_mempool_transactions }); - sleep(Duration::from_millis(500)).await; + stream_opened_rx + .await + .expect("the collector task opens the stream"); mockchain.mine_blocks(1);