PoC: replace the mempool mirror with Bitcoin Core's TxCollection

.github/workflows/integration-tests.yaml

@@ -35,6 +35,38 @@ jobs:
         if: matrix.os == 'macos-latest'
         run: brew install capnp
 
+      # TEMPORARY: build Bitcoin Core from the bitcoin/bitcoin#35671 PR branch
+      # (TxCollection) until an official v32 release with TxCollection support is
+      # available. Remove these steps (and BITCOIN_CORE_V32_BINARY below) once v32
+      # follows the standard release-binary download flow.
+      - name: Install Bitcoin Core build dependencies (Ubuntu)
+        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get install -y build-essential cmake pkgconf python3 libevent-dev libboost-dev ccache
+
+      - name: Install Bitcoin Core build dependencies (macOS)
+        if: matrix.os == 'macos-latest'
+        run: brew install cmake boost pkgconf ccache
+
+      - name: ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ matrix.os == 'macos-latest' && '~/Library/Caches/ccache' || '~/.cache/ccache' }}
+          key: ccache-bitcoin-${{ runner.os }}-${{ github.sha }}
+          restore-keys: ccache-bitcoin-${{ runner.os }}-
+
+      - name: Checkout Bitcoin Core (bitcoin/bitcoin#35671)
+        run: |
+          git clone --depth 1 https://github.com/bitcoin/bitcoin.git
+          cd bitcoin
+          git fetch --depth 1 origin pull/35671/head:pr-35671
+          git checkout pr-35671
+
+      - name: Build Bitcoin Core
+        run: |
+          cd bitcoin
+          cmake -B build -DENABLE_WALLET=ON -DBUILD_TESTS=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+          cmake --build build -j $(nproc 2>/dev/null || sysctl -n hw.ncpu)
+
       # Install cargo-binstall so tool installs below use prebuilt binaries.
       - name: Install cargo-binstall
         uses: cargo-bins/cargo-binstall@v1.20.0
@@ -49,5 +81,7 @@ jobs:
         run: cargo binstall --no-confirm --disable-telemetry --disable-strategies quick-install --locked cargo-nextest@0.9.100
 
       - name: Integration Tests
+        env:
+          BITCOIN_CORE_V32_BINARY: ${{ github.workspace }}/bitcoin/build/bin/bitcoin-node
         run: |
          RUST_BACKTRACE=1 RUST_LOG=debug cargo nextest run --manifest-path=integration-tests/Cargo.toml --nocapture

bitcoin-core-sv2/Cargo.toml

@@ -23,6 +23,12 @@ bitcoin_capnp_types_v30 = { package = "bitcoin-capnp-types", version = "0.1.2" }
 # Bitcoin Core v31.x IPC dependencies
 bitcoin_capnp_types_v31 = { package = "bitcoin-capnp-types", version = "0.2.1" }
 
+# todo: fetch from crates.io
+# Bitcoin Core v32.x IPC dependencies
+# TEMPORARY: pinned to https://github.com/2140-dev/bitcoin-capnp-types/pull/29 (TxCollection)
+# switch back to 2140-dev once that PR is merged and released
+bitcoin_capnp_types_v32 = { package = "bitcoin-capnp-types", git = "https://github.com/Sjors/bitcoin-capnp-types.git", branch = "2026/07/tx-collection" }
+
 # fetching from github enables synchronizing development workflows across sv2-apps and stratum repos
 # it MUST be changed before bitcoin-core-sv2 is published to crates.io
 # with the proper version of stratum-core being fetched from crates.io as well

bitcoin-core-sv2/README.md

@@ -8,11 +8,12 @@ A Rust library that integrates [Bitcoin Core](https://bitcoin.org/en/bitcoin-cor
 - building Sv2 applications that act as a Client under the Template Distribution Protocol (e.g.: Pool or JDC) while connecting directly to the Bitcoin Core node.
 - building a Sv2 Template Provider application that acts as a Template Distribution Protocol Server while creating templates from a Bitcoin Core node.
 
-The crate exposes three module families:
+The crate exposes four module families:
 
 - `bitcoin_core_sv2::common` - version-agnostic enum-dispatch runtime handles and protocol-specific `new(version, ...)` factories.
 - `bitcoin_core_sv2::unix_capnp::v30x` - Bitcoin Core v30.x IPC implementation.
 - `bitcoin_core_sv2::unix_capnp::v31x` - Bitcoin Core v31.x IPC implementation.
+- `bitcoin_core_sv2::unix_capnp::v32x` - Bitcoin Core v32.x IPC implementation.
 
 ### Flavor naming rationale
 
@@ -59,6 +60,7 @@ The `min_interval` parameter (in seconds) determines the minimum amount of time
 
 - `tdp_logger_v30x` - Template Distribution Protocol logger wired to `bitcoin_core_sv2::unix_capnp::v30x`.
 - `tdp_logger_v31x` - Template Distribution Protocol logger wired to `bitcoin_core_sv2::unix_capnp::v31x`.
+- `tdp_logger_v32x` - Template Distribution Protocol logger wired to `bitcoin_core_sv2::unix_capnp::v32x`.
 
 ## License
 

bitcoin-core-sv2/examples/tdp_logger_v32x.rs

@@ -0,0 +1,178 @@
+//! A simple example of how to use `BitcoinCoreSv2TDP`.
+//!
+//! This example demonstrates the pattern used in applications where `BitcoinCoreSv2TDP` is
+//! spawned in a dedicated thread with its own Tokio runtime and `LocalSet`. This allows the
+//! main application to run in a separate async context while `BitcoinCoreSv2TDP` runs in its
+//! own isolated thread.
+//!
+//! We connect to the Bitcoin Core UNIX socket, and log the received Sv2 Template Distribution
+//! Protocol messages.
+//!
+//! We send a `CoinbaseOutputConstraints` message to the `BitcoinCoreSv2TDP` instance once at
+//! startup.
+//!
+//! `BitcoinCoreSv2TDP` will not start distributing new templates until it receives the first
+//! `CoinbaseOutputConstraints` message.
+
+use bitcoin_core_sv2::unix_capnp::v32x::template_distribution_protocol::BitcoinCoreSv2TDP;
+use std::path::Path;
+
+use async_channel::unbounded;
+use stratum_core::{
+    parsers_sv2::TemplateDistribution,
+    template_distribution_sv2::{CoinbaseOutputConstraints, RequestTransactionData},
+};
+use tokio_util::sync::CancellationToken;
+use tracing::{error, info};
+
+#[tokio::main]
+async fn main() {
+    tracing_subscriber::fmt::init();
+
+    // the user must provide the path to the Bitcoin Core UNIX socket
+    let args: Vec<String> = std::env::args().collect();
+    if args.len() != 2 {
+        eprintln!("Usage: {} <bitcoin_core_unix_socket_path>", args[0]);
+        eprintln!("Example: {} /path/to/bitcoin/regtest/node.sock", args[0]);
+        std::process::exit(1);
+    }
+
+    let bitcoin_core_unix_socket_path = Path::new(&args[1]);
+
+    // `BitcoinCoreSv2TDP` uses this to cancel internally spawned tasks
+    let cancellation_token = CancellationToken::new();
+
+    // get new templates whenever the mempool has changed by more than 100 sats
+    let fee_threshold = 100;
+
+    // the minimum interval between template updates in seconds
+    let min_interval = 5;
+
+    // these messages are sent into the `BitcoinCoreSv2TDP` instance
+    let (msg_sender_into_bitcoin_core_sv2, msg_receiver_into_bitcoin_core_sv2) = unbounded();
+    // these messages are received from the `BitcoinCoreSv2TDP` instance
+    let (msg_sender_from_bitcoin_core_sv2, msg_receiver_from_bitcoin_core_sv2) = unbounded();
+
+    // clone so we can move it into the thread
+    let cancellation_token_clone = cancellation_token.clone();
+    let bitcoin_core_unix_socket_path_clone = bitcoin_core_unix_socket_path.to_path_buf();
+
+    // spawn a dedicated thread to run the BitcoinCoreSv2TDP instance
+    // because we're limited to tokio::task::LocalSet
+    //
+    // please note that it's important to keep a reference to the join handle so we can wait for it
+    // to finish shutdown this will no longer be a pre-requisite once https://github.com/bitcoin/bitcoin/pull/33676 lands in a release
+    // see https://github.com/stratum-mining/sv2-apps/issues/81 for more details
+    let join_handle = std::thread::spawn(move || {
+        // we need a dedicated runtime so we can spawn an async task inside the LocalSet
+        let rt = match tokio::runtime::Runtime::new() {
+            Ok(rt) => rt,
+            Err(e) => {
+                error!("Failed to create Tokio runtime: {:?}", e);
+                cancellation_token_clone.cancel();
+                return;
+            }
+        };
+        let tokio_local_set = tokio::task::LocalSet::new();
+
+        tokio_local_set.block_on(&rt, async move {
+            // create a new `BitcoinCoreSv2TDP` instance
+            let mut sv2_bitcoin_core = match BitcoinCoreSv2TDP::new(
+                &bitcoin_core_unix_socket_path_clone,
+                fee_threshold,
+                min_interval,
+                msg_receiver_into_bitcoin_core_sv2,
+                msg_sender_from_bitcoin_core_sv2,
+                cancellation_token_clone.clone(),
+            )
+            .await
+            {
+                Ok(sv2_bitcoin_core) => sv2_bitcoin_core,
+                Err(e) => {
+                    error!("Failed to create BitcoinCoreToSv2: {:?}", e);
+                    cancellation_token_clone.cancel();
+                    return;
+                }
+            };
+
+            // run the `BitcoinCoreSv2TDP` instance, which will block until the cancellation token
+            // is activated
+            sv2_bitcoin_core.run().await;
+        });
+    });
+
+    // clone so we can move it
+    let cancellation_token_clone = cancellation_token.clone();
+
+    // clone so we can move it
+    let msg_sender_into_bitcoin_core_sv2_clone = msg_sender_into_bitcoin_core_sv2.clone();
+
+    // a task to consume and log the received Sv2 Template Distribution Protocol messages
+    tokio::spawn(async move {
+        loop {
+            tokio::select! {
+                // monitor for Ctrl+C, activating the cancellation token and exiting the loop
+                _ = tokio::signal::ctrl_c() => {
+                    info!("Ctrl+C received");
+                    cancellation_token_clone.cancel();
+                    return;
+                }
+                // monitor potential internal activations of the cancellation token for exiting the loop
+                _ = cancellation_token_clone.cancelled() => {
+                    info!("Cancellation token activated");
+                    return;
+                }
+                // monitor for Sv2 Template Distribution Protocol messages
+                // coming from `BitcoinCoreSv2TDP`
+                Ok(template_distribution_message) = msg_receiver_from_bitcoin_core_sv2.recv() => {
+                    // log the message
+                    info!("Message received: {}", template_distribution_message);
+
+                    // send a RequestTransactionData every time a NewTemplate message is received
+                    if let TemplateDistribution::NewTemplate(new_template) = template_distribution_message {
+                        let template_id = new_template.template_id;
+                        let request_transaction_data = TemplateDistribution::RequestTransactionData(RequestTransactionData {
+                            template_id,
+                        });
+
+                        match msg_sender_into_bitcoin_core_sv2_clone.send(request_transaction_data).await {
+                            Ok(_) => (),
+                            Err(e) => {
+                                error!("Failed to send request transaction data: {}", e);
+                                cancellation_token_clone.cancel();
+                                return;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    });
+
+    // send CoinbaseOutputConstraints once at startup
+    //
+    // `BitcoinCoreSv2TDP` will not start distributing new templates until it receives the first
+    // `CoinbaseOutputConstraints` message.
+    let new_coinbase_output_constraints =
+        TemplateDistribution::CoinbaseOutputConstraints(CoinbaseOutputConstraints {
+            coinbase_output_max_additional_size: 2,
+            coinbase_output_max_additional_sigops: 2,
+        });
+
+    if let Err(e) = msg_sender_into_bitcoin_core_sv2
+        .send(new_coinbase_output_constraints)
+        .await
+    {
+        error!("Failed to send coinbase output constraints: {}", e);
+        cancellation_token.cancel();
+    }
+    info!("Sent CoinbaseOutputConstraints");
+
+    // wait for the cancellation token to be activated
+    cancellation_token.cancelled().await;
+    info!("Shutting down...");
+
+    // wait for the dedicated thread to finish shutdown
+    join_handle.join().unwrap();
+    info!("BitcoinCoreSv2TDP dedicated thread shutdown complete.");
+}

bitcoin-core-sv2/src/common/job_declaration_protocol/io.rs

@@ -1,14 +1,20 @@
 //! Request / response types exchanged between `jd-server` and the Bitcoin Core IPC thread.
 
-use stratum_core::{
-    bitcoin::{BlockHash, CompactTarget, Transaction, Txid, Wtxid, block::Version},
-    job_declaration_sv2::PushSolution,
+use stratum_core::bitcoin::{
+    BlockHash, CompactTarget, Transaction, TxMerkleNode, Wtxid, block::Version,
 };
 use tokio::sync::oneshot;
 
-/// Snapshot of the template parameters used by the validator at decision time.
+/// Identifies a downstream client of `jd-server`.
+pub type DownstreamId = usize;
+
+/// Identifies a `DeclareMiningJob` request within a downstream connection.
+pub type RequestId = u32;
+
+/// Snapshot of the template parameters used by the mirror-based validators (v30.x/v31.x)
+/// at decision time.
 ///
-/// This lets callers distinguish stale-tip races from other validation failures.
+/// This lets those backends distinguish stale-tip races from other validation failures.
 ///
 /// Please check https://github.com/stratum-mining/sv2-apps/issues/364
 /// for more details on the regression that motivated this field.
@@ -22,41 +28,73 @@ pub struct ValidationContext {
 /// A request sent from `jd-server` to the [`BitcoinCoreSv2JDP`](super::BitcoinCoreSv2JDP) IPC
 /// thread.
 ///
-/// Built from a `DeclareMiningJob` (plus an optional `ProvideMissingTransactionsSuccess`)
-/// or a `PushSolution`.
+/// `DeclareMiningJob` is built from a `DeclareMiningJob` message (plus an optional
+/// `ProvideMissingTransactionsSuccess`), `PushSolution` from a `PushSolution` message. The
+/// remaining variants let `jd-server` mirror its job lifecycle so backends that retain
+/// per-job state (the v32.x `TxCollection` backend keeps a validated `BlockTemplate` per
+/// declared job) can release it.
 pub enum JdRequest {
-    /// Validate a declared mining job via Bitcoin Core's `checkBlock`.
+    /// Validate a declared mining job.
+    ///
+    /// Invariants (enforced by `jd-server`): `wtxid_list` contains no duplicates and every
+    /// transaction in `missing_txs` is part of `wtxid_list`. The v32.x `TxCollection`
+    /// backend relies on these; Bitcoin Core rejects violations with RPC-level errors.
     DeclareMiningJob {
+        downstream_id: DownstreamId,
+        request_id: RequestId,
         version: Version,
         coinbase_tx: Transaction,
         wtxid_list: Vec<Wtxid>,
         missing_txs: Vec<Transaction>,
         response_tx: oneshot::Sender<JdResponse>,
     },
-    /// Submit a mining solution to Bitcoin Core (fire-and-forget).
+    /// Submit a solution for a previously declared job to Bitcoin Core (fire-and-forget).
+    ///
+    /// `downstream_id`/`request_id` identify the `DeclareMiningJob` the solution belongs
+    /// to. `coinbase_tx` is the declared coinbase with the solution's extranonce applied.
     PushSolution {
-        push_solution: PushSolution<'static>,
+        downstream_id: DownstreamId,
+        request_id: RequestId,
+        version: u32,
+        ntime: u32,
+        nonce: u32,
+        coinbase_tx: Transaction,
     },
+    /// Release any backend state retained for a declared job that will never be used
+    /// (consumed with an error, expired, or superseded). Fire-and-forget; harmless when the
+    /// backend retained nothing.
+    ReleaseDeclaredJob {
+        downstream_id: DownstreamId,
+        request_id: RequestId,
+    },
+    /// Release all backend state retained for a disconnected downstream (fire-and-forget).
+    CleanupDownstream { downstream_id: DownstreamId },
 }
 
 /// The result of trying to handle a DeclareMiningJob request.
+///
+/// `Error` and `MissingTransactions` carry only the chain tip (`prev_hash`) the validator
+/// operated against: stale-tip classification is restricted to `prev_hash` comparison
+/// (see https://github.com/stratum-mining/sv2-apps/issues/597).
 #[derive(Debug, Clone)]
 pub enum JdResponse {
     Success {
         prev_hash: BlockHash,
         nbits: CompactTarget,
         min_ntime: u32,
-        /// Txids for all transactions (excluding coinbase), in the same order as the declared
-        /// wtxid_list. Enables the caller to build the txid merkle tree for validating
-        /// SetCustomMiningJob.merkle_path.
-        txid_list: Vec<Txid>,
+        /// Coinbase merkle branch (sibling hashes from leaf to root at position 0) in the
+        /// txid merkle tree, used by `jd-server` to validate a `SetCustomMiningJob`'s
+        /// `merkle_path`. It does not depend on the coinbase transaction itself.
+        merkle_path: Vec<TxMerkleNode>,
     },
     Error {
         error_code: &'static str,
-        validation_context: ValidationContext,
+        /// Chain tip at decision time; `None` when the failure happened before the validator
+        /// could establish one (e.g. internal IPC errors).
+        prev_hash: Option<BlockHash>,
     },
     MissingTransactions {
         missing_wtxids: Vec<Wtxid>,
-        validation_context: ValidationContext,
+        prev_hash: BlockHash,
     },
 }