From 4fa75f3fed1c2a3fe7f94129386999bdb2f9737f Mon Sep 17 00:00:00 2001 From: Blake Emerson Date: Sat, 11 Jul 2026 09:31:56 +0000 Subject: [PATCH 1/5] Route Sapling-output sends to the fused path A send that pays a Sapling output must not take the cached-proving-key path: that path's extractor is handed no Sapling verifying key, so it rejects a Sapling bundle. Divert any send with a Sapling-output recipient to the fused build path, which builds its own keys. This is about paying a Sapling recipient, not spending Sapling notes, which zecd never does on the cached path. Co-authored-by: AI --- src/wallet/actor.rs | 137 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 128 insertions(+), 9 deletions(-) diff --git a/src/wallet/actor.rs b/src/wallet/actor.rs index d7fcd75..59785a5 100644 --- a/src/wallet/actor.rs +++ b/src/wallet/actor.rs @@ -35,7 +35,7 @@ use zcash_primitives::transaction::builder::{BuildConfig, Builder}; use zcash_primitives::transaction::fees::zip317::FeeRule as Zip317FeeRule; use zcash_primitives::transaction::Transaction; use zcash_proofs::prover::LocalTxProver; -use zcash_protocol::consensus::{BlockHeight, BranchId}; +use zcash_protocol::consensus::{BlockHeight, BranchId, Parameters}; use zcash_protocol::value::Zatoshis; use zcash_protocol::{PoolType, ShieldedProtocol, TxId}; use zcash_transparent::address::TransparentAddress; @@ -2785,10 +2785,13 @@ impl WalletActor { } } - /// Whether sends on this wallet use the cached-Orchard PCZT path (so prove and store are + /// Whether sends on this wallet *may* use the cached-Orchard PCZT path (so prove and store are /// separable). True for the default Orchard-only wallet with `cache_proving_key` on. A /// Sapling-spending wallet (or `cache_proving_key` off) uses the fused path, which has no - /// prove/store seam - see [`Self::do_send_fused`]. + /// prove/store seam - see [`Self::do_send_fused`]. NB this gates on the *wallet's* pools, not + /// the send's recipients: even when this is true, an individual send that pays a Sapling output + /// is still diverted to the fused path (`request_pays_sapling_output`), because the cached + /// path's extractor is handed no Sapling verifying key. fn cached_pczt_path(&self) -> bool { self.orchard_keys.is_some() && !self.enabled_pools.contains(Pool::Sapling) } @@ -2953,7 +2956,10 @@ impl WalletActor { // note selection; the synchronous sends pass `None` and use the configured policy. let policy = confirmations.unwrap_or(self.confirmations_policy); - if !self.cached_pczt_path() { + // The cached-Orchard PCZT path can't finalize a Sapling output: its extractor is handed no + // Sapling verifying key. A send to a Sapling-only recipient therefore takes the fused path, + // which proves and verifies Sapling outputs itself. + if !self.cached_pczt_path() || request_pays_sapling_output(&self.network, &request) { return self.do_send_fused(usk, request, policy, privacy).await; } @@ -3090,7 +3096,14 @@ impl WalletActor { // `AllowFullyTransparent` sends are handled inline by `do_send` (they build via the // transparent Builder, not the cached-Orchard PCZT prove path that pipelining accelerates), // so never queue them for off-actor proving. - if privacy == SendPrivacy::AllowFullyTransparent || !self.pipeline_eligible() { + // + // A Sapling-output send can't ride the pipeline either (it commits via the same PCZT + // extractor that has no Sapling verifying key). Route it through `do_send`, which diverts + // it to the fused path. + if privacy == SendPrivacy::AllowFullyTransparent + || !self.pipeline_eligible() + || request_pays_sapling_output(&self.network, &request) + { let res = self.do_send(request, confirmations, privacy).await; let _ = reply.send(res); return; @@ -4015,6 +4028,37 @@ fn now_unix() -> i64 { .as_secs() as i64 } +/// Whether any recipient in `request` forces a **Sapling output**: a shielded address carrying a +/// Sapling receiver but *no* Orchard receiver (a bare `zs…`, or a UA whose only shielded receiver +/// is Sapling). This is the sole way a send on the Orchard-only cached PCZT path produces a PCZT +/// with a non-empty Sapling bundle - spends are always Orchard and change goes to Orchard (Sapling +/// isn't an enabled pool on that path, or `cached_pczt_path` is already false). The PCZT extractor +/// (`extract_and_store_transaction_from_pczt`) rejects such a bundle without a Sapling *verifying* +/// key, and the cached path passes none - so `do_send`/`begin_or_queue_send` divert these sends to +/// the fused `create_proposed_transactions` path, which builds, proves, and verifies Sapling +/// outputs itself. A dual Orchard+Sapling UA is deliberately *not* flagged: an Orchard-only wallet +/// routes the payment to the UA's Orchard receiver, so no Sapling output is produced. Keyed off the +/// recipient address (not the built proposal) so the routing decision is made before any build; on +/// this path that is equivalent, since a Sapling-only recipient is the only Sapling-output source. +fn request_pays_sapling_output(net: &ZNetwork, request: &TransactionRequest) -> bool { + let net_type = net.network_type(); + request.payments().values().any(|p| { + match p + .recipient_address() + .clone() + .convert_if_network::
(net_type) + { + Ok(addr) => { + crate::address::has_shielded_receiver(&addr) + && !crate::address::has_orchard_receiver(&addr) + } + // Unparseable on this network (already rejected at the RPC layer): don't reroute; the + // normal build path surfaces the error. + Err(_) => false, + } + }) +} + /// Prove (Orchard, plus Sapling outputs if any) and sign the Orchard spends with the account's /// key, returning the signed PCZT ready to extract+store. This is the **pure-CPU** half of a PCZT /// send (phase B): it touches no DB, so it can run off the single-writer actor (see @@ -4080,10 +4124,15 @@ fn prove_sign_pczt( /// Finalize + persist a proven, signed PCZT (phase C): records the tx, its spends/change, and /// marks inputs spent - the same wallet bookkeeping `create_proposed_transactions` does. A DB -/// write, so it runs on the single-writer actor. The cached verifying key avoids regenerating it -/// per send; no Sapling verifying key is needed since zecd never produces Sapling spends. `N` (the -/// note-ref type) is otherwise unconstrained here - it only appears in the error type - so pin it -/// to our `WalletDb`'s note ref, as `error::ProposalError` does for the fused path. +/// write, so it runs on the single-writer actor. The cached Orchard verifying key avoids +/// regenerating it per send. The Sapling verifying key is `None` because this path is only reached +/// for sends with **no Sapling output** - the extractor rejects a Sapling bundle without one, so +/// `do_send`/`begin_or_queue_send` divert any send to a Sapling-only recipient to the fused path +/// before reaching here (the guard is `request_pays_sapling_output`). Note this is about Sapling +/// *outputs*, not spends: zecd never spends Sapling notes on this path, but a send can still *pay* +/// a Sapling recipient. `N` (the note-ref type) is otherwise unconstrained here - it only appears +/// in the error type - so pin it to our `WalletDb`'s note ref, as `error::ProposalError` does for +/// the fused path. fn store_pczt( db: &mut WriteDb, pczt: pczt::Pczt, @@ -4699,6 +4748,76 @@ mod tests { ); } + /// The cached-Orchard PCZT extractor is handed no Sapling verifying key, so a send that pays a + /// Sapling output must be diverted to the fused path. `request_pays_sapling_output` + /// is that routing predicate: it flags a bare Sapling recipient (a Sapling-only UA too), but not + /// an Orchard recipient or a dual Orchard+Sapling UA - an Orchard-only wallet pays such a UA on + /// its Orchard receiver, producing no Sapling output. Without the divert, + /// `extract_and_store_transaction_from_pczt` rejects the PCZT and the send fails after proving. + #[test] + fn request_pays_sapling_output_flags_only_sapling_only_recipients() { + use super::request_pays_sapling_output; + use crate::network::ZNetwork; + use zcash_address::ZcashAddress; + use zcash_keys::address::{Address, UnifiedAddress}; + use zcash_keys::keys::{ReceiverRequirement::*, UnifiedAddressRequest, UnifiedSpendingKey}; + use zcash_protocol::value::Zatoshis; + use zip32::{AccountId, DiversifierIndex}; + use zip321::{Payment, TransactionRequest}; + + let net = ZNetwork::Test; + // A UA carrying both shielded receivers (Sapling + Orchard), no transparent. + let both = UnifiedAddressRequest::unsafe_custom(Require, Require, Omit); + let ufvk = UnifiedSpendingKey::from_seed(&net, &[7u8; 32], AccountId::ZERO) + .unwrap() + .to_unified_full_viewing_key(); + let (ua, _) = ufvk.find_address(DiversifierIndex::new(), both).unwrap(); + + // Three recipient shapes built from that one address's receivers. + let sapling_only = Address::Sapling(ua.sapling().cloned().unwrap()); + let orchard_only = Address::Unified( + UnifiedAddress::from_receivers(ua.orchard().cloned(), None, None).unwrap(), + ); + let dual = Address::Unified(ua.clone()); + + // A one-payment request paying `addr` 1 ZEC. + let request_to = |addr: &Address| { + let zaddr = ZcashAddress::try_from_encoded(&addr.encode(&net)).unwrap(); + let payment = Payment::without_memo(zaddr, Zatoshis::const_from_u64(100_000_000)); + TransactionRequest::new(vec![payment]).unwrap() + }; + + assert!( + request_pays_sapling_output(&net, &request_to(&sapling_only)), + "a bare Sapling recipient forces a Sapling output → must divert to the fused path" + ); + assert!( + !request_pays_sapling_output(&net, &request_to(&orchard_only)), + "an Orchard recipient stays on the cached PCZT path" + ); + assert!( + !request_pays_sapling_output(&net, &request_to(&dual)), + "a dual Orchard+Sapling UA is paid on its Orchard receiver → no Sapling output" + ); + + // A multi-recipient send diverts if *any* payment forces a Sapling output. + let mixed = TransactionRequest::new(vec![ + Payment::without_memo( + ZcashAddress::try_from_encoded(&orchard_only.encode(&net)).unwrap(), + Zatoshis::const_from_u64(1), + ), + Payment::without_memo( + ZcashAddress::try_from_encoded(&sapling_only.encode(&net)).unwrap(), + Zatoshis::const_from_u64(1), + ), + ]) + .unwrap(); + assert!( + request_pays_sapling_output(&net, &mixed), + "one Sapling recipient among several diverts the whole send" + ); + } + /// The enhancement backlog counts only requests zecd can actually service: full-tx /// `Enhancement` and `GetStatus`. The transparent-address variant (which zecd has no source /// for and skips) must be excluded, or it would pin `pending_enhancements` above zero forever From 58df8c69117bcb6e66293c1724f7cff21053994a Mon Sep 17 00:00:00 2001 From: Blake Emerson Date: Sat, 11 Jul 2026 09:36:53 +0000 Subject: [PATCH 2/5] Default /readyz to synced readiness and surface scan lag `/readyz` now defaults to "synced": ready only once the wallet has scanned to within `max_scan_lag` blocks of the tip and drained the enhancement backlog, so a client routed by readiness never treats an empty or stale balance as authoritative while the wallet is still scanning. Set `readiness = "connected"` for the old reachability-only behavior. Both modes now surface a per-wallet `scan_lag` (the block gap between the chain tip and the last fully-scanned height) on `/readyz` and `/status`. Co-authored-by: AI --- README.md | 32 +++++++++++++++++++------------- deploy/zecd.toml | 24 ++++++++++++++---------- docs/OPERATIONS.md | 28 +++++++++++++++++----------- src/config.rs | 39 ++++++++++++++++++++++++++++++++++----- src/health.rs | 10 ++++++++++ zecd.example.toml | 24 ++++++++++++++---------- 6 files changed, 108 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 3f19d00..7e9c02f 100644 --- a/README.md +++ b/README.md @@ -197,8 +197,9 @@ format = "text" # "text" | "json" (structured, for log aggregat enabled = true bind = "127.0.0.1" # set 0.0.0.0 for Kubernetes/LB probes port = 9233 -readiness = "connected" # "connected" (ready when the backend is live, past birthday) - # or "synced" (ready only once scanned to near the tip) +readiness = "synced" # default; ready only once scanned to near the tip, so reads + # are never stale. "connected" = ready when the backend is live + # (past birthday) but the scan may still be catching up. max_scan_lag = 4 # "synced" mode: max chain_tip - fully_scanned block gap ``` @@ -268,18 +269,23 @@ With `[health] enabled` (default), zecd serves unauthenticated probes on a separ - `GET /healthz`: liveness. `200 ok` while the process is running. - `GET /readyz`: readiness. `200`/`503`, gated by `[health] readiness`: - - `"connected"` (default): ready as soon as the backend is connected and its chain tip is past - the wallet's birthday height (a sanity check that we're on the right, live network). It does + - `"synced"` (**default**): ready only once every wallet is connected, within + `[health] max_scan_lag` blocks of the chain tip, **and** with an empty transaction-enhancement + backlog. Strict - a from-birthday restore stays not-ready until it has scanned to its own funds + *and* finished backfilling memos (see below). This is the default so a client routed by + readiness never sees an empty or stale balance/history as authoritative while the wallet is + still scanning: an exchange or any balance-sensitive deployment should keep it. + - `"connected"`: ready as soon as the backend is connected and its chain tip is past the + wallet's birthday height (a sanity check that we're on the right, live network). It does **not** wait for the wallet to finish scanning, so RPC clients can reach zecd while it catches - up and readiness doesn't flap during a long sync - reads may lag the tip until caught up. - - `"synced"`: ready only once every wallet is connected, within `[health] max_scan_lag` - blocks of the chain tip, **and** with an empty transaction-enhancement backlog. Strict - a - from-birthday restore stays not-ready until it has scanned to its own funds *and* finished - backfilling memos (see below). - - Body is JSON with per-wallet detail; when not ready it carries a `reason` (`"upstream_down"`, - `"actor_down"`, `"enhancing"`, or `"syncing"`) so alerting can tell an unreachable zebra apart - from a dead writer, from backfilling memos, from normal block catch-up. + up and readiness doesn't flap during a long sync - but reads may lag the tip until caught up. + Choose this only when reachability matters more than balance freshness; the per-wallet + `scan_lag` in the JSON body (and `/status`) shows how far behind reads may be. + + Body is JSON with per-wallet detail (including `scan_lag`, the `chain_tip - fully_scanned` block + gap); when not ready it carries a `reason` (`"upstream_down"`, `"actor_down"`, `"enhancing"`, or + `"syncing"`) so alerting can tell an unreachable zebra apart from a dead writer, from backfilling + memos, from normal block catch-up. - `GET /status`: JSON snapshot of per-wallet sync state, including the active `server` endpoint, `conn_state` (`down` | `syncing` | `ready`), and the per-wallet `pending_enhancements` count. `getpeerinfo` reflects the same active upstream. diff --git a/deploy/zecd.toml b/deploy/zecd.toml index c75e577..63af52b 100644 --- a/deploy/zecd.toml +++ b/deploy/zecd.toml @@ -38,16 +38,20 @@ interval_secs = 20 bind = "0.0.0.0" port = 9233 # What /readyz gates on: -# "connected" (default) - ready as soon as the backend is connected and its chain tip is past -# this wallet's birthday height (a sanity check that we're on the right, live network). Does -# NOT wait for the wallet to finish scanning, so RPC clients can reach zecd while it catches -# up, and readiness doesn't flap during a long sync. Reads may lag the tip until caught up. -# "synced" - ready only once the wallet is connected AND within `max_scan_lag` blocks of the -# chain tip. Strict: a from-birthday restore stays not-ready until it catches up. The height -# gap is the meaningful "caught up" signal - librustzcash's note-weighted progress ratio -# reaches 1.0 while historical ranges are still being scanned, so it would report ready long -# before the wallet has scanned to its own funds. -# readiness = "connected" +# "synced" (default) - ready only once the wallet is connected AND within `max_scan_lag` blocks +# of the chain tip (and the enhancement backlog has drained). Strict: a from-birthday restore +# stays not-ready until it catches up. This is the default so a client routed by readiness +# never sees an empty or stale balance/history as authoritative while the wallet is still +# scanning - keep it for any balance-sensitive deployment. The height gap is the meaningful +# "caught up" signal - librustzcash's note-weighted progress ratio reaches 1.0 while +# historical ranges are still being scanned, so it would report ready long before the wallet +# has scanned to its own funds. +# "connected" - ready as soon as the backend is connected and its chain tip is past this wallet's +# birthday height (a sanity check that we're on the right, live network). Does NOT wait for +# the wallet to finish scanning, so RPC clients can reach zecd while it catches up, and +# readiness doesn't flap during a long sync. Reads may lag the tip until caught up (the +# per-wallet `scan_lag` in /readyz and /status shows by how much). +# readiness = "synced" # max_scan_lag only applies in "synced" mode. # max_scan_lag = 4 diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index b879991..643a013 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -146,18 +146,24 @@ it reveals the wallet's entire transaction graph, though it cannot spend. ## Monitoring - `GET /healthz` (health port, default 9233) - liveness. -- `GET /readyz` - readiness, gated by `[health] readiness`. In `"connected"` mode - (default) it's 200 as soon as zebra is connected and its tip is past the wallet's - birthday - it does NOT wait for the scan to finish, so it stays ready (no flapping) - while the wallet catches up. In `"synced"` mode it's 200 only once connected, - within `[health] max_scan_lag` blocks of the tip, AND with the transaction-enhancement - backlog drained (see below). When 503, the body's `reason` distinguishes `upstream_down` - (zebra unreachable - page someone) from `syncing` (normal block catch-up), `enhancing` - (scanned to tip, still backfilling memos), and `actor_down` (a dead writer - restart - the process). +- `GET /readyz` - readiness, gated by `[health] readiness`. In `"synced"` mode + (**default**) it's 200 only once connected, within `[health] max_scan_lag` blocks of the + tip, AND with the transaction-enhancement backlog drained (see below) - so a client + routed by readiness never sees an empty or stale balance/history as authoritative while + the wallet is still scanning. **Keep this default for any balance-sensitive deployment** + (an exchange, an accounting integration): a from-birthday restore or fresh sync stays 503 + until it has actually caught up to its own funds. In `"connected"` mode it's 200 as soon as + zebra is connected and its tip is past the wallet's birthday - it does NOT wait for the scan + to finish, so it stays ready (no flapping) while the wallet catches up, but reads may lag the + tip; choose it only when reachability matters more than balance freshness, and watch the + per-wallet `scan_lag` to see how far behind reads may be. When 503, the body's `reason` + distinguishes `upstream_down` (zebra unreachable - page someone) from `syncing` (normal block + catch-up), `enhancing` (scanned to tip, still backfilling memos), and `actor_down` (a dead + writer - restart the process). - `GET /status` - per-wallet sync state, active upstream endpoint, `conn_state` - (`down` | `syncing` | `ready`), and `pending_enhancements` (the enhancement backlog). - Alert if `conn_state` stays `down`. + (`down` | `syncing` | `ready`), `scan_lag` (the `chain_tip - fully_scanned` block gap - how + far behind reads may be, reported in both readiness modes), and `pending_enhancements` (the + enhancement backlog). Alert if `conn_state` stays `down`. - **Enhancement backlog (`pending_enhancements`).** The compact-block scan reaching the tip (`scan_progress: 1.0`, `scanning: false` for the block scan) is NOT "ready to serve full history". Compact blocks carry no memos, so a per-transaction enhancement pass then diff --git a/src/config.rs b/src/config.rs index fd1400c..6612efb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -164,12 +164,14 @@ impl Default for PoolsConfig { pub enum ReadinessMode { /// Ready only once the wallet has actually scanned to (near) the chain tip: connected and /// within `max_scan_lag` blocks of the tip. Strict - a from-birthday restore stays "not - /// ready" until it catches up. Use when a client must not see stale balances/history. + /// ready" until it catches up. **This is the default**: a client must not see an empty or + /// stale balance/history as authoritative while the wallet is still scanning. Synced, /// Ready as soon as the backend is connected and its chain tip is past the wallet's birthday /// (a cheap sanity check that we're talking to the right, live network). Does NOT wait for /// the wallet to finish scanning, so RPC clients can reach zecd while it catches up - at the - /// cost of reads possibly lagging the tip. Avoids readiness flapping during long scans. + /// cost of reads possibly lagging the tip. Avoids readiness flapping during long scans. Opt in + /// (`readiness = "connected"`) only when reachability matters more than balance freshness. Connected, } @@ -1066,14 +1068,17 @@ impl AppConfig { .parse() .context("parsing health bind address")?, port: health_file.port.unwrap_or(defaults.health_port), - // Default to "connected": be generous about how synced we are so RPC clients can - // reach zecd in most situations, and avoid readiness flapping during long scans. + // Default to "synced": `/readyz` reports ready only once the wallet has actually + // scanned to (near) the tip, so a client routed by readiness never sees an empty or + // stale balance/history as authoritative during initial sync or a `--restore` (audit + // 3.15). Deployments that prefer reachability-over-freshness - reaching zecd while it + // catches up, at the cost of possibly-lagging reads - can set `readiness = "connected"`. readiness: health_file .readiness .as_deref() .map(ReadinessMode::parse) .transpose()? - .unwrap_or(ReadinessMode::Connected), + .unwrap_or(ReadinessMode::Synced), max_scan_lag: health_file.max_scan_lag.unwrap_or(4), }; @@ -1694,6 +1699,30 @@ mod tests { assert!(cfg.backend.allow_remote_cleartext); } + #[test] + fn readiness_defaults_to_synced_and_is_overridable() { + use clap::Parser as _; + // Absent [health] readiness resolves to Synced: `/readyz` waits for the scan so a client + // routed by readiness never trusts an empty/stale balance mid-sync. + let cli = Cli::parse_from(["zecd"]); + let cfg = AppConfig::resolve(&cli).unwrap(); + assert_eq!(cfg.health.readiness, ReadinessMode::Synced); + + // A [health] block that omits `readiness` still defaults to Synced... + let dir = tempfile::tempdir().unwrap(); + let conf = dir.path().join("zecd.toml"); + std::fs::write(&conf, "[health]\nport = 9999\n").unwrap(); + let cli = Cli::parse_from(["zecd", "--conf", conf.to_str().unwrap()]); + let cfg = AppConfig::resolve(&cli).unwrap(); + assert_eq!(cfg.health.readiness, ReadinessMode::Synced); + + // ...and the lenient mode is still opt-in from the file. + std::fs::write(&conf, "[health]\nreadiness = \"connected\"\n").unwrap(); + let cli = Cli::parse_from(["zecd", "--conf", conf.to_str().unwrap()]); + let cfg = AppConfig::resolve(&cli).unwrap(); + assert_eq!(cfg.health.readiness, ReadinessMode::Connected); + } + #[test] fn sync_interval_secs_is_clamped_to_at_least_one() { use clap::Parser as _; diff --git a/src/health.rs b/src/health.rs index 4a92f26..ed8a10a 100644 --- a/src/health.rs +++ b/src/health.rs @@ -147,6 +147,15 @@ fn snapshot(state: &AppState) -> Snapshot { if !st.scanning && st.pending_enhancements > 0 { any_enhancing = true; } + // The block-height gap between the tip and the last fully-scanned height - the + // meaningful "how far behind" signal. Surfaced on every wallet regardless of the + // configured readiness mode, so an operator on `readiness = "connected"` (which reports + // ready before the scan finishes) can still see how stale the wallet's reads may be. + // `null` until both heights are known. + let scan_lag = match (st.chain_tip, st.fully_scanned) { + (Some(tip), Some(scanned)) => Some(tip.saturating_sub(scanned)), + _ => None, + }; wallets.insert( name, json!({ @@ -156,6 +165,7 @@ fn snapshot(state: &AppState) -> Snapshot { "conn_state": st.conn_state.as_str(), "chain_tip": st.chain_tip, "fully_scanned": st.fully_scanned, + "scan_lag": scan_lag, "birthday": st.birthday, "scan_progress": st.scan_progress, "scanning": st.scanning, diff --git a/zecd.example.toml b/zecd.example.toml index 45b1584..5c136d6 100644 --- a/zecd.example.toml +++ b/zecd.example.toml @@ -277,15 +277,19 @@ enabled = true bind = "127.0.0.1" # set 0.0.0.0 to expose probes off-host port = 9233 # What /readyz means: -# "connected" (default) - ready as soon as the backend is connected and its chain tip is past -# this wallet's birthday height (a sanity check that we're on the right, live network). It -# does NOT wait for the wallet to finish scanning, so RPC clients can reach zecd while it -# catches up, and readiness doesn't flap during a long sync. Reads may lag the tip until -# the wallet is caught up. -# "synced" - ready only once connected AND `fully_scanned` is within `max_scan_lag` blocks of -# the chain tip. Strict: a from-birthday restore stays not-ready until it has scanned to its -# own funds. (The height gap is used rather than librustzcash's note-weighted scan progress, -# which reaches 100% long before the wallet has actually caught up to the tip.) -readiness = "connected" +# "synced" (default) - ready only once connected AND `fully_scanned` is within `max_scan_lag` +# blocks of the chain tip (and the enhancement backlog has drained). Strict: a from-birthday +# restore stays not-ready until it has scanned to its own funds. This is the default so a +# client routed by readiness never sees an empty or stale balance/history as authoritative +# while the wallet is still scanning - keep it for any balance-sensitive deployment. (The +# height gap is used rather than librustzcash's note-weighted scan progress, which reaches +# 100% long before the wallet has actually caught up to the tip.) +# "connected" - ready as soon as the backend is connected and its chain tip is past this +# wallet's birthday height (a sanity check that we're on the right, live network). It does +# NOT wait for the wallet to finish scanning, so RPC clients can reach zecd while it catches +# up, and readiness doesn't flap during a long sync. Reads may lag the tip until the wallet +# is caught up - the per-wallet `scan_lag` in /readyz and /status shows by how much. Choose +# this only when reachability matters more than balance freshness. +readiness = "synced" # Only consulted in "synced" mode: max_scan_lag = 4 From 29e64061aa73a51a3d585039c23c250be4ed7e9a Mon Sep 17 00:00:00 2001 From: Blake Emerson Date: Sat, 11 Jul 2026 09:41:20 +0000 Subject: [PATCH 3/5] Document the datadir lock's host-locality and stamp the holder The single-instance datadir lock is an advisory `flock`, so it only guards against a second zecd on the same host. It does not span hosts: two machines sharing one datadir over a network filesystem (NFS, SMB, a Kubernetes ReadWriteMany volume) would each be granted the lock independently. Document this limitation and stamp the lockfile with its holder (hostname and pid) as a breadcrumb for diagnosing a suspected shared-datadir conflict. Co-authored-by: AI --- README.md | 10 ++++++ docs/OPERATIONS.md | 10 ++++++ src/lock.rs | 87 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7e9c02f..ee7b461 100644 --- a/README.md +++ b/README.md @@ -825,6 +825,16 @@ lockfile to delete: if the error appears and no zecd is running, just retry. The `zecd export-ufvk` is exempt (it only reads the wallet DB), so you can export a UFVK while the daemon is running. +> **The datadir must be host-local.** The lock is an OS advisory lock, enforced by the local +> kernel, so it only guards against a second zecd on the *same host*. It does **not** span hosts: +> on a network filesystem shared read-write by two machines (NFS, SMB, a Kubernetes +> `ReadWriteMany` volume), each host's kernel grants the lock independently, so two zecd on +> different hosts could both write the same wallet DB and corrupt it. Put the datadir on a local +> disk, or on a volume mounted so exactly one node can write it (Kubernetes `ReadWriteOnce`), and +> never share it read-write across hosts. For diagnostics the lockfile records the holder's +> `hostname:pid`. There is no cross-host guard today; a shared-storage deployment would need a +> host-independent lease (in the DB or an external lock service), which is not implemented. + ## Security Three key-custody models - the first two mirror bitcoind's unencrypted/encrypted wallet states, diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 643a013..f520206 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -73,6 +73,16 @@ For a cloud deployment: put `keys.toml` (and `identity.txt`, if used) in a read- and point `ZECD_KEYS_FILE` / `ZECD_AGE_IDENTITY` at the mount. `blocks/` is always disposable - excluding it from any image/volume snapshot is the single biggest space win. +> **The data directory must be host-local.** zecd's single-instance lock on `/.lock` is +> an OS advisory lock enforced by the *local* kernel, so it only prevents a second zecd on the +> *same host*. It does **not** span hosts: a datadir on a network filesystem shared read-write by +> two machines (NFS, SMB, a Kubernetes `ReadWriteMany` volume) can be locked independently by each +> host's kernel, so two zecd on different hosts could both write the wallet DB and corrupt it. Keep +> the datadir on a local disk or a single-writer volume (Kubernetes `ReadWriteOnce`), and never +> share it read-write across hosts - the `keys.toml`/`identity.txt` Secret can be mounted read-only +> by many pods, but each pod needs its **own** writable datadir. The lockfile records the holder's +> `hostname:pid` for diagnostics. No cross-host lease is implemented. + ### Bootstrapping a disposable data directory With `[keys] bootstrap_from_keys` (default on), an empty data directory next to a present diff --git a/src/lock.rs b/src/lock.rs index 57c5d2d..b59e675 100644 --- a/src/lock.rs +++ b/src/lock.rs @@ -15,6 +15,21 @@ //! - `export-ufvk` - reads the wallet DB read-only (short-lived connection, WAL), so it is safe //! to run while the daemon holds the datadir; locking it would needlessly refuse a UFVK export //! from a live wallet. +//! +//! # Limitation: the lock is host-local +//! +//! The advisory file lock is enforced by the *local* kernel, so it only guards against a second +//! zecd on the **same host**. It does **not** span hosts: on a network filesystem (NFS, SMB, a +//! Kubernetes `ReadWriteMany` volume) mounted by two machines, each host's kernel grants the lock +//! independently, so two zecd on different hosts sharing one datadir would both believe they hold +//! it and then corrupt the wallet DB with concurrent SQLite writers. **The data directory must +//! therefore be host-local** - a local disk, or a volume mounted so exactly one node can write it +//! (Kubernetes `ReadWriteOnce`) - and must never be shared read-write across hosts. There is no +//! cross-host guard today; a host-independent lease (in the DB, or an external lock service) would +//! be required for a shared-storage deployment and is out of scope. As a diagnostic aid the +//! lockfile records the holder's `hostname:pid`, so an operator investigating suspected corruption +//! can see which host last wrote a datadir. This is a breadcrumb, not enforcement: because the +//! lock doesn't conflict across hosts, a second host would overwrite the stamp, not be refused. use std::fs; use std::path::Path; @@ -29,6 +44,9 @@ use anyhow::{anyhow, Context as _}; /// /// Returns an error if the lock is already held by another process (the "already running" case), /// or on an I/O failure creating/reading the lockfile. +/// +/// NB the lock is **host-local** - see the module docs. It does not protect a datadir shared +/// read-write across hosts (a network volume); that datadir must be host-local. pub fn lock_datadir(datadir: &Path) -> anyhow::Result> { // `init` may be the first thing ever run against this datadir, so make sure it (and thus the // lockfile's parent) exists before creating the lockfile. @@ -36,20 +54,54 @@ pub fn lock_datadir(datadir: &Path) -> anyhow::Result> { .with_context(|| format!("creating data directory {}", datadir.display()))?; let lockfile = datadir.join(".lock"); - // Ensure the lockfile exists on disk before we try to lock it (an empty marker; the advisory - // OS lock on it is what actually enforces single-instance access). - let _ = fs::File::create(&lockfile) + // Ensure the lockfile exists before we try to lock it (the advisory OS lock on it is what + // actually enforces single-instance access). Create-if-absent *without* truncating: a failed + // lock attempt must not clobber the current holder's diagnostic stamp. + fs::OpenOptions::new() + .create(true) + .append(true) + .open(&lockfile) .with_context(|| format!("creating lockfile {}", lockfile.display()))?; - fmutex::try_lock_exclusive_path(&lockfile) + let guard = fmutex::try_lock_exclusive_path(&lockfile) .with_context(|| format!("reading lockfile {}", lockfile.display()))? .ok_or_else(|| { anyhow!( - "Cannot lock data directory {}. Another zecd is already running; \ - the lock clears when it exits, so just retry (no lockfile to delete).", + "Cannot lock data directory {}. Another zecd is already running on this host; \ + the lock clears when it exits, so just retry (no lockfile to delete). \ + Note the lock is host-local: it does NOT protect a datadir shared across hosts \ + (e.g. a network/ReadWriteMany volume) - keep the datadir host-local.", datadir.display() ) - }) + })?; + + // We own the lock: record a best-effort diagnostic stamp so an operator can see which host/pid + // holds (or last held) this datadir. This does NOT enforce anything across hosts - the advisory + // lock is host-local (see the module docs) - it is purely a breadcrumb for diagnosing suspected + // shared-datadir corruption. Written on a separate fd, which does not disturb the lock the + // guard holds; a write failure is ignored (the lock, not the stamp, is what matters). + let stamp = format!("{}:{}\n", hostname(), std::process::id()); + let _ = fs::write(&lockfile, stamp); + + Ok(guard) +} + +/// Best-effort local hostname for the lockfile diagnostic stamp; `"unknown"` if it can't be read. +fn hostname() -> String { + let mut buf = [0u8; 256]; + // SAFETY: `gethostname` writes at most `buf.len()` bytes into `buf` and NUL-terminates when + // there is room. We pass the real buffer length and read back only up to the first NUL. + let rc = unsafe { libc::gethostname(buf.as_mut_ptr() as *mut libc::c_char, buf.len()) }; + if rc != 0 { + return "unknown".to_string(); + } + let end = buf.iter().position(|&b| b == 0).unwrap_or(buf.len()); + let name = String::from_utf8_lossy(&buf[..end]).into_owned(); + if name.is_empty() { + "unknown".to_string() + } else { + name + } } #[cfg(test)] @@ -78,6 +130,27 @@ mod tests { lock_datadir(dir.path()).expect("lock is free after the guard is dropped"); } + #[test] + fn lock_records_a_hostname_pid_diagnostic_stamp() { + // The lockfile carries a best-effort `hostname:pid` breadcrumb for diagnosing suspected + // shared-datadir corruption. It is diagnostic only - the host-local advisory + // lock, not this stamp, is what enforces single-instance access. + let dir = tempfile::tempdir().unwrap(); + let _guard = lock_datadir(dir.path()).expect("lock succeeds"); + + let contents = std::fs::read_to_string(dir.path().join(".lock")).expect("read lockfile"); + let stamp = contents.trim(); + let (host, pid) = stamp + .rsplit_once(':') + .unwrap_or_else(|| panic!("stamp should be hostname:pid, got {stamp:?}")); + assert!(!host.is_empty(), "hostname part is present: {stamp:?}"); + assert_eq!( + pid.parse::().ok(), + Some(std::process::id()), + "the stamp records this process's pid: {stamp:?}" + ); + } + #[test] fn lock_creates_the_datadir_if_missing() { // `zecd init` can run before the datadir exists; locking must create it. From 47a9e7b805aae10f991264ace0cbbad482d13cf0 Mon Sep 17 00:00:00 2001 From: Blake Emerson Date: Sun, 12 Jul 2026 01:23:04 -0500 Subject: [PATCH 4/5] Bump the zebra pin to 6.0.0 Update the compose and regtest zebra pins to the current verified release. Co-authored-by: AI --- .github/workflows/regtest.yml | 2 +- deploy/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml index ca1e665..49a5d2d 100644 --- a/.github/workflows/regtest.yml +++ b/.github/workflows/regtest.yml @@ -71,7 +71,7 @@ on: # environment variables as config overrides and exits with "unknown field" on ones it doesn't # recognise. (The harness also scrubs ZEBRA_* before spawning zebrad, as a second line of defence.) env: - ZEBRAD_IMAGE_PINNED: "5.0.0" + ZEBRAD_IMAGE_PINNED: "6.0.0" concurrency: group: regtest-${{ github.event_name }}-${{ github.ref }} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index b668191..cce5471 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -16,7 +16,7 @@ services: zebra: - image: zfnd/zebra:5.0.0 # pin to a verified release (zebra tags have no 'v' prefix) + image: zfnd/zebra:6.0.0 # pin to a verified release (zebra tags have no 'v' prefix) restart: unless-stopped command: ["zebrad", "-c", "/etc/zebra/zebrad.toml", "start"] volumes: From 4db176cb795ebc77fe03e552df4793130e88c5c7 Mon Sep 17 00:00:00 2001 From: Blake Emerson Date: Sun, 12 Jul 2026 01:54:20 -0500 Subject: [PATCH 5/5] Enable crates.io publishing Set publish to true and add the repository and homepage links, so the crate can be uploaded to crates.io and installed with `cargo install zecd`. Co-authored-by: AI --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a1c2924..411d244 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,8 +4,10 @@ version = "0.4.2" edition = "2021" rust-version = "1.88" license = "MIT OR Apache-2.0" -publish = false +publish = true description = "A Bitcoin-Core-style JSON-RPC server for Orchard-only Zcash, built on librustzcash" +repository = "https://github.com/zecrocks/zecd" +homepage = "https://zecd.org" [[bin]] name = "zecd"