Release 0.4.3 candidate: audit fixes + zebra 6.0.0

.github/workflows/regtest.yml

@@ -71,7 +71,7 @@ on:
 # environment variables as config overrides and exits with "unknown field" on ones it doesn't
 # recognise. (The harness also scrubs ZEBRA_* before spawning zebrad, as a second line of defence.)
 env:
-  ZEBRAD_IMAGE_PINNED: "5.0.0"
+  ZEBRAD_IMAGE_PINNED: "6.0.0"
 
 concurrency:
   group: regtest-${{ github.event_name }}-${{ github.ref }}

Cargo.toml

@@ -4,8 +4,10 @@ version = "0.4.2"
 edition = "2021"
 rust-version = "1.88"
 license = "MIT OR Apache-2.0"
-publish = false
+publish = true
 description = "A Bitcoin-Core-style JSON-RPC server for Orchard-only Zcash, built on librustzcash"
+repository = "https://github.com/zecrocks/zecd"
+homepage = "https://zecd.org"
 
 [[bin]]
 name = "zecd"

README.md

@@ -197,8 +197,9 @@ format = "text"                  # "text" | "json" (structured, for log aggregat
 enabled = true
 bind = "127.0.0.1"               # set 0.0.0.0 for Kubernetes/LB probes
 port = 9233
-readiness = "connected"          # "connected" (ready when the backend is live, past birthday)
-                                 # or "synced" (ready only once scanned to near the tip)
+readiness = "synced"             # default; ready only once scanned to near the tip, so reads
+                                 # are never stale. "connected" = ready when the backend is live
+                                 # (past birthday) but the scan may still be catching up.
 max_scan_lag = 4                 # "synced" mode: max chain_tip - fully_scanned block gap
 ```
 
@@ -268,18 +269,23 @@ With `[health] enabled` (default), zecd serves unauthenticated probes on a separ
 
 - `GET /healthz`: liveness. `200 ok` while the process is running.
 - `GET /readyz`: readiness. `200`/`503`, gated by `[health] readiness`:
-  - `"connected"` (default): ready as soon as the backend is connected and its chain tip is past
-    the wallet's birthday height (a sanity check that we're on the right, live network). It does
+  - `"synced"` (**default**): ready only once every wallet is connected, within
+    `[health] max_scan_lag` blocks of the chain tip, **and** with an empty transaction-enhancement
+    backlog. Strict - a from-birthday restore stays not-ready until it has scanned to its own funds
+    *and* finished backfilling memos (see below). This is the default so a client routed by
+    readiness never sees an empty or stale balance/history as authoritative while the wallet is
+    still scanning: an exchange or any balance-sensitive deployment should keep it.
+  - `"connected"`: ready as soon as the backend is connected and its chain tip is past the
+    wallet's birthday height (a sanity check that we're on the right, live network). It does
     **not** wait for the wallet to finish scanning, so RPC clients can reach zecd while it catches
-    up and readiness doesn't flap during a long sync - reads may lag the tip until caught up.
-  - `"synced"`: ready only once every wallet is connected, within `[health] max_scan_lag`
-    blocks of the chain tip, **and** with an empty transaction-enhancement backlog. Strict - a
-    from-birthday restore stays not-ready until it has scanned to its own funds *and* finished
-    backfilling memos (see below).
-
-  Body is JSON with per-wallet detail; when not ready it carries a `reason` (`"upstream_down"`,
-  `"actor_down"`, `"enhancing"`, or `"syncing"`) so alerting can tell an unreachable zebra apart
-  from a dead writer, from backfilling memos, from normal block catch-up.
+    up and readiness doesn't flap during a long sync - but reads may lag the tip until caught up.
+    Choose this only when reachability matters more than balance freshness; the per-wallet
+    `scan_lag` in the JSON body (and `/status`) shows how far behind reads may be.
+
+  Body is JSON with per-wallet detail (including `scan_lag`, the `chain_tip - fully_scanned` block
+  gap); when not ready it carries a `reason` (`"upstream_down"`, `"actor_down"`, `"enhancing"`, or
+  `"syncing"`) so alerting can tell an unreachable zebra apart from a dead writer, from backfilling
+  memos, from normal block catch-up.
 - `GET /status`: JSON snapshot of per-wallet sync state, including the active `server` endpoint,
   `conn_state` (`down` | `syncing` | `ready`), and the per-wallet `pending_enhancements` count.
   `getpeerinfo` reflects the same active upstream.
@@ -819,6 +825,16 @@ lockfile to delete: if the error appears and no zecd is running, just retry. The
 `zecd export-ufvk` is exempt (it only reads the wallet DB), so you can export a UFVK while the
 daemon is running.
 
+> **The datadir must be host-local.** The lock is an OS advisory lock, enforced by the local
+> kernel, so it only guards against a second zecd on the *same host*. It does **not** span hosts:
+> on a network filesystem shared read-write by two machines (NFS, SMB, a Kubernetes
+> `ReadWriteMany` volume), each host's kernel grants the lock independently, so two zecd on
+> different hosts could both write the same wallet DB and corrupt it. Put the datadir on a local
+> disk, or on a volume mounted so exactly one node can write it (Kubernetes `ReadWriteOnce`), and
+> never share it read-write across hosts. For diagnostics the lockfile records the holder's
+> `hostname:pid`. There is no cross-host guard today; a shared-storage deployment would need a
+> host-independent lease (in the DB or an external lock service), which is not implemented.
+
 ## Security
 
 Three key-custody models - the first two mirror bitcoind's unencrypted/encrypted wallet states,

deploy/docker-compose.yml

@@ -16,7 +16,7 @@
 
 services:
   zebra:
-    image: zfnd/zebra:5.0.0                 # pin to a verified release (zebra tags have no 'v' prefix)
+    image: zfnd/zebra:6.0.0                 # pin to a verified release (zebra tags have no 'v' prefix)
     restart: unless-stopped
     command: ["zebrad", "-c", "/etc/zebra/zebrad.toml", "start"]
     volumes:

deploy/zecd.toml

@@ -38,16 +38,20 @@ interval_secs = 20
 bind = "0.0.0.0"
 port = 9233
 # What /readyz gates on:
-#   "connected" (default) - ready as soon as the backend is connected and its chain tip is past
-#       this wallet's birthday height (a sanity check that we're on the right, live network). Does
-#       NOT wait for the wallet to finish scanning, so RPC clients can reach zecd while it catches
-#       up, and readiness doesn't flap during a long sync. Reads may lag the tip until caught up.
-#   "synced" - ready only once the wallet is connected AND within `max_scan_lag` blocks of the
-#       chain tip. Strict: a from-birthday restore stays not-ready until it catches up. The height
-#       gap is the meaningful "caught up" signal - librustzcash's note-weighted progress ratio
-#       reaches 1.0 while historical ranges are still being scanned, so it would report ready long
-#       before the wallet has scanned to its own funds.
-# readiness = "connected"
+#   "synced" (default) - ready only once the wallet is connected AND within `max_scan_lag` blocks
+#       of the chain tip (and the enhancement backlog has drained). Strict: a from-birthday restore
+#       stays not-ready until it catches up. This is the default so a client routed by readiness
+#       never sees an empty or stale balance/history as authoritative while the wallet is still
+#       scanning - keep it for any balance-sensitive deployment. The height gap is the meaningful
+#       "caught up" signal - librustzcash's note-weighted progress ratio reaches 1.0 while
+#       historical ranges are still being scanned, so it would report ready long before the wallet
+#       has scanned to its own funds.
+#   "connected" - ready as soon as the backend is connected and its chain tip is past this wallet's
+#       birthday height (a sanity check that we're on the right, live network). Does NOT wait for
+#       the wallet to finish scanning, so RPC clients can reach zecd while it catches up, and
+#       readiness doesn't flap during a long sync. Reads may lag the tip until caught up (the
+#       per-wallet `scan_lag` in /readyz and /status shows by how much).
+# readiness = "synced"
 # max_scan_lag only applies in "synced" mode.
 # max_scan_lag = 4
 

docs/OPERATIONS.md

@@ -73,6 +73,16 @@ For a cloud deployment: put `keys.toml` (and `identity.txt`, if used) in a read-
 and point `ZECD_KEYS_FILE` / `ZECD_AGE_IDENTITY` at the mount. `blocks/` is always disposable -
 excluding it from any image/volume snapshot is the single biggest space win.
 
+> **The data directory must be host-local.** zecd's single-instance lock on `<datadir>/.lock` is
+> an OS advisory lock enforced by the *local* kernel, so it only prevents a second zecd on the
+> *same host*. It does **not** span hosts: a datadir on a network filesystem shared read-write by
+> two machines (NFS, SMB, a Kubernetes `ReadWriteMany` volume) can be locked independently by each
+> host's kernel, so two zecd on different hosts could both write the wallet DB and corrupt it. Keep
+> the datadir on a local disk or a single-writer volume (Kubernetes `ReadWriteOnce`), and never
+> share it read-write across hosts - the `keys.toml`/`identity.txt` Secret can be mounted read-only
+> by many pods, but each pod needs its **own** writable datadir. The lockfile records the holder's
+> `hostname:pid` for diagnostics. No cross-host lease is implemented.
+
 ### Bootstrapping a disposable data directory
 
 With `[keys] bootstrap_from_keys` (default on), an empty data directory next to a present
@@ -146,18 +156,24 @@ it reveals the wallet's entire transaction graph, though it cannot spend.
 ## Monitoring
 
 - `GET /healthz` (health port, default 9233) - liveness.
-- `GET /readyz` - readiness, gated by `[health] readiness`. In `"connected"` mode
-  (default) it's 200 as soon as zebra is connected and its tip is past the wallet's
-  birthday - it does NOT wait for the scan to finish, so it stays ready (no flapping)
-  while the wallet catches up. In `"synced"` mode it's 200 only once connected,
-  within `[health] max_scan_lag` blocks of the tip, AND with the transaction-enhancement
-  backlog drained (see below). When 503, the body's `reason` distinguishes `upstream_down`
-  (zebra unreachable - page someone) from `syncing` (normal block catch-up), `enhancing`
-  (scanned to tip, still backfilling memos), and `actor_down` (a dead writer - restart
-  the process).
+- `GET /readyz` - readiness, gated by `[health] readiness`. In `"synced"` mode
+  (**default**) it's 200 only once connected, within `[health] max_scan_lag` blocks of the
+  tip, AND with the transaction-enhancement backlog drained (see below) - so a client
+  routed by readiness never sees an empty or stale balance/history as authoritative while
+  the wallet is still scanning. **Keep this default for any balance-sensitive deployment**
+  (an exchange, an accounting integration): a from-birthday restore or fresh sync stays 503
+  until it has actually caught up to its own funds. In `"connected"` mode it's 200 as soon as
+  zebra is connected and its tip is past the wallet's birthday - it does NOT wait for the scan
+  to finish, so it stays ready (no flapping) while the wallet catches up, but reads may lag the
+  tip; choose it only when reachability matters more than balance freshness, and watch the
+  per-wallet `scan_lag` to see how far behind reads may be. When 503, the body's `reason`
+  distinguishes `upstream_down` (zebra unreachable - page someone) from `syncing` (normal block
+  catch-up), `enhancing` (scanned to tip, still backfilling memos), and `actor_down` (a dead
+  writer - restart the process).
 - `GET /status` - per-wallet sync state, active upstream endpoint, `conn_state`
-  (`down` | `syncing` | `ready`), and `pending_enhancements` (the enhancement backlog).
-  Alert if `conn_state` stays `down`.
+  (`down` | `syncing` | `ready`), `scan_lag` (the `chain_tip - fully_scanned` block gap - how
+  far behind reads may be, reported in both readiness modes), and `pending_enhancements` (the
+  enhancement backlog). Alert if `conn_state` stays `down`.
 - **Enhancement backlog (`pending_enhancements`).** The compact-block scan reaching the
   tip (`scan_progress: 1.0`, `scanning: false` for the block scan) is NOT "ready to serve
   full history". Compact blocks carry no memos, so a per-transaction enhancement pass then

src/config.rs

@@ -164,12 +164,14 @@ impl Default for PoolsConfig {
 pub enum ReadinessMode {
     /// Ready only once the wallet has actually scanned to (near) the chain tip: connected and
     /// within `max_scan_lag` blocks of the tip. Strict - a from-birthday restore stays "not
-    /// ready" until it catches up. Use when a client must not see stale balances/history.
+    /// ready" until it catches up. **This is the default**: a client must not see an empty or
+    /// stale balance/history as authoritative while the wallet is still scanning.
     Synced,
     /// Ready as soon as the backend is connected and its chain tip is past the wallet's birthday
     /// (a cheap sanity check that we're talking to the right, live network). Does NOT wait for
     /// the wallet to finish scanning, so RPC clients can reach zecd while it catches up - at the
-    /// cost of reads possibly lagging the tip. Avoids readiness flapping during long scans.
+    /// cost of reads possibly lagging the tip. Avoids readiness flapping during long scans. Opt in
+    /// (`readiness = "connected"`) only when reachability matters more than balance freshness.
     Connected,
 }
 
@@ -1066,14 +1068,17 @@ impl AppConfig {
                 .parse()
                 .context("parsing health bind address")?,
             port: health_file.port.unwrap_or(defaults.health_port),
-            // Default to "connected": be generous about how synced we are so RPC clients can
-            // reach zecd in most situations, and avoid readiness flapping during long scans.
+            // Default to "synced": `/readyz` reports ready only once the wallet has actually
+            // scanned to (near) the tip, so a client routed by readiness never sees an empty or
+            // stale balance/history as authoritative during initial sync or a `--restore` (audit
+            // 3.15). Deployments that prefer reachability-over-freshness - reaching zecd while it
+            // catches up, at the cost of possibly-lagging reads - can set `readiness = "connected"`.
             readiness: health_file
                 .readiness
                 .as_deref()
                 .map(ReadinessMode::parse)
                 .transpose()?
-                .unwrap_or(ReadinessMode::Connected),
+                .unwrap_or(ReadinessMode::Synced),
             max_scan_lag: health_file.max_scan_lag.unwrap_or(4),
         };
 
@@ -1694,6 +1699,30 @@ mod tests {
         assert!(cfg.backend.allow_remote_cleartext);
     }
 
+    #[test]
+    fn readiness_defaults_to_synced_and_is_overridable() {
+        use clap::Parser as _;
+        // Absent [health] readiness resolves to Synced: `/readyz` waits for the scan so a client
+        // routed by readiness never trusts an empty/stale balance mid-sync.
+        let cli = Cli::parse_from(["zecd"]);
+        let cfg = AppConfig::resolve(&cli).unwrap();
+        assert_eq!(cfg.health.readiness, ReadinessMode::Synced);
+
+        // A [health] block that omits `readiness` still defaults to Synced...
+        let dir = tempfile::tempdir().unwrap();
+        let conf = dir.path().join("zecd.toml");
+        std::fs::write(&conf, "[health]\nport = 9999\n").unwrap();
+        let cli = Cli::parse_from(["zecd", "--conf", conf.to_str().unwrap()]);
+        let cfg = AppConfig::resolve(&cli).unwrap();
+        assert_eq!(cfg.health.readiness, ReadinessMode::Synced);
+
+        // ...and the lenient mode is still opt-in from the file.
+        std::fs::write(&conf, "[health]\nreadiness = \"connected\"\n").unwrap();
+        let cli = Cli::parse_from(["zecd", "--conf", conf.to_str().unwrap()]);
+        let cfg = AppConfig::resolve(&cli).unwrap();
+        assert_eq!(cfg.health.readiness, ReadinessMode::Connected);
+    }
+
     #[test]
     fn sync_interval_secs_is_clamped_to_at_least_one() {
         use clap::Parser as _;

src/health.rs

@@ -147,6 +147,15 @@ fn snapshot(state: &AppState) -> Snapshot {
             if !st.scanning && st.pending_enhancements > 0 {
                 any_enhancing = true;
             }
+            // The block-height gap between the tip and the last fully-scanned height - the
+            // meaningful "how far behind" signal. Surfaced on every wallet regardless of the
+            // configured readiness mode, so an operator on `readiness = "connected"` (which reports
+            // ready before the scan finishes) can still see how stale the wallet's reads may be.
+            // `null` until both heights are known.
+            let scan_lag = match (st.chain_tip, st.fully_scanned) {
+                (Some(tip), Some(scanned)) => Some(tip.saturating_sub(scanned)),
+                _ => None,
+            };
             wallets.insert(
                 name,
                 json!({
@@ -156,6 +165,7 @@ fn snapshot(state: &AppState) -> Snapshot {
                     "conn_state": st.conn_state.as_str(),
                     "chain_tip": st.chain_tip,
                     "fully_scanned": st.fully_scanned,
+                    "scan_lag": scan_lag,
                     "birthday": st.birthday,
                     "scan_progress": st.scan_progress,
                     "scanning": st.scanning,