fix(state): audit and self-repair the zakura header store at startup

CHANGELOG.md

@@ -35,6 +35,13 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Fixed
 
+- Zebra can now audit the Zakura header store when the state database opens and
+  self-repair any incoherence (broken linkage, hash↔height index mismatches,
+  gaps with stranded rows above them, stale rows at committed heights) by
+  truncating the Zakura column families to the last coherent height in bounded
+  batches, then letting header sync re-download the truncated suffix. Operators
+  can opt in with `state.repair_zakura_header_store_on_startup = true`; each
+  repair emits a warning and the `state.zakura.header_store.incoherent` metric.
 - Zakura header-store consensus reads now verify store invariants as they
   walk: the difficulty-context walk checks that every stored header is the
   block its hash row names and links to the row below it, and the anchor

zebra-state/src/config.rs

@@ -102,6 +102,15 @@ pub struct Config {
     ///       Zebra's last non-finalized state before it shut down.
     pub should_backup_non_finalized_state: bool,
 
+    /// Whether to audit and repair the Zakura header store when the state database opens.
+    ///
+    /// Set to `false` by default. When this is `true`, writable state opens scan
+    /// the full Zakura header-store frontier and delete any incoherent suffix or
+    /// stale committed-height rows so header sync can re-download them. This can
+    /// be expensive while syncing from genesis, because the header frontier can
+    /// contain millions of rows above the finalized tip.
+    pub repair_zakura_header_store_on_startup: bool,
+
     /// Whether committed full blocks should seed the Zakura header-only store.
     ///
     /// This is enabled by zebrad only for the Zakura v2 path. It is skipped in
@@ -427,6 +436,7 @@ impl Default for Config {
             cache_dir: default_cache_dir(),
             ephemeral: false,
             should_backup_non_finalized_state: true,
+            repair_zakura_header_store_on_startup: false,
             enable_zakura_header_seed_from_committed_blocks: false,
             checkpoint_sync: true,
             vct_fast_sync: true,
@@ -455,11 +465,20 @@ mod tests {
             Config::default().vct_fast_sync,
             "VCT fast sync is enabled by default when checkpoint sync and embedded frontiers are available"
         );
+        assert!(
+            !Config::default().repair_zakura_header_store_on_startup,
+            "Zakura header-store startup repair is opt-in because it scans the full header frontier"
+        );
 
         let archive: Config = toml::from_str(r#"storage_mode = "archive""#)
             .expect("archive storage mode deserializes from a string");
         assert_eq!(archive.storage_mode, StorageMode::Archive);
 
+        let repair_enabled: Config =
+            toml::from_str(r#"repair_zakura_header_store_on_startup = true"#)
+                .expect("startup repair config deserializes from a bool");
+        assert!(repair_enabled.repair_zakura_header_store_on_startup);
+
         let pruned: Config = toml::from_str(r#"storage_mode = "pruned""#)
             .expect("pruned storage mode deserializes from a string");
         assert_eq!(

zebra-state/src/service/finalized_state/zebra_db.rs

@@ -175,6 +175,16 @@ impl ZebraDb {
             )
         }
 
+        // Optionally audit the zakura header store's on-disk invariants and
+        // truncate any incoherent suffix. This can scan a large header frontier
+        // while syncing from genesis, so operators opt in when they need a
+        // startup repair. Read-only instances cannot repair; their reads surface
+        // any corruption as explicit `StoreIncoherentError`s instead.
+        if !read_only && config.repair_zakura_header_store_on_startup {
+            db.audit_and_repair_zakura_header_store()
+                .expect("startup header-store repair write failed: RocksDB is unavailable");
+        }
+
         db.spawn_format_change(format_change);
 
         db

zebra-state/src/service/finalized_state/zebra_db/block.rs

@@ -57,6 +57,8 @@ use crate::{
 #[cfg(feature = "indexer")]
 use crate::request::Spend;
 
+mod startup_audit;
+
 #[cfg(test)]
 mod tests;