fix(ledger): stop one unreconcilable row from hiding every row after it

apps/loopover-ui/public/openapi.json

@@ -20760,7 +20760,7 @@
         "summary": "Verify a window of the hash-chained decision ledger (resumable via afterSeq)",
         "responses": {
           "200": {
-            "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)."
+            "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)."
           },
           "409": {
             "description": "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering."

src/openapi/spec.ts

@@ -1882,7 +1882,7 @@ export function buildOpenApiSpec() {
     tags: ["Public"],
     summary: "Verify a window of the hash-chained decision ledger (resumable via afterSeq)",
     responses: {
-      200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." },
+      200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." },
       409: { description: "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." },
     },
   });

src/review/decision-record.ts

@@ -443,12 +443,28 @@ export async function verifyDecisionLedger(
   env: Env,
   afterSeq = 0,
   limit = 500,
-): Promise<{ ok: boolean; checked: number; nextAfterSeq: number | null; tipSeq: number; tipHash: string; totalCount: number; prunedRecords: number; break?: LedgerBreak }> {
+): Promise<{ ok: boolean; checked: number; nextAfterSeq: number | null; tipSeq: number; tipHash: string; totalCount: number; prunedRecords: number; contentMismatches: number; break?: LedgerBreak }> {
   const bounded = Math.max(1, Math.min(1000, limit));
   // #9474: rows whose record preimage was legitimately pruned by the published retention policy (see the
   // missing-record branch below). Surfaced in the result so "the chain is clean but N old preimages are no
   // longer independently checkable" is an explicit, countable statement rather than silent.
   let prunedRecords = 0;
+  // #9850: content mismatches no longer ABORT the scan, they accumulate. Returning at the first one made a
+  // single unreconcilable row a denial-of-verification for everything after it: found on a live instance
+  // carrying 83 rows (seq 5-257) from before #9123 replaced the record-overwriting UPDATE with the revision
+  // scheme, where verification stopped at seq 5 and never examined the remaining 1,649 rows. Real tampering
+  // at seq 900 would have been invisible behind permanent, historical damage at seq 5.
+  //
+  // Safe to continue precisely BECAUSE the chain checks above already passed for this row: sequence,
+  // predecessor and row_hash all reconciled, so `prevHash` for the next row is sound. A content mismatch says
+  // "this row's preimage no longer matches what the chain committed to", which is a statement about that row
+  // alone. A STRUCTURAL break is different -- a sequence gap or predecessor mismatch means everything after
+  // it is unverifiable, so those still return immediately.
+  //
+  // `ok` is still false and the first mismatch is still reported as `break`, so nothing about the verdict is
+  // softened; the scan simply keeps going and reports how many there are.
+  let contentMismatches = 0;
+  let firstContentMismatch: LedgerBreak | null = null;
   const decisionRecordsPruneCutoff = retentionCutoffIsoForTable("decision_records");
   const [totalRow, globalTip, prior] = await Promise.all([
     env.DB.prepare("SELECT COUNT(*) AS n FROM decision_ledger").first<{ n: number }>(),
@@ -461,7 +477,7 @@ export async function verifyDecisionLedger(
   const tipSeq = globalTip?.seq ?? 0;
   const tipHash = globalTip?.rowHash ?? LEDGER_GENESIS_HASH;
   // `== null` deliberately: D1 drivers disagree on .first() returning null vs undefined for no-row.
-  if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } };
+  if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } };
   let prevHash = prior?.rowHash ?? LEDGER_GENESIS_HASH;
   let expectedSeq = afterSeq + 1;
   const { results } = await env.DB.prepare(
@@ -487,10 +503,10 @@ export async function verifyDecisionLedger(
   // from) so a call that finds ZERO new rows still has an anchor to reconcile against.
   let lastVerifiedCreatedAt = prior?.createdAt ?? null;
   for (const row of results) {
-    if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } };
-    if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "predecessor_mismatch", atSeq: row.seq } };
+    if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } };
+    if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "predecessor_mismatch", atSeq: row.seq } };
     const recomputed = await ledgerRowHash(prevHash, { seq: row.seq, recordId: row.recordId, recordDigest: row.recordDigest, createdAt: row.createdAt });
-    if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "row_hash_mismatch", atSeq: row.seq } };
+    if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "row_hash_mismatch", atSeq: row.seq } };
     // #9078: the promised record/ledger reconciliation — a row_hash chained cleanly can still commit to a
     // digest whose CONTENT has since been rewritten (or whose preimage is simply gone). Neither is visible to
     // the chain-only checks above, since those only ever compare ledger rows against each other.
@@ -508,7 +524,7 @@ export async function verifyDecisionLedger(
       if (decisionRecordsPruneCutoff !== null && row.createdAt < decisionRecordsPruneCutoff) {
         prunedRecords += 1;
       } else {
-        return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } };
+        return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } };
       }
     } else {
       let recomputedContentDigest: string | null = null;
@@ -518,7 +534,10 @@ export async function verifyDecisionLedger(
         // Unparseable record_json is itself proof the content no longer matches whatever the chain committed to.
         recomputedContentDigest = null;
       }
-      if (recomputedContentDigest !== row.recordDigest) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "content_mismatch", atSeq: row.seq, recordId: row.recordId } };
+      if (recomputedContentDigest !== row.recordDigest) {
+        contentMismatches += 1;
+        firstContentMismatch ??= { kind: "content_mismatch", atSeq: row.seq, recordId: row.recordId };
+      }
     }
     prevHash = row.rowHash;
     lastVerifiedCreatedAt = row.createdAt;
@@ -569,14 +588,20 @@ export async function verifyDecisionLedger(
         tipHash,
         totalCount,
         prunedRecords,
+        contentMismatches,
         break:
           orphan.createdAt > lastVerifiedCreatedAt
             ? { kind: "short_tail", atSeq: expectedSeq - 1 }
             : { kind: "unchained_record", atSeq: expectedSeq - 1, recordId: orphan.id },
       };
     }
   }
-  return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords };
+  // A content mismatch is still a FAILED verification -- the scan continuing does not soften the verdict, it
+  // only stops one bad row from hiding every row after it. The first is reported as `break` exactly as before.
+  if (firstContentMismatch !== null) {
+    return { ok: false, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: firstContentMismatch };
+  }
+  return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches };
 }
 
 /** One ledger row, exactly as chained -- the shape `GET /v1/public/decision-ledger/row/:seq` returns. */

test/unit/decision-record.test.ts

@@ -384,7 +384,7 @@ describe("decision ledger (#8837)", () => {
   it("verifying a completely empty ledger returns ok:true with a zero tip, and skips the tail-truncation check (nothing to anchor against yet)", async () => {
     const env = createTestEnv();
     const verified = await verifyDecisionLedger(env);
-    expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0 });
+    expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0, contentMismatches: 0 });
   });
 
   it("TAIL TRUNCATION now breaks verify instead of passing clean (#9122): dropping the newest ledger rows leaves an orphaned decision_records tail", async () => {
@@ -640,3 +640,76 @@ describe("verifier vs absence (#9474 pruned records, #9489 grace + interior orph
     expect(verified.break).toBeUndefined();
   });
 });
+
+// #9850: a content mismatch used to ABORT verification, so one unreconcilable row was a denial-of-
+// verification for every row after it. Found live: 83 rows (seq 5-257) left by the record-overwriting UPDATE
+// that #9123 replaced, with verification stopping at seq 5 and never examining the remaining 1,649 rows --
+// real tampering at seq 900 would have been invisible behind permanent historical damage at seq 5.
+describe("content mismatches do not mask later rows (#9850)", () => {
+  const seedChained = async (env: Env, count: number) => {
+    for (let i = 1; i <= count; i += 1) {
+      const { record, recordDigest } = await buildDecisionRecord(recordInput({ pullNumber: i }));
+      await persistDecisionRecord(env, record, recordDigest);
+    }
+  };
+  /** Rewrite one record's stored body so its digest no longer matches what the chain committed to -- exactly
+   *  the state the pre-#9123 UPDATE left behind. */
+  const corruptRecordAt = async (env: Env, seq: number) => {
+    const row = await env.DB.prepare("SELECT record_id AS recordId FROM decision_ledger WHERE seq = ?").bind(seq).first<{ recordId: string }>();
+    await env.DB.prepare("UPDATE decision_records SET record_json = ? WHERE id = ?").bind('{"tampered":true}', row!.recordId).run();
+  };
+
+  it("REGRESSION: keeps verifying past a content mismatch instead of stopping at it", async () => {
+    const env = createTestEnv();
+    await seedChained(env, 6);
+    await corruptRecordAt(env, 2);
+
+    const verified = await verifyDecisionLedger(env);
+
+    expect(verified.ok).toBe(false);
+    expect(verified.break).toMatchObject({ kind: "content_mismatch", atSeq: 2 });
+    // The whole window was examined, not just the two rows before the break.
+    expect(verified.checked).toBe(6);
+  });
+
+  it("REGRESSION: a STRUCTURAL break after a content mismatch is still found -- it used to be unreachable", async () => {
+    // This is the security consequence, not just a cosmetic one: a historical unreconcilable row at seq 2 hid
+    // a genuinely broken chain further along.
+    const env = createTestEnv();
+    await seedChained(env, 6);
+    await corruptRecordAt(env, 2);
+    await env.DB.prepare("UPDATE decision_ledger SET row_hash = ? WHERE seq = 5").bind("f".repeat(64)).run();
+
+    const verified = await verifyDecisionLedger(env);
+
+    // The structural break wins the `break` slot: everything after it is unverifiable, so it is the more
+    // serious finding and the scan stops there.
+    expect(verified.break).toMatchObject({ kind: "row_hash_mismatch", atSeq: 5 });
+    expect(verified.contentMismatches).toBe(1); // ...and the earlier content mismatch is still counted
+  });
+
+  it("counts EVERY content mismatch, not just the first", async () => {
+    const env = createTestEnv();
+    await seedChained(env, 6);
+    for (const seq of [2, 4, 5]) await corruptRecordAt(env, seq);
+
+    const verified = await verifyDecisionLedger(env);
+
+    expect(verified.contentMismatches).toBe(3);
+    expect(verified.break).toMatchObject({ atSeq: 2 }); // the first is still what `break` names
+  });
+
+  it("INVARIANT: the verdict is not softened -- any mismatch still means ok:false", async () => {
+    const env = createTestEnv();
+    await seedChained(env, 3);
+    await corruptRecordAt(env, 3);
+    expect((await verifyDecisionLedger(env)).ok).toBe(false);
+  });
+
+  it("reports contentMismatches: 0 on a clean chain, so the field is a fact and not only an error signal", async () => {
+    const env = createTestEnv();
+    await seedChained(env, 4);
+    const verified = await verifyDecisionLedger(env);
+    expect(verified).toMatchObject({ ok: true, contentMismatches: 0 });
+  });
+});