diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 53ed127f68..cd014bafe2 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -20760,7 +20760,7 @@ "summary": "Verify a window of the hash-chained decision ledger (resumable via afterSeq)", "responses": { "200": { - "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." + "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, "409": { "description": "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index 7dcb7c84fe..2c62a216ed 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -1882,7 +1882,7 @@ export function buildOpenApiSpec() { tags: ["Public"], summary: "Verify a window of the hash-chained decision ledger (resumable via afterSeq)", responses: { - 200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, + 200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, 409: { description: "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." }, }, }); diff --git a/src/review/decision-record.ts b/src/review/decision-record.ts index 3584396a92..bc8a1dfbbf 100644 --- a/src/review/decision-record.ts +++ b/src/review/decision-record.ts @@ -443,12 +443,28 @@ export async function verifyDecisionLedger( env: Env, afterSeq = 0, limit = 500, -): Promise<{ ok: boolean; checked: number; nextAfterSeq: number | null; tipSeq: number; tipHash: string; totalCount: number; prunedRecords: number; break?: LedgerBreak }> { +): Promise<{ ok: boolean; checked: number; nextAfterSeq: number | null; tipSeq: number; tipHash: string; totalCount: number; prunedRecords: number; contentMismatches: number; break?: LedgerBreak }> { const bounded = Math.max(1, Math.min(1000, limit)); // #9474: rows whose record preimage was legitimately pruned by the published retention policy (see the // missing-record branch below). Surfaced in the result so "the chain is clean but N old preimages are no // longer independently checkable" is an explicit, countable statement rather than silent. let prunedRecords = 0; + // #9850: content mismatches no longer ABORT the scan, they accumulate. Returning at the first one made a + // single unreconcilable row a denial-of-verification for everything after it: found on a live instance + // carrying 83 rows (seq 5-257) from before #9123 replaced the record-overwriting UPDATE with the revision + // scheme, where verification stopped at seq 5 and never examined the remaining 1,649 rows. Real tampering + // at seq 900 would have been invisible behind permanent, historical damage at seq 5. + // + // Safe to continue precisely BECAUSE the chain checks above already passed for this row: sequence, + // predecessor and row_hash all reconciled, so `prevHash` for the next row is sound. A content mismatch says + // "this row's preimage no longer matches what the chain committed to", which is a statement about that row + // alone. A STRUCTURAL break is different -- a sequence gap or predecessor mismatch means everything after + // it is unverifiable, so those still return immediately. + // + // `ok` is still false and the first mismatch is still reported as `break`, so nothing about the verdict is + // softened; the scan simply keeps going and reports how many there are. + let contentMismatches = 0; + let firstContentMismatch: LedgerBreak | null = null; const decisionRecordsPruneCutoff = retentionCutoffIsoForTable("decision_records"); const [totalRow, globalTip, prior] = await Promise.all([ env.DB.prepare("SELECT COUNT(*) AS n FROM decision_ledger").first<{ n: number }>(), @@ -461,7 +477,7 @@ export async function verifyDecisionLedger( const tipSeq = globalTip?.seq ?? 0; const tipHash = globalTip?.rowHash ?? LEDGER_GENESIS_HASH; // `== null` deliberately: D1 drivers disagree on .first() returning null vs undefined for no-row. - if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } }; + if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } }; let prevHash = prior?.rowHash ?? LEDGER_GENESIS_HASH; let expectedSeq = afterSeq + 1; const { results } = await env.DB.prepare( @@ -487,10 +503,10 @@ export async function verifyDecisionLedger( // from) so a call that finds ZERO new rows still has an anchor to reconcile against. let lastVerifiedCreatedAt = prior?.createdAt ?? null; for (const row of results) { - if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } }; - if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "predecessor_mismatch", atSeq: row.seq } }; + if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } }; + if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "predecessor_mismatch", atSeq: row.seq } }; const recomputed = await ledgerRowHash(prevHash, { seq: row.seq, recordId: row.recordId, recordDigest: row.recordDigest, createdAt: row.createdAt }); - if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "row_hash_mismatch", atSeq: row.seq } }; + if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "row_hash_mismatch", atSeq: row.seq } }; // #9078: the promised record/ledger reconciliation — a row_hash chained cleanly can still commit to a // digest whose CONTENT has since been rewritten (or whose preimage is simply gone). Neither is visible to // the chain-only checks above, since those only ever compare ledger rows against each other. @@ -508,7 +524,7 @@ export async function verifyDecisionLedger( if (decisionRecordsPruneCutoff !== null && row.createdAt < decisionRecordsPruneCutoff) { prunedRecords += 1; } else { - return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } }; + return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } }; } } else { let recomputedContentDigest: string | null = null; @@ -518,7 +534,10 @@ export async function verifyDecisionLedger( // Unparseable record_json is itself proof the content no longer matches whatever the chain committed to. recomputedContentDigest = null; } - if (recomputedContentDigest !== row.recordDigest) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "content_mismatch", atSeq: row.seq, recordId: row.recordId } }; + if (recomputedContentDigest !== row.recordDigest) { + contentMismatches += 1; + firstContentMismatch ??= { kind: "content_mismatch", atSeq: row.seq, recordId: row.recordId }; + } } prevHash = row.rowHash; lastVerifiedCreatedAt = row.createdAt; @@ -569,6 +588,7 @@ export async function verifyDecisionLedger( tipHash, totalCount, prunedRecords, + contentMismatches, break: orphan.createdAt > lastVerifiedCreatedAt ? { kind: "short_tail", atSeq: expectedSeq - 1 } @@ -576,7 +596,12 @@ export async function verifyDecisionLedger( }; } } - return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords }; + // A content mismatch is still a FAILED verification -- the scan continuing does not soften the verdict, it + // only stops one bad row from hiding every row after it. The first is reported as `break` exactly as before. + if (firstContentMismatch !== null) { + return { ok: false, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: firstContentMismatch }; + } + return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches }; } /** One ledger row, exactly as chained -- the shape `GET /v1/public/decision-ledger/row/:seq` returns. */ diff --git a/test/unit/decision-record.test.ts b/test/unit/decision-record.test.ts index 76cce78e10..f5bd9b29e8 100644 --- a/test/unit/decision-record.test.ts +++ b/test/unit/decision-record.test.ts @@ -384,7 +384,7 @@ describe("decision ledger (#8837)", () => { it("verifying a completely empty ledger returns ok:true with a zero tip, and skips the tail-truncation check (nothing to anchor against yet)", async () => { const env = createTestEnv(); const verified = await verifyDecisionLedger(env); - expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0 }); + expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0, contentMismatches: 0 }); }); it("TAIL TRUNCATION now breaks verify instead of passing clean (#9122): dropping the newest ledger rows leaves an orphaned decision_records tail", async () => { @@ -640,3 +640,76 @@ describe("verifier vs absence (#9474 pruned records, #9489 grace + interior orph expect(verified.break).toBeUndefined(); }); }); + +// #9850: a content mismatch used to ABORT verification, so one unreconcilable row was a denial-of- +// verification for every row after it. Found live: 83 rows (seq 5-257) left by the record-overwriting UPDATE +// that #9123 replaced, with verification stopping at seq 5 and never examining the remaining 1,649 rows -- +// real tampering at seq 900 would have been invisible behind permanent historical damage at seq 5. +describe("content mismatches do not mask later rows (#9850)", () => { + const seedChained = async (env: Env, count: number) => { + for (let i = 1; i <= count; i += 1) { + const { record, recordDigest } = await buildDecisionRecord(recordInput({ pullNumber: i })); + await persistDecisionRecord(env, record, recordDigest); + } + }; + /** Rewrite one record's stored body so its digest no longer matches what the chain committed to -- exactly + * the state the pre-#9123 UPDATE left behind. */ + const corruptRecordAt = async (env: Env, seq: number) => { + const row = await env.DB.prepare("SELECT record_id AS recordId FROM decision_ledger WHERE seq = ?").bind(seq).first<{ recordId: string }>(); + await env.DB.prepare("UPDATE decision_records SET record_json = ? WHERE id = ?").bind('{"tampered":true}', row!.recordId).run(); + }; + + it("REGRESSION: keeps verifying past a content mismatch instead of stopping at it", async () => { + const env = createTestEnv(); + await seedChained(env, 6); + await corruptRecordAt(env, 2); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(false); + expect(verified.break).toMatchObject({ kind: "content_mismatch", atSeq: 2 }); + // The whole window was examined, not just the two rows before the break. + expect(verified.checked).toBe(6); + }); + + it("REGRESSION: a STRUCTURAL break after a content mismatch is still found -- it used to be unreachable", async () => { + // This is the security consequence, not just a cosmetic one: a historical unreconcilable row at seq 2 hid + // a genuinely broken chain further along. + const env = createTestEnv(); + await seedChained(env, 6); + await corruptRecordAt(env, 2); + await env.DB.prepare("UPDATE decision_ledger SET row_hash = ? WHERE seq = 5").bind("f".repeat(64)).run(); + + const verified = await verifyDecisionLedger(env); + + // The structural break wins the `break` slot: everything after it is unverifiable, so it is the more + // serious finding and the scan stops there. + expect(verified.break).toMatchObject({ kind: "row_hash_mismatch", atSeq: 5 }); + expect(verified.contentMismatches).toBe(1); // ...and the earlier content mismatch is still counted + }); + + it("counts EVERY content mismatch, not just the first", async () => { + const env = createTestEnv(); + await seedChained(env, 6); + for (const seq of [2, 4, 5]) await corruptRecordAt(env, seq); + + const verified = await verifyDecisionLedger(env); + + expect(verified.contentMismatches).toBe(3); + expect(verified.break).toMatchObject({ atSeq: 2 }); // the first is still what `break` names + }); + + it("INVARIANT: the verdict is not softened -- any mismatch still means ok:false", async () => { + const env = createTestEnv(); + await seedChained(env, 3); + await corruptRecordAt(env, 3); + expect((await verifyDecisionLedger(env)).ok).toBe(false); + }); + + it("reports contentMismatches: 0 on a clean chain, so the field is a fact and not only an error signal", async () => { + const env = createTestEnv(); + await seedChained(env, 4); + const verified = await verifyDecisionLedger(env); + expect(verified).toMatchObject({ ok: true, contentMismatches: 0 }); + }); +});