diff --git a/content/skills/loaf-reference/SKILL.md b/content/skills/loaf-reference/SKILL.md index bc4d5a9ab..b4273ca60 100644 --- a/content/skills/loaf-reference/SKILL.md +++ b/content/skills/loaf-reference/SKILL.md @@ -62,7 +62,7 @@ Names and one-line purposes only. Run `loaf --help` for options, argum | `loaf docs` | Manage docs/ indexing | index | | `loaf change` | Shape-first Change artifacts: git-canonical work context under docs/changes/ | init, check, list | | `loaf render` | Maintain committed durable Markdown renders | sweep | -| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | +| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate alias-orphans, migrate journal-duplicates, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | | `loaf journal` | Record and read the project-scoped journal (the durable record across all conversations) | log, recent, search, show, context, export, defer | | `loaf project` | Manage durable project identity | list, show, identity, rename, move, delete | | `loaf migrate` | Run native migration workflows | markdown, storage-home, schema, lifecycle-statuses, journal-first, worktree-storage | diff --git a/dist/amp/skills/loaf-reference/SKILL.md b/dist/amp/skills/loaf-reference/SKILL.md index 30e68ef1a..b053dfb4c 100644 --- a/dist/amp/skills/loaf-reference/SKILL.md +++ b/dist/amp/skills/loaf-reference/SKILL.md @@ -68,7 +68,7 @@ Names and one-line purposes only. Run `loaf --help` for options, argum | `loaf docs` | Manage docs/ indexing | index | | `loaf change` | Shape-first Change artifacts: git-canonical work context under docs/changes/ | init, check, list | | `loaf render` | Maintain committed durable Markdown renders | sweep | -| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | +| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate alias-orphans, migrate journal-duplicates, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | | `loaf journal` | Record and read the project-scoped journal (the durable record across all conversations) | log, recent, search, show, context, export, defer | | `loaf project` | Manage durable project identity | list, show, identity, rename, move, delete | | `loaf migrate` | Run native migration workflows | markdown, storage-home, schema, lifecycle-statuses, journal-first, worktree-storage | diff --git a/dist/codex/skills/loaf-reference/SKILL.md b/dist/codex/skills/loaf-reference/SKILL.md index 30e68ef1a..b053dfb4c 100644 --- a/dist/codex/skills/loaf-reference/SKILL.md +++ b/dist/codex/skills/loaf-reference/SKILL.md @@ -68,7 +68,7 @@ Names and one-line purposes only. Run `loaf --help` for options, argum | `loaf docs` | Manage docs/ indexing | index | | `loaf change` | Shape-first Change artifacts: git-canonical work context under docs/changes/ | init, check, list | | `loaf render` | Maintain committed durable Markdown renders | sweep | -| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | +| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate alias-orphans, migrate journal-duplicates, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | | `loaf journal` | Record and read the project-scoped journal (the durable record across all conversations) | log, recent, search, show, context, export, defer | | `loaf project` | Manage durable project identity | list, show, identity, rename, move, delete | | `loaf migrate` | Run native migration workflows | markdown, storage-home, schema, lifecycle-statuses, journal-first, worktree-storage | diff --git a/dist/cursor/skills/loaf-reference/SKILL.md b/dist/cursor/skills/loaf-reference/SKILL.md index 30e68ef1a..b053dfb4c 100644 --- a/dist/cursor/skills/loaf-reference/SKILL.md +++ b/dist/cursor/skills/loaf-reference/SKILL.md @@ -68,7 +68,7 @@ Names and one-line purposes only. Run `loaf --help` for options, argum | `loaf docs` | Manage docs/ indexing | index | | `loaf change` | Shape-first Change artifacts: git-canonical work context under docs/changes/ | init, check, list | | `loaf render` | Maintain committed durable Markdown renders | sweep | -| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | +| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate alias-orphans, migrate journal-duplicates, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | | `loaf journal` | Record and read the project-scoped journal (the durable record across all conversations) | log, recent, search, show, context, export, defer | | `loaf project` | Manage durable project identity | list, show, identity, rename, move, delete | | `loaf migrate` | Run native migration workflows | markdown, storage-home, schema, lifecycle-statuses, journal-first, worktree-storage | diff --git a/dist/opencode/skills/loaf-reference/SKILL.md b/dist/opencode/skills/loaf-reference/SKILL.md index 30e68ef1a..b053dfb4c 100644 --- a/dist/opencode/skills/loaf-reference/SKILL.md +++ b/dist/opencode/skills/loaf-reference/SKILL.md @@ -68,7 +68,7 @@ Names and one-line purposes only. Run `loaf --help` for options, argum | `loaf docs` | Manage docs/ indexing | index | | `loaf change` | Shape-first Change artifacts: git-canonical work context under docs/changes/ | init, check, list | | `loaf render` | Maintain committed durable Markdown renders | sweep | -| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | +| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate alias-orphans, migrate journal-duplicates, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | | `loaf journal` | Record and read the project-scoped journal (the durable record across all conversations) | log, recent, search, show, context, export, defer | | `loaf project` | Manage durable project identity | list, show, identity, rename, move, delete | | `loaf migrate` | Run native migration workflows | markdown, storage-home, schema, lifecycle-statuses, journal-first, worktree-storage | diff --git a/dist/skills/loaf-reference/SKILL.md b/dist/skills/loaf-reference/SKILL.md index 01979679a..74a3a9d66 100644 --- a/dist/skills/loaf-reference/SKILL.md +++ b/dist/skills/loaf-reference/SKILL.md @@ -67,7 +67,7 @@ Names and one-line purposes only. Run `loaf --help` for options, argum | `loaf docs` | Manage docs/ indexing | index | | `loaf change` | Shape-first Change artifacts: git-canonical work context under docs/changes/ | init, check, list | | `loaf render` | Maintain committed durable Markdown renders | sweep | -| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | +| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate alias-orphans, migrate journal-duplicates, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | | `loaf journal` | Record and read the project-scoped journal (the durable record across all conversations) | log, recent, search, show, context, export, defer | | `loaf project` | Manage durable project identity | list, show, identity, rename, move, delete | | `loaf migrate` | Run native migration workflows | markdown, storage-home, schema, lifecycle-statuses, journal-first, worktree-storage | diff --git a/docs/changes/20260807-state-dedupe/receipts/verify.json b/docs/changes/20260807-state-dedupe/receipts/verify.json new file mode 100644 index 000000000..1835aec7f --- /dev/null +++ b/docs/changes/20260807-state-dedupe/receipts/verify.json @@ -0,0 +1,122 @@ +{ + "schema_version": 2, + "change": "state-dedupe", + "verified_commit": "2f4a9e5680fbf14933ec3f6e24d1a3e38c82bff4", + "verified_root_tree": "2883589a39c16f1cc4cd19c182944dcaf21247d0", + "verified_at": "2026-08-09T01:40:34Z", + "criteria_digest": "9947e96459453f0a5ee2f965cf50f0b7ef770444f928562a5a93a0b4889c0ebb", + "scope_digest": "f69f338b54e4fe1109009a45bd844845f9739da2336394255c8e8f49aaa61daa", + "scope_sections": { + ".agents": "0cdcb499203f22e1faebbef89e211e4b6d4b253bc5486866ed28a50e15c2dd9e", + ".claude": "8dc123c00b953695edd9ccc4bb966e986bcea05607d9a125e71038a4c2ebb6be", + ".github": "1620d60edd726d78c03e1810a690c54669d4dbca40699b4489125696f1b28525", + ".gitignore": "4ec1c977b3002200cfe2c9a652cbbe1cb31e934d12a7c0029b1dd6cdc668099a", + ".serena": "749668b92acde130ae6a4514f59b9e08bf7095c8bdd47b6b2e5929b0729ba91f", + "AGENTS.md": "96a27c8deab6683f3c9ae669520ed0ed8cd2f22bc71231af8e00e205a7ea71e6", + "README.md": "32cecbbf375e1b689ac4702acc72a81f5ad89be495f87680524b1fcd76f5089b", + "cli": "cccb406db757aa65127b9657fde0c8a20c722c9ec9e3ae16269d1c0b3c21208a", + "cmd": "3d4f455f2eea30cecbef4ea78280e6b86b5b48033074369f476a1805d513d217", + "config": "ed88cb91f75fb35580ffe31433b1664efc91f31253abba6a27a645ddb009ae09", + "content": "869587ab6ca70cb7620dfe09ada7a87a8155898b8473aa17ddd32e510ab73828", + "docs": "c9f38d5ed12702698ddd94f14d5a84e9746fff71895f1fc6aeafc8184bb0851d", + "go.mod": "b9247690761e43ce92335c553e1f916132580b2a1e7f434c7d99742e56fb8b21", + "go.sum": "44a5d68a99a05f3d9aff3321f4b5645d0870cc1afd36e183fbc08c07d8550d6b", + "internal": "093982eb7c220d000b84b8eb643661d4b9e339a06d114163f5283bf6e2f163fb", + "package-lock.json": "9a3e1b4d99a42b2fefa998dd5c753c1f00ee12a43d1a477b674fe04e6c779f01" + }, + "exclusions": [ + "docs/changes/*/receipts/**", + "docs/changes/*/reports/**", + "package.json", + ".claude-plugin/marketplace.json", + "CHANGELOG.md", + "dist/**", + "plugins/**", + "bin/**" + ], + "digest_spec": "v1", + "tool_version": "0.2.20", + "toolchain": { + "go": "1.26.5", + "os": "darwin", + "arch": "arm64" + }, + "worktree_clean": true, + "results": [ + { + "id": "V1", + "command": "go test ./internal/state -run 'AliasOrphan' -count=1", + "exit_code": 0, + "output_digest": "7440b3f39d37c66c45ccfe537c30b2416ce08190ec2a806d056f380c4ef7350e", + "ok": true, + "expect": "exit 0.", + "expect_checks": [ + { + "kind": "exit", + "value": "0", + "ok": true + } + ] + }, + { + "id": "V2", + "command": "go test ./internal/state -run 'ImportAliasFirst' -count=1", + "exit_code": 0, + "output_digest": "e845c5b2606eb00052422cb37dae96c5123333538541ce34d42492e543b7efaa", + "ok": true, + "expect": "exit 0.", + "expect_checks": [ + { + "kind": "exit", + "value": "0", + "ok": true + } + ] + }, + { + "id": "V3", + "command": "go test ./... -run 'AliasParity' -count=1", + "exit_code": 0, + "output_digest": "1b3e6a16b7b8079aa560fb2c833280990e9a5dae2e13cd17c51e14ade000802d", + "ok": true, + "expect": "exit 0.", + "expect_checks": [ + { + "kind": "exit", + "value": "0", + "ok": true + } + ] + }, + { + "id": "V4", + "command": "go test ./...", + "exit_code": 0, + "output_digest": "41852648a8b4ea3df781fcb34395bfebf21796b861376e9175b73463ac53833c", + "ok": true, + "expect": "exit 0.", + "expect_checks": [ + { + "kind": "exit", + "value": "0", + "ok": true + } + ] + }, + { + "id": "V5", + "command": "go test ./internal/state -run 'JournalDuplicate' -count=1", + "exit_code": 0, + "output_digest": "2c8c42a2d80f0c91ed00a3a451c2e1848fabc97650639bdd4ce5e24040685fe5", + "ok": true, + "expect": "exit 0.", + "expect_checks": [ + { + "kind": "exit", + "value": "0", + "ok": true + } + ] + } + ] +} diff --git a/docs/changes/20260807-state-dedupe/shape.md b/docs/changes/20260807-state-dedupe/shape.md new file mode 100644 index 000000000..7f668e0c5 --- /dev/null +++ b/docs/changes/20260807-state-dedupe/shape.md @@ -0,0 +1,165 @@ + + +# State Dedupe + +## Problem + +The 2026-06-24 markdown import ran after migration 3 had rekeyed the project from its legacy path-hash ID to an opaque `proj_…` ID. Entity primary keys are derived — `sha256(kind \0 project_id \0 alias)` — so every re-derived ID missed its existing row, the importer inserted a full second copy of every artifact, and the alias upsert (`ON CONFLICT(project_id, namespace, alias) DO UPDATE SET entity_id`) re-pointed each alias to the new twin. The June-13 originals became alias-orphans: invisible to every list command (all list queries INNER JOIN through `aliases`) yet fully visible to the housekeeping scanner (raw `WHERE project_id = ?`). + +The two surfaces now disagree about how much work exists — 299/38/14 scanner vs 233/26/11 canonical for tasks/specs/reports — and the blast radius extends beyond the brief: ideas (51 orphans), sparks (56), brainstorms (3), ~230 duplicated `sources` rows, and one dangling alias (literal `[]`, pointing at a nonexistent task row). `loaf task list --status done --json` returned zero rows while 66 done tasks existed, because the done rows were exactly the orphans. The 2026-08-07 housekeeping pass archived ghosts: all 66 `done → archived` events were written against orphan rows reached by raw internal-ID resolution. + +Both halves of the defect remain live in code: the importer still trusts derived IDs alone, and nothing — no schema constraint, no doctor check — detects alias-orphaning. Separately, one report row ("Transitional TypeScript Surfaces — Do Not Deepen", status `active`, out-of-vocabulary) holds its alias but has no body and cites a source file with no git history — unrecoverable evidence. + +## Hypothesis + +If alias-orphans are classified and retired by an audited migration, the importer resolves identity through aliases before deriving IDs, and doctor checks alias parity, then the scanner and the list commands agree on counts for every entity table in every project, `--json` list output can be trusted by agents and workflows again, and no future rekey, merge, or import can silently orphan an aliased entity — divergence becomes detectable the day it happens instead of discoverable by accident at housekeeping. (One documented, alias-safe exception: a rekey re-import of a verbatim-duplicated spark line duplicates those spark rows — every copy stays alias-reachable, so nothing is hidden; see the resolveImportedSparkID comment.) + +## Scope + +**In** + +- An alias-orphan repair migration (`loaf state migrate alias-orphans`) with the full preview → backup → manifest → apply → verify → rollback ceremony, covering all seven aliased entity tables (tasks, specs, reports, ideas, sparks, brainstorms, shaping drafts — the seventh added in review, because the housekeeping scanner counts it and the importer aliases it), orphaned `sources` rows, dangling alias rows, and the reference-table sweep (events, entity_tags, bundle_members, backend_mappings, exports, relationships, artifact bodies/FTS) for every retired row. +- A journal-duplicates repair migration (`loaf state migrate journal-duplicates`) on the same triad, retiring the ~1,020 June-13 journal rows whose `(entry_type, scope, message)` twins were re-minted at the June-24 instant — journal rows carry no aliases, so this rides natural-key window classification instead of alias reachability (expansion by operator direction; see Decision 7). +- Importer identity fix: markdown import resolves `(project_id, namespace, alias)` against the aliases table first and reuses the existing entity ID; derivation only mints IDs for genuinely new entities. Journal entries, which have no aliases, resolve by natural identity (`entry_type`, scope, message) before deriving; sparks whose message normalizes to an empty slug receive a deterministic content-hash alias so no row is born orphaned. +- `loaf state doctor` gains an alias-parity diagnostic: per-project, per-table raw counts vs alias-reachable counts, plus dangling-alias detection. +- Explicit disposition of the broken-evidence report row: archive as moot with an event recording why (evidence unrecoverable; SPEC-047 already shipped the simplification this report guarded against deepening). +- The production repair ceremony, including the never-run `loaf state migrate lifecycle-statuses --apply` sequenced after the dedupe. + +**Out** (deferred, not rejected) + +- Write-time status-vocabulary enforcement and set-status verbs — TASK-408 / SPEC-049 territory. +- Sanctioned body-edit paths — TASK-407 / SPEC-055 territory. +- `specTaskCounts` joining only the spec alias (spec_list.go) and raw status literals in task_archive.go/spec_list.go — post-dedupe these count truthfully; revisit if the doctor parity check ever shows drift. +- Stale source *paths* (rows citing files renamed on disk, e.g. the sidecar-audit report) — path drift is not duplication damage. + +**Cut** (explicitly rejected) + +- Re-derivation of entity IDs anywhere — not in `rekeyLegacyProjectTx`, not as a one-time repair. Derived IDs are mint-once opaque keys; identity lives in the aliases table. All 27 project rows already carry opaque IDs, so the rekey trigger is extinct in this database, and the importer fix neutralizes it everywhere else. +- Fabricating replacement content for the broken-evidence report. +- Changing list-surface JOIN semantics (e.g. LEFT JOIN to display orphans) — an alias-orphan is damage to repair, not a display state. +- New schema-level unique constraints on entity tables — the aliases table's `UNIQUE (project_id, namespace, alias)` is the identity registry. + +## Observable Workflow + +``` +$ loaf state migrate alias-orphans # preview (default), all projects + project proj_7afeb3fc… (loaf): + tasks: 66 orphans — 63 retire (twin proven), 3 unproven (operator disposition required) + specs: 12 orphans — 12 retire + reports: 3 orphans — 3 retire + ideas: 51 retire (derivation); sparks: 46 retire + 10 unproven (June-24-born collision victims — realias, not retire); brainstorms: 3 retire + sources: N orphan-referenced rows to retire; aliases: 1 dead to delete + dispositions: report:7644bb23… → archive-as-moot (evidence unrecoverable) + +$ loaf state migrate alias-orphans --retire … --realias … # dispositions rehearse in preview too +$ loaf state migrate alias-orphans --apply --retire … --realias … # backup first, manifest written, verify after + +$ loaf state migrate journal-duplicates # preview: ~1,020 June-13/June-24 twin pairs +$ loaf state migrate journal-duplicates --apply # same backup/manifest/verify ceremony + +$ loaf state doctor # alias-parity section green: raw == reachable, 0 dead aliases +$ loaf housekeeping # scanner counts now equal list counts +$ loaf task list --status done --json # returns every done task that exists + +$ loaf state migrate lifecycle-statuses --apply # existing tool, first run, after dedupe +``` + +## Rabbit Holes and No-Gos + +- The migration is a fixed-classification repair for this damage class, not a general database fsck. New damage classes get new migrations. +- Twin-ship is proven by legacy-salt ID recomputation, with content identity (title equality within the event's timestamp cluster) as a distinctly-labeled fallback — never timestamp alone, per the brief. Unproven rows are refused, surfaced, and left for explicit operator disposition; the migration never guesses. +- Do not chase stale source paths, missing bodies on live rows, or status semantics beyond what the existing lifecycle-statuses migration already implements. +- Do not add code that resolves entities by recomputing `stableMigrationID` — that pattern is the root cause. Recomputation appears exactly once, inside the migration's twin-proof, against historical salts. +- The lifecycle-statuses run may surface out-of-vocabulary free-text statuses it cannot map; record their handling in the ceremony and leave any needed set-status verb to TASK-408. Do not extend the vocabulary migration here. + +## Decisions + +Provenance: operator interview during shaping (2026-08-07, four structured questions), on top of the captured brief and a full code/database investigation (see Problem). + +1. **Full blast radius.** One migration repairs all seven aliased entity tables plus sources and dangling aliases. Same mechanism, same surgery, one backup — splitting would mean operating on the production database twice. Forecloses a tasks/specs/reports-only partial repair. +2. **Prevention is importer alias-first resolution plus doctor parity; re-derivation is rejected.** With the importer resolving identity through aliases, a rekey can no longer cause orphaning at the next import, so re-deriving IDs (in rekey or as a one-time repair) buys insurance against a neutralized scenario at the cost of rewriting IDs across ~10 tables in one transaction. Forecloses ID-rewriting sweeps permanently. +3. **The broken-evidence report is archived as moot.** Status normalized from out-of-vocabulary `active` and archived by the migration as a named per-row disposition, with an event recording that the evidence is unrecoverable and the guardrail moot (SPEC-047 shipped the simplification it guarded). Forecloses both fabricated replacement content and a permanently-`active` bodyless row. +4. **The lifecycle-statuses migration runs as part of the ceremony**, after dedupe so no effort is spent normalizing rows about to be retired. Zero new code; closes the vocabulary half of the housekeeping finding. +5. **Canonical rows are the alias-holders** (confirmed from the brief with a correction: status vocabulary is *not* a discriminator — both copies carry raw vocab because lifecycle normalization never ran). The orphans retire; the twins survive. +6. **The migration sweeps every project in the global database, not just this one.** All 27 projects were rekeyed by migration 3; any of them with pre-rekey markdown imports carries the same damage. Preview reports per project before any apply, so the blast radius is visible first. (Shaper's decomposition call — flagged for review rather than interviewed.) +7. **The ~1,020 duplicated journal rows fold into this Change as TASK-005** (operator direction, 2026-08-08): the June-24 fork also re-minted journal entries, invisible to the alias-orphan lens because journal rows carry no aliases. Repair work of this kind takes a plan, not a shaping ceremony — the packet carries the plan; a separate `journal-duplicates` migration rides the same triad and runs in the same TASK-004 ceremony. Forecloses both leaving the duplicates permanently and bolting journal classification onto the alias-orphans migration. +8. **Two review-proven calibrations bind the proofs** (round-4 confirmation, reproduced on a production copy): retirement classification iterates to a fixed point, because content-identity and source proofs are orphan-count-sensitive and a retirement can unlock a proof mid-run; and the derivation proof must never gain a body-fingerprint guard — all 132 production derivation-proven pairs have mismatched fingerprints (June-13 originals and June-24 re-imports genuinely differ in content), so title + recency is the calibrated design, recorded as a load-bearing code comment. + +## Planning Contract + +### Approach + +Model the migration on `lifecycle_status_migration.go` — the existing preview/apply/rollback triad: preview runs against a temp copy (`copySQLiteDatabase`), apply takes a mandatory `Backup` first, writes a JSON rollback manifest next to the backup, applies in one transaction, verifies after, and `rollback` restores from the manifest. Register it in the `stateMigrateSources` registry (cli.go:3190) as `alias-orphans`. + +Classification, per project, per entity table: + +- **Orphan** = entity row with no `aliases` row matching `(project_id, entity_kind, entity_id, namespace)`. +- **Retire (twin proven):** recompute `stableMigrationID(kind, legacy_project_id, alias)` for every alias in the project, where legacy project IDs derive from `hex(sha256(path))` over the project's current *and historical* paths (`project_paths`); an ID match plus title equality and orphan-older ordering proves the alias-holder is the twin (`derivation`). Fallback proof (`content-identity`): exact title match with exactly one candidate on each side, the holder created inside the June-24 reimport window, and equal body fingerprints — where both-bodyless counts as equal only when the orphan sits in the June-13 original-import window. A third labeled proof (`source-derivation`) binds source-keyed twins under the same window, ordering, and both-side uniqueness gates. Anything short of a proof is `unproven`. +- **Unproven:** orphans with neither proof are listed, refused by default, and require explicit per-row operator disposition supplied as repeatable apply flags — `--retire ` and `--realias =` — recorded verbatim in the manifest. No disposition, no touch. +- **Dangling aliases** are deleted when they are dead: the entity row is missing *and* nothing in the project still names that entity. An alias the importer forward-declares for a referenced-but-unimported artifact (a `depends_on` naming a task with no file) keeps a live relationship edge and is a reference, not damage — the detector and the repair both pass over it, or import → repair → import never converges. The edge goes when the reference leaves the markdown, and the alias it left behind is then collected. (Refinement discovered in review: the production `[]` alias is dead by exactly this test.) +- **Orphaned sources:** `sources` rows referenced only by retired rows retire with them. +- **Named dispositions:** special-cased rows (the broken-evidence report) carry their disposition in the plan and manifest. + +Retirement reuses the reference-table sweep from `spec_delete.go:90-132` (artifact bodies, FTS, events, entity_tags, bundle_members, backend_mappings, exports, relationships, then the row) under `PRAGMA defer_foreign_keys = ON`, generalized across entity kinds. Events written against retired rows are deleted with them, consistent with `spec delete`; the manifest preserves every deleted row for rollback and audit. + +The importer fix inverts identity resolution in `markdown_import.go`: before using a derived ID, look up `(project_id, namespace, alias)`; if the alias names an existing entity of that kind, use that entity's ID so `ON CONFLICT(id) DO UPDATE` fires. Regression test simulates the full historical sequence — import under one project ID, rekey, re-import — and asserts zero new entity rows and zero orphans. + +The doctor diagnostic is read-only: for each project and entity table, compare raw counts to alias-joined counts, count dangling aliases, and report per-table parity. It detects; the migration repairs. No `--fix`. + +### Idempotency and safety + +The migration is re-runnable: a second preview after apply classifies zero orphans; a second apply is a no-op. Rides Recovery Tiers (ARCHITECTURE.md): mandatory backup, isolated preview, manifest rollback, post-apply verification. All tests isolate via temp DBs (`t.Setenv`/`LOAF_DB`); only the ceremony (TASK-004) touches the production database, deliberately. + +### Journal duplicates + +`loaf state migrate journal-duplicates` (TASK-005) repairs the unaliased half of the same event: pairs are identical `(entry_type, scope, message)` triples with one row in the June-13 import window and one in the June-24 reimport window (the same named window constants as the alias-orphans cluster gates); the June-24 row survives, ambiguous multi-candidate matches classify unproven and are refused without an explicit `--retire`. The full plan lives in the TASK-005 packet. + +### Sequencing + +TASK-001 (migration) → TASK-002 (importer), TASK-003 (doctor), and TASK-005 (journal duplicates) are independent of each other; TASK-004 (ceremony) is blocked by all four — it runs the shipped code against the production database and uses the doctor check as its verification surface. + +## Implementation Units + +- **TASK-001 — Alias-orphan repair migration.** `loaf state migrate alias-orphans` preview/apply/rollback with classification, twin proofs, reference-table sweep, named dispositions, manifest, and tests. +- **TASK-002 — Importer alias-first identity resolution.** Markdown import resolves aliases before deriving IDs; simulated-rekey regression test. +- **TASK-003 — Doctor alias-parity diagnostic.** Read-only per-project, per-table parity section in `loaf state doctor`, with tests. +- **TASK-004 — Production repair ceremony.** Backup, rehearsed preview with dispositions, alias-orphans apply, journal-duplicates apply, doctor verification, lifecycle-statuses run, count-agreement receipts, journal entries. +- **TASK-005 — Journal-duplicates repair migration.** `loaf state migrate journal-duplicates` on the same triad: window-gated natural-key pairing, June-13-copy retirement with reference sweep and FTS parity, refuse-by-default ambiguity handling, and tests. + +## Verification Contract + +- **V1.** Migration classification, apply, rollback, and idempotency tests pass. Command: `go test ./internal/state -run 'AliasOrphan' -count=1`. Expect: exit 0. +- **V2.** Importer resolves identity through aliases; simulated rekey + re-import creates zero new rows. Command: `go test ./internal/state -run 'ImportAliasFirst' -count=1`. Expect: exit 0. +- **V3.** Doctor reports alias parity and dangling aliases. Command: `go test ./... -run 'AliasParity' -count=1`. Expect: exit 0. +- **V4.** The whole suite stays green. Command: `go test ./...`. Expect: exit 0. +- **V5.** Journal-duplicates pairing, refusal, apply/rollback, and FTS-parity tests pass. Command: `go test ./internal/state -run 'JournalDuplicate' -count=1`. Expect: exit 0. + + + +- **H1.** Ceremony receipts: backup ID, preview output for all projects, apply manifest path, post-apply doctor parity green, scanner-vs-list equality for all seven aliased tables, lifecycle-statuses manifest, journal entries. +- **H2.** The broken-evidence report row is archived with its moot-rationale event; the unrecoverable evidence is documented, not fabricated. +- **H3.** The three unproven task orphans (66 orphans vs 63 title twins) received explicit manifest-recorded dispositions. + +## Definition of Done + +- V1–V5 green in CI. +- On the production database: for every project and every entity table, raw row counts equal alias-reachable counts, and zero dead aliases remain (doctor parity green). +- Housekeeping scanner counts equal canonical list counts — the brief's acceptance signal. +- Zero `(entry_type, scope, message)` journal twins remain across the June-13/June-24 import windows. +- The broken-evidence report is archived with recorded rationale. +- Backup and rollback manifests retained per Recovery Tiers; ceremony receipts journaled. + +## Durable Outputs + +- ADR candidate: entity identity lives in the aliases table; derived entity IDs are mint-once opaque keys, never recomputed for resolution. (Supersedes the implicit stable-derivation assumption that caused this event.) +- CHANGELOG entry for the 0.2.x line covering the repair migration, importer fix, and doctor diagnostic. +- Post-ceremony journal synthesis: final per-table counts, dispositions of unproven rows, lifecycle-statuses OOV leftovers if any. + +## Open Questions + + + +- [KU] Do the other 26 projects carry alias-orphans, and does each have a recomputable legacy ID (`sha256(current_path)`)? → **Resolved by the production-copy rehearsal:** all 191 orphans belong to this project; post-apply doctor reports alias-parity clear across all 27 projects and 189 table checks. The other projects carry no alias-orphan damage. +- [KU] What are the three task orphans without title twins? → TASK-001 preview classifies them as unproven; operator dispositions in TASK-004, manifest-recorded. +- [KU] Which out-of-vocabulary free-text statuses can the lifecycle-statuses migration not map? → surfaced by its preview in TASK-004; handling recorded in ceremony receipts; any needed set-status verb routes to TASK-408, not this Change. +- [KU] How many of the ~1,020 journal twin pairs are ambiguous (multi-candidate) and need explicit `--retire` dispositions? → **Resolved by the production-copy rehearsal:** 1,019 duplicated triples decompose into 866 clean 1:1 pairs (retired automatically) and 153 ambiguous groups spanning 614 rows, which refuse by default. The Definition of Done's zero-twins line is reachable only through ceremony dispositions — scriptable from the preview JSON — not a bare apply. diff --git a/docs/changes/20260807-state-dedupe/tasks/TASK-001-alias-orphan-migration.md b/docs/changes/20260807-state-dedupe/tasks/TASK-001-alias-orphan-migration.md new file mode 100644 index 000000000..37cf8d011 --- /dev/null +++ b/docs/changes/20260807-state-dedupe/tasks/TASK-001-alias-orphan-migration.md @@ -0,0 +1,48 @@ +--- +change: state-dedupe +id: TASK-001 +title: Alias-orphan repair migration +blocks: + - TASK-004 +--- + +# TASK-001 — Alias-orphan repair migration + +## Objective + +`loaf state migrate alias-orphans` exists with the full preview → backup → manifest → apply → verify → rollback ceremony: it classifies alias-orphaned entity rows across all seven aliased entity tables in every project, retires proven duplicates with their reference-table residue, deletes dangling aliases, refuses unproven rows without explicit disposition, and executes named per-row dispositions (the broken-evidence report archives as moot). + +## Scope boundaries + +**In:** New migration file in `internal/state/` (e.g. `alias_orphan_migration.go`) plus its tests; registration in the `stateMigrateSources` registry (`internal/cli/cli.go:3190` area); a kind-generic retirement sweep derived from `spec_delete.go:90-132`. + +**Out:** The importer (TASK-002), doctor (TASK-003), any run against the production database (TASK-004), lifecycle-status semantics (existing migration owns them), list-command queries, schema changes. + +## Context pointers + +- Contract: `shape.md` — Planning Contract (Approach, Idempotency and safety), Decisions 1/3/5/6 +- Template code: `internal/state/lifecycle_status_migration.go` (triad + manifest), `internal/state/spec_delete.go:90-132` (reference-table sweep), `internal/state/markdown_import.go:1120-1127` (`stableMigrationID`, used only for twin proof against historical salts) + +## Acquisition + +```bash +loaf journal log "skill(implement): TASK-001 — alias-orphan repair migration" +# Read lifecycle_status_migration.go end to end before writing anything; +# the triad, backup call, manifest write, and registry entry are the pattern to follow. +export LOAF_DB="$(mktemp -d)/loaf.sqlite" # never touch the production DB from tests or smokes +``` + +## Steps + +- [x] Classification: per project, per entity table (tasks, specs, reports, ideas, sparks, brainstorms), find entity rows with no matching `aliases` row; prove twins by recomputing `stableMigrationID(kind, hex(sha256(current_path)), alias)` for the project's aliases, with exact-title match in the event's timestamp cluster as a distinctly-labeled `content-identity` fallback; everything else is `unproven` +- [x] Preview: run classification against a temp copy; report per project and per table — retire/unproven/dangling-alias/orphaned-source counts and the named dispositions +- [x] Apply: mandatory `Backup` first, JSON rollback manifest beside it (every deleted row preserved), retirement sweep per row (bodies, FTS, events, entity_tags, bundle_members, backend_mappings, exports, relationships, then the row) under `PRAGMA defer_foreign_keys = ON`, dangling aliases deleted, unproven rows untouched unless an explicit per-row disposition is supplied via repeatable flags (`--retire `, `--realias =`) recorded verbatim in the manifest +- [x] Named disposition: the broken-evidence report (`report:7644bb23d2664de93b6cb6a5`) archives as moot — status normalized and an event recording the unrecoverable evidence and SPEC-047 rationale +- [x] Rollback: restore deleted rows from the manifest; verify round-trip in tests +- [x] Tests (`TestAliasOrphan*`): classification correctness on a fixture reproducing the June-24 shape (rekey + re-import), derivation vs content-identity vs unproven labeling, apply/rollback round-trip, idempotency (second preview classifies zero, second apply no-ops), reference-table residue fully removed, unproven rows refused without disposition + +## Verification + +- `go test ./internal/state -run 'AliasOrphan' -count=1` exits 0 +- `go test ./...` exits 0 +- Preview against a fixture DB shows per-project classification and touches nothing diff --git a/docs/changes/20260807-state-dedupe/tasks/TASK-002-importer-alias-first-identity.md b/docs/changes/20260807-state-dedupe/tasks/TASK-002-importer-alias-first-identity.md new file mode 100644 index 000000000..9e2c5a946 --- /dev/null +++ b/docs/changes/20260807-state-dedupe/tasks/TASK-002-importer-alias-first-identity.md @@ -0,0 +1,53 @@ +--- +change: state-dedupe +id: TASK-002 +title: Importer alias-first identity resolution +blocks: + - TASK-004 +--- + +# TASK-002 — Importer alias-first identity resolution + +## Objective + +The markdown importer resolves entity identity through the aliases table before deriving an ID: when `(project_id, namespace, alias)` already names an entity of the imported kind, the importer reuses that entity's ID so `ON CONFLICT(id) DO UPDATE` fires, making alias re-pointing and orphan creation impossible regardless of project-ID changes. + +## Scope boundaries + +**In:** `internal/state/markdown_import.go` upsert paths for specs, tasks, reports, ideas, sparks, brainstorms, and sources where they key off derived IDs; regression tests. + +**Out:** The repair migration (TASK-001), doctor (TASK-003), `rekeyLegacyProjectTx` (re-derivation is Cut by the contract), alias derivation rules (`firstNonEmpty(frontmatter id, path alias, stem)` stay as they are — a renamed file is a new artifact by design). + +## Context pointers + +- Contract: `shape.md` — Planning Contract (Approach), Decision 2, Cut list +- Root cause mechanics: `internal/state/markdown_import.go:218-261` (importSpecs), `:724-734` (task upsert `ON CONFLICT(id)`), `:967-974` (alias upsert that re-points `entity_id`), `:1120-1127` (`stableMigrationID`) + +## Acquisition + +```bash +loaf journal log "skill(implement): TASK-002 — importer alias-first identity resolution" +export LOAF_DB="$(mktemp -d)/loaf.sqlite" +``` + +## Steps + +- [x] Add an alias-first lookup on the import path: resolve `(project_id, namespace, alias)` to an existing entity ID of the imported kind before falling back to `stableMigrationID` for genuinely new entities +- [x] Ensure the alias upsert can no longer re-point an alias away from a live entity row as a side effect of import (with alias-first resolution the `entity_id` it writes is the resolved one; assert this in tests rather than trusting it) +- [x] Apply the same resolution to `sources` rows so source doubling cannot recur +- [x] Regression test (`TestImportAliasFirst*`): import a markdown tree under one project ID, rewrite `project_id` columns exactly as `rekeyLegacyProjectTx` does, re-import — assert zero new entity rows, zero new source rows, zero alias-orphans, and stable alias→entity mappings +- [x] Idempotency test: re-import with no changes is byte-stable (no row churn, `updated_at` semantics preserved as today) + +## Verification + +- `go test ./internal/state -run 'ImportAliasFirst' -count=1` exits 0 +- `go test ./...` exits 0 + +## Delivered variances + +Recorded post-review for provenance — the shipped resolution is alias-first for generic entities, with these kind-specific mechanisms the original packet did not enumerate: + +- Sparks resolve by exact text and source with an exactly-one-candidate rule; colliding sparks receive numbered aliases instead of stealing; a spark whose message normalizes to an empty slug gets a deterministic content-hash alias. A rekey re-import of a verbatim-duplicated spark line duplicates those rows (documented at the resolveImportedSparkID comment) — alias-safe, never orphaning. +- Journal entries (unaliased) resolve by natural identity, applied to markdown-origin rows. +- Sources resolve by `(project_id, path)`. +- `shaping_draft` resolution shipped alongside the seven aliased kinds though this packet's scope named six. diff --git a/docs/changes/20260807-state-dedupe/tasks/TASK-003-doctor-alias-parity.md b/docs/changes/20260807-state-dedupe/tasks/TASK-003-doctor-alias-parity.md new file mode 100644 index 000000000..83cf287c3 --- /dev/null +++ b/docs/changes/20260807-state-dedupe/tasks/TASK-003-doctor-alias-parity.md @@ -0,0 +1,42 @@ +--- +change: state-dedupe +id: TASK-003 +title: Doctor alias-parity diagnostic +blocks: + - TASK-004 +--- + +# TASK-003 — Doctor alias-parity diagnostic + +## Objective + +`loaf state doctor` reports alias parity: for every project and each of the seven aliased entity tables, raw row counts vs alias-reachable counts, plus dangling-alias counts — so a future identity fork is detected the day it happens instead of discovered by accident at housekeeping. + +## Scope boundaries + +**In:** The `loaf state doctor` surface (`internal/cli/cli.go:2368-2450` and the state-layer inspection it calls); tests. + +**Out:** Any repair behavior — the diagnostic is read-only and never gains a `--fix` for this damage class (the migration from TASK-001 is the repair); `loaf doctor` (harness content files, unrelated); housekeeping scanner internals. + +## Context pointers + +- Contract: `shape.md` — Planning Contract (Approach), Decision 2 +- Divergence mechanics: `internal/state/housekeeping.go:104-132` (raw count query) vs `internal/state/task_list.go:63-85` (alias INNER JOIN) — parity means these two shapes agree + +## Acquisition + +```bash +loaf journal log "skill(implement): TASK-003 — doctor alias-parity diagnostic" +export LOAF_DB="$(mktemp -d)/loaf.sqlite" +``` + +## Steps + +- [x] Add an alias-parity section to `loaf state doctor` output (human and JSON): per project, per entity table — raw count, alias-reachable count, orphan delta, dangling aliases +- [x] Green state is exact parity and zero dangling aliases; any delta renders as a finding that names `loaf state migrate alias-orphans` as the repair +- [x] Tests (`TestStateDoctorAliasParity*`): parity on a clean fixture; orphan and dangling-alias fixtures produce the finding with correct counts; diagnostic performs no writes + +## Verification + +- `go test ./... -run 'AliasParity' -count=1` exits 0 +- `go test ./...` exits 0 diff --git a/docs/changes/20260807-state-dedupe/tasks/TASK-004-production-repair-ceremony.md b/docs/changes/20260807-state-dedupe/tasks/TASK-004-production-repair-ceremony.md new file mode 100644 index 000000000..26fce8c95 --- /dev/null +++ b/docs/changes/20260807-state-dedupe/tasks/TASK-004-production-repair-ceremony.md @@ -0,0 +1,54 @@ +--- +change: state-dedupe +id: TASK-004 +title: Production repair ceremony +blocked-by: + - TASK-001 + - TASK-002 + - TASK-003 + - TASK-005 +--- + +# TASK-004 — Production repair ceremony + +## Objective + +The production database is repaired and proven: backup taken, preview read for all projects, unproven rows dispositioned explicitly, dedupe applied, doctor parity green, the lifecycle-statuses migration run for the first time, and scanner-vs-list count agreement demonstrated — with receipts journaled. + +## Scope boundaries + +**In:** Operator ceremony against `~/.local/share/loaf/loaf.sqlite` using the built binary from this branch; journal entries; ceremony receipts for H1–H3 review. + +**Out:** Any code changes (if the ceremony reveals a defect, it routes back to TASK-001/002/003); dispositions of lifecycle OOV statuses beyond recording them (set-status verbs are TASK-408 territory). + +## Context pointers + +- Contract: `shape.md` — Observable Workflow, Decisions 3/4/6, Definition of Done, Open Questions (all three resolve here) +- Recovery discipline: `docs/ARCHITECTURE.md` — Recovery Tiers and Restore Safety + +## Acquisition + +```bash +loaf journal log "skill(implement): TASK-004 — production alias-orphan repair ceremony" +npm run build # ceremony runs the binary built from this branch — no LOAF_DB isolation, deliberately +``` + +## Steps + +- [ ] `loaf state backup` and record the backup ID (Recovery Tier: local rollback) +- [ ] `loaf state migrate alias-orphans` (preview): read per-project classification for all projects; record counts +- [ ] Disposition the unproven rows explicitly — expected: 23 rows (rehearsed on a production copy during review). 3 task orphans without title twins (`--retire` or `--realias` per row), and 20 sparks forming 10 both-orphan message pairs: one copy per import instant, neither holding an alias, no surviving twin to bind to. Each pair takes one `--realias` (the member that lives on) and one `--retire`; which member survives is ceremony judgment — the June-24-survives convention from entity twins is the sensible default +- [ ] Rehearse the exact apply invocation as a preview first: `loaf state migrate alias-orphans --retire … --realias …` (dispositions are accepted in preview and reflected in its totals) — the rehearsed and applied invocations must be identical +- [ ] `loaf state migrate alias-orphans --apply --retire … --realias …`; record the manifest path; first run must exit 0 with post-apply verification passing and a truthful non-zero `orphaned_sources` figure +- [ ] `loaf state migrate journal-duplicates` (preview): read pair counts and ambiguous matches; disposition ambiguities via `--retire`; then `--apply`; record the manifest path +- [ ] `loaf state doctor`: alias-parity section green — raw == reachable for every project and table, zero dead aliases +- [ ] Confirm the broken-evidence report is archived with its moot-rationale event +- [ ] `loaf state migrate lifecycle-statuses` preview, then `--apply`; record OOV statuses it could not map, if any +- [ ] Demonstrate count agreement: `loaf housekeeping` totals equal list-command counts for all seven aliased tables; `loaf task list --status done --json` returns exactly the done rows that exist +- [ ] Journal the ceremony: `decision(state)` with counts and dispositions; `discover(state)` for anything the preview revealed about other projects + +## Verification + +- Doctor alias-parity green on the production database +- Housekeeping scanner counts equal canonical list counts (the brief's acceptance signal) +- Backup + both manifests retained; receipts sufficient for H1–H3 review diff --git a/docs/changes/20260807-state-dedupe/tasks/TASK-005-journal-duplicates-repair.md b/docs/changes/20260807-state-dedupe/tasks/TASK-005-journal-duplicates-repair.md new file mode 100644 index 000000000..d5b11e3d5 --- /dev/null +++ b/docs/changes/20260807-state-dedupe/tasks/TASK-005-journal-duplicates-repair.md @@ -0,0 +1,57 @@ +--- +change: state-dedupe +id: TASK-005 +title: Journal-duplicates repair migration +blocks: + - TASK-004 +--- + +# TASK-005 — Journal-duplicates repair migration + +## Objective + +`loaf state migrate journal-duplicates` exists as a sibling of alias-orphans on the same preview/backup/manifest/apply/verify/rollback triad: it classifies the ~1,020 journal rows duplicated by the June-24 identity fork (exact `(entry_type, scope, message)` twins across the two import instants), retires the June-13 copies, and leaves the journal timeline single-voiced without touching any legitimately repeated entry. + +## Scope boundaries + +**In:** New migration in `internal/state/` (e.g. `journal_duplicate_migration.go`) plus tests; registration in `stateMigrateSources`; sweep of journal-adjacent reference surfaces (`journal_search` FTS parity, `journal_origins`, `journal_deferrals`, and any FK reference to `journal_entries.id` — enumerate from the schema before writing). + +**Out:** The alias-orphans migration (TASK-001 owns aliased entities), the importer identity fix (landed with the consolidated fix round), any deduplication of rows that repeat legitimately at different times, and any rewrite of `created_at` semantics. + +## Plan + +The damage signature, verified against production: 1,348 rows created in the June-13 import window (`2026-06-13T01:39`) and 1,156 in the June-24 reimport window (`2026-06-24T13:03`); ~1,020 `(entry_type, scope, message)` triples appear in both windows. Journal rows carry no aliases, so classification is by natural key and window membership, not alias reachability: + +1. **Duplicate pair** = identical `(entry_type, scope, message)` where one row's `created_at` falls in the June-13 window and the other's in the June-24 window (named constants shared with the alias-orphans migration's cluster gates). The June-24 row survives — its ID derives from the current project ID, consistent with the canonical-twin rule for aliased entities. Rows matching more than one candidate on either side classify `unproven` and are refused by default (same refuse-by-default posture and `--retire` disposition flag as alias-orphans; `--realias` has no meaning here and is rejected). +2. **Retirement** deletes the June-13 row and its reference residue. Before coding, enumerate every table referencing `journal_entries.id` from the migrations schema; known candidates: `journal_search` (FTS — delete or rebuild via the existing `RepairJournalSearch` parity machinery), `journal_origins`, `journal_deferrals`, `intent_*` tables if they cite entry IDs. The manifest preserves every deleted row and reference edge for rollback. +3. **Ceremony wiring:** TASK-004 runs this migration's preview → apply immediately after alias-orphans and before lifecycle-statuses, in the same backup session. +4. **Acceptance:** post-apply, zero `(entry_type, scope, message)` triples duplicated across the two windows; `loaf journal recent`/`search` return single copies; journal-search parity check green; total row count drops by exactly the retired count; second apply is a no-op. + +## Context pointers + +- Contract: `shape.md` — Planning Contract, Decisions (journal-dupes expansion) +- Pattern: `internal/state/alias_orphan_migration.go` (post-fix-round state: cluster-gate constants, manifest fsync, verbatim flag recording, fixed-point retirement), `internal/state/lifecycle_status_migration.go` (triad origin), `internal/state/journal_search_integrity.go` and `RepairJournalSearch` (FTS parity) +- Damage evidence: journal `finding(state)` entry of 2026-08-08 (the ~1,020-pair discovery, with window timestamps) + +## Acquisition + +```bash +loaf journal log "skill(implement): TASK-005 — journal-duplicates repair migration" +export LOAF_DB="$(mktemp -d)/loaf.sqlite" # tests and smokes never touch the production DB +``` + +## Steps + +- [x] Enumerate every schema reference to `journal_entries.id` and record the sweep list in the migration's doc comment +- [x] Classification: window-gated natural-key pairing with `unproven` refusal for ambiguous matches; preview reports pair counts, unproven counts, and per-window row totals +- [x] Apply: mandatory backup, fsynced JSON rollback manifest, June-13-copy retirement with full reference sweep, `--retire` dispositions recorded verbatim, `--realias` rejected +- [x] Rollback: restore rows and reference edges from the manifest; round-trip test +- [x] FTS: keep `journal_search` consistent (targeted deletes or a post-apply `RepairJournalSearch` rebuild — pick one, justify in the code) +- [x] Tests (`TestJournalDuplicate*`): fixture reproducing the two-window duplication, pairing correctness, ambiguity refusal, apply/rollback round-trip, idempotency, FTS parity after apply +- [x] Wire into TASK-004's ceremony sequence (alias-orphans → journal-duplicates → lifecycle-statuses) + +## Verification + +- `go test ./internal/state -run 'JournalDuplicate' -count=1` exits 0 +- `go test ./...` exits 0 +- Preview against a copy of the production database reports ~1,020 retire pairs and touches nothing diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 6e963bd5e..1103469dc 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -1696,6 +1696,8 @@ func writeStateMigrateHelp(out io.Writer) { fmt.Fprintln(out, " lifecycle-statuses Normalize legacy lifecycle statuses in SQLite") fmt.Fprintln(out, " journal-first Transform the global database to the journal-first model") fmt.Fprintln(out, " deferrals Convert historical journal deferrals into canonical deferred Intents") + fmt.Fprintln(out, " alias-orphans Retire alias-orphaned entity rows with backup and rollback") + fmt.Fprintln(out, " journal-duplicates Retire June-13/June-24 journal natural-key twins with backup and rollback") fmt.Fprintln(out) fmt.Fprintln(out, "Options:") fmt.Fprintln(out, " -h, --help Show help") @@ -2374,7 +2376,7 @@ func (r Runner) runStateDoctor(args []string, out io.Writer, runtime state.Runti } return err } - status, err := r.inspectState(runtime) + status, err := r.inspectStateWithOptions(runtime, state.InspectOptions{AliasParity: true}) if err != nil { if jsonOutput { return writeJSONCommandError(out, "state doctor", err) @@ -3205,9 +3207,11 @@ var stateMigrateSources = map[string]stateMigrateSource{ }, help: writeStateMigrateSchemaHelp, }, - "markdown": {run: Runner.runStateMigrateMarkdown, help: writeStateMigrateMarkdownHelp}, - "storage-home": {run: Runner.runStateMigrateStorageHome, help: writeStateMigrateStorageHomeHelp}, - "deferrals": {run: Runner.runStateMigrateDeferrals, help: writeStateMigrateDeferralsHelp}, + "markdown": {run: Runner.runStateMigrateMarkdown, help: writeStateMigrateMarkdownHelp}, + "storage-home": {run: Runner.runStateMigrateStorageHome, help: writeStateMigrateStorageHomeHelp}, + "deferrals": {run: Runner.runStateMigrateDeferrals, help: writeStateMigrateDeferralsHelp}, + "alias-orphans": {run: Runner.runStateMigrateAliasOrphans, help: writeStateMigrateAliasOrphansHelp}, + "journal-duplicates": {run: Runner.runStateMigrateJournalDuplicates, help: writeStateMigrateJournalDuplicatesHelp}, } // stateMigrateSourceHelp derives the `loaf state migrate --help` @@ -3246,6 +3250,14 @@ func writeStateMigrateLifecycleStatusesHelp(out io.Writer) { writeUsageHelp(out, "loaf state migrate lifecycle-statuses [--dry-run|--apply|--rollback ] [--json]", "Normalize legacy lifecycle statuses in SQLite with a backup and rollback manifest.", "--dry-run Preview on a temporary database copy", "--apply Normalize live SQLite statuses after creating a backup", "--rollback Restore statuses from a lifecycle-statuses rollback manifest", "--json Output migration contract, project context, counts, backup, and rollback fields as JSON") } +func writeStateMigrateAliasOrphansHelp(out io.Writer) { + writeUsageHelp(out, "loaf state migrate alias-orphans [--dry-run|--apply|--rollback ] [--retire ]... [--realias =]... [--json]", "Retire alias-orphaned entity rows across every project with a backup and rollback manifest.", "--dry-run Preview classification on a temporary database copy (default)", "--apply Apply the repair after creating a backup", "--rollback Restore deleted rows from an alias-orphans rollback manifest", "--retire Force-retire an unproven orphan (repeatable)", "--realias = Attach an alias to an unproven orphan (repeatable)", "--json Output migration contract, per-project classification, counts, backup, and rollback fields as JSON") +} + +func writeStateMigrateJournalDuplicatesHelp(out io.Writer) { + writeUsageHelp(out, "loaf state migrate journal-duplicates [--dry-run|--apply|--rollback ] [--retire ]... [--json]", "Retire June-13/June-24 journal natural-key twins across every project with a backup and rollback manifest.", "--dry-run Preview classification on a temporary database copy (default)", "--apply Apply the repair after creating a backup", "--rollback Restore deleted rows from a journal-duplicates rollback manifest", "--retire Force-retire an unproven multi-candidate journal row (repeatable)", "--json Output migration contract, per-project classification, counts, backup, and rollback fields as JSON") +} + func writeStateMigrateSchemaHelp(out io.Writer) { writeUsageHelp(out, "loaf state migrate schema [--dry-run|--apply] [--json]", "Preview or apply pending SQLite schema upgrades with a verified backup before mutation.", "--dry-run Preview pending schema upgrades without writing", "--apply Apply pending schema upgrades after creating and verifying a backup", "--json Output schema upgrade action, versions, pending migrations, backup, and verification as JSON") } @@ -3409,6 +3421,94 @@ func (r Runner) runStateMigrateLifecycleStatuses(args []string, out io.Writer, r return r.runLifecycleStatusMigration(args, out, runtime, "loaf state migrate lifecycle-statuses") } +func (r Runner) runStateMigrateAliasOrphans(args []string, out io.Writer, runtime state.Runtime) error { + return r.runAliasOrphanMigration(args, out, runtime, "loaf state migrate alias-orphans") +} + +func (r Runner) runStateMigrateJournalDuplicates(args []string, out io.Writer, runtime state.Runtime) error { + return r.runJournalDuplicateMigration(args, out, runtime, "loaf state migrate journal-duplicates") +} + +func (r Runner) runJournalDuplicateMigration(args []string, out io.Writer, runtime state.Runtime, displayCommand string) error { + command := strings.TrimPrefix(displayCommand, "loaf ") + jsonRequested := hasFlag(args, "--json") + options, err := parseJournalDuplicateMigrationArgs(args, command) + if err != nil { + if jsonRequested { + return writeJSONCommandError(out, command, err) + } + return err + } + projectRoot, err := project.ResolveRoot(runtime.RootPath()) + if err != nil { + if options.jsonOutput { + return writeJSONCommandError(out, command, err) + } + return err + } + resolver := state.PathResolver{StateHome: r.StateHome} + var result state.JournalDuplicateMigrationResult + switch { + case options.rollbackPath != "": + result, err = state.RollbackJournalDuplicateMigration(context.Background(), projectRoot, resolver, options.rollbackPath) + case options.apply: + result, err = state.ApplyJournalDuplicateMigration(context.Background(), projectRoot, resolver, options.applyOptions) + default: + result, err = state.PreviewJournalDuplicateMigration(context.Background(), projectRoot, resolver, options.applyOptions) + } + if err != nil { + if options.jsonOutput { + return writeJSONCommandError(out, command, err) + } + return err + } + if options.jsonOutput { + return writeJSON(out, result) + } + writeJournalDuplicateMigrationHuman(out, displayCommand, result) + return nil +} + +func (r Runner) runAliasOrphanMigration(args []string, out io.Writer, runtime state.Runtime, displayCommand string) error { + command := strings.TrimPrefix(displayCommand, "loaf ") + jsonRequested := hasFlag(args, "--json") + options, err := parseAliasOrphanMigrationArgs(args, command) + if err != nil { + if jsonRequested { + return writeJSONCommandError(out, command, err) + } + return err + } + projectRoot, err := project.ResolveRoot(runtime.RootPath()) + if err != nil { + if options.jsonOutput { + return writeJSONCommandError(out, command, err) + } + return err + } + resolver := state.PathResolver{StateHome: r.StateHome} + var result state.AliasOrphanMigrationResult + switch { + case options.rollbackPath != "": + result, err = state.RollbackAliasOrphanMigration(context.Background(), projectRoot, resolver, options.rollbackPath) + case options.apply: + result, err = state.ApplyAliasOrphanMigration(context.Background(), projectRoot, resolver, options.applyOptions) + default: + result, err = state.PreviewAliasOrphanMigration(context.Background(), projectRoot, resolver, options.applyOptions) + } + if err != nil { + if options.jsonOutput { + return writeJSONCommandError(out, command, err) + } + return err + } + if options.jsonOutput { + return writeJSON(out, result) + } + writeAliasOrphanMigrationHuman(out, displayCommand, result) + return nil +} + func (r Runner) runLifecycleStatusMigration(args []string, out io.Writer, runtime state.Runtime, displayCommand string) error { command := strings.TrimPrefix(displayCommand, "loaf ") jsonRequested := hasFlag(args, "--json") @@ -3730,6 +3830,159 @@ func writeStorageHomeMigrationPlan(out io.Writer, plan state.StorageHomeMigratio } } +func aliasOrphanTitleSuffix(title string) string { + title = strings.TrimSpace(title) + if title == "" { + return "" + } + if len(title) > 72 { + title = title[:69] + "..." + } + return " — " + title +} + +func aliasOrphanDispositionSuffix(disposition string) string { + if disposition == "" { + return "" + } + return " [" + disposition + "]" +} + +func writeJournalDuplicateMigrationHuman(out io.Writer, displayCommand string, result state.JournalDuplicateMigrationResult) { + switch result.Action { + case state.JournalDuplicateMigrationActionApply: + fmt.Fprintf(out, "%s --apply\n", displayCommand) + case state.JournalDuplicateMigrationActionRollback: + fmt.Fprintf(out, "%s --rollback %s\n", displayCommand, result.RollbackManifestPath) + default: + fmt.Fprintf(out, "%s --dry-run\n", displayCommand) + } + fmt.Fprintf(out, "scope: %s database, journal-duplicate migration\n", result.DatabaseScope) + fmt.Fprintf(out, "database: %s\n", result.DatabasePath) + fmt.Fprintf(out, "action: %s\n", result.Action) + fmt.Fprintf(out, "applied: %t\n", result.Applied) + fmt.Fprintf(out, "copy run: %t\n", result.CopyRun) + if result.BackupPath != "" { + fmt.Fprintf(out, "backup: %s\n", result.BackupPath) + } + if result.RollbackManifestPath != "" { + fmt.Fprintf(out, "rollback manifest: %s\n", result.RollbackManifestPath) + } + fmt.Fprintf(out, "totals: june13=%d june24=%d pairs=%d retire=%d unproven=%d entries_retired=%d\n", + result.Totals.June13Rows, result.Totals.June24Rows, result.Totals.Pairs, result.Totals.Retire, result.Totals.Unproven, result.Totals.EntriesRetired) + for _, project := range result.Projects { + if project.Counts.Pairs == 0 && project.Counts.Unproven == 0 && project.Counts.Retire == 0 { + continue + } + fmt.Fprintf(out, "project %s (%s): june13=%d june24=%d pairs=%d retire=%d unproven=%d\n", + project.ProjectID, project.ProjectName, project.Counts.June13Rows, project.Counts.June24Rows, project.Counts.Pairs, project.Counts.Retire, project.Counts.Unproven) + for _, c := range project.Classifications { + if c.Proof != "unproven" { + continue + } + fmt.Fprintf(out, " unproven: %s [%s] %s(%s): %s%s\n", c.EntryID, c.Window, c.EntryType, c.Scope, truncateForDisplay(c.Message, 60), aliasOrphanDispositionSuffix(c.Disposition)) + } + } + for _, warning := range result.Warnings { + fmt.Fprintf(out, "warning: %s\n", warning) + } + switch result.Action { + case state.JournalDuplicateMigrationActionDryRun: + if result.Totals.Retire > 0 { + fmt.Fprintln(out, "next: rerun with --apply to repair after a backup; pass --retire for unproven multi-candidate rows") + } else if result.Totals.Unproven > 0 { + fmt.Fprintln(out, "next: unproven multi-candidate matches require explicit --retire on --apply") + } else { + fmt.Fprintln(out, "next: no journal-duplicate repair is needed") + } + case state.JournalDuplicateMigrationActionApply: + if result.RollbackManifestPath != "" { + fmt.Fprintln(out, "next: keep the rollback manifest until the migration is verified") + } + case state.JournalDuplicateMigrationActionRollback: + fmt.Fprintln(out, "next: inspect state before rerunning journal-duplicate migration") + } +} + +func truncateForDisplay(value string, max int) string { + if max <= 0 || len(value) <= max { + return value + } + if max <= 3 { + return value[:max] + } + return value[:max-3] + "..." +} + +func writeAliasOrphanMigrationHuman(out io.Writer, displayCommand string, result state.AliasOrphanMigrationResult) { + switch result.Action { + case state.AliasOrphanMigrationActionApply: + fmt.Fprintf(out, "%s --apply\n", displayCommand) + case state.AliasOrphanMigrationActionRollback: + fmt.Fprintf(out, "%s --rollback %s\n", displayCommand, result.RollbackManifestPath) + default: + fmt.Fprintf(out, "%s --dry-run\n", displayCommand) + } + fmt.Fprintf(out, "scope: %s database, alias-orphan migration\n", result.DatabaseScope) + fmt.Fprintf(out, "database: %s\n", result.DatabasePath) + fmt.Fprintf(out, "action: %s\n", result.Action) + fmt.Fprintf(out, "applied: %t\n", result.Applied) + fmt.Fprintf(out, "copy run: %t\n", result.CopyRun) + if result.BackupPath != "" { + fmt.Fprintf(out, "backup: %s\n", result.BackupPath) + } + if result.RollbackManifestPath != "" { + fmt.Fprintf(out, "rollback manifest: %s\n", result.RollbackManifestPath) + } + fmt.Fprintf(out, "totals: orphans=%d retire=%d unproven=%d dangling_aliases=%d orphaned_sources=%d\n", + result.Totals.Orphans, result.Totals.Retire, result.Totals.Unproven, result.Totals.DanglingAliases, result.Totals.OrphanedSources) + for _, project := range result.Projects { + if project.Counts.Orphans == 0 && project.Counts.DanglingAliases == 0 && project.Counts.NamedDispositions == 0 { + continue + } + fmt.Fprintf(out, "project %s (%s):\n", project.ProjectID, project.ProjectName) + for _, table := range project.Tables { + if table.Orphans == 0 && table.DanglingAliases == 0 { + continue + } + fmt.Fprintf(out, " %s: %d orphans — %d retire, %d unproven; dangling_aliases=%d; sources=%d orphan-referenced rows to retire\n", + table.Table, table.Orphans, table.Retire, table.Unproven, table.DanglingAliases, table.OrphanedSources) + // The operator has to name these on --apply, so they cannot live in + // --json alone. + for _, c := range table.Classifications { + if c.Proof != "unproven" { + continue + } + fmt.Fprintf(out, " unproven: %s%s%s\n", c.EntityID, aliasOrphanTitleSuffix(c.Title), aliasOrphanDispositionSuffix(c.Disposition)) + } + } + for _, d := range project.Dispositions { + if d.Action == "archive-as-moot" { + fmt.Fprintf(out, " dispositions: %s → archive-as-moot\n", d.EntityID) + } + } + } + for _, warning := range result.Warnings { + fmt.Fprintf(out, "warning: %s\n", warning) + } + switch result.Action { + case state.AliasOrphanMigrationActionDryRun: + if result.Totals.Retire > 0 || result.Totals.DanglingAliases > 0 || result.Totals.NamedDispositions > 0 { + fmt.Fprintln(out, "next: rerun with --apply to repair after a backup; pass --retire/--realias for unproven rows") + } else if result.Totals.Unproven > 0 { + fmt.Fprintln(out, "next: unproven orphans require explicit --retire or --realias on --apply") + } else { + fmt.Fprintln(out, "next: no alias-orphan repair is needed") + } + case state.AliasOrphanMigrationActionApply: + if result.RollbackManifestPath != "" { + fmt.Fprintln(out, "next: keep the rollback manifest until the migration is verified") + } + case state.AliasOrphanMigrationActionRollback: + fmt.Fprintln(out, "next: inspect state before rerunning alias-orphan migration") + } +} + func writeLifecycleStatusMigrationHuman(out io.Writer, displayCommand string, result state.LifecycleStatusMigrationResult) { switch result.Action { case state.LifecycleStatusMigrationActionApply: @@ -3813,11 +4066,15 @@ func writeSchemaUpgradeHuman(out io.Writer, displayCommand string, result state. } func (r Runner) inspectState(runtime state.Runtime) (state.Status, error) { + return r.inspectStateWithOptions(runtime, state.InspectOptions{}) +} + +func (r Runner) inspectStateWithOptions(runtime state.Runtime, options state.InspectOptions) (state.Status, error) { projectRoot, err := project.ResolveRoot(runtime.RootPath()) if err != nil { return state.Status{}, err } - return state.Inspect(projectRoot, state.PathResolver{StateHome: r.StateHome}) + return state.InspectWithOptions(projectRoot, state.PathResolver{StateHome: r.StateHome}, options) } func (r Runner) initializeState(runtime state.Runtime) (state.Status, error) { @@ -13396,6 +13653,22 @@ type lifecycleStatusMigrationOptions struct { rollbackPath string } +type aliasOrphanMigrationOptions struct { + jsonOutput bool + apply bool + dryRun bool + rollbackPath string + applyOptions state.AliasOrphanApplyOptions +} + +type journalDuplicateMigrationOptions struct { + jsonOutput bool + apply bool + dryRun bool + rollbackPath string + applyOptions state.JournalDuplicateApplyOptions +} + type relationshipOriginRepairOptions struct { jsonOutput bool apply bool @@ -13516,6 +13789,181 @@ func parseLifecycleStatusMigrationArgs(args []string, command string) (lifecycle return options, nil } +func parseAliasOrphanMigrationArgs(args []string, command string) (aliasOrphanMigrationOptions, error) { + var options aliasOrphanMigrationOptions + options.applyOptions.Realias = map[string]string{} + retireSeen := map[string]struct{}{} + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "--dry-run": + options.dryRun = true + case arg == "--json": + options.jsonOutput = true + case arg == "--apply": + options.apply = true + case arg == "--rollback": + if i+1 >= len(args) { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s requires --rollback ", command) + } + i++ + options.rollbackPath = args[i] + case arg == "--retire": + if i+1 >= len(args) { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s requires --retire ", command) + } + i++ + entityID := strings.TrimSpace(args[i]) + if entityID == "" { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s --retire requires a non-empty entity id", command) + } + if err := recordAliasOrphanRetire(&options, retireSeen, entityID, command); err != nil { + return aliasOrphanMigrationOptions{}, err + } + options.applyOptions.Flags = append(options.applyOptions.Flags, arg, args[i]) + case strings.HasPrefix(arg, "--retire="): + entityID := strings.TrimSpace(strings.TrimPrefix(arg, "--retire=")) + if entityID == "" { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s --retire requires a non-empty entity id", command) + } + if err := recordAliasOrphanRetire(&options, retireSeen, entityID, command); err != nil { + return aliasOrphanMigrationOptions{}, err + } + options.applyOptions.Flags = append(options.applyOptions.Flags, arg) + case arg == "--realias": + if i+1 >= len(args) { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s requires --realias =", command) + } + i++ + entityID, alias, err := parseAliasOrphanRealiasValue(args[i]) + if err != nil { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s: %w", command, err) + } + if err := recordAliasOrphanRealias(&options, retireSeen, entityID, alias, command); err != nil { + return aliasOrphanMigrationOptions{}, err + } + options.applyOptions.Flags = append(options.applyOptions.Flags, arg, args[i]) + case strings.HasPrefix(arg, "--realias="): + entityID, alias, err := parseAliasOrphanRealiasValue(strings.TrimPrefix(arg, "--realias=")) + if err != nil { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s: %w", command, err) + } + if err := recordAliasOrphanRealias(&options, retireSeen, entityID, alias, command); err != nil { + return aliasOrphanMigrationOptions{}, err + } + options.applyOptions.Flags = append(options.applyOptions.Flags, arg) + default: + return aliasOrphanMigrationOptions{}, fmt.Errorf("unknown option %q", arg) + } + } + if options.apply && options.dryRun { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s cannot combine --apply and --dry-run", command) + } + if options.rollbackPath != "" && (options.apply || options.dryRun) { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s cannot combine --rollback with --apply or --dry-run", command) + } + if options.rollbackPath != "" && (len(options.applyOptions.Retire) > 0 || len(options.applyOptions.Realias) > 0) { + return aliasOrphanMigrationOptions{}, fmt.Errorf("%s cannot combine --rollback with --retire or --realias", command) + } + return options, nil +} + +func parseJournalDuplicateMigrationArgs(args []string, command string) (journalDuplicateMigrationOptions, error) { + var options journalDuplicateMigrationOptions + retireSeen := map[string]struct{}{} + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "--dry-run": + options.dryRun = true + case arg == "--json": + options.jsonOutput = true + case arg == "--apply": + options.apply = true + case arg == "--rollback": + if i+1 >= len(args) { + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s requires --rollback ", command) + } + i++ + options.rollbackPath = args[i] + case arg == "--retire": + if i+1 >= len(args) { + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s requires --retire ", command) + } + i++ + entryID := strings.TrimSpace(args[i]) + if entryID == "" { + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s --retire requires a non-empty entry id", command) + } + if _, ok := retireSeen[entryID]; !ok { + retireSeen[entryID] = struct{}{} + options.applyOptions.Retire = append(options.applyOptions.Retire, entryID) + } + options.applyOptions.Flags = append(options.applyOptions.Flags, arg, args[i]) + case strings.HasPrefix(arg, "--retire="): + entryID := strings.TrimSpace(strings.TrimPrefix(arg, "--retire=")) + if entryID == "" { + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s --retire requires a non-empty entry id", command) + } + if _, ok := retireSeen[entryID]; !ok { + retireSeen[entryID] = struct{}{} + options.applyOptions.Retire = append(options.applyOptions.Retire, entryID) + } + options.applyOptions.Flags = append(options.applyOptions.Flags, arg) + case arg == "--realias", strings.HasPrefix(arg, "--realias="): + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s does not support --realias; journal rows carry no aliases — use --retire for unproven multi-candidate matches", command) + default: + return journalDuplicateMigrationOptions{}, fmt.Errorf("unknown option %q", arg) + } + } + if options.apply && options.dryRun { + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s cannot combine --apply and --dry-run", command) + } + if options.rollbackPath != "" && (options.apply || options.dryRun) { + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s cannot combine --rollback with --apply or --dry-run", command) + } + if options.rollbackPath != "" && len(options.applyOptions.Retire) > 0 { + return journalDuplicateMigrationOptions{}, fmt.Errorf("%s cannot combine --rollback with --retire", command) + } + return options, nil +} + +func recordAliasOrphanRetire(options *aliasOrphanMigrationOptions, retireSeen map[string]struct{}, entityID, command string) error { + if existing, ok := options.applyOptions.Realias[entityID]; ok { + return fmt.Errorf("%s: conflicting dispositions for %s: --retire and --realias %s=%s", command, entityID, entityID, existing) + } + if _, ok := retireSeen[entityID]; ok { + return nil + } + retireSeen[entityID] = struct{}{} + options.applyOptions.Retire = append(options.applyOptions.Retire, entityID) + return nil +} + +func recordAliasOrphanRealias(options *aliasOrphanMigrationOptions, retireSeen map[string]struct{}, entityID, alias, command string) error { + if _, ok := retireSeen[entityID]; ok { + return fmt.Errorf("%s: conflicting dispositions for %s: --retire and --realias %s=%s", command, entityID, entityID, alias) + } + if existing, ok := options.applyOptions.Realias[entityID]; ok { + if existing == alias { + return nil + } + return fmt.Errorf("%s: conflicting --realias for %s: %s and %s", command, entityID, existing, alias) + } + options.applyOptions.Realias[entityID] = alias + return nil +} + +func parseAliasOrphanRealiasValue(value string) (string, string, error) { + entityID, alias, ok := strings.Cut(value, "=") + entityID = strings.TrimSpace(entityID) + alias = strings.TrimSpace(alias) + if !ok || entityID == "" || alias == "" { + return "", "", fmt.Errorf("--realias requires =") + } + return entityID, alias, nil +} + func parseLegacyProjectDatabaseRepairArgs(args []string) (legacyProjectDatabaseRepairOptions, error) { var options legacyProjectDatabaseRepairOptions for _, arg := range args { diff --git a/internal/cli/cli_reference.go b/internal/cli/cli_reference.go index 88e2fdc40..c33e23fa6 100644 --- a/internal/cli/cli_reference.go +++ b/internal/cli/cli_reference.go @@ -221,6 +221,21 @@ func cliReferenceCommands() []cliReferenceCommand { {Flags: "--rollback ", Description: "Restore statuses from a lifecycle-statuses rollback manifest"}, {Flags: "--json", Description: "Output migration contract, project context, counts, backup, and rollback fields as JSON"}, }}, + {Name: "migrate alias-orphans", Description: "Retire alias-orphaned entity rows across every project with a backup and rollback manifest", Options: []cliReferenceOption{ + {Flags: "--dry-run", Description: "Preview classification on a temporary database copy (default)"}, + {Flags: "--apply", Description: "Apply the repair after creating a backup"}, + {Flags: "--rollback ", Description: "Restore deleted rows from an alias-orphans rollback manifest"}, + {Flags: "--retire ", Description: "Force-retire an unproven orphan (repeatable)"}, + {Flags: "--realias =", Description: "Attach an alias to an unproven orphan (repeatable)"}, + {Flags: "--json", Description: "Output migration contract, per-project classification, counts, backup, and rollback fields as JSON"}, + }}, + {Name: "migrate journal-duplicates", Description: "Retire June-13/June-24 journal natural-key twins across every project with a backup and rollback manifest", Options: []cliReferenceOption{ + {Flags: "--dry-run", Description: "Preview classification on a temporary database copy (default)"}, + {Flags: "--apply", Description: "Apply the repair after creating a backup"}, + {Flags: "--rollback ", Description: "Restore deleted rows from a journal-duplicates rollback manifest"}, + {Flags: "--retire ", Description: "Force-retire an unproven multi-candidate journal row (repeatable)"}, + {Flags: "--json", Description: "Output migration contract, per-project classification, counts, backup, and rollback fields as JSON"}, + }}, {Name: "migrate journal-first", Description: "Transform the global database to the journal-first model: purge lifecycle noise, drop the session entity, rekey journal search; destructive by consent", Options: []cliReferenceOption{ {Flags: "--dry-run", Description: "Preview counts against a temporary database copy without mutation or backup"}, {Flags: "--apply", Description: "Take a mandatory backup, then apply the migration to the live database"}, diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 8e6501a89..0b276fdfd 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -3451,6 +3451,12 @@ VALUES ('task-linear-typo', ?, NULL, 'Linear typo task', 'todo', 'P2', NULL, '20 t.Fatalf("insert task fixture error = %v", err) } if _, err := db.Exec(` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES ('alias-task-linear-typo', ?, 'task', 'task-linear-typo', 'task', 'TASK-LINEAR-TYPO', '2026-06-13T10:00:00Z', '2026-06-13T10:00:00Z') +`, initialized.ProjectID); err != nil { + t.Fatalf("insert task alias fixture error = %v", err) + } + if _, err := db.Exec(` INSERT INTO backend_mappings (id, project_id, backend, entity_kind, entity_id, external_kind, external_id, external_url, sync_status, created_at, updated_at) VALUES ('backend-mapping-linear-typo', ?, 'linear', 'task', 'task-linear-typo', 'issue', 'ENG-126', 'https://linear.app/workspace/issue/ENG-126', 'lnked', '2026-06-13T10:00:00Z', '2026-06-13T10:00:00Z') `, initialized.ProjectID); err != nil { @@ -3493,6 +3499,12 @@ VALUES ('task-active-unmapped', ?, NULL, 'Active unmapped task', 'todo', 'P2', N `, initialized.ProjectID); err != nil { t.Fatalf("insert task fixture error = %v", err) } + if _, err := db.Exec(` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES ('alias-task-active-unmapped', ?, 'task', 'task-active-unmapped', 'task', 'TASK-ACTIVE-UNMAPPED', '2026-06-13T10:00:00Z', '2026-06-13T10:00:00Z') +`, initialized.ProjectID); err != nil { + t.Fatalf("insert task alias fixture error = %v", err) + } closeCLITestDB(t, db) return workingDir, stateHome }, @@ -4458,6 +4470,12 @@ VALUES ('task-active-unmapped', ?, NULL, 'Active unmapped task', 'todo', 'P2', N `, initialized.ProjectID); err != nil { t.Fatalf("insert task fixture error = %v", err) } + if _, err := db.Exec(` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES ('alias-task-active-unmapped', ?, 'task', 'task-active-unmapped', 'task', 'TASK-ACTIVE-UNMAPPED', '2026-06-13T10:00:00Z', '2026-06-13T10:00:00Z') +`, initialized.ProjectID); err != nil { + t.Fatalf("insert task alias fixture error = %v", err) + } closeCLITestDB(t, db) var humanOut bytes.Buffer diff --git a/internal/state/alias_orphan_migration.go b/internal/state/alias_orphan_migration.go new file mode 100644 index 000000000..29d97ca19 --- /dev/null +++ b/internal/state/alias_orphan_migration.go @@ -0,0 +1,2317 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/levifig/loaf/internal/project" +) + +const ( + AliasOrphanMigrationActionDryRun = "dry-run" + AliasOrphanMigrationActionApply = "apply" + AliasOrphanMigrationActionRollback = "rollback" + + aliasOrphanMigrationName = "alias-orphans" + + aliasOrphanProofDerivation = "derivation" + // aliasOrphanProofSourceDerivation recomputes the orphan's own source row + // under a legacy salt and then binds it to a holder by content. The salt half + // is derivation; the binding half is content identity, so the manifest labels + // it as its own class instead of borrowing the stronger name. + aliasOrphanProofSourceDerivation = "source-derivation" + aliasOrphanProofContentIdentity = "content-identity" + aliasOrphanProofUnproven = "unproven" + + aliasOrphanDispositionRetire = "retire" + aliasOrphanDispositionRealias = "realias" + aliasOrphanDispositionArchiveMoot = "archive-as-moot" + aliasOrphanDispositionDeleteDangle = "delete-dangling-alias" + + // brokenEvidenceReportID is the named per-row disposition for the bodyless + // report whose evidence is unrecoverable (SPEC-047 already shipped the + // simplification it guarded). + brokenEvidenceReportID = "report:7644bb23d2664de93b6cb6a5" + + aliasOrphanArchiveMootEventType = "status_normalized" + aliasOrphanArchiveMootNote = "evidence unrecoverable; archived as moot — SPEC-047 shipped the simplification this report guarded against deepening" + + // The 2026-06-24 re-import wrote its rows in one tight instant cluster; rows created later that + // same day are real work, not re-import twins, so membership is a window and not a date prefix. + june24ReimportWindowStart = "2026-06-24T13:03:00Z" + june24ReimportWindowEnd = "2026-06-24T13:04:00Z" + + // The 2026-06-13 original import wrote at one instant; bodyless content-identity twins use this + // window so two empty fingerprints are not treated as equal evidence outside the import event. + june13OriginalImportWindowStart = "2026-06-13T01:39:00Z" + june13OriginalImportWindowEnd = "2026-06-13T01:46:00Z" +) + +// AliasOrphanMigrationResult is the preview/apply/rollback outcome for the +// alias-orphan repair migration. Classification spans every project in the +// global database. +type AliasOrphanMigrationResult struct { + ContractVersion int `json:"contract_version"` + DatabaseScope string `json:"database_scope"` + DatabasePath string `json:"database_path"` + ProjectID string `json:"project_id,omitempty"` + ProjectName string `json:"project_name,omitempty"` + ProjectCurrentPath string `json:"project_current_path,omitempty"` + Action string `json:"action"` + Applied bool `json:"applied"` + CopyRun bool `json:"copy_run"` + BackupPath string `json:"backup_path,omitempty"` + RollbackManifestPath string `json:"rollback_manifest_path,omitempty"` + Projects []AliasOrphanProjectSummary `json:"projects"` + Totals AliasOrphanCounts `json:"totals"` + Dispositions []AliasOrphanDisposition `json:"dispositions,omitempty"` + OperatorFlags []string `json:"operator_flags,omitempty"` + Warnings []string `json:"warnings,omitempty"` + RowsRestored int `json:"rows_restored,omitempty"` +} + +// AliasOrphanProjectSummary reports classification for one project. +type AliasOrphanProjectSummary struct { + ProjectID string `json:"project_id"` + ProjectName string `json:"project_name,omitempty"` + ProjectCurrentPath string `json:"project_current_path,omitempty"` + LegacyProjectID string `json:"legacy_project_id,omitempty"` + LegacyProjectIDs []string `json:"legacy_project_ids,omitempty"` + LegacyPaths []string `json:"legacy_paths,omitempty"` + Tables []AliasOrphanTableSummary `json:"tables"` + Counts AliasOrphanCounts `json:"counts"` + Dispositions []AliasOrphanDisposition `json:"dispositions,omitempty"` +} + +// AliasOrphanTableSummary reports per-entity-table classification counts. +// Retire counts the rows this run would retire — proven twins plus any row the +// operator named with --retire. Unproven counts rows no proof reached, whatever +// disposition the operator then supplied for them, so the two overlap by design. +type AliasOrphanTableSummary struct { + Kind string `json:"kind"` + Table string `json:"table"` + Orphans int `json:"orphans"` + Retire int `json:"retire"` + Unproven int `json:"unproven"` + DanglingAliases int `json:"dangling_aliases"` + OrphanedSources int `json:"orphaned_sources,omitempty"` + Classifications []AliasOrphanRowClassify `json:"classifications,omitempty"` + DanglingAliasIDs []string `json:"dangling_alias_ids,omitempty"` +} + +// AliasOrphanCounts aggregates migration action counts. +type AliasOrphanCounts struct { + Orphans int `json:"orphans"` + Retire int `json:"retire"` + Unproven int `json:"unproven"` + DanglingAliases int `json:"dangling_aliases"` + OrphanedSources int `json:"orphaned_sources"` + NamedDispositions int `json:"named_dispositions,omitempty"` + OperatorRetire int `json:"operator_retire,omitempty"` + OperatorRealias int `json:"operator_realias,omitempty"` + EntitiesRetired int `json:"entities_retired,omitempty"` + AliasesDeleted int `json:"aliases_deleted,omitempty"` + SourcesDeleted int `json:"sources_deleted,omitempty"` + StatusesChanged int `json:"statuses_changed,omitempty"` + AliasesInserted int `json:"aliases_inserted,omitempty"` +} + +// AliasOrphanRowClassify is one orphan entity's classification. +type AliasOrphanRowClassify struct { + ProjectID string `json:"project_id"` + Kind string `json:"kind"` + Table string `json:"table"` + EntityID string `json:"entity_id"` + Title string `json:"title,omitempty"` + Proof string `json:"proof"` + TwinID string `json:"twin_id,omitempty"` + TwinAlias string `json:"twin_alias,omitempty"` + LegacyProjectID string `json:"legacy_project_id,omitempty"` + LegacyPath string `json:"legacy_path,omitempty"` + Disposition string `json:"disposition,omitempty"` +} + +// AliasOrphanDisposition is a planned action against a specific row. +type AliasOrphanDisposition struct { + ProjectID string `json:"project_id"` + Kind string `json:"kind,omitempty"` + EntityID string `json:"entity_id"` + Action string `json:"action"` + Alias string `json:"alias,omitempty"` + Proof string `json:"proof,omitempty"` + TwinID string `json:"twin_id,omitempty"` + TwinAlias string `json:"twin_alias,omitempty"` + LegacyProjectID string `json:"legacy_project_id,omitempty"` + LegacyPath string `json:"legacy_path,omitempty"` + Note string `json:"note,omitempty"` + Flag string `json:"flag,omitempty"` +} + +// AliasOrphanApplyOptions carries explicit per-row operator dispositions for apply. +type AliasOrphanApplyOptions struct { + Retire []string // entity IDs + Realias map[string]string // entity ID → alias + Flags []string // verbatim flag strings for the manifest +} + +// AliasOrphanRollbackManifest preserves every deleted/changed row for rollback. +type AliasOrphanRollbackManifest struct { + ContractVersion int `json:"contract_version"` + Migration string `json:"migration"` + CreatedAt string `json:"created_at"` + DatabaseScope string `json:"database_scope"` + DatabasePath string `json:"database_path"` + OperatorFlags []string `json:"operator_flags,omitempty"` + OperatorDispositions []AliasOrphanDisposition `json:"operator_dispositions,omitempty"` + Retirements []AliasOrphanDisposition `json:"retirements,omitempty"` + DeletedRows []AliasOrphanDeletedRow `json:"deleted_rows"` + StatusChanges []AliasOrphanStatusChange `json:"status_changes,omitempty"` + AliasInserts []AliasOrphanAliasInsert `json:"alias_inserts,omitempty"` + Unlinks []AliasOrphanUnlink `json:"unlinks,omitempty"` + Counts AliasOrphanCounts `json:"counts"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// AliasOrphanDeletedRow is one full row snapshot for rollback restore. +type AliasOrphanDeletedRow struct { + Table string `json:"table"` + Columns []string `json:"columns"` + Values []any `json:"values"` + Order int `json:"order"` + Meta map[string]string `json:"meta,omitempty"` +} + +// AliasOrphanStatusChange records a status rewrite for rollback. +type AliasOrphanStatusChange struct { + ProjectID string `json:"project_id"` + Table string `json:"table"` + Kind string `json:"kind"` + EntityID string `json:"entity_id"` + PreviousStatus string `json:"previous_status"` + PreviousUpdatedAt string `json:"previous_updated_at,omitempty"` + NewStatus string `json:"new_status"` + EventID string `json:"event_id"` + EventNote string `json:"event_note"` +} + +// AliasOrphanAliasInsert records an alias created by --realias for rollback. +type AliasOrphanAliasInsert struct { + ProjectID string `json:"project_id"` + AliasID string `json:"alias_id"` + EntityKind string `json:"entity_kind"` + EntityID string `json:"entity_id"` + Namespace string `json:"namespace"` + Alias string `json:"alias"` +} + +// AliasOrphanUnlink records a reference rewrite for rollback restore. NewID is +// empty when the column was nulled and carries the twin's ID when the reference +// was repointed at the surviving row. +type AliasOrphanUnlink struct { + Table string `json:"table"` + ProjectID string `json:"project_id"` + Column string `json:"column"` + KeyColumn string `json:"key_column,omitempty"` + RowID string `json:"row_id"` + PreviousID string `json:"previous_id"` + NewID string `json:"new_id,omitempty"` +} + +type aliasOrphanEntityTable struct { + kind string + table string + titleColumn string + sourceColumn string + namespace string +} + +var aliasOrphanEntityTables = []aliasOrphanEntityTable{ + {kind: "task", table: "tasks", titleColumn: "title", sourceColumn: "body_source_id", namespace: "task"}, + {kind: "spec", table: "specs", titleColumn: "title", sourceColumn: "body_source_id", namespace: "spec"}, + {kind: "report", table: "reports", titleColumn: "title", sourceColumn: "body_source_id", namespace: "report"}, + {kind: "idea", table: "ideas", titleColumn: "title", sourceColumn: "body_source_id", namespace: "idea"}, + {kind: "spark", table: "sparks", titleColumn: "text", sourceColumn: "source_id", namespace: "spark"}, + {kind: "brainstorm", table: "brainstorms", titleColumn: "title", sourceColumn: "body_source_id", namespace: "brainstorm"}, + {kind: "shaping_draft", table: "shaping_drafts", titleColumn: "title", sourceColumn: "body_source_id", namespace: "shaping_draft"}, +} + +// aliasOrphanDeadAliasPredicate distinguishes a dead alias from a forward +// reference. An alias with no entity row is damage only when nothing in the +// project still names that entity: the importer legitimately registers the alias +// of a referenced-but-unimported artifact (a `depends_on` naming a task with no +// file) so the reference renders as `TASK-000` instead of an opaque ID, and it +// deletes the edge when the reference leaves the markdown. What survives that +// deletion — an alias nothing points at, its row long gone — is the wreckage +// this migration collects. Placeholders are formatted with the entity table. +const aliasOrphanDeadAliasPredicate = ` + AND NOT EXISTS ( + SELECT 1 FROM %s AS e + WHERE e.project_id = a.project_id AND e.id = a.entity_id + ) + AND NOT EXISTS ( + SELECT 1 FROM relationships AS r + WHERE r.project_id = a.project_id + AND ((r.from_entity_kind = a.entity_kind AND r.from_entity_id = a.entity_id) + OR (r.to_entity_kind = a.entity_kind AND r.to_entity_id = a.entity_id)) + )` + +const aliasOrphanDanglingAliasQuery = ` +SELECT a.id +FROM aliases AS a +WHERE a.project_id = ? + AND a.entity_kind = ? + AND a.namespace = ?` + aliasOrphanDeadAliasPredicate + ` +ORDER BY a.id +` + +// aliasOrphanTableForKind returns the entity-table descriptor this migration +// classifies for a kind. +func aliasOrphanTableForKind(kind string) (aliasOrphanEntityTable, bool) { + for _, table := range aliasOrphanEntityTables { + if table.kind == kind { + return table, true + } + } + return aliasOrphanEntityTable{}, false +} + +// PreviewAliasOrphanMigration classifies alias-orphans against a temporary copy. +// Options may include operator --retire / --realias dispositions so the ceremony +// invocation can be rehearsed before --apply. +func PreviewAliasOrphanMigration(ctx context.Context, root project.Root, resolver PathResolver, options AliasOrphanApplyOptions) (AliasOrphanMigrationResult, error) { + status, err := requireAliasOrphanMigrationStatus(root, resolver) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + source, err := OpenStoreReadOnly(status.DatabasePath) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + defer source.Close() + + tempDir, err := os.MkdirTemp("", "loaf-alias-orphan-migration-*") + if err != nil { + return AliasOrphanMigrationResult{}, fmt.Errorf("create alias-orphan migration temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + copyPath := filepath.Join(tempDir, "state.sqlite") + if err := copySQLiteDatabase(ctx, source, copyPath, 0o600); err != nil { + return AliasOrphanMigrationResult{}, err + } + copyStore, err := OpenStore(copyPath) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + defer copyStore.Close() + + result, manifest, err := planAliasOrphanMigration(ctx, copyStore, aliasOrphanMigrationBaseResult(status, AliasOrphanMigrationActionDryRun), options, aliasOrphanExecutedDispositions{}) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + result.OperatorFlags = append([]string{}, options.Flags...) + // The copy is disposable, so the repair runs against it for real. That is + // the only way the preview can report the source rows the retire set will + // strand — the blast radius the go/no-go decision reads. + if err := applyAliasOrphanMigrationManifest(ctx, copyStore, &manifest, nil); err != nil { + return AliasOrphanMigrationResult{}, fmt.Errorf("simulate alias-orphan migration: %w", err) + } + applyAliasOrphanSimulationProjection(&result, manifest) + result.CopyRun = true + return result, nil +} + +// applyAliasOrphanSimulationProjection folds the simulated run's collateral back +// into the plan, per table and in total: the source rows the retire set strands, +// and the aliases the retirements themselves kill. Neither is visible to +// classification — both only exist once the repair has run — so the preview +// reads them off the simulation instead, and the operator sees the same blast +// radius apply will produce. +func applyAliasOrphanSimulationProjection(result *AliasOrphanMigrationResult, manifest AliasOrphanRollbackManifest) { + sourcesByTable := map[string]int{} + aliasesByTable := map[string][]string{} + for _, row := range manifest.DeletedRows { + projectID := rowValueString(row, "project_id") + switch row.Table { + case "sources": + sourcesByTable[projectID+"\x00"+row.Meta["entity_kind"]]++ + case "aliases": + key := projectID + "\x00" + rowValueString(row, "entity_kind") + aliasesByTable[key] = append(aliasesByTable[key], rowValueString(row, "id")) + } + } + result.Totals.OrphanedSources = manifest.Counts.OrphanedSources + result.Totals.SourcesDeleted = manifest.Counts.SourcesDeleted + result.Totals.DanglingAliases = 0 + result.Dispositions = nil + for i := range result.Projects { + project := &result.Projects[i] + project.Counts.DanglingAliases = 0 + for j := range project.Tables { + table := &project.Tables[j] + key := project.ProjectID + "\x00" + table.Kind + table.OrphanedSources = sourcesByTable[key] + project.Counts.OrphanedSources += table.OrphanedSources + + known := map[string]struct{}{} + for _, aliasID := range table.DanglingAliasIDs { + known[aliasID] = struct{}{} + } + for _, aliasID := range aliasesByTable[key] { + if _, seen := known[aliasID]; seen { + continue + } + known[aliasID] = struct{}{} + table.DanglingAliasIDs = append(table.DanglingAliasIDs, aliasID) + project.Dispositions = append(project.Dispositions, AliasOrphanDisposition{ + ProjectID: project.ProjectID, + Kind: table.Kind, + EntityID: aliasID, + Action: aliasOrphanDispositionDeleteDangle, + }) + } + sort.Strings(table.DanglingAliasIDs) + table.DanglingAliases = len(table.DanglingAliasIDs) + project.Counts.DanglingAliases += table.DanglingAliases + } + result.Totals.DanglingAliases += project.Counts.DanglingAliases + result.Dispositions = append(result.Dispositions, project.Dispositions...) + } +} + +// ApplyAliasOrphanMigration backs up, writes a rollback manifest, and repairs alias-orphans. +func ApplyAliasOrphanMigration(ctx context.Context, root project.Root, resolver PathResolver, options AliasOrphanApplyOptions) (AliasOrphanMigrationResult, error) { + status, err := requireAliasOrphanMigrationStatus(root, resolver) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + backup, err := Backup(ctx, root, resolver) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + store, err := openInitializedStore(root, resolver) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + defer store.Close() + + executed, err := readAliasOrphanExecutedDispositions(filepath.Dir(backup.BackupPath)) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + result, manifest, err := planAliasOrphanMigration(ctx, store, aliasOrphanMigrationBaseResult(status, AliasOrphanMigrationActionApply), options, executed) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + result.BackupPath = backup.BackupPath + result.OperatorFlags = append([]string{}, options.Flags...) + // A disposition that names nothing and was never carried out is a typo, and + // silently applying the rest would leave the operator believing a row was + // handled. + if len(result.Warnings) > 0 { + return result, fmt.Errorf("alias-orphan dispositions matched no rows: %s", strings.Join(result.Warnings, "; ")) + } + result.Applied = true + + // The row snapshots are captured inside the transaction and the rollback + // manifest is written to disk before COMMIT, so no deletion is ever visible + // without its restore record: backup → manifest → apply. + manifestPath := "" + if err := applyAliasOrphanMigrationManifest(ctx, store, &manifest, func(final AliasOrphanRollbackManifest) error { + if !aliasOrphanManifestHasWork(final) { + return nil + } + path, err := writeAliasOrphanRollbackManifest(final, filepath.Dir(backup.BackupPath), time.Now().UTC()) + if err != nil { + return err + } + manifestPath = path + return nil + }); err != nil { + if manifestPath != "" { + os.Remove(manifestPath) + } + // Nothing committed: the backup is the only artifact this run left. + result.Applied = false + return result, err + } + result.RollbackManifestPath = manifestPath + result.Totals.EntitiesRetired = manifest.Counts.EntitiesRetired + result.Totals.AliasesDeleted = manifest.Counts.AliasesDeleted + result.Totals.SourcesDeleted = manifest.Counts.SourcesDeleted + result.Totals.OrphanedSources = manifest.Counts.OrphanedSources + result.Totals.StatusesChanged = manifest.Counts.StatusesChanged + result.Totals.AliasesInserted = manifest.Counts.AliasesInserted + + // The repair is committed from here on, so every failure has to hand back the + // backup and the rollback manifest — they are the operator's way out. + verify, _, err := planAliasOrphanMigration(ctx, store, aliasOrphanMigrationBaseResult(status, AliasOrphanMigrationActionApply), AliasOrphanApplyOptions{}, executed) + if err != nil { + return result, fmt.Errorf("post-apply verification: %w (backup %s, rollback manifest %s)", err, result.BackupPath, result.RollbackManifestPath) + } + if verify.Totals.Retire > 0 || verify.Totals.DanglingAliases > 0 { + return result, fmt.Errorf("post-apply verification failed: %d retire-class orphans and %d dangling aliases remain (backup %s, rollback manifest %s)", verify.Totals.Retire, verify.Totals.DanglingAliases, result.BackupPath, result.RollbackManifestPath) + } + return result, nil +} + +// RollbackAliasOrphanMigration restores rows recorded in an alias-orphan rollback manifest. +func RollbackAliasOrphanMigration(ctx context.Context, root project.Root, resolver PathResolver, manifestPath string) (AliasOrphanMigrationResult, error) { + if manifestPath == "" { + return AliasOrphanMigrationResult{}, fmt.Errorf("alias-orphan rollback requires a manifest path") + } + status, err := requireAliasOrphanMigrationStatus(root, resolver) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + manifest, err := readAliasOrphanRollbackManifest(manifestPath) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + backup, err := Backup(ctx, root, resolver) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + store, err := openInitializedStore(root, resolver) + if err != nil { + return AliasOrphanMigrationResult{}, err + } + defer store.Close() + + result := aliasOrphanMigrationBaseResult(status, AliasOrphanMigrationActionRollback) + result.Applied = true + result.BackupPath = backup.BackupPath + result.RollbackManifestPath = manifestPath + result.OperatorFlags = append([]string{}, manifest.OperatorFlags...) + if err := rollbackAliasOrphanMigrationManifest(ctx, store, manifest, &result); err != nil { + return AliasOrphanMigrationResult{}, err + } + return result, nil +} + +func aliasOrphanMigrationBaseResult(status Status, action string) AliasOrphanMigrationResult { + return AliasOrphanMigrationResult{ + ContractVersion: StateJSONContractVersion, + DatabaseScope: "global", + DatabasePath: status.DatabasePath, + ProjectID: status.ProjectID, + ProjectName: status.ProjectName, + ProjectCurrentPath: status.ProjectCurrentPath, + Action: action, + Projects: []AliasOrphanProjectSummary{}, + } +} + +func requireAliasOrphanMigrationStatus(root project.Root, resolver PathResolver) (Status, error) { + status, err := Inspect(root, resolver) + if err != nil { + return Status{}, err + } + switch status.Mode { + case ModeSQLiteReady: + return status, nil + case ModeMarkdownOnly: + return Status{}, fmt.Errorf("SQLite state database is not initialized; run `loaf state migrate markdown --apply` first") + case ModeInvalid: + return Status{}, fmt.Errorf("state database is invalid; run `loaf state doctor`") + default: + return Status{}, fmt.Errorf("state database is not ready; run `loaf state status`") + } +} + +func planAliasOrphanMigration(ctx context.Context, store *Store, result AliasOrphanMigrationResult, options AliasOrphanApplyOptions, executed aliasOrphanExecutedDispositions) (AliasOrphanMigrationResult, AliasOrphanRollbackManifest, error) { + manifest := AliasOrphanRollbackManifest{ + ContractVersion: StateJSONContractVersion, + Migration: aliasOrphanMigrationName, + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + DatabaseScope: result.DatabaseScope, + DatabasePath: result.DatabasePath, + OperatorFlags: append([]string{}, options.Flags...), + DeletedRows: []AliasOrphanDeletedRow{}, + Metadata: map[string]string{ + "broken_evidence_report_id": brokenEvidenceReportID, + }, + } + + retireSet, realiasSet := aliasOrphanOperatorSets(options) + for _, id := range sortedKeys(retireSet) { + manifest.OperatorDispositions = append(manifest.OperatorDispositions, AliasOrphanDisposition{ + EntityID: id, + Action: aliasOrphanDispositionRetire, + Flag: "--retire " + id, + }) + } + for _, id := range sortedKeys(realiasSet) { + manifest.OperatorDispositions = append(manifest.OperatorDispositions, AliasOrphanDisposition{ + EntityID: id, + Action: aliasOrphanDispositionRealias, + Alias: realiasSet[id], + Flag: "--realias " + id + "=" + realiasSet[id], + }) + } + + projects, err := store.ListProjects(ctx) + if err != nil { + return result, manifest, err + } + + matched := map[string]struct{}{} + for _, project := range projects.Projects { + salts, err := aliasOrphanLegacySalts(ctx, store.db, project) + if err != nil { + return result, manifest, err + } + summary, err := classifyAliasOrphansForProject(ctx, store.db, project, salts, retireSet, realiasSet) + if err != nil { + return result, manifest, err + } + result.Projects = append(result.Projects, summary) + result.Totals.Orphans += summary.Counts.Orphans + result.Totals.Retire += summary.Counts.Retire + result.Totals.Unproven += summary.Counts.Unproven + result.Totals.DanglingAliases += summary.Counts.DanglingAliases + result.Totals.NamedDispositions += summary.Counts.NamedDispositions + result.Totals.OperatorRetire += summary.Counts.OperatorRetire + result.Totals.OperatorRealias += summary.Counts.OperatorRealias + result.Dispositions = append(result.Dispositions, summary.Dispositions...) + for _, table := range summary.Tables { + for _, c := range table.Classifications { + matched[c.EntityID] = struct{}{} + } + } + } + warnings, err := aliasOrphanUnmatchedDispositionWarnings(ctx, store.db, retireSet, realiasSet, matched, executed) + if err != nil { + return result, manifest, err + } + result.Warnings = append(result.Warnings, warnings...) + + manifest.Counts = AliasOrphanCounts{ + Orphans: result.Totals.Orphans, + Retire: result.Totals.Retire, + Unproven: result.Totals.Unproven, + DanglingAliases: result.Totals.DanglingAliases, + NamedDispositions: result.Totals.NamedDispositions, + OperatorRetire: result.Totals.OperatorRetire, + OperatorRealias: result.Totals.OperatorRealias, + } + return result, manifest, nil +} + +func aliasOrphanOperatorSets(options AliasOrphanApplyOptions) (map[string]struct{}, map[string]string) { + retireSet := map[string]struct{}{} + for _, id := range options.Retire { + if id = strings.TrimSpace(id); id != "" { + retireSet[id] = struct{}{} + } + } + realiasSet := map[string]string{} + for id, alias := range options.Realias { + id = strings.TrimSpace(id) + alias = strings.TrimSpace(alias) + if id != "" && alias != "" { + realiasSet[id] = alias + } + } + return retireSet, realiasSet +} + +// aliasOrphanUnmatchedDispositionWarnings names every operator flag that no +// classified orphan answered, so a typo is reported instead of ignored. A flag +// an earlier run already carried out is not a typo: rerunning the exact command +// the ceremony recorded has to be a no-op, so a disposition whose end state is +// already in place is passed over silently. +func aliasOrphanUnmatchedDispositionWarnings(ctx context.Context, q aliasOrphanQuerier, retireSet map[string]struct{}, realiasSet map[string]string, matched map[string]struct{}, executed aliasOrphanExecutedDispositions) ([]string, error) { + var warnings []string + for _, id := range sortedKeys(retireSet) { + if _, ok := matched[id]; ok { + continue + } + done, err := executed.retirementCarriedOut(ctx, q, id) + if err != nil { + return nil, err + } + if done { + continue + } + warnings = append(warnings, fmt.Sprintf("--retire %s matched no alias-orphan row", id)) + } + for _, id := range sortedKeys(realiasSet) { + if _, ok := matched[id]; ok { + continue + } + done, err := aliasOrphanAliasNames(ctx, q, id, realiasSet[id]) + if err != nil { + return nil, err + } + if done { + continue + } + warnings = append(warnings, fmt.Sprintf("--realias %s=%s matched no alias-orphan row", id, realiasSet[id])) + } + return warnings, nil +} + +// aliasOrphanExecutedDispositions is the set of operator dispositions earlier +// runs of this migration recorded in their rollback manifests. The manifests are +// the only durable evidence that a row was retired on purpose — the row itself +// is gone — so they are what separates a rerun from a typo. +type aliasOrphanExecutedDispositions struct { + retired map[string]struct{} +} + +// readAliasOrphanExecutedDispositions collects the entity IDs earlier alias-orphan +// runs retired, from the rollback manifests kept beside the backups. A directory +// that cannot be read yields an empty set: no evidence is not evidence of a typo +// either way, and the caller still refuses the disposition. +func readAliasOrphanExecutedDispositions(dir string) (aliasOrphanExecutedDispositions, error) { + executed := aliasOrphanExecutedDispositions{retired: map[string]struct{}{}} + if dir == "" { + return executed, nil + } + paths, err := filepath.Glob(filepath.Join(dir, "alias-orphan-rollback-*.json")) + if err != nil { + return executed, fmt.Errorf("list alias-orphan rollback manifests: %w", err) + } + for _, path := range paths { + payload, err := os.ReadFile(path) + if err != nil { + return executed, fmt.Errorf("read alias-orphan rollback manifest %s: %w", path, err) + } + var manifest struct { + Retirements []AliasOrphanDisposition `json:"retirements"` + } + if err := json.Unmarshal(payload, &manifest); err != nil { + // A manifest this migration cannot parse is not proof of anything; + // leaving it out only makes the disposition check stricter. + continue + } + for _, retirement := range manifest.Retirements { + executed.retired[retirement.EntityID] = struct{}{} + } + } + return executed, nil +} + +// retirementCarriedOut reports whether an earlier run retired this entity and +// the row is still gone. +func (e aliasOrphanExecutedDispositions) retirementCarriedOut(ctx context.Context, q aliasOrphanQuerier, entityID string) (bool, error) { + if _, ok := e.retired[entityID]; !ok { + return false, nil + } + exists, err := aliasOrphanEntityRowExists(ctx, q, entityID) + if err != nil { + return false, err + } + return !exists, nil +} + +// aliasOrphanEntityRowExists looks for an entity row with this ID in any table +// the migration classifies, in any project. +func aliasOrphanEntityRowExists(ctx context.Context, q aliasOrphanQuerier, entityID string) (bool, error) { + for _, table := range aliasOrphanEntityTables { + exists, err := sqliteTableExistsQ(ctx, q, table.table) + if err != nil { + return false, err + } + if !exists { + continue + } + var found int + query := fmt.Sprintf(`SELECT EXISTS(SELECT 1 FROM %s WHERE id = ?)`, quoteSQLiteIdentifier(table.table)) + if err := q.QueryRowContext(ctx, query, entityID).Scan(&found); err != nil { + return false, fmt.Errorf("look for %s row %s: %w", table.table, entityID, err) + } + if found == 1 { + return true, nil + } + } + return false, nil +} + +// aliasOrphanAliasNames reports whether the alias already names this entity — +// the end state --realias asks for. +func aliasOrphanAliasNames(ctx context.Context, q aliasOrphanQuerier, entityID string, alias string) (bool, error) { + var found int + if err := q.QueryRowContext(ctx, ` +SELECT EXISTS(SELECT 1 FROM aliases WHERE entity_id = ? AND alias = ?) +`, entityID, alias).Scan(&found); err != nil { + return false, fmt.Errorf("look for alias %s on %s: %w", alias, entityID, err) + } + return found == 1, nil +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +// aliasOrphanQuerier is satisfied by both *sql.DB and *sql.Tx so classification +// runs from exactly one implementation on the preview and apply paths. +type aliasOrphanQuerier interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// aliasOrphanLegacySalt is one historical project ID the entity IDs of this +// project may have been derived under, with the path that produced it. +type aliasOrphanLegacySalt struct { + projectID string + path string +} + +func classifyAliasOrphansForProject(ctx context.Context, q aliasOrphanQuerier, project ProjectIdentity, salts []aliasOrphanLegacySalt, retireSet map[string]struct{}, realiasSet map[string]string) (AliasOrphanProjectSummary, error) { + summary := AliasOrphanProjectSummary{ + ProjectID: project.ID, + ProjectName: project.FriendlyName, + ProjectCurrentPath: project.CurrentPath, + Tables: []AliasOrphanTableSummary{}, + Dispositions: []AliasOrphanDisposition{}, + } + for _, salt := range salts { + summary.LegacyProjectIDs = append(summary.LegacyProjectIDs, salt.projectID) + summary.LegacyPaths = append(summary.LegacyPaths, salt.path) + if salt.path == project.CurrentPath { + summary.LegacyProjectID = salt.projectID + } + } + if summary.LegacyProjectID == "" && len(salts) > 0 { + summary.LegacyProjectID = salts[0].projectID + } + + for _, table := range aliasOrphanEntityTables { + exists, err := sqliteTableExistsQ(ctx, q, table.table) + if err != nil { + return summary, err + } + if !exists { + continue + } + tableSummary, err := classifyAliasOrphansForTable(ctx, q, project.ID, salts, table, retireSet, realiasSet) + if err != nil { + return summary, err + } + summary.Tables = append(summary.Tables, tableSummary) + summary.Counts.Orphans += tableSummary.Orphans + summary.Counts.Retire += tableSummary.Retire + summary.Counts.Unproven += tableSummary.Unproven + summary.Counts.DanglingAliases += tableSummary.DanglingAliases + for _, c := range tableSummary.Classifications { + switch { + case c.Disposition == aliasOrphanDispositionRetire && c.Proof != aliasOrphanProofUnproven: + summary.Dispositions = append(summary.Dispositions, aliasOrphanRetireDisposition(c)) + case c.Disposition == aliasOrphanDispositionRetire: + summary.Counts.OperatorRetire++ + d := aliasOrphanRetireDisposition(c) + d.Flag = "--retire " + c.EntityID + summary.Dispositions = append(summary.Dispositions, d) + case c.Disposition == aliasOrphanDispositionRealias: + summary.Counts.OperatorRealias++ + summary.Dispositions = append(summary.Dispositions, AliasOrphanDisposition{ + ProjectID: project.ID, + Kind: c.Kind, + EntityID: c.EntityID, + Action: aliasOrphanDispositionRealias, + Alias: realiasSet[c.EntityID], + Proof: c.Proof, + Flag: "--realias " + c.EntityID + "=" + realiasSet[c.EntityID], + }) + } + } + for _, aliasID := range tableSummary.DanglingAliasIDs { + summary.Dispositions = append(summary.Dispositions, AliasOrphanDisposition{ + ProjectID: project.ID, + Kind: table.kind, + EntityID: aliasID, + Action: aliasOrphanDispositionDeleteDangle, + }) + } + } + + named, err := classifyBrokenEvidenceReport(ctx, q, project.ID) + if err != nil { + return summary, err + } + if named != nil { + summary.Dispositions = append(summary.Dispositions, *named) + summary.Counts.NamedDispositions++ + } + return summary, nil +} + +func aliasOrphanRetireDisposition(c AliasOrphanRowClassify) AliasOrphanDisposition { + return AliasOrphanDisposition{ + ProjectID: c.ProjectID, + Kind: c.Kind, + EntityID: c.EntityID, + Action: aliasOrphanDispositionRetire, + Proof: c.Proof, + TwinID: c.TwinID, + TwinAlias: c.TwinAlias, + LegacyProjectID: c.LegacyProjectID, + LegacyPath: c.LegacyPath, + Note: c.TwinAlias, + } +} + +// aliasOrphanRow is an entity row on either side of a twin proof. +type aliasOrphanRow struct { + entityID string + alias string + title string + createdAt string + sourceID string + sourcePath string +} + +func classifyAliasOrphansForTable(ctx context.Context, q aliasOrphanQuerier, projectID string, salts []aliasOrphanLegacySalt, table aliasOrphanEntityTable, retireSet map[string]struct{}, realiasSet map[string]string) (AliasOrphanTableSummary, error) { + summary := AliasOrphanTableSummary{ + Kind: table.kind, + Table: table.table, + Classifications: []AliasOrphanRowClassify{}, + } + + orphans, err := readAliasOrphanRows(ctx, q, projectID, table, false) + if err != nil { + return summary, err + } + holders, err := readAliasOrphanRows(ctx, q, projectID, table, true) + if err != nil { + return summary, err + } + + holdersByTitle := map[string][]aliasOrphanRow{} + // derivedIDs maps a legacy-salt recomputation of an alias holder's ID onto + // the holder it proves. This is the only recomputation in the codebase and + // it runs against historical salts, never to resolve a live entity. + derivedIDs := map[string]aliasOrphanDerivedTwin{} + for _, h := range holders { + holdersByTitle[h.title] = append(holdersByTitle[h.title], h) + for _, salt := range salts { + derived := stableMigrationID(table.kind, salt.projectID, h.alias) + if derived == h.entityID { + continue + } + if _, taken := derivedIDs[derived]; taken { + continue + } + derivedIDs[derived] = aliasOrphanDerivedTwin{holder: h, salt: salt} + } + } + + // Uniqueness for content-identity and source-derivation is computed over the + // orphans not yet retiring. Retiring one row can unlock another, so the + // classification iterates to a fixed point: each pass may promote more rows + // into the retiring set, then counts recompute over the smaller pool. + // Operator --retire seeds that set immediately; --realias never does, so a + // row named for preservation cannot free a competitor's uniqueness. + retiringForCounts := map[string]struct{}{} + for id := range retireSet { + retiringForCounts[id] = struct{}{} + } + frozen := map[string]AliasOrphanRowClassify{} + + for { + orphanTitleCounts := map[string]int{} + orphanSourceKeyCounts := map[string]int{} + for _, orphan := range orphans { + if _, retiring := retiringForCounts[orphan.entityID]; retiring { + continue + } + orphanTitleCounts[orphan.title]++ + orphanSourceKeyCounts[aliasOrphanSourceKey(orphan)]++ + } + + promoted := 0 + pass := map[string]AliasOrphanRowClassify{} + for _, orphan := range orphans { + if existing, ok := frozen[orphan.entityID]; ok { + pass[orphan.entityID] = existing + continue + } + classify, err := classifyOneAliasOrphan(ctx, q, projectID, table, orphan, holders, holdersByTitle, derivedIDs, salts, orphanTitleCounts, orphanSourceKeyCounts, retireSet, realiasSet) + if err != nil { + return summary, err + } + pass[orphan.entityID] = classify + if classify.Disposition != aliasOrphanDispositionRetire { + continue + } + // --realias outranks automatic retire; disposition would not be retire. + if _, already := retiringForCounts[orphan.entityID]; !already { + promoted++ + } + frozen[orphan.entityID] = classify + retiringForCounts[orphan.entityID] = struct{}{} + } + if promoted == 0 { + summary.Orphans = len(orphans) + for _, orphan := range orphans { + classify := pass[orphan.entityID] + if classify.Proof == aliasOrphanProofUnproven { + summary.Unproven++ + } + if classify.Disposition == aliasOrphanDispositionRetire { + summary.Retire++ + } + summary.Classifications = append(summary.Classifications, classify) + } + break + } + } + + dangling, err := readDeadAliasIDs(ctx, q, projectID, table) + if err != nil { + return summary, err + } + summary.DanglingAliasIDs = dangling + summary.DanglingAliases = len(summary.DanglingAliasIDs) + + return summary, nil +} + +func classifyOneAliasOrphan( + ctx context.Context, + q aliasOrphanQuerier, + projectID string, + table aliasOrphanEntityTable, + orphan aliasOrphanRow, + holders []aliasOrphanRow, + holdersByTitle map[string][]aliasOrphanRow, + derivedIDs map[string]aliasOrphanDerivedTwin, + salts []aliasOrphanLegacySalt, + orphanTitleCounts map[string]int, + orphanSourceKeyCounts map[string]int, + retireSet map[string]struct{}, + realiasSet map[string]string, +) (AliasOrphanRowClassify, error) { + classify := AliasOrphanRowClassify{ + ProjectID: projectID, + Kind: table.kind, + Table: table.table, + EntityID: orphan.entityID, + Title: orphan.title, + Proof: aliasOrphanProofUnproven, + } + if twin, ok := aliasOrphanDerivationTwin(orphan, derivedIDs); ok { + classify.Proof = aliasOrphanProofDerivation + classify.TwinID = twin.holder.entityID + classify.TwinAlias = twin.holder.alias + classify.LegacyProjectID = twin.salt.projectID + classify.LegacyPath = twin.salt.path + } else if twin, salt, ok := aliasOrphanSourceSaltTwin(orphan, holders, salts, orphanSourceKeyCounts); ok { + classify.Proof = aliasOrphanProofSourceDerivation + classify.TwinID = twin.entityID + classify.TwinAlias = twin.alias + classify.LegacyProjectID = salt.projectID + classify.LegacyPath = salt.path + } else { + twin, ok, err := aliasOrphanContentIdentityTwin(ctx, q, projectID, table, orphan, holdersByTitle, orphanTitleCounts) + if err != nil { + return classify, err + } + if ok { + classify.Proof = aliasOrphanProofContentIdentity + classify.TwinID = twin.entityID + classify.TwinAlias = twin.alias + } + } + // An explicit operator disposition outranks the automatic one. A proof + // that the row has a twin is not a licence to delete a row the operator + // named for preservation. + _, retireRequested := retireSet[orphan.entityID] + switch { + case realiasSet[orphan.entityID] != "": + classify.Disposition = aliasOrphanDispositionRealias + case retireRequested: + classify.Disposition = aliasOrphanDispositionRetire + case classify.Proof != aliasOrphanProofUnproven: + classify.Disposition = aliasOrphanDispositionRetire + } + return classify, nil +} + +// readDeadAliasIDs lists the dead aliases of one entity table: no entity row and +// nothing left in the project naming the entity. +func readDeadAliasIDs(ctx context.Context, q aliasOrphanQuerier, projectID string, table aliasOrphanEntityTable) ([]string, error) { + rows, err := q.QueryContext(ctx, fmt.Sprintf(aliasOrphanDanglingAliasQuery, quoteSQLiteIdentifier(table.table)), projectID, table.kind, table.namespace) + if err != nil { + return nil, fmt.Errorf("scan %s dangling aliases: %w", table.table, err) + } + defer rows.Close() + var ids []string + for rows.Next() { + var aliasID string + if err := rows.Scan(&aliasID); err != nil { + return nil, fmt.Errorf("scan %s dangling alias: %w", table.table, err) + } + ids = append(ids, aliasID) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("scan %s dangling aliases: %w", table.table, err) + } + return ids, nil +} + +// readAliasOrphanRows returns either the alias-orphaned rows of a table or the +// rows that hold an alias, with their source path attached. +func readAliasOrphanRows(ctx context.Context, q aliasOrphanQuerier, projectID string, table aliasOrphanEntityTable, aliasHolders bool) ([]aliasOrphanRow, error) { + title := quoteSQLiteIdentifier(table.titleColumn) + entity := quoteSQLiteIdentifier(table.table) + source := quoteSQLiteIdentifier(table.sourceColumn) + var query string + if aliasHolders { + query = fmt.Sprintf(` +SELECT a.entity_id, a.alias, e.%s, e.created_at, COALESCE(e.%s, ''), COALESCE(s.path, '') +FROM aliases AS a +JOIN %s AS e ON e.project_id = a.project_id AND e.id = a.entity_id +LEFT JOIN sources AS s ON s.project_id = e.project_id AND s.id = e.%s +WHERE a.project_id = ? AND a.entity_kind = ? AND a.namespace = ? +ORDER BY a.alias +`, title, source, entity, source) + } else { + query = fmt.Sprintf(` +SELECT e.id, '', e.%s, e.created_at, COALESCE(e.%s, ''), COALESCE(s.path, '') +FROM %s AS e +LEFT JOIN sources AS s ON s.project_id = e.project_id AND s.id = e.%s +WHERE e.project_id = ? + AND NOT EXISTS ( + SELECT 1 FROM aliases AS a + WHERE a.project_id = e.project_id + AND a.entity_kind = ? + AND a.entity_id = e.id + AND a.namespace = ? + ) +ORDER BY e.id +`, title, source, entity, source) + } + rows, err := q.QueryContext(ctx, query, projectID, table.kind, table.namespace) + if err != nil { + return nil, fmt.Errorf("scan %s rows: %w", table.table, err) + } + defer rows.Close() + var out []aliasOrphanRow + for rows.Next() { + var row aliasOrphanRow + if err := rows.Scan(&row.entityID, &row.alias, &row.title, &row.createdAt, &row.sourceID, &row.sourcePath); err != nil { + return nil, fmt.Errorf("scan %s row: %w", table.table, err) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("scan %s rows: %w", table.table, err) + } + return out, nil +} + +// aliasOrphanDerivedTwin binds a legacy-salt recomputation to the alias holder +// it proves, and the salt that produced the match. +type aliasOrphanDerivedTwin struct { + holder aliasOrphanRow + salt aliasOrphanLegacySalt +} + +// aliasOrphanDerivationTwin accepts the salt recomputation as a twin proof only +// when the two rows are the same artifact, not merely the same alias. +// +// Recomputation proves the orphan was minted under a legacy salt *for this +// alias* — it says nothing about whether the row now holding that alias is the +// orphan's duplicate. Alias numbers get reused: delete TASK-042 and the next +// task file to claim the number recomputes to exactly the orphan's ID, and ID +// match alone would then retire a row that is nobody's duplicate. Two guards +// close that, both from data already in hand: the rows must agree on their +// title, and the orphan must predate the row it would retire into — the damage +// this migration repairs always leaves the pre-rekey original as the older of +// the pair. A reuse victim fails one or both and becomes an operator decision +// instead of a silent deletion. +// +// Do not add body-fingerprint equality to this proof. The calibrated production +// case has derivation-proven pairs whose stored bodies genuinely differ: the +// artifacts changed between the June-13 originals and the June-24 re-imports. +// Title equality plus orphan-predates-twin is the guard; a content check would +// refuse every one of those pairs and break the repair. +func aliasOrphanDerivationTwin(orphan aliasOrphanRow, derivedIDs map[string]aliasOrphanDerivedTwin) (aliasOrphanDerivedTwin, bool) { + twin, ok := derivedIDs[orphan.entityID] + if !ok { + return aliasOrphanDerivedTwin{}, false + } + if orphan.title != twin.holder.title { + return aliasOrphanDerivedTwin{}, false + } + if orphan.createdAt == "" || orphan.createdAt >= twin.holder.createdAt { + return aliasOrphanDerivedTwin{}, false + } + return twin, true +} + +// aliasOrphanSourceKey identifies a row by the content and file that produced +// it, the pair the source-salt proof binds an orphan to its twin with. +func aliasOrphanSourceKey(row aliasOrphanRow) string { + return row.title + "\x00" + row.sourcePath +} + +// aliasOrphanSourceSaltTwin proves twin-ship for rows whose IDs were never +// derived from an alias — sparks are minted from (path, line) — by recomputing +// the row's own source ID under each historical salt. A match proves the orphan +// was minted before the rekey; the surviving twin is then the unique alias +// holder that carries the same content from the same source path. Uniqueness is +// required on both sides: many orphans collapsing onto one holder is a merge, +// not a twin proof, and stays unproven. The holder must also sit in the June-24 +// re-import window and the orphan must predate it — the same gates the other +// auto-retiring proofs apply. +func aliasOrphanSourceSaltTwin(orphan aliasOrphanRow, holders []aliasOrphanRow, salts []aliasOrphanLegacySalt, orphanSourceKeyCounts map[string]int) (aliasOrphanRow, aliasOrphanLegacySalt, bool) { + if orphan.sourceID == "" || orphan.sourcePath == "" || strings.TrimSpace(orphan.title) == "" { + return aliasOrphanRow{}, aliasOrphanLegacySalt{}, false + } + if orphanSourceKeyCounts[aliasOrphanSourceKey(orphan)] != 1 { + return aliasOrphanRow{}, aliasOrphanLegacySalt{}, false + } + var matched aliasOrphanLegacySalt + found := false + for _, salt := range salts { + if stableMigrationID("source", salt.projectID, orphan.sourcePath) == orphan.sourceID { + matched = salt + found = true + break + } + } + if !found { + return aliasOrphanRow{}, aliasOrphanLegacySalt{}, false + } + var twin aliasOrphanRow + matches := 0 + for _, h := range holders { + if h.title != orphan.title || h.sourcePath != orphan.sourcePath { + continue + } + twin = h + matches++ + } + if matches != 1 { + return aliasOrphanRow{}, aliasOrphanLegacySalt{}, false + } + if !inTimestampWindow(twin.createdAt, june24ReimportWindowStart, june24ReimportWindowEnd) { + return aliasOrphanRow{}, aliasOrphanLegacySalt{}, false + } + if orphan.createdAt == "" || orphan.createdAt >= twin.createdAt { + return aliasOrphanRow{}, aliasOrphanLegacySalt{}, false + } + return twin, matched, true +} + +// aliasOrphanContentIdentityTwin is the distinctly-labeled fallback proof. It +// requires the surviving alias holder to be a member of the 2026-06-24 +// re-import window, the orphan to predate it, exactly one candidate on each +// side, a non-empty title, and matching body evidence. Bodyless pairs need the +// orphan in the June-13 original-import window; bodyful pairs need equal +// fingerprints. Anything short of that stays unproven. +func aliasOrphanContentIdentityTwin(ctx context.Context, q aliasOrphanQuerier, projectID string, table aliasOrphanEntityTable, orphan aliasOrphanRow, holdersByTitle map[string][]aliasOrphanRow, orphanTitleCounts map[string]int) (aliasOrphanRow, bool, error) { + if strings.TrimSpace(orphan.title) == "" { + return aliasOrphanRow{}, false, nil + } + if orphanTitleCounts[orphan.title] != 1 { + return aliasOrphanRow{}, false, nil + } + holders := holdersByTitle[orphan.title] + if len(holders) != 1 { + return aliasOrphanRow{}, false, nil + } + twin := holders[0] + if !inTimestampWindow(twin.createdAt, june24ReimportWindowStart, june24ReimportWindowEnd) { + return aliasOrphanRow{}, false, nil + } + if orphan.createdAt == "" || orphan.createdAt >= twin.createdAt { + return aliasOrphanRow{}, false, nil + } + orphanBodies, err := aliasOrphanBodyFingerprint(ctx, q, projectID, table.kind, orphan.entityID) + if err != nil { + return aliasOrphanRow{}, false, err + } + twinBodies, err := aliasOrphanBodyFingerprint(ctx, q, projectID, table.kind, twin.entityID) + if err != nil { + return aliasOrphanRow{}, false, err + } + orphanBodyless := orphanBodies == "" + twinBodyless := twinBodies == "" + switch { + case orphanBodyless && twinBodyless: + if !inTimestampWindow(orphan.createdAt, june13OriginalImportWindowStart, june13OriginalImportWindowEnd) { + return aliasOrphanRow{}, false, nil + } + case !orphanBodyless && !twinBodyless: + if orphanBodies != twinBodies { + return aliasOrphanRow{}, false, nil + } + default: + return aliasOrphanRow{}, false, nil + } + return twin, true, nil +} + +// aliasOrphanBodyFingerprint canonicalizes an entity's stored bodies as +// body_kind=content_hash pairs so two rows can be compared for content +// identity. An entity with no bodies fingerprints as the empty string. +func aliasOrphanBodyFingerprint(ctx context.Context, q aliasOrphanQuerier, projectID string, kind string, entityID string) (string, error) { + rows, err := q.QueryContext(ctx, ` +SELECT body_kind, COALESCE(content_hash, '') +FROM artifact_bodies +WHERE project_id = ? AND entity_kind = ? AND entity_id = ? +ORDER BY body_kind +`, projectID, kind, entityID) + if err != nil { + return "", fmt.Errorf("read %s body fingerprint: %w", kind, err) + } + defer rows.Close() + var parts []string + for rows.Next() { + var bodyKind, hash string + if err := rows.Scan(&bodyKind, &hash); err != nil { + return "", fmt.Errorf("scan %s body fingerprint: %w", kind, err) + } + parts = append(parts, bodyKind+"="+hash) + } + if err := rows.Err(); err != nil { + return "", fmt.Errorf("read %s body fingerprint: %w", kind, err) + } + sort.Strings(parts) + return strings.Join(parts, "\n"), nil +} + +func classifyBrokenEvidenceReport(ctx context.Context, q aliasOrphanQuerier, projectID string) (*AliasOrphanDisposition, error) { + var status string + err := q.QueryRowContext(ctx, ` +SELECT status FROM reports WHERE project_id = ? AND id = ? +`, projectID, brokenEvidenceReportID).Scan(&status) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read broken-evidence report: %w", err) + } + if status == LifecycleStatusArchived { + return nil, nil + } + return &AliasOrphanDisposition{ + ProjectID: projectID, + Kind: "report", + EntityID: brokenEvidenceReportID, + Action: aliasOrphanDispositionArchiveMoot, + Note: aliasOrphanArchiveMootNote, + }, nil +} + +// inTimestampWindow reports whether timestamp is in the half-open interval +// [start, end). Values that fail RFC3339 parse are not in the window. +func inTimestampWindow(timestamp, start, end string) bool { + ts, err := time.Parse(time.RFC3339, timestamp) + if err != nil { + return false + } + startAt, err := time.Parse(time.RFC3339, start) + if err != nil { + return false + } + endAt, err := time.Parse(time.RFC3339, end) + if err != nil { + return false + } + return !ts.Before(startAt) && ts.Before(endAt) +} + +// aliasOrphanLegacySalts returns every historical project ID this project's +// rows could have been derived under: the current path plus every path the +// project has ever been recorded at. +func aliasOrphanLegacySalts(ctx context.Context, q aliasOrphanQuerier, project ProjectIdentity) ([]aliasOrphanLegacySalt, error) { + paths := []string{} + seen := map[string]struct{}{} + add := func(path string) { + path = strings.TrimSpace(path) + if path == "" { + return + } + if _, ok := seen[path]; ok { + return + } + seen[path] = struct{}{} + paths = append(paths, path) + } + add(project.CurrentPath) + + exists, err := sqliteTableExistsQ(ctx, q, "project_paths") + if err != nil { + return nil, err + } + if exists { + rows, err := q.QueryContext(ctx, `SELECT path FROM project_paths WHERE project_id = ? ORDER BY is_current DESC, path`, project.ID) + if err != nil { + return nil, fmt.Errorf("list historical project paths: %w", err) + } + defer rows.Close() + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, fmt.Errorf("scan historical project path: %w", err) + } + add(path) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list historical project paths: %w", err) + } + } + + salts := make([]aliasOrphanLegacySalt, 0, len(paths)) + for _, path := range paths { + salts = append(salts, aliasOrphanLegacySalt{projectID: legacyProjectIDFromPath(path), path: path}) + } + return salts, nil +} + +func legacyProjectIDFromPath(path string) string { + sum := sha256.Sum256([]byte(path)) + return hex.EncodeToString(sum[:]) +} + +func aliasOrphanManifestHasWork(manifest AliasOrphanRollbackManifest) bool { + return len(manifest.DeletedRows) > 0 || + len(manifest.StatusChanges) > 0 || + len(manifest.AliasInserts) > 0 || + len(manifest.Unlinks) > 0 || + manifest.Counts.EntitiesRetired > 0 || + manifest.Counts.AliasesDeleted > 0 || + manifest.Counts.SourcesDeleted > 0 || + manifest.Counts.StatusesChanged > 0 || + manifest.Counts.AliasesInserted > 0 +} + +// applyAliasOrphanMigrationManifest performs the repair in one transaction. +// beforeCommit runs with the fully-populated manifest still inside the +// transaction, so the rollback record is durable on disk before any deletion +// becomes visible; a failure there aborts the repair. +func applyAliasOrphanMigrationManifest(ctx context.Context, store *Store, manifest *AliasOrphanRollbackManifest, beforeCommit func(AliasOrphanRollbackManifest) error) error { + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin alias-orphan migration: %w", err) + } + defer tx.Rollback() + + if _, err := tx.ExecContext(ctx, `PRAGMA defer_foreign_keys = ON`); err != nil { + return fmt.Errorf("defer foreign keys: %w", err) + } + + now := time.Now().UTC().Format(time.RFC3339Nano) + order := 0 + + // Re-classify inside the transaction against live state and apply. + projects, err := listProjectsTx(ctx, tx, store.path) + if err != nil { + return err + } + + retireSet := map[string]struct{}{} + realiasSet := map[string]string{} + for _, d := range manifest.OperatorDispositions { + switch d.Action { + case aliasOrphanDispositionRetire: + retireSet[d.EntityID] = struct{}{} + case aliasOrphanDispositionRealias: + realiasSet[d.EntityID] = d.Alias + } + } + + // Dead aliases go first so a --realias target freed by this same run is + // available, and so realias never has to distinguish a live claim from a + // dead one. + if err := collectDeadAliasesTx(ctx, tx, projects, manifest, &order); err != nil { + return err + } + + for _, project := range projects { + salts, err := aliasOrphanLegacySalts(ctx, tx, project) + if err != nil { + return err + } + + // Named disposition: archive broken-evidence report as moot. + if err := applyBrokenEvidenceArchiveTx(ctx, tx, project.ID, now, manifest); err != nil { + return err + } + + for _, table := range aliasOrphanEntityTables { + exists, err := sqliteTableExistsQ(ctx, tx, table.table) + if err != nil { + return err + } + if !exists { + continue + } + + summary, err := classifyAliasOrphansForTable(ctx, tx, project.ID, salts, table, retireSet, realiasSet) + if err != nil { + return err + } + + for _, c := range summary.Classifications { + switch c.Disposition { + case aliasOrphanDispositionRetire: + if err := retireEntityWithResidueTx(ctx, tx, project.ID, table, c, manifest, &order); err != nil { + return err + } + manifest.Retirements = append(manifest.Retirements, aliasOrphanRetireDisposition(c)) + manifest.Counts.EntitiesRetired++ + case aliasOrphanDispositionRealias: + alias := realiasSet[c.EntityID] + if err := realiasEntityTx(ctx, tx, project.ID, table, c.EntityID, alias, now, manifest); err != nil { + return err + } + manifest.Counts.AliasesInserted++ + } + } + } + } + + // Retiring an orphan deletes the relationship edges that were keeping a + // forward-declared alias alive, so an alias this run itself kills is + // invisible to any sweep that ran before the retirement — including a later + // table's, since the table order is fixed. Collect once more after every + // retirement, across every project and table. Deleting an alias cannot kill + // another one, so this second pass is the fixed point and post-apply + // verification is guaranteed clean on a correct run. + if err := collectDeadAliasesTx(ctx, tx, projects, manifest, &order); err != nil { + return err + } + + if beforeCommit != nil { + if err := beforeCommit(*manifest); err != nil { + return err + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit alias-orphan migration: %w", err) + } + return nil +} + +func applyBrokenEvidenceArchiveTx(ctx context.Context, tx *sql.Tx, projectID string, now string, manifest *AliasOrphanRollbackManifest) error { + var previous, previousUpdatedAt string + err := tx.QueryRowContext(ctx, `SELECT status, COALESCE(updated_at, '') FROM reports WHERE project_id = ? AND id = ?`, projectID, brokenEvidenceReportID).Scan(&previous, &previousUpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("read broken-evidence report status: %w", err) + } + if previous == LifecycleStatusArchived { + return nil + } + if _, err := tx.ExecContext(ctx, `UPDATE reports SET status = ?, updated_at = ? WHERE project_id = ? AND id = ?`, LifecycleStatusArchived, now, projectID, brokenEvidenceReportID); err != nil { + return fmt.Errorf("archive broken-evidence report: %w", err) + } + eventID := stableMigrationID("event", projectID, "report", brokenEvidenceReportID, aliasOrphanArchiveMootEventType, previous, LifecycleStatusArchived, "moot") + result, err := tx.ExecContext(ctx, ` +INSERT OR IGNORE INTO events (id, project_id, entity_kind, entity_id, event_type, from_status, to_status, note, created_at, updated_at) +VALUES (?, ?, 'report', ?, ?, ?, ?, ?, ?, ?) +`, eventID, projectID, brokenEvidenceReportID, aliasOrphanArchiveMootEventType, previous, LifecycleStatusArchived, aliasOrphanArchiveMootNote, now, now) + if err != nil { + return fmt.Errorf("record broken-evidence archive event: %w", err) + } + recordedEventID := "" + if rows, rowsErr := result.RowsAffected(); rowsErr == nil && rows > 0 { + recordedEventID = eventID + } + manifest.StatusChanges = append(manifest.StatusChanges, AliasOrphanStatusChange{ + ProjectID: projectID, + Table: "reports", + Kind: "report", + EntityID: brokenEvidenceReportID, + PreviousStatus: previous, + PreviousUpdatedAt: previousUpdatedAt, + NewStatus: LifecycleStatusArchived, + EventID: recordedEventID, + EventNote: aliasOrphanArchiveMootNote, + }) + manifest.Counts.StatusesChanged++ + return nil +} + +func realiasEntityTx(ctx context.Context, tx *sql.Tx, projectID string, table aliasOrphanEntityTable, entityID string, alias string, now string, manifest *AliasOrphanRollbackManifest) error { + // Re-pointing a claimed alias at a different row is the exact mechanic that + // created this damage class. Refuse it: the incumbent keeps its identity and + // the operator picks a free alias. + var incumbent string + err := tx.QueryRowContext(ctx, ` +SELECT entity_id FROM aliases WHERE project_id = ? AND namespace = ? AND alias = ? +`, projectID, table.namespace, alias).Scan(&incumbent) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("read alias %s:%s: %w", table.namespace, alias, err) + } + if err == nil && incumbent != entityID { + return fmt.Errorf("realias %s: alias %s:%s already names %s; pick an unclaimed alias or retire the incumbent first", entityID, table.namespace, alias, incumbent) + } + + aliasID := stableMigrationID("alias", projectID, table.namespace, alias) + if _, err := tx.ExecContext(ctx, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(project_id, namespace, alias) DO UPDATE SET + entity_kind = excluded.entity_kind, + entity_id = excluded.entity_id, + updated_at = excluded.updated_at +`, aliasID, projectID, table.kind, entityID, table.namespace, alias, now, now); err != nil { + return fmt.Errorf("realias %s %s as %s: %w", table.kind, entityID, alias, err) + } + manifest.AliasInserts = append(manifest.AliasInserts, AliasOrphanAliasInsert{ + ProjectID: projectID, + AliasID: aliasID, + EntityKind: table.kind, + EntityID: entityID, + Namespace: table.namespace, + Alias: alias, + }) + return nil +} + +// collectDeadAliasesTx deletes every dead alias in the database, across all +// projects and all entity tables, recording each one in the manifest. +func collectDeadAliasesTx(ctx context.Context, tx *sql.Tx, projects []ProjectIdentity, manifest *AliasOrphanRollbackManifest, order *int) error { + for _, project := range projects { + for _, table := range aliasOrphanEntityTables { + exists, err := sqliteTableExistsQ(ctx, tx, table.table) + if err != nil { + return err + } + if !exists { + continue + } + aliasIDs, err := readDeadAliasIDs(ctx, tx, project.ID, table) + if err != nil { + return err + } + for _, aliasID := range aliasIDs { + if err := deleteDanglingAliasTx(ctx, tx, project.ID, aliasID, manifest, order); err != nil { + return err + } + manifest.Counts.AliasesDeleted++ + } + } + } + return nil +} + +func deleteDanglingAliasTx(ctx context.Context, tx *sql.Tx, projectID string, aliasID string, manifest *AliasOrphanRollbackManifest, order *int) error { + if err := captureRowsTx(ctx, tx, "aliases", `SELECT * FROM aliases WHERE project_id = ? AND id = ?`, []any{projectID, aliasID}, manifest, order, nil); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM aliases WHERE project_id = ? AND id = ?`, projectID, aliasID); err != nil { + return fmt.Errorf("delete dangling alias %s: %w", aliasID, err) + } + return nil +} + +// entityReferenceSweep is one polymorphic (entity_kind, entity_id) reference site. +type entityReferenceSweep struct { + table string + where string + args []any +} + +// polymorphicEntityReferenceSweeps returns the six polymorphic tables that can +// cite an entity by (entity_kind, entity_id). where clauses are suitable for +// both captureAndDeleteTx and captureAndDeleteJournalDuplicateTx. +func polymorphicEntityReferenceSweeps(projectID, kind, entityID string) []entityReferenceSweep { + return []entityReferenceSweep{ + {"events", `WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {"entity_tags", `WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {"bundle_members", `WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {"backend_mappings", `WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {"exports", `WHERE project_id = ? AND source_entity_kind = ? AND source_entity_id = ?`, []any{projectID, kind, entityID}}, + {"relationships", `WHERE project_id = ? AND ((from_entity_kind = ? AND from_entity_id = ?) OR (to_entity_kind = ? AND to_entity_id = ?))`, []any{projectID, kind, entityID, kind, entityID}}, + } +} + +func retireEntityWithResidueTx(ctx context.Context, tx *sql.Tx, projectID string, table aliasOrphanEntityTable, classify AliasOrphanRowClassify, manifest *AliasOrphanRollbackManifest, order *int) error { + entityID := classify.EntityID + // Capture and delete artifact bodies (FTS included via delete helper after capture). + if err := captureRowsTx(ctx, tx, "artifact_bodies", ` +SELECT * FROM artifact_bodies WHERE project_id = ? AND entity_kind = ? AND entity_id = ? +`, []any{projectID, table.kind, entityID}, manifest, order, nil); err != nil { + return err + } + if _, _, err := deleteArtifactBodiesForEntityTx(ctx, tx, projectID, table.kind, entityID); err != nil { + return err + } + + sweeps := polymorphicEntityReferenceSweeps(projectID, table.kind, entityID) + for _, op := range sweeps { + quoted := quoteSQLiteIdentifier(op.table) + if err := captureRowsTx(ctx, tx, op.table, fmt.Sprintf(`SELECT * FROM %s %s`, quoted, op.where), op.args, manifest, order, nil); err != nil { + return err + } + } + for _, op := range sweeps { + quoted := quoteSQLiteIdentifier(op.table) + if _, err := execCountTx(ctx, tx, fmt.Sprintf(`DELETE FROM %s %s`, quoted, op.where), op.args...); err != nil { + return fmt.Errorf("delete %s rows for %s %s: %w", op.table, table.kind, entityID, err) + } + } + + if err := retireKindSpecificResidueTx(ctx, tx, projectID, table.kind, entityID, classify.TwinID, manifest, order); err != nil { + return err + } + + if err := unlinkReferencesToEntityTx(ctx, tx, projectID, table.kind, entityID, manifest); err != nil { + return err + } + + // Capture entity source id before deleting the entity row. + var bodySourceID sql.NullString + sourceQuery := fmt.Sprintf(`SELECT %s FROM %s WHERE project_id = ? AND id = ?`, quoteSQLiteIdentifier(table.sourceColumn), quoteSQLiteIdentifier(table.table)) + if err := tx.QueryRowContext(ctx, sourceQuery, projectID, entityID).Scan(&bodySourceID); err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("read %s source: %w", table.kind, err) + } + + if err := captureRowsTx(ctx, tx, table.table, fmt.Sprintf(`SELECT * FROM %s WHERE project_id = ? AND id = ?`, quoteSQLiteIdentifier(table.table)), []any{projectID, entityID}, manifest, order, nil); err != nil { + return err + } + if _, err := execCountTx(ctx, tx, fmt.Sprintf(`DELETE FROM %s WHERE project_id = ? AND id = ?`, quoteSQLiteIdentifier(table.table)), projectID, entityID); err != nil { + return fmt.Errorf("delete %s row %s: %w", table.kind, entityID, err) + } + + candidateSources := map[string]struct{}{} + if bodySourceID.Valid && bodySourceID.String != "" { + candidateSources[bodySourceID.String] = struct{}{} + } + // Also consider sources captured from artifact_bodies for this entity. + for _, row := range manifest.DeletedRows { + if row.Table != "artifact_bodies" { + continue + } + if sid := rowValueString(row, "source_id"); sid != "" { + if rowValueString(row, "entity_id") == entityID && rowValueString(row, "entity_kind") == table.kind { + candidateSources[sid] = struct{}{} + } + } + } + for sid := range candidateSources { + referenced, err := sourceStillReferencedTx(ctx, tx, projectID, sid) + if err != nil { + return err + } + if referenced { + continue + } + if err := captureRowsTx(ctx, tx, "sources", `SELECT * FROM sources WHERE project_id = ? AND id = ?`, []any{projectID, sid}, manifest, order, map[string]string{"entity_kind": table.kind, "entity_id": entityID}); err != nil { + return err + } + count, err := execCountTx(ctx, tx, `DELETE FROM sources WHERE project_id = ? AND id = ?`, projectID, sid) + if err != nil { + return fmt.Errorf("delete source %s: %w", sid, err) + } + manifest.Counts.SourcesDeleted += count + manifest.Counts.OrphanedSources += count + } + return nil +} + +// retireKindSpecificResidueTx clears the references no polymorphic +// (entity_kind, entity_id) sweep can reach: child rows bound by a NOT NULL +// foreign key, and NOT NULL soft references that carry no constraint at all. +// Both would otherwise abort the whole migration at COMMIT or leave the row +// dangling with nothing to detect it. +func retireKindSpecificResidueTx(ctx context.Context, tx *sql.Tx, projectID string, kind string, entityID string, twinID string, manifest *AliasOrphanRollbackManifest, order *int) error { + switch kind { + case "report": + // verdicts hang off findings, findings hang off the report; both FKs + // are NOT NULL, so the subtree retires with its root. Finding/verdict + // aliases, relationship edges, events, and entity_tags are not covered + // by the report-endpoint polymorphic sweep and must go with them. + if err := retireReportSubtreeResidueTx(ctx, tx, projectID, entityID, manifest, order); err != nil { + return err + } + if err := captureAndDeleteTx(ctx, tx, "verdicts", ` +WHERE project_id = ? AND finding_id IN (SELECT id FROM findings WHERE project_id = ? AND report_id = ?) +`, []any{projectID, projectID, entityID}, manifest, order); err != nil { + return err + } + return captureAndDeleteTx(ctx, tx, "findings", `WHERE project_id = ? AND report_id = ?`, []any{projectID, entityID}, manifest, order) + case "spark": + // spark_id is NOT NULL in both tables and carries no foreign key, so a + // retirement leaves silent dangling provenance. Repoint at the proven + // twin where possible; delete the row only when there is nowhere to go. + for _, ref := range []aliasOrphanSoftRef{ + {table: "journal_deferrals", column: "spark_id", keyColumn: "operation_key", unique: true}, + {table: "intent_operations", column: "spark_id", keyColumn: "operation_key"}, + } { + if err := repointOrDeleteSoftRefTx(ctx, tx, projectID, ref, entityID, twinID, manifest, order); err != nil { + return err + } + } + } + return nil +} + +// aliasOrphanSoftRef is a NOT NULL reference to an entity that the schema does +// not enforce. unique marks columns whose value cannot be shared, so a repoint +// has to yield when the twin already holds one. +type aliasOrphanSoftRef struct { + table string + column string + keyColumn string + unique bool +} + +func repointOrDeleteSoftRefTx(ctx context.Context, tx *sql.Tx, projectID string, ref aliasOrphanSoftRef, entityID string, twinID string, manifest *AliasOrphanRollbackManifest, order *int) error { + exists, err := sqliteTableExistsQ(ctx, tx, ref.table) + if err != nil { + return err + } + if !exists { + return nil + } + quotedTable := quoteSQLiteIdentifier(ref.table) + quotedColumn := quoteSQLiteIdentifier(ref.column) + quotedKey := quoteSQLiteIdentifier(ref.keyColumn) + + rows, err := tx.QueryContext(ctx, fmt.Sprintf(`SELECT %s FROM %s WHERE project_id = ? AND %s = ? ORDER BY %s`, quotedKey, quotedTable, quotedColumn, quotedKey), projectID, entityID) + if err != nil { + return fmt.Errorf("list %s.%s references: %w", ref.table, ref.column, err) + } + var keys []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + rows.Close() + return fmt.Errorf("scan %s.%s reference: %w", ref.table, ref.column, err) + } + keys = append(keys, key) + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("list %s.%s references: %w", ref.table, ref.column, err) + } + rows.Close() + + for _, key := range keys { + repoint := twinID != "" + if repoint && ref.unique { + var taken int + if err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT EXISTS(SELECT 1 FROM %s WHERE %s = ?)`, quotedTable, quotedColumn), twinID).Scan(&taken); err != nil { + return fmt.Errorf("check %s.%s availability: %w", ref.table, ref.column, err) + } + repoint = taken == 0 + } + if repoint { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET %s = ? WHERE project_id = ? AND %s = ?`, quotedTable, quotedColumn, quotedKey), twinID, projectID, key); err != nil { + return fmt.Errorf("repoint %s.%s: %w", ref.table, ref.column, err) + } + manifest.Unlinks = append(manifest.Unlinks, AliasOrphanUnlink{ + Table: ref.table, + ProjectID: projectID, + Column: ref.column, + KeyColumn: ref.keyColumn, + RowID: key, + PreviousID: entityID, + NewID: twinID, + }) + continue + } + if err := captureAndDeleteTx(ctx, tx, ref.table, fmt.Sprintf(`WHERE project_id = ? AND %s = ?`, quotedKey), []any{projectID, key}, manifest, order); err != nil { + return err + } + } + return nil +} + +// captureAndDeleteTx snapshots every row a predicate selects into the rollback +// manifest and then deletes it. +func captureAndDeleteTx(ctx context.Context, tx *sql.Tx, table string, where string, args []any, manifest *AliasOrphanRollbackManifest, order *int) error { + quoted := quoteSQLiteIdentifier(table) + if err := captureRowsTx(ctx, tx, table, fmt.Sprintf(`SELECT * FROM %s %s`, quoted, where), args, manifest, order, nil); err != nil { + return err + } + if _, err := execCountTx(ctx, tx, fmt.Sprintf(`DELETE FROM %s %s`, quoted, where), args...); err != nil { + return fmt.Errorf("delete %s rows: %w", table, err) + } + return nil +} + +func unlinkReferencesToEntityTx(ctx context.Context, tx *sql.Tx, projectID string, kind string, entityID string, manifest *AliasOrphanRollbackManifest) error { + type unlinkSpec struct { + table string + column string + } + var unlinkSpecs []unlinkSpec + switch kind { + case "spec": + unlinkSpecs = []unlinkSpec{ + {"tasks", "spec_id"}, + {"journal_entries", "spec_id"}, + {"plans", "spec_id"}, + {"councils", "spec_id"}, + } + case "task": + unlinkSpecs = []unlinkSpec{ + {"journal_entries", "task_id"}, + {"handoffs", "task_id"}, + } + default: + return nil + } + for _, spec := range unlinkSpecs { + exists, err := sqliteTableExistsQ(ctx, tx, spec.table) + if err != nil { + return err + } + if !exists { + continue + } + rows, err := tx.QueryContext(ctx, fmt.Sprintf(`SELECT id FROM %s WHERE project_id = ? AND %s = ?`, quoteSQLiteIdentifier(spec.table), quoteSQLiteIdentifier(spec.column)), projectID, entityID) + if err != nil { + return fmt.Errorf("list %s.%s unlinks: %w", spec.table, spec.column, err) + } + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return err + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + for _, id := range ids { + manifest.Unlinks = append(manifest.Unlinks, AliasOrphanUnlink{ + Table: spec.table, + ProjectID: projectID, + Column: spec.column, + KeyColumn: "id", + RowID: id, + PreviousID: entityID, + }) + } + if len(ids) > 0 { + if _, err := execCountTx(ctx, tx, fmt.Sprintf(`UPDATE %s SET %s = NULL WHERE project_id = ? AND %s = ?`, quoteSQLiteIdentifier(spec.table), quoteSQLiteIdentifier(spec.column), quoteSQLiteIdentifier(spec.column)), projectID, entityID); err != nil { + return fmt.Errorf("unlink %s.%s: %w", spec.table, spec.column, err) + } + } + } + return nil +} + +func rollbackAliasOrphanMigrationManifest(ctx context.Context, store *Store, manifest AliasOrphanRollbackManifest, result *AliasOrphanMigrationResult) error { + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin alias-orphan rollback: %w", err) + } + defer tx.Rollback() + + if _, err := tx.ExecContext(ctx, `PRAGMA defer_foreign_keys = ON`); err != nil { + return fmt.Errorf("defer foreign keys: %w", err) + } + + // Undo alias inserts from --realias first. + for _, insert := range manifest.AliasInserts { + if _, err := tx.ExecContext(ctx, `DELETE FROM aliases WHERE project_id = ? AND id = ?`, insert.ProjectID, insert.AliasID); err != nil { + return fmt.Errorf("rollback alias insert %s: %w", insert.AliasID, err) + } + } + + // Undo status changes: delete archive events this run inserted and restore previous status. + for _, change := range manifest.StatusChanges { + if change.EventID != "" { + if _, err := tx.ExecContext(ctx, `DELETE FROM events WHERE project_id = ? AND id = ?`, change.ProjectID, change.EventID); err != nil { + return fmt.Errorf("rollback status event %s: %w", change.EventID, err) + } + } + if change.PreviousUpdatedAt != "" { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET status = ?, updated_at = ? WHERE project_id = ? AND id = ?`, quoteSQLiteIdentifier(change.Table)), change.PreviousStatus, change.PreviousUpdatedAt, change.ProjectID, change.EntityID); err != nil { + return fmt.Errorf("rollback status for %s %s: %w", change.Kind, change.EntityID, err) + } + continue + } + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET status = ? WHERE project_id = ? AND id = ?`, quoteSQLiteIdentifier(change.Table)), change.PreviousStatus, change.ProjectID, change.EntityID); err != nil { + return fmt.Errorf("rollback status for %s %s: %w", change.Kind, change.EntityID, err) + } + } + + // Restore deleted rows in reverse capture order so parents come before children when needed. + rows := append([]AliasOrphanDeletedRow{}, manifest.DeletedRows...) + sort.SliceStable(rows, func(i, j int) bool { return rows[i].Order > rows[j].Order }) + // Prefer restoring entity tables and sources before reference tables? Actually + // reverse order of deletion: we deleted residue first then entity then sources. + // Capture order: bodies, polymorphic, entity, sources. Reverse: sources, entity, polymorphic, bodies. + // Restoring entity before polymorphic is required for FK if not deferred; with defer, any order works at commit. + // Restore non-archive-event rows; archive events were deleted above via StatusChanges. + archiveEventIDs := map[string]struct{}{} + for _, change := range manifest.StatusChanges { + archiveEventIDs[change.EventID] = struct{}{} + } + for _, row := range rows { + if row.Table == "events" { + if id := rowValueString(row, "id"); id != "" { + if _, skip := archiveEventIDs[id]; skip { + continue + } + } + } + if err := insertDeletedRowTx(ctx, tx, row); err != nil { + return err + } + if row.Table == "artifact_bodies" { + if err := restoreArtifactSearchForRowTx(ctx, tx, row); err != nil { + return err + } + } + result.RowsRestored++ + } + + // Restore nulled and repointed references. + for _, unlink := range manifest.Unlinks { + keyColumn := firstNonEmpty(unlink.KeyColumn, "id") + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET %s = ? WHERE project_id = ? AND %s = ?`, quoteSQLiteIdentifier(unlink.Table), quoteSQLiteIdentifier(unlink.Column), quoteSQLiteIdentifier(keyColumn)), unlink.PreviousID, unlink.ProjectID, unlink.RowID); err != nil { + return fmt.Errorf("restore unlink %s.%s: %w", unlink.Table, unlink.Column, err) + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit alias-orphan rollback: %w", err) + } + return nil +} + +func restoreArtifactSearchForRowTx(ctx context.Context, tx *sql.Tx, row AliasOrphanDeletedRow) error { + return restoreArtifactSearchTx(ctx, tx, + rowValueString(row, "project_id"), + rowValueString(row, "entity_kind"), + rowValueString(row, "entity_id"), + rowValueString(row, "body_kind"), + rowValueString(row, "content"), + ) +} + +func restoreArtifactSearchTx(ctx context.Context, tx *sql.Tx, projectID, entityKind, entityID, bodyKind, content string) error { + if projectID == "" || entityKind == "" || entityID == "" { + return nil + } + kind := firstNonEmpty(bodyKind, ArtifactBodyKindMarkdown) + rowID, err := artifactBodyRowID(ctx, tx, projectID, entityKind, entityID, kind) + if err != nil { + return err + } + return upsertArtifactSearchTx(ctx, tx, artifactSearchRow{}, false, rowID, projectID, entityKind, entityID, kind, content) +} + +func insertDeletedRowTx(ctx context.Context, tx *sql.Tx, row AliasOrphanDeletedRow) error { + if len(row.Columns) == 0 || len(row.Columns) != len(row.Values) { + return fmt.Errorf("deleted row for %s has mismatched columns/values", row.Table) + } + quoted := make([]string, len(row.Columns)) + placeholders := make([]string, len(row.Columns)) + args := make([]any, len(row.Values)) + for i, col := range row.Columns { + quoted[i] = quoteSQLiteIdentifier(col) + placeholders[i] = "?" + args[i] = normalizeManifestValue(row.Values[i]) + } + query := fmt.Sprintf(`INSERT OR REPLACE INTO %s (%s) VALUES (%s)`, quoteSQLiteIdentifier(row.Table), strings.Join(quoted, ", "), strings.Join(placeholders, ", ")) + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("restore row into %s: %w", row.Table, err) + } + return nil +} + +func normalizeManifestValue(value any) any { + switch v := value.(type) { + case float64: + if v == float64(int64(v)) { + return int64(v) + } + return v + case json.Number: + if i, err := v.Int64(); err == nil { + return i + } + if f, err := v.Float64(); err == nil { + return f + } + return v.String() + default: + return v + } +} + +func captureRowsTx(ctx context.Context, tx *sql.Tx, table string, query string, args []any, manifest *AliasOrphanRollbackManifest, order *int, meta map[string]string) error { + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("capture %s rows: %w", table, err) + } + defer rows.Close() + scanned, err := scanRows(rows) + if err != nil { + return fmt.Errorf("scan %s rows for capture: %w", table, err) + } + for _, row := range scanned { + columns := make([]string, 0, len(row)) + // Stable column order for deterministic manifests. + for col := range row { + columns = append(columns, col) + } + sort.Strings(columns) + values := make([]any, len(columns)) + for i, col := range columns { + values[i] = row[col] + } + *order++ + entry := AliasOrphanDeletedRow{ + Table: table, + Columns: columns, + Values: values, + Order: *order, + } + if len(meta) > 0 { + entry.Meta = meta + } + manifest.DeletedRows = append(manifest.DeletedRows, entry) + } + return nil +} + +func rowValueString(row AliasOrphanDeletedRow, column string) string { + for i, col := range row.Columns { + if col == column { + if i >= len(row.Values) || row.Values[i] == nil { + return "" + } + switch v := row.Values[i].(type) { + case string: + return v + case []byte: + return string(v) + default: + return fmt.Sprint(v) + } + } + } + return "" +} + +func listProjectsTx(ctx context.Context, tx *sql.Tx, databasePath string) ([]ProjectIdentity, error) { + rows, err := tx.QueryContext(ctx, ` +SELECT + projects.id, + COALESCE(NULLIF(projects.friendly_name, ''), projects.id), + COALESCE(current_path.path, projects.current_path, ''), + COALESCE(projects.last_seen_at, '') +FROM projects +LEFT JOIN project_paths AS current_path + ON current_path.project_id = projects.id + AND current_path.is_current = 1 +ORDER BY lower(COALESCE(NULLIF(projects.friendly_name, ''), projects.id)), projects.id +`) + if err != nil { + return nil, fmt.Errorf("list projects: %w", err) + } + defer rows.Close() + var projects []ProjectIdentity + for rows.Next() { + identity := ProjectIdentity{ContractVersion: StateJSONContractVersion, DatabaseScope: "global", DatabasePath: databasePath} + if err := rows.Scan(&identity.ID, &identity.FriendlyName, &identity.CurrentPath, &identity.LastSeenAt); err != nil { + return nil, fmt.Errorf("scan project identity: %w", err) + } + projects = append(projects, identity) + } + if err := rows.Err(); err != nil { + return nil, err + } + return projects, nil +} + +func sqliteTableExistsQ(ctx context.Context, q aliasOrphanQuerier, table string) (bool, error) { + var name string + err := q.QueryRowContext(ctx, `SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`, table).Scan(&name) + if err == nil { + return true, nil + } + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return false, fmt.Errorf("inspect table %s: %w", table, err) +} + +func writeAliasOrphanRollbackManifest(manifest AliasOrphanRollbackManifest, dir string, now time.Time) (string, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("create alias-orphan rollback manifest directory: %w", err) + } + for i := 0; i < 100; i++ { + suffix := "" + if i > 0 { + suffix = fmt.Sprintf("-%02d", i) + } + path := filepath.Join(dir, fmt.Sprintf("alias-orphan-rollback-%s%s.json", now.Format("20060102T150405Z"), suffix)) + if _, err := os.Stat(path); err == nil { + continue + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("stat alias-orphan rollback manifest: %w", err) + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return "", fmt.Errorf("encode alias-orphan rollback manifest: %w", err) + } + payload = append(payload, '\n') + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, os.ErrExist) { + continue + } + return "", fmt.Errorf("create alias-orphan rollback manifest: %w", err) + } + if _, err := file.Write(payload); err != nil { + _ = file.Close() + _ = os.Remove(path) + return "", fmt.Errorf("write alias-orphan rollback manifest: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + _ = os.Remove(path) + return "", fmt.Errorf("sync alias-orphan rollback manifest: %w", err) + } + if err := file.Close(); err != nil { + _ = os.Remove(path) + return "", fmt.Errorf("close alias-orphan rollback manifest: %w", err) + } + if err := syncDirectory(dir); err != nil { + _ = os.Remove(path) + return "", fmt.Errorf("sync alias-orphan rollback manifest directory: %w", err) + } + return path, nil + } + return "", fmt.Errorf("create alias-orphan rollback manifest: exhausted timestamp suffixes") +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + if err := directory.Sync(); err != nil { + _ = directory.Close() + return err + } + return directory.Close() +} + +// retireReportSubtreeResidueTx captures and deletes aliases, relationships, +// events, and entity_tags that belong to findings and verdicts under a report +// being retired. The report-endpoint polymorphic sweep does not reach these +// rows because findings are not in aliasOrphanEntityTables. +func retireReportSubtreeResidueTx(ctx context.Context, tx *sql.Tx, projectID string, reportID string, manifest *AliasOrphanRollbackManifest, order *int) error { + findingIDs, err := listIDsTx(ctx, tx, `SELECT id FROM findings WHERE project_id = ? AND report_id = ? ORDER BY id`, projectID, reportID) + if err != nil { + return fmt.Errorf("list findings for report %s: %w", reportID, err) + } + if len(findingIDs) == 0 { + return nil + } + verdictIDs, err := listIDsTx(ctx, tx, ` +SELECT id FROM verdicts +WHERE project_id = ? AND finding_id IN (SELECT id FROM findings WHERE project_id = ? AND report_id = ?) +ORDER BY id +`, projectID, projectID, reportID) + if err != nil { + return fmt.Errorf("list verdicts for report %s: %w", reportID, err) + } + + for _, findingID := range findingIDs { + if err := captureAndDeleteTx(ctx, tx, "aliases", ` +WHERE project_id = ? AND entity_kind = 'finding' AND entity_id = ? +`, []any{projectID, findingID}, manifest, order); err != nil { + return err + } + if err := captureAndDeleteEntityRefsTx(ctx, tx, projectID, "finding", findingID, manifest, order); err != nil { + return err + } + } + for _, verdictID := range verdictIDs { + if err := captureAndDeleteTx(ctx, tx, "aliases", ` +WHERE project_id = ? AND entity_kind = 'verdict' AND entity_id = ? +`, []any{projectID, verdictID}, manifest, order); err != nil { + return err + } + if err := captureAndDeleteEntityRefsTx(ctx, tx, projectID, "verdict", verdictID, manifest, order); err != nil { + return err + } + } + return nil +} + +func captureAndDeleteEntityRefsTx(ctx context.Context, tx *sql.Tx, projectID string, kind string, entityID string, manifest *AliasOrphanRollbackManifest, order *int) error { + ops := []struct { + table string + where string + args []any + }{ + {"events", `WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {"entity_tags", `WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {"relationships", `WHERE project_id = ? AND ((from_entity_kind = ? AND from_entity_id = ?) OR (to_entity_kind = ? AND to_entity_id = ?))`, []any{projectID, kind, entityID, kind, entityID}}, + } + for _, op := range ops { + if err := captureAndDeleteTx(ctx, tx, op.table, op.where, op.args, manifest, order); err != nil { + return err + } + } + return nil +} + +func listIDsTx(ctx context.Context, tx *sql.Tx, query string, args ...any) ([]string, error) { + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return ids, nil +} + +func readAliasOrphanRollbackManifest(path string) (AliasOrphanRollbackManifest, error) { + payload, err := os.ReadFile(path) + if err != nil { + return AliasOrphanRollbackManifest{}, fmt.Errorf("read alias-orphan rollback manifest: %w", err) + } + var manifest AliasOrphanRollbackManifest + dec := json.NewDecoder(strings.NewReader(string(payload))) + dec.UseNumber() + if err := dec.Decode(&manifest); err != nil { + return AliasOrphanRollbackManifest{}, fmt.Errorf("decode alias-orphan rollback manifest: %w", err) + } + if manifest.Migration != aliasOrphanMigrationName { + return AliasOrphanRollbackManifest{}, fmt.Errorf("rollback manifest migration %q is not %s", manifest.Migration, aliasOrphanMigrationName) + } + return manifest, nil +} diff --git a/internal/state/alias_orphan_migration_proof_test.go b/internal/state/alias_orphan_migration_proof_test.go new file mode 100644 index 000000000..1b57a1138 --- /dev/null +++ b/internal/state/alias_orphan_migration_proof_test.go @@ -0,0 +1,864 @@ +package state + +import ( + "context" + "encoding/hex" + "strings" + "testing" + + "github.com/levifig/loaf/internal/project" +) + +// A same-titled row that predates the damaging import is not a twin. The proof +// has to name the surviving alias holder as the 2026-06-24 re-import, not merely +// find the date somewhere in the pair. +func TestAliasOrphanContentIdentityRequiresTheHolderToBeTheReimport(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + + seedTask(t, stateHome, root, projectID, "task:incumbentlive00000001", "Untitled", "todo", "2024-01-05T00:00:00Z", true, "TASK-500") + unrelated := "task:unrelatedlive0000000001" + seedTask(t, stateHome, root, projectID, unrelated, "Untitled", "todo", "2026-06-24T13:03:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + got := aliasOrphanClassification(t, preview, unrelated) + if got.Proof != aliasOrphanProofUnproven || got.Disposition != "" { + t.Fatalf("classification = %#v, want unproven with no disposition", got) + } + if preview.Totals.Retire != 0 { + t.Fatalf("preview retire = %d, want 0", preview.Totals.Retire) + } +} + +// Two bodyless rows match on empty fingerprints only when the orphan sits in +// the June-13 original-import window; later same-day bodyless rows stay unproven. +func TestAliasOrphanContentIdentityBodylessRequiresOriginalImportWindow(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + + twinID := "task:bodylesstwin0000000001" + outsideWindow := "task:bodylesslate0000000001" + seedTask(t, stateHome, root, projectID, twinID, "Bodyless Title", "todo", "2026-06-24T13:03:37Z", true, "TASK-BODYLESS") + seedTask(t, stateHome, root, projectID, outsideWindow, "Bodyless Title", "todo", "2026-06-13T14:28:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if got := aliasOrphanClassification(t, preview, outsideWindow); got.Proof != aliasOrphanProofUnproven { + t.Fatalf("outside June-13 window classification = %#v, want unproven", got) + } + + inWindow := "task:bodylessorig0000000001" + seedTask(t, stateHome, root, projectID, inWindow, "Bodyless Title", "todo", "2026-06-13T01:39:42Z", false, "") + // Two orphans with the same title make orphan-side uniqueness fail; retire + // the late one from the fixture by removing it before the in-window proof. + mustExecOpen(t, stateHome, root, `DELETE FROM tasks WHERE project_id = ? AND id = ?`, projectID, outsideWindow) + + agreed, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("second PreviewAliasOrphanMigration() error = %v", err) + } + got := aliasOrphanClassification(t, agreed, inWindow) + if got.Proof != aliasOrphanProofContentIdentity || got.TwinID != twinID { + t.Fatalf("in-window bodyless classification = %#v, want content-identity against %s", got, twinID) + } +} + +// Equal titles are not equal content: rows whose stored bodies differ stay +// unproven, and rows that agree on both keep the fallback proof. +func TestAliasOrphanContentIdentityComparesBodies(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + + twinID := "spec:bodytwin000000000001" + orphanID := "spec:bodyorphan00000000001" + seedSpec(t, stateHome, root, projectID, twinID, "Same Title", "active", "2026-06-24T13:03:00Z", true, "SPEC-BODY") + seedSpec(t, stateHome, root, projectID, orphanID, "Same Title", "active", "2026-06-13T10:00:00Z", false, "") + seedArtifactBody(t, stateHome, root, projectID, "spec", twinID, "twin body", "hash-twin") + seedArtifactBody(t, stateHome, root, projectID, "spec", orphanID, "orphan body", "hash-orphan") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if got := aliasOrphanClassification(t, preview, orphanID); got.Proof != aliasOrphanProofUnproven { + t.Fatalf("differing bodies classification = %#v, want unproven", got) + } + + mustExecOpen(t, stateHome, root, `UPDATE artifact_bodies SET content = 'twin body', content_hash = 'hash-twin' WHERE project_id = ? AND entity_id = ?`, projectID, orphanID) + agreed, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("second PreviewAliasOrphanMigration() error = %v", err) + } + got := aliasOrphanClassification(t, agreed, orphanID) + if got.Proof != aliasOrphanProofContentIdentity || got.TwinID != twinID { + t.Fatalf("matching bodies classification = %#v, want content-identity against %s", got, twinID) + } +} + +// Recomputation proves the orphan was minted for this alias, not that the row +// now holding the alias is its duplicate. A reused alias number recomputes to +// exactly the same ID, so a distinct artifact would be deleted on ID match alone. +func TestAliasOrphanDerivationRefusesAReusedAlias(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-042" + reuser := "task:reusedaliasholder00001" + orphanID := stableMigrationID("task", legacyID, alias) + seedTask(t, stateHome, root, projectID, reuser, "Refactor the database layer", "todo", "2027-02-01T00:00:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "Add login screen", "todo", "2026-06-13T10:00:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + got := aliasOrphanClassification(t, preview, orphanID) + if got.Proof != aliasOrphanProofUnproven || got.Disposition != "" { + t.Fatalf("classification = %#v, want unproven with no disposition", got) + } + + if _, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}); err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if !entityExists(t, stateHome, root, "tasks", orphanID) { + t.Fatal("the alias-reuse victim was retired against an unrelated row") + } +} + +// An orphan that is newer than the row it would retire into is not the pre-rekey +// original this migration repairs. +func TestAliasOrphanDerivationRefusesAnOrphanNewerThanItsHolder(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-043" + holderID := stableMigrationID("task", projectID, alias) + orphanID := stableMigrationID("task", legacyID, alias) + seedTask(t, stateHome, root, projectID, holderID, "Same Title", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "Same Title", "todo", "2026-07-01T10:00:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if got := aliasOrphanClassification(t, preview, orphanID); got.Proof != aliasOrphanProofUnproven { + t.Fatalf("classification = %#v, want unproven", got) + } +} + +// Retiring an orphan deletes the relationship edges that were keeping a +// forward-declared alias alive. A sweep that runs before the retirement cannot +// see the alias its own run just killed, and post-apply verification then fails +// on a repair that actually succeeded. +func TestAliasOrphanCollectsTheAliasesItsOwnRetirementsKill(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-901" + twinID := stableMigrationID("task", projectID, alias) + orphanID := stableMigrationID("task", legacyID, alias) + seedTask(t, stateHome, root, projectID, twinID, "Forward Referencing Task", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "Forward Referencing Task", "todo", "2026-06-13T10:00:00Z", false, "") + + // TASK-900 is forward-declared: the importer registered the alias for a + // referenced-but-unimported artifact, so the row is missing while the + // orphan's depends_on edge still names it. It is a live reference at + // classification time and wreckage the moment the orphan retires. + forwardID := "task:forwarddeclared0000001" + forwardAliasID := stableMigrationID("alias", projectID, "task", "TASK-900") + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'task', ?, 'task', 'TASK-900', ?, ?) +`, forwardAliasID, projectID, forwardID, "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") + mustExecOpen(t, stateHome, root, ` +INSERT INTO relationships (id, project_id, from_entity_kind, from_entity_id, to_entity_kind, to_entity_id, relationship_type, reason, created_at, updated_at) +VALUES (?, ?, 'task', ?, 'task', ?, 'depends_on', 'fixture', ?, ?) +`, "relationship:forward0000001", projectID, orphanID, forwardID, "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") + + // The preview simulates the whole repair, so the go/no-go number already + // counts the alias the retirement will strand. + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if preview.Totals.DanglingAliases != 1 { + t.Fatalf("preview dangling aliases = %d, want 1", preview.Totals.DanglingAliases) + } + if !entityExists(t, stateHome, root, "aliases", forwardAliasID) { + t.Fatal("preview simulation leaked onto the live database") + } + + // One apply, one green verification: the error path this used to take names + // the rollback manifest, inviting the operator to undo a correct repair. + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if applied.Totals.AliasesDeleted != 1 { + t.Fatalf("aliases deleted = %d, want 1", applied.Totals.AliasesDeleted) + } + if entityExists(t, stateHome, root, "aliases", forwardAliasID) { + t.Fatal("the alias this run's own retirement killed survived it") + } + + // Rollback still restores it: the sweep runs before the manifest is written. + if _, err := RollbackAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath); err != nil { + t.Fatalf("RollbackAliasOrphanMigration() error = %v", err) + } + if !entityExists(t, stateHome, root, "aliases", forwardAliasID) { + t.Fatal("rollback did not restore the collected alias") + } +} + +// A project that moved after the damaging import still has a recomputable +// legacy ID — from project_paths, not only from its current path. +func TestAliasOrphanDerivationUsesHistoricalProjectPaths(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + + oldPath := "/previous/home/of/this/project" + mustExecOpen(t, stateHome, root, ` +INSERT INTO project_paths (id, project_id, path, is_current, first_seen_at, last_seen_at, created_at, updated_at) +VALUES (?, ?, ?, 0, ?, ?, ?, ?) +`, "projpath:historical00000001", projectID, oldPath, "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") + + legacyID := hex.EncodeToString(sha256Sum(oldPath)) + alias := "TASK-MOVED" + twinID := stableMigrationID("task", projectID, alias) + orphanID := stableMigrationID("task", legacyID, alias) + seedTask(t, stateHome, root, projectID, twinID, "Moved Project Task", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "Moved Project Task", "todo", "2026-06-13T10:00:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + got := aliasOrphanClassification(t, preview, orphanID) + if got.Proof != aliasOrphanProofDerivation { + t.Fatalf("classification = %#v, want derivation from the historical path", got) + } + if got.LegacyPath != oldPath || got.LegacyProjectID != legacyID { + t.Fatalf("classification legacy salt = %q/%q, want %q/%q", got.LegacyPath, got.LegacyProjectID, oldPath, legacyID) + } +} + +// Sparks are minted from (path, line), never from an alias, so alias-salt +// recomputation can never reach them. Their own source ID carries the salt, and +// the manifest labels the resulting proof for what it is: half salt, half content. +func TestAliasOrphanSparkEarnsSourceDerivationProof(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + relPath := ".agents/sessions/20260613-session.md" + + legacySourceID := stableMigrationID("source", legacyID, relPath) + currentSourceID := stableMigrationID("source", projectID, relPath) + seedSource(t, stateHome, root, projectID, legacySourceID, relPath) + seedSource(t, stateHome, root, projectID, currentSourceID, relPath) + + orphanID := stableMigrationID("spark", legacyID, relPath, "12") + twinID := stableMigrationID("spark", projectID, relPath, "12") + seedSpark(t, stateHome, root, projectID, twinID, "dedupe the state tables one day", currentSourceID, "2026-06-24T13:03:00Z", "SPARK-dedupe") + seedSpark(t, stateHome, root, projectID, orphanID, "dedupe the state tables one day", legacySourceID, "2026-06-13T10:00:00Z", "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + got := aliasOrphanClassification(t, preview, orphanID) + if got.Proof != aliasOrphanProofSourceDerivation || got.TwinID != twinID { + t.Fatalf("spark classification = %#v, want source-derivation against %s", got, twinID) + } +} + +// Source-salt proof shares the June-24 holder window and orphan-predates-twin +// ordering gates with the other auto-retiring proofs. +func TestAliasOrphanSourceSaltRequiresReimportHolderAndOrdering(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + relPath := ".agents/sessions/20260613-salt-gates.md" + + legacySourceID := stableMigrationID("source", legacyID, relPath) + currentSourceID := stableMigrationID("source", projectID, relPath) + seedSource(t, stateHome, root, projectID, legacySourceID, relPath) + seedSource(t, stateHome, root, projectID, currentSourceID, relPath) + + text := "gate the source salt proof" + // Holder outside the re-import window. + lateHolder := stableMigrationID("spark", projectID, relPath, "12") + orphanID := stableMigrationID("spark", legacyID, relPath, "12") + seedSpark(t, stateHome, root, projectID, lateHolder, text, currentSourceID, "2026-06-24T18:02:00Z", "SPARK-gate") + seedSpark(t, stateHome, root, projectID, orphanID, text, legacySourceID, "2026-06-13T10:00:00Z", "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if got := aliasOrphanClassification(t, preview, orphanID); got.Proof != aliasOrphanProofUnproven { + t.Fatalf("late holder classification = %#v, want unproven", got) + } + + // Newer orphan against an in-window holder is also unproven. + mustExecOpen(t, stateHome, root, `DELETE FROM sparks WHERE project_id = ? AND id IN (?, ?)`, projectID, lateHolder, orphanID) + mustExecOpen(t, stateHome, root, `DELETE FROM aliases WHERE project_id = ? AND entity_id = ?`, projectID, lateHolder) + inWindowHolder := stableMigrationID("spark", projectID, relPath, "20") + newerOrphan := stableMigrationID("spark", legacyID, relPath, "20") + seedSpark(t, stateHome, root, projectID, inWindowHolder, text, currentSourceID, "2026-06-24T13:03:37Z", "SPARK-gate") + seedSpark(t, stateHome, root, projectID, newerOrphan, text, legacySourceID, "2026-07-01T10:00:00Z", "") + + ordered, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("second PreviewAliasOrphanMigration() error = %v", err) + } + if got := aliasOrphanClassification(t, ordered, newerOrphan); got.Proof != aliasOrphanProofUnproven { + t.Fatalf("newer orphan classification = %#v, want unproven", got) + } +} + +// Two pre-rekey sparks with identical text from one file against a single alias +// holder is a merge, not a twin proof. Both rows stay unproven and untouched. +func TestAliasOrphanSourceSaltRefusesManyOrphansToOneHolder(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + relPath := ".agents/sessions/20260613-merge.md" + + legacySourceID := stableMigrationID("source", legacyID, relPath) + currentSourceID := stableMigrationID("source", projectID, relPath) + seedSource(t, stateHome, root, projectID, legacySourceID, relPath) + seedSource(t, stateHome, root, projectID, currentSourceID, relPath) + + text := "dedupe the state tables one day" + holderID := stableMigrationID("spark", projectID, relPath, "12") + firstOrphan := stableMigrationID("spark", legacyID, relPath, "12") + secondOrphan := stableMigrationID("spark", legacyID, relPath, "31") + seedSpark(t, stateHome, root, projectID, holderID, text, currentSourceID, "2026-06-24T13:03:00Z", "SPARK-dedupe") + seedSpark(t, stateHome, root, projectID, firstOrphan, text, legacySourceID, "2026-06-13T10:00:00Z", "") + seedSpark(t, stateHome, root, projectID, secondOrphan, text, legacySourceID, "2026-06-13T10:05:00Z", "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + for _, orphanID := range []string{firstOrphan, secondOrphan} { + got := aliasOrphanClassification(t, preview, orphanID) + if got.Proof != aliasOrphanProofUnproven || got.Disposition != "" { + t.Fatalf("classification for %s = %#v, want unproven with no disposition", orphanID, got) + } + } + + if _, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}); err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + for _, orphanID := range []string{firstOrphan, secondOrphan} { + if !entityExists(t, stateHome, root, "sparks", orphanID) { + t.Fatalf("unproven spark %s was retired against a shared holder", orphanID) + } + } +} + +// Retiring a report used to abort the whole migration at COMMIT: findings hold +// a NOT NULL foreign key that no polymorphic sweep reaches. +func TestAliasOrphanRetiresReportFindingsAndVerdicts(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "report-findings" + twinID := stableMigrationID("report", projectID, alias) + orphanID := stableMigrationID("report", legacyID, alias) + + seedReport(t, stateHome, root, projectID, twinID, "Findings Report", "2026-06-24T13:03:00Z", alias) + seedReport(t, stateHome, root, projectID, orphanID, "Findings Report", "2026-06-13T10:00:00Z", "") + mustExecOpen(t, stateHome, root, ` +INSERT INTO findings (id, project_id, report_id, title, status, severity, confidence, created_at, updated_at) +VALUES (?, ?, ?, 'Orphan finding', 'open', 'high', 'confirmed', ?, ?) +`, "finding:orphan00000000001", projectID, orphanID, "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") + mustExecOpen(t, stateHome, root, ` +INSERT INTO verdicts (id, project_id, finding_id, outcome, rationale, created_at, updated_at) +VALUES (?, ?, ?, 'confirmed', 'fixture', ?, ?) +`, "verdict:orphan00000000001", projectID, "finding:orphan00000000001", "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") + + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if entityExists(t, stateHome, root, "reports", orphanID) { + t.Fatal("orphan report survived apply") + } + if entityExists(t, stateHome, root, "findings", "finding:orphan00000000001") { + t.Fatal("finding for retired report survived apply") + } + if entityExists(t, stateHome, root, "verdicts", "verdict:orphan00000000001") { + t.Fatal("verdict for retired finding survived apply") + } + + if _, err := RollbackAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath); err != nil { + t.Fatalf("RollbackAliasOrphanMigration() error = %v", err) + } + if !entityExists(t, stateHome, root, "findings", "finding:orphan00000000001") { + t.Fatal("finding not restored by rollback") + } + if !entityExists(t, stateHome, root, "verdicts", "verdict:orphan00000000001") { + t.Fatal("verdict not restored by rollback") + } +} + +// journal_deferrals.spark_id and intent_operations.spark_id are NOT NULL with no +// foreign key: retiring a spark without touching them leaves silent dangling +// provenance that nothing detects. +func TestAliasOrphanRepointsSparkProvenanceAtTheTwin(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + relPath := ".agents/sessions/20260613-deferral.md" + + legacySourceID := stableMigrationID("source", legacyID, relPath) + currentSourceID := stableMigrationID("source", projectID, relPath) + seedSource(t, stateHome, root, projectID, legacySourceID, relPath) + seedSource(t, stateHome, root, projectID, currentSourceID, relPath) + + orphanID := stableMigrationID("spark", legacyID, relPath, "4") + twinID := stableMigrationID("spark", projectID, relPath, "4") + seedSpark(t, stateHome, root, projectID, twinID, "defer this thought", currentSourceID, "2026-06-24T13:03:00Z", "SPARK-defer") + seedSpark(t, stateHome, root, projectID, orphanID, "defer this thought", legacySourceID, "2026-06-13T10:00:00Z", "") + + mustExecOpen(t, stateHome, root, ` +INSERT INTO journal_entries (id, project_id, entry_type, scope, message, created_at, updated_at) +VALUES (?, ?, 'spark', 'scope', 'defer this thought', ?, ?) +`, "journal:deferral0000000001", projectID, "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") + mustExecOpen(t, stateHome, root, ` +INSERT INTO journal_deferrals (project_id, operation_key, journal_entry_id, spark_id, stored_digest, created_at) +VALUES (?, 'op-defer-1', ?, ?, ?, ?) +`, projectID, "journal:deferral0000000001", orphanID, strings.Repeat("a", 64), "2026-06-13T10:00:00Z") + + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if entityExists(t, stateHome, root, "sparks", orphanID) { + t.Fatal("orphan spark survived apply") + } + if got := deferralSparkID(t, stateHome, root, projectID, "op-defer-1"); got != twinID { + t.Fatalf("journal_deferrals.spark_id = %q, want the surviving twin %q", got, twinID) + } + + if _, err := RollbackAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath); err != nil { + t.Fatalf("RollbackAliasOrphanMigration() error = %v", err) + } + if got := deferralSparkID(t, stateHome, root, projectID, "op-defer-1"); got != orphanID { + t.Fatalf("journal_deferrals.spark_id after rollback = %q, want %q", got, orphanID) + } +} + +// --realias onto a claimed alias would manufacture a brand-new alias-orphan of +// exactly the class this migration exists to repair. +func TestAliasOrphanRealiasRefusesAClaimedAlias(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + + incumbent := "task:incumbent000000000001" + thief := "task:orphanthief00000000001" + seedTask(t, stateHome, root, projectID, incumbent, "Incumbent", "todo", "2026-05-01T00:00:00Z", true, "TASK-777") + seedTask(t, stateHome, root, projectID, thief, "Orphan Thief", "todo", "2026-05-01T00:00:00Z", false, "") + + _, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{ + Realias: map[string]string{thief: "TASK-777"}, + Flags: []string{"--realias " + thief + "=TASK-777"}, + }) + if err == nil { + t.Fatal("ApplyAliasOrphanMigration() accepted a realias onto a claimed alias") + } + if !strings.Contains(err.Error(), incumbent) { + t.Fatalf("error = %v, want it to name the incumbent %s", err, incumbent) + } + if !aliasPointsTo(t, stateHome, root, projectID, "task", "TASK-777", incumbent) { + t.Fatal("TASK-777 no longer names the incumbent") + } + if !entityExists(t, stateHome, root, "tasks", thief) { + t.Fatal("the refused orphan was mutated") + } +} + +// A mistyped disposition names nothing; applying the rest of the run silently +// would leave the operator believing a row was handled. +func TestAliasOrphanApplyRejectsDispositionsThatMatchNothing(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + seedTask(t, stateHome, root, projectID, "task:realorphan0000000001", "Real Orphan", "todo", "2026-05-01T00:00:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if len(preview.Warnings) != 0 { + t.Fatalf("preview warnings = %v, want none", preview.Warnings) + } + + refused, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{ + Retire: []string{"task:doesnotexist00000001"}, + Flags: []string{"--retire task:doesnotexist00000001"}, + }) + if err == nil { + t.Fatal("ApplyAliasOrphanMigration() silently ignored a disposition that matched nothing") + } + if !strings.Contains(err.Error(), "task:doesnotexist00000001") { + t.Fatalf("error = %v, want it to name the unmatched id", err) + } + // The refusal happens after the backup, so the result has to name it — an + // error that hides the artifact it just created is an artifact nobody cleans up. + if refused.Applied || refused.BackupPath == "" { + t.Fatalf("refused result = %#v, want applied=false with the backup path", refused) + } + if !entityExists(t, stateHome, root, "tasks", "task:realorphan0000000001") { + t.Fatal("apply mutated rows despite the rejected disposition") + } +} + +// An explicit --realias outranks the automatic classification: a row the +// operator named for preservation is preserved, whatever the proof says. +func TestAliasOrphanRealiasOutranksAProvenRetirement(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-777" + twinID := stableMigrationID("task", projectID, alias) + orphanID := stableMigrationID("task", legacyID, alias) + seedTask(t, stateHome, root, projectID, twinID, "Proven Twin", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "Proven Twin", "todo", "2026-06-13T10:00:00Z", false, "") + + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{ + Realias: map[string]string{orphanID: "TASK-KEEPME"}, + Flags: []string{"--realias " + orphanID + "=TASK-KEEPME"}, + }) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if !entityExists(t, stateHome, root, "tasks", orphanID) { + t.Fatal("--realias on a proven orphan deleted the row the operator asked to keep") + } + if !aliasPointsTo(t, stateHome, root, projectID, "task", "TASK-KEEPME", orphanID) { + t.Fatal("--realias did not attach the requested alias") + } + if applied.Totals.EntitiesRetired != 0 { + t.Fatalf("entities retired = %d, want 0", applied.Totals.EntitiesRetired) + } + for _, disposition := range applied.Dispositions { + if disposition.EntityID == orphanID && disposition.Action != aliasOrphanDispositionRealias { + t.Fatalf("manifest disposition for %s = %q, want realias", orphanID, disposition.Action) + } + } +} + +// The ceremony records the exact apply command; running it again has to be a +// no-op, not a hard error about flags the first run already carried out. +func TestAliasOrphanSecondApplyWithTheSameDispositionsIsANoOp(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + retireID := "task:rerunretire00000000001" + realiasID := "task:rerunrealias0000000001" + seedTask(t, stateHome, root, projectID, retireID, "Retire On Rerun", "todo", "2026-05-01T00:00:00Z", false, "") + seedTask(t, stateHome, root, projectID, realiasID, "Realias On Rerun", "todo", "2026-05-01T00:00:00Z", false, "") + + options := AliasOrphanApplyOptions{ + Retire: []string{retireID}, + Realias: map[string]string{realiasID: "TASK-RERUN"}, + Flags: []string{"--retire " + retireID, "--realias " + realiasID + "=TASK-RERUN"}, + } + if _, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, options); err != nil { + t.Fatalf("first ApplyAliasOrphanMigration() error = %v", err) + } + + second, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, options) + if err != nil { + t.Fatalf("second ApplyAliasOrphanMigration() error = %v", err) + } + if len(second.Warnings) != 0 { + t.Fatalf("second apply warnings = %v, want none", second.Warnings) + } + if second.Totals.EntitiesRetired != 0 || second.Totals.AliasesInserted != 0 { + t.Fatalf("second apply totals = %#v, want a no-op", second.Totals) + } + if entityExists(t, stateHome, root, "tasks", retireID) { + t.Fatal("the retired row came back") + } + if !aliasPointsTo(t, stateHome, root, projectID, "task", "TASK-RERUN", realiasID) { + t.Fatal("the realiased row lost its alias") + } +} + +// Classification iterates to a fixed point inside the shared classifier: a +// second orphan that only becomes unique after its title-twin retires is +// retire-class in one plan, so a single bare apply passes post-apply +// verification and a second preview reports zero retire-class rows. +func TestAliasOrphanClassificationReachesFixedPoint(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-777" + + holder := stableMigrationID("task", projectID, alias) + provenOrphan := stableMigrationID("task", legacyID, alias) + secondOrphan := "task:secondorphan000000001" + + seedTask(t, stateHome, root, projectID, holder, "Shared Title", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, provenOrphan, "Shared Title", "todo", "2026-06-13T10:00:00Z", false, "") + // Bodyless content-identity needs the original-import window once uniqueness unlocks. + seedTask(t, stateHome, root, projectID, secondOrphan, "Shared Title", "todo", "2026-06-13T01:39:42Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if got := aliasOrphanClassification(t, preview, provenOrphan); got.Proof != aliasOrphanProofDerivation || got.Disposition != aliasOrphanDispositionRetire { + t.Fatalf("derivation orphan = %#v, want derivation/retire", got) + } + if got := aliasOrphanClassification(t, preview, secondOrphan); got.Proof != aliasOrphanProofContentIdentity || got.Disposition != aliasOrphanDispositionRetire { + t.Fatalf("unlocked orphan = %#v, want content-identity/retire after fixed point", got) + } + if preview.Totals.Retire < 2 { + t.Fatalf("preview retire = %d, want both orphans retire-class", preview.Totals.Retire) + } + + if _, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}); err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if entityExists(t, stateHome, root, "tasks", provenOrphan) || entityExists(t, stateHome, root, "tasks", secondOrphan) { + t.Fatal("both orphans should be retired by a single apply") + } + if !entityExists(t, stateHome, root, "tasks", holder) { + t.Fatal("holder was retired") + } + + after, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("second PreviewAliasOrphanMigration() error = %v", err) + } + if after.Totals.Retire != 0 { + t.Fatalf("second preview retire = %d, want 0", after.Totals.Retire) + } +} + +// Preview reports the source rows the retire set will strand — the ceremony's +// go/no-go reads that number before any apply. Apply surfaces the same total. +func TestAliasOrphanPreviewReportsOrphanedSources(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "SPEC-SOURCES" + twinID := stableMigrationID("spec", projectID, alias) + orphanID := stableMigrationID("spec", legacyID, alias) + + seedSpec(t, stateHome, root, projectID, twinID, "Sourced Spec", "active", "2026-06-24T13:03:00Z", true, alias) + seedSpec(t, stateHome, root, projectID, orphanID, "Sourced Spec", "active", "2026-06-13T10:00:00Z", false, "") + seedSpecResidue(t, stateHome, root, projectID, orphanID) + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if preview.Totals.OrphanedSources != 1 { + t.Fatalf("preview orphaned sources = %d, want 1", preview.Totals.OrphanedSources) + } + found := false + for _, project := range preview.Projects { + for _, table := range project.Tables { + if table.Table == "specs" && table.OrphanedSources == 1 { + found = true + } + } + } + if !found { + t.Fatalf("specs table did not report its orphaned source: %#v", preview.Projects) + } + if !entityExists(t, stateHome, root, "specs", orphanID) { + t.Fatal("preview simulation leaked onto the live database") + } + if !entityExists(t, stateHome, root, "sources", stableMigrationID("source", projectID, "specs/"+orphanID+".md")) { + t.Fatal("preview simulation deleted a live source row") + } + + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if applied.Totals.OrphanedSources != 1 { + t.Fatalf("apply orphaned sources = %d, want 1", applied.Totals.OrphanedSources) + } +} + +// Rollback restores the archived report byte-identically, updated_at included. +func TestAliasOrphanRollbackRestoresUpdatedAt(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + seedReport(t, stateHome, root, projectID, brokenEvidenceReportID, "Transitional TypeScript Surfaces — Do Not Deepen", "2026-06-20T00:00:00Z", "transitional-surfaces-do-not-deepen") + mustExecOpen(t, stateHome, root, `UPDATE reports SET status = 'active', updated_at = ? WHERE id = ?`, "2026-06-20T00:00:00Z", brokenEvidenceReportID) + + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if _, err := RollbackAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath); err != nil { + t.Fatalf("RollbackAliasOrphanMigration() error = %v", err) + } + status, updatedAt := reportStatusAndUpdatedAt(t, stateHome, root, brokenEvidenceReportID) + if status != "active" || updatedAt != "2026-06-20T00:00:00Z" { + t.Fatalf("report after rollback = %q/%q, want active/2026-06-20T00:00:00Z", status, updatedAt) + } +} + +// The housekeeping scanner counts shaping drafts and the importer gives them +// aliases, so they can orphan exactly like the other six tables. Detector and +// repair have to reach them or the count-agreement receipt is not a receipt. +func TestAliasOrphanCoversShapingDrafts(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "shape-token-rotation" + twinID := stableMigrationID("shaping_draft", projectID, alias) + orphanID := stableMigrationID("shaping_draft", legacyID, alias) + + seedShapingDraft(t, stateHome, root, projectID, twinID, "Token Rotation Shape", "2026-06-24T13:03:00Z", alias) + seedShapingDraft(t, stateHome, root, projectID, orphanID, "Token Rotation Shape", "2026-06-13T10:00:00Z", "") + + store := openTestStore(t, root, stateHome) + parity, err := InspectAliasParity(ctx, store) + store.Close() + if err != nil { + t.Fatalf("InspectAliasParity() error = %v", err) + } + drafts := findAliasParityTable(t, parity, projectID, "shaping_drafts") + if drafts.RawCount != 2 || drafts.AliasReachableCount != 1 || drafts.OrphanDelta != 1 { + t.Fatalf("shaping_drafts parity = %#v, want raw=2 reachable=1 orphan=1", drafts) + } + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if got := aliasOrphanClassification(t, preview, orphanID); got.Proof != aliasOrphanProofDerivation || got.TwinID != twinID { + t.Fatalf("shaping draft classification = %#v, want derivation against %s", got, twinID) + } + + if _, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}); err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if entityExists(t, stateHome, root, "shaping_drafts", orphanID) { + t.Fatal("orphan shaping draft survived apply") + } + if !entityExists(t, stateHome, root, "shaping_drafts", twinID) { + t.Fatal("the alias-holding shaping draft was retired") + } +} + +// --- fixture helpers --- + +func seedShapingDraft(t *testing.T, stateHome string, root project.Root, projectID, id, title, createdAt, alias string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO shaping_drafts (id, project_id, title, status, body_source_id, created_at, updated_at) +VALUES (?, ?, ?, 'draft', NULL, ?, ?) +`, id, projectID, title, createdAt, createdAt) + if alias != "" { + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'shaping_draft', ?, 'shaping_draft', ?, ?, ?) +`, stableMigrationID("alias", projectID, "shaping_draft", alias), projectID, id, alias, createdAt, createdAt) + } +} + +func aliasOrphanClassification(t *testing.T, result AliasOrphanMigrationResult, entityID string) AliasOrphanRowClassify { + t.Helper() + for _, project := range result.Projects { + for _, table := range project.Tables { + for _, c := range table.Classifications { + if c.EntityID == entityID { + return c + } + } + } + } + t.Fatalf("no classification for %s", entityID) + return AliasOrphanRowClassify{} +} + +func seedArtifactBody(t *testing.T, stateHome string, root project.Root, projectID, kind, entityID, content, hash string) { + t.Helper() + bodyID := stableMigrationID("artifact_body", projectID, kind, entityID, "markdown") + mustExecOpen(t, stateHome, root, ` +INSERT INTO artifact_bodies (id, project_id, entity_kind, entity_id, body_kind, content, content_hash, source_id, created_at, updated_at) +VALUES (?, ?, ?, ?, 'markdown', ?, ?, NULL, ?, ?) +`, bodyID, projectID, kind, entityID, content, hash, "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") + store := openTestStore(t, root, stateHome) + defer store.Close() + var rowID int64 + if err := store.db.QueryRow(`SELECT rowid FROM artifact_bodies WHERE id = ?`, bodyID).Scan(&rowID); err != nil { + t.Fatalf("read body rowid: %v", err) + } + if _, err := store.db.Exec(`INSERT INTO artifact_search(rowid, project_id, entity_kind, entity_id, body_kind, content) VALUES (?, ?, ?, ?, 'markdown', ?)`, rowID, projectID, kind, entityID, content); err != nil { + t.Fatalf("insert artifact_search: %v", err) + } +} + +func seedSource(t *testing.T, stateHome string, root project.Root, projectID, id, path string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO sources (id, project_id, source_kind, path, hash, imported_at, created_at, updated_at) +VALUES (?, ?, 'markdown', ?, 'hash', ?, ?, ?) +`, id, projectID, path, "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z", "2026-06-13T10:00:00Z") +} + +func seedSpark(t *testing.T, stateHome string, root project.Root, projectID, id, text, sourceID, createdAt, alias string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO sparks (id, project_id, scope, status, text, source_id, created_at, updated_at) +VALUES (?, ?, 'scope', 'open', ?, ?, ?, ?) +`, id, projectID, text, sourceID, createdAt, createdAt) + if alias != "" { + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'spark', ?, 'spark', ?, ?, ?) +`, stableMigrationID("alias", projectID, "spark", alias), projectID, id, alias, createdAt, createdAt) + } +} + +func seedReport(t *testing.T, stateHome string, root project.Root, projectID, id, title, createdAt, alias string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO reports (id, project_id, report_kind, title, status, body_source_id, created_at, updated_at) +VALUES (?, ?, 'audit', ?, 'final', NULL, ?, ?) +`, id, projectID, title, createdAt, createdAt) + if alias != "" { + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'report', ?, 'report', ?, ?, ?) +`, stableMigrationID("alias", projectID, "report", alias), projectID, id, alias, createdAt, createdAt) + } +} + +func deferralSparkID(t *testing.T, stateHome string, root project.Root, projectID, operationKey string) string { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var sparkID string + if err := store.db.QueryRow(`SELECT spark_id FROM journal_deferrals WHERE project_id = ? AND operation_key = ?`, projectID, operationKey).Scan(&sparkID); err != nil { + t.Fatalf("read journal_deferrals.spark_id: %v", err) + } + return sparkID +} + +func reportStatusAndUpdatedAt(t *testing.T, stateHome string, root project.Root, id string) (string, string) { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var status, updatedAt string + if err := store.db.QueryRow(`SELECT status, updated_at FROM reports WHERE id = ?`, id).Scan(&status, &updatedAt); err != nil { + t.Fatalf("read report: %v", err) + } + return status, updatedAt +} diff --git a/internal/state/alias_orphan_migration_test.go b/internal/state/alias_orphan_migration_test.go new file mode 100644 index 000000000..fae1d9271 --- /dev/null +++ b/internal/state/alias_orphan_migration_test.go @@ -0,0 +1,441 @@ +package state + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/levifig/loaf/internal/project" +) + +func TestAliasOrphanClassificationProofs(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-190" + twinID := stableMigrationID("task", projectID, alias) + orphanDerivedID := stableMigrationID("task", legacyID, alias) + if twinID == orphanDerivedID { + t.Fatalf("fixture requires distinct twin and derived orphan ids; both %s", twinID) + } + + seedTask(t, stateHome, root, projectID, twinID, "Archive duplicate work", "done", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanDerivedID, "Archive duplicate work", "done", "2026-06-13T10:00:00Z", false, "") + + contentOrphanID := "task:contentidentity0000001" + seedTask(t, stateHome, root, projectID, "task:content-twin00000000001", "Content Identity Twin", "todo", "2026-06-24T13:03:00Z", true, "TASK-CONTENT") + seedTask(t, stateHome, root, projectID, contentOrphanID, "Content Identity Twin", "todo", "2026-06-13T01:39:42Z", false, "") + + unprovenID := "task:unproven00000000000001" + seedTask(t, stateHome, root, projectID, unprovenID, "Unproven Orphan Title", "todo", "2026-06-13T12:00:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("PreviewAliasOrphanMigration() error = %v", err) + } + if !preview.CopyRun || preview.Applied { + t.Fatalf("preview copy_run/applied = %t/%t, want true/false", preview.CopyRun, preview.Applied) + } + + byID := map[string]AliasOrphanRowClassify{} + for _, project := range preview.Projects { + for _, table := range project.Tables { + for _, c := range table.Classifications { + byID[c.EntityID] = c + } + } + } + if got := byID[orphanDerivedID]; got.Proof != aliasOrphanProofDerivation || got.Disposition != aliasOrphanDispositionRetire { + t.Fatalf("derived orphan classify = %#v, want derivation/retire", got) + } + if got := byID[contentOrphanID]; got.Proof != aliasOrphanProofContentIdentity || got.Disposition != aliasOrphanDispositionRetire { + t.Fatalf("content-identity orphan classify = %#v, want content-identity/retire", got) + } + if got := byID[unprovenID]; got.Proof != aliasOrphanProofUnproven || got.Disposition != "" { + t.Fatalf("unproven orphan classify = %#v, want unproven with empty disposition", got) + } + if preview.Totals.Retire < 2 || preview.Totals.Unproven < 1 { + t.Fatalf("preview totals = %#v, want retire>=2 unproven>=1", preview.Totals) + } + + // Live DB untouched by preview. + if !entityExists(t, stateHome, root, "tasks", orphanDerivedID) { + t.Fatal("preview deleted derived orphan on live DB") + } + if !entityExists(t, stateHome, root, "tasks", unprovenID) { + t.Fatal("preview deleted unproven orphan on live DB") + } +} + +func TestAliasOrphanApplyRollbackAndIdempotency(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "SPEC-001" + twinID := stableMigrationID("spec", projectID, alias) + orphanID := stableMigrationID("spec", legacyID, alias) + + seedSpec(t, stateHome, root, projectID, twinID, "Canonical Spec", "active", "2026-06-24T13:03:00Z", true, alias) + seedSpec(t, stateHome, root, projectID, orphanID, "Canonical Spec", "active", "2026-06-13T10:00:00Z", false, "") + seedSpecResidue(t, stateHome, root, projectID, orphanID) + + danglingAliasID := "alias:dangling000000000001" + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'task', 'task:missing0000000000001', 'task', 'TASK-MISSING', ?, ?) +`, danglingAliasID, projectID, "2026-06-24T13:03:00Z", "2026-06-24T13:03:00Z") + + unprovenID := "spec:unproven0000000000001" + seedSpec(t, stateHome, root, projectID, unprovenID, "Unproven Spec", "active", "2026-06-13T12:00:00Z", false, "") + + // Seed broken-evidence report (named disposition). + mustExecOpen(t, stateHome, root, ` +INSERT INTO reports (id, project_id, report_kind, title, status, body_source_id, created_at, updated_at) +VALUES (?, ?, 'audit', 'Transitional TypeScript Surfaces — Do Not Deepen', 'active', NULL, ?, ?) +`, brokenEvidenceReportID, projectID, "2026-06-20T00:00:00Z", "2026-06-20T00:00:00Z") + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'report', ?, 'report', 'transitional-surfaces-do-not-deepen', ?, ?) +`, stableMigrationID("alias", projectID, "report", "transitional-surfaces-do-not-deepen"), projectID, brokenEvidenceReportID, "2026-06-20T00:00:00Z", "2026-06-20T00:00:00Z") + + // Apply without disposition must leave unproven alone and retire proven rows. + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if !applied.Applied || applied.BackupPath == "" || applied.RollbackManifestPath == "" { + t.Fatalf("apply result incomplete: %#v", applied) + } + if _, err := os.Stat(applied.BackupPath); err != nil { + t.Fatalf("stat backup: %v", err) + } + if _, err := os.Stat(applied.RollbackManifestPath); err != nil { + t.Fatalf("stat manifest: %v", err) + } + if entityExists(t, stateHome, root, "specs", orphanID) { + t.Fatal("derived orphan still present after apply") + } + if !entityExists(t, stateHome, root, "specs", twinID) { + t.Fatal("twin was retired; want preserved") + } + if !entityExists(t, stateHome, root, "specs", unprovenID) { + t.Fatal("unproven orphan was retired without disposition") + } + if entityExists(t, stateHome, root, "aliases", danglingAliasID) { + t.Fatal("dangling alias still present after apply") + } + if residueCount(t, stateHome, root, projectID, "spec", orphanID) != 0 { + t.Fatalf("reference residue remains for retired orphan") + } + if got := aliasOrphanReportStatus(t, stateHome, root, brokenEvidenceReportID); got != LifecycleStatusArchived { + t.Fatalf("broken-evidence status = %q, want archived", got) + } + if mootEventCount(t, stateHome, root, projectID, brokenEvidenceReportID) != 1 { + t.Fatal("expected one moot archive event for broken-evidence report") + } + + // Idempotency: second preview classifies zero retire/dangling; second apply no-ops. + secondPreview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("second PreviewAliasOrphanMigration() error = %v", err) + } + if secondPreview.Totals.Retire != 0 || secondPreview.Totals.DanglingAliases != 0 { + t.Fatalf("second preview totals = %#v, want zero retire and dangling", secondPreview.Totals) + } + if secondPreview.Totals.Unproven < 1 { + t.Fatalf("second preview unproven = %d, want the untouched unproven orphan", secondPreview.Totals.Unproven) + } + secondApply, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("second ApplyAliasOrphanMigration() error = %v", err) + } + if secondApply.Totals.EntitiesRetired != 0 || secondApply.Totals.AliasesDeleted != 0 { + t.Fatalf("second apply should no-op retire/delete, got %#v", secondApply.Totals) + } + + // Rollback restores the retired orphan and residue. + rolled, err := RollbackAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath) + if err != nil { + t.Fatalf("RollbackAliasOrphanMigration() error = %v", err) + } + if !rolled.Applied || rolled.RowsRestored == 0 { + t.Fatalf("rollback result = %#v, want restored rows", rolled) + } + if !entityExists(t, stateHome, root, "specs", orphanID) { + t.Fatal("orphan not restored after rollback") + } + if residueCount(t, stateHome, root, projectID, "spec", orphanID) == 0 { + t.Fatal("expected residue restored with orphan") + } + if got := aliasOrphanReportStatus(t, stateHome, root, brokenEvidenceReportID); got != "active" { + t.Fatalf("broken-evidence status after rollback = %q, want active", got) + } +} + +func TestAliasOrphanOperatorDispositions(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + + retireID := "task:operatorretire00000001" + realiasID := "task:operatorrealias0000001" + seedTask(t, stateHome, root, projectID, retireID, "Operator Retire Me", "todo", "2026-05-01T00:00:00Z", false, "") + seedTask(t, stateHome, root, projectID, realiasID, "Operator Realias Me", "todo", "2026-05-01T00:00:00Z", false, "") + + // Without disposition, unproven remain. + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("preview error = %v", err) + } + if preview.Totals.Unproven < 2 || preview.Totals.Retire != 0 { + t.Fatalf("preview totals = %#v, want unproven>=2 retire=0", preview.Totals) + } + + applied, err := ApplyAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{ + Retire: []string{retireID}, + Realias: map[string]string{realiasID: "TASK-REALIAS"}, + Flags: []string{"--retire " + retireID, "--realias " + realiasID + "=TASK-REALIAS"}, + }) + if err != nil { + t.Fatalf("apply with dispositions error = %v", err) + } + if entityExists(t, stateHome, root, "tasks", retireID) { + t.Fatal("operator --retire left the row in place") + } + if !entityExists(t, stateHome, root, "tasks", realiasID) { + t.Fatal("operator --realias deleted the row") + } + if !aliasPointsTo(t, stateHome, root, projectID, "task", "TASK-REALIAS", realiasID) { + t.Fatal("operator --realias did not attach alias") + } + if applied.RollbackManifestPath == "" { + t.Fatal("expected rollback manifest for operator dispositions") + } + + // After dispositions, no unproven remain for these IDs. + after, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("post-disposition preview error = %v", err) + } + for _, project := range after.Projects { + for _, table := range project.Tables { + for _, c := range table.Classifications { + if c.EntityID == retireID || c.EntityID == realiasID { + t.Fatalf("classified %s after disposition: %#v", c.EntityID, c) + } + } + } + } +} + +func TestAliasOrphanPreviewIsolatesAllProjects(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-MULTI" + twinID := stableMigrationID("task", projectID, alias) + orphanID := stableMigrationID("task", legacyID, alias) + seedTask(t, stateHome, root, projectID, twinID, "Multi Project Task", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "Multi Project Task", "todo", "2026-06-13T10:00:00Z", false, "") + + preview, err := PreviewAliasOrphanMigration(ctx, root, PathResolver{StateHome: stateHome}, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("preview error = %v", err) + } + if len(preview.Projects) == 0 { + t.Fatal("preview reported no projects") + } + found := false + for _, project := range preview.Projects { + if project.ProjectID == projectID { + found = true + if project.LegacyProjectID != legacyID { + t.Fatalf("legacy project id = %q, want %q", project.LegacyProjectID, legacyID) + } + if project.Counts.Retire < 1 { + t.Fatalf("project counts = %#v, want retire>=1", project.Counts) + } + } + } + if !found { + t.Fatalf("preview missing project %s", projectID) + } + if !entityExists(t, stateHome, root, "tasks", orphanID) { + t.Fatal("preview mutated live orphan") + } +} + +// --- fixture helpers --- + +func seedAliasOrphanFixtureBase(t *testing.T) (project.Root, string, string, string) { + t.Helper() + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + dbPath := filepath.Join(stateHome, "loaf", "loaf.sqlite") + t.Setenv("LOAF_DB", dbPath) + status, err := Initialize(ctx, root, PathResolver{StateHome: stateHome}) + if err != nil { + t.Fatalf("Initialize() error = %v", err) + } + return root, stateHome, status.ProjectID, status.ProjectCurrentPath +} + +func seedTask(t *testing.T, stateHome string, root project.Root, projectID, id, title, status, createdAt string, withAlias bool, alias string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO tasks (id, project_id, spec_id, title, status, priority, body_source_id, created_at, updated_at) +VALUES (?, ?, NULL, ?, ?, NULL, NULL, ?, ?) +`, id, projectID, title, status, createdAt, createdAt) + if withAlias { + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'task', ?, 'task', ?, ?, ?) +`, stableMigrationID("alias", projectID, "task", alias), projectID, id, alias, createdAt, createdAt) + } +} + +func seedSpec(t *testing.T, stateHome string, root project.Root, projectID, id, title, status, createdAt string, withAlias bool, alias string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO specs (id, project_id, title, status, body_source_id, created_at, updated_at) +VALUES (?, ?, ?, ?, NULL, ?, ?) +`, id, projectID, title, status, createdAt, createdAt) + if withAlias { + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'spec', ?, 'spec', ?, ?, ?) +`, stableMigrationID("alias", projectID, "spec", alias), projectID, id, alias, createdAt, createdAt) + } +} + +func seedSpecResidue(t *testing.T, stateHome string, root project.Root, projectID, specID string) { + t.Helper() + now := "2026-06-13T10:00:00Z" + sourceID := stableMigrationID("source", projectID, "specs/"+specID+".md") + mustExecOpen(t, stateHome, root, ` +INSERT INTO sources (id, project_id, source_kind, path, hash, line_start, line_end, imported_at, created_at, updated_at) +VALUES (?, ?, 'markdown', ?, 'hash', NULL, NULL, ?, ?, ?) +`, sourceID, projectID, ".agents/specs/"+specID+".md", now, now, now) + mustExecOpen(t, stateHome, root, `UPDATE specs SET body_source_id = ? WHERE id = ?`, sourceID, specID) + bodyID := stableMigrationID("artifact_body", projectID, "spec", specID, "markdown") + mustExecOpen(t, stateHome, root, ` +INSERT INTO artifact_bodies (id, project_id, entity_kind, entity_id, body_kind, content, content_hash, source_id, created_at, updated_at) +VALUES (?, ?, 'spec', ?, 'markdown', 'body', 'hash', ?, ?, ?) +`, bodyID, projectID, specID, sourceID, now, now) + // Index FTS for the body. + store := openTestStore(t, root, stateHome) + defer store.Close() + var rowID int64 + if err := store.db.QueryRow(`SELECT rowid FROM artifact_bodies WHERE id = ?`, bodyID).Scan(&rowID); err != nil { + t.Fatalf("read body rowid: %v", err) + } + if _, err := store.db.Exec(`INSERT INTO artifact_search(rowid, project_id, entity_kind, entity_id, body_kind, content) VALUES (?, ?, 'spec', ?, 'markdown', 'body')`, rowID, projectID, specID); err != nil { + t.Fatalf("insert artifact_search: %v", err) + } + mustExecOpen(t, stateHome, root, ` +INSERT INTO events (id, project_id, entity_kind, entity_id, event_type, from_status, to_status, note, created_at, updated_at) +VALUES (?, ?, 'spec', ?, 'status_changed', 'active', 'archived', 'housekeeping', ?, ?) +`, stableMigrationID("event", projectID, "spec", specID, "archived"), projectID, specID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO tags (id, project_id, name, created_at, updated_at) VALUES (?, ?, 'orphan-tag', ?, ?) +`, stableMigrationID("tag", projectID, "orphan-tag"), projectID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO entity_tags (id, project_id, tag_id, entity_kind, entity_id, created_at, updated_at) +VALUES (?, ?, ?, 'spec', ?, ?, ?) +`, stableMigrationID("entity_tag", projectID, "orphan-tag", "spec", specID), projectID, stableMigrationID("tag", projectID, "orphan-tag"), specID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO relationships (id, project_id, from_entity_kind, from_entity_id, to_entity_kind, to_entity_id, relationship_type, reason, created_at, updated_at) +VALUES (?, ?, 'spec', ?, 'spec', ?, 'related_to', 'fixture', ?, ?) +`, stableMigrationID("relationship", projectID, "spec", specID, "related_to", "spec", specID), projectID, specID, specID, now, now) +} + +func mustExecOpen(t *testing.T, stateHome string, root project.Root, query string, args ...any) { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + if _, err := store.db.ExecContext(context.Background(), query, args...); err != nil { + t.Fatalf("exec %q: %v", query, err) + } +} + +func entityExists(t *testing.T, stateHome string, root project.Root, table, id string) bool { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var count int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM `+quoteSQLiteIdentifier(table)+` WHERE id = ?`, id).Scan(&count); err != nil { + t.Fatalf("count %s %s: %v", table, id, err) + } + return count > 0 +} + +func residueCount(t *testing.T, stateHome string, root project.Root, projectID, kind, entityID string) int { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + total := 0 + for _, q := range []struct { + sql string + args []any + }{ + {`SELECT COUNT(*) FROM artifact_bodies WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {`SELECT COUNT(*) FROM events WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {`SELECT COUNT(*) FROM entity_tags WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, kind, entityID}}, + {`SELECT COUNT(*) FROM relationships WHERE project_id = ? AND ((from_entity_kind = ? AND from_entity_id = ?) OR (to_entity_kind = ? AND to_entity_id = ?))`, []any{projectID, kind, entityID, kind, entityID}}, + } { + var n int + if err := store.db.QueryRow(q.sql, q.args...).Scan(&n); err != nil { + t.Fatalf("residue count: %v", err) + } + total += n + } + return total +} + +func aliasOrphanReportStatus(t *testing.T, stateHome string, root project.Root, id string) string { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var status string + if err := store.db.QueryRow(`SELECT status FROM reports WHERE id = ?`, id).Scan(&status); err != nil { + t.Fatalf("report status: %v", err) + } + return status +} + +func mootEventCount(t *testing.T, stateHome string, root project.Root, projectID, entityID string) int { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var n int + if err := store.db.QueryRow(` +SELECT COUNT(*) FROM events +WHERE project_id = ? AND entity_id = ? AND event_type = ? AND note = ? +`, projectID, entityID, aliasOrphanArchiveMootEventType, aliasOrphanArchiveMootNote).Scan(&n); err != nil { + t.Fatalf("moot event count: %v", err) + } + return n +} + +func aliasPointsTo(t *testing.T, stateHome string, root project.Root, projectID, namespace, alias, entityID string) bool { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var got string + err := store.db.QueryRow(` +SELECT entity_id FROM aliases WHERE project_id = ? AND namespace = ? AND alias = ? +`, projectID, namespace, alias).Scan(&got) + if err != nil { + return false + } + return got == entityID +} + +func sha256Sum(s string) []byte { + sum := sha256.Sum256([]byte(s)) + return sum[:] +} diff --git a/internal/state/alias_parity.go b/internal/state/alias_parity.go new file mode 100644 index 000000000..f1e4f8af0 --- /dev/null +++ b/internal/state/alias_parity.go @@ -0,0 +1,355 @@ +package state + +import ( + "context" + "database/sql" + "fmt" +) + +// AliasParityDivergenceCode is the stable diagnostic code when raw entity +// counts diverge from alias-reachable counts, or dangling aliases exist. +const AliasParityDivergenceCode = "alias-parity-diverged" + +// AliasParityMultiAliasCode is the stable diagnostic code when an entity holds +// more than one alias. Multi-alias is countable and warning-only; it does not +// gate Ready and names no repair command. +const AliasParityMultiAliasCode = "alias-multi-alias" + +// AliasParityClearCode is the info-severity receipt when every project/table is at parity. +const AliasParityClearCode = "alias-parity-clear" + +// AliasParityRepairCommand is the preview-form migration that repairs alias orphans. +const AliasParityRepairCommand = "loaf state migrate alias-orphans" + +// AliasParityTable holds raw vs alias-reachable counts for one project entity +// table. AliasReachableCount mirrors what the list surfaces return: they INNER +// JOIN through aliases, so their cardinality is the number of alias rows that +// resolve, not the number of entities that happen to hold one. AliasedEntities +// counts the distinct entities behind those rows, so both directions of +// divergence — entities with no alias, entities with more than one — are +// visible instead of cancelling out. +type AliasParityTable struct { + ProjectID string `json:"project_id"` + Kind string `json:"kind"` + Table string `json:"table"` + Namespace string `json:"namespace"` + RawCount int `json:"raw_count"` + AliasReachableCount int `json:"alias_reachable_count"` + AliasedEntities int `json:"aliased_entities"` + OrphanDelta int `json:"orphan_delta"` + MultiAlias int `json:"multi_alias"` + DanglingAliases int `json:"dangling_aliases"` + // Inconsistent is true when the table's counts fail the internal identity + // check (orphan_delta != raw_count - aliased_entities). Counted as + // divergence rather than an inspect error so Mode stays ready. + Inconsistent bool `json:"inconsistent,omitempty"` + Inconsistency string `json:"inconsistency,omitempty"` +} + +// AliasParity is the read-only doctor report for entity/alias identity parity. +type AliasParity struct { + Tables []AliasParityTable `json:"tables"` + ProjectsChecked int `json:"projects_checked"` + TablesChecked int `json:"tables_checked"` + RawCount int `json:"raw_count"` + AliasReachableCount int `json:"alias_reachable_count"` + AliasedEntities int `json:"aliased_entities"` + OrphanDelta int `json:"orphan_delta"` + MultiAlias int `json:"multi_alias"` + DanglingAliases int `json:"dangling_aliases"` + Ready bool `json:"ready"` + Inconsistencies []string `json:"inconsistencies,omitempty"` +} + +// InspectAliasParity compares raw entity row counts to alias-joined counts and +// counts dangling aliases for every project and each entity table. Read-only. +func InspectAliasParity(ctx context.Context, store *Store) (AliasParity, error) { + if store == nil || store.db == nil { + return AliasParity{}, fmt.Errorf("inspect alias parity: store is nil") + } + + projectIDs, err := listAliasParityProjectIDs(ctx, store) + if err != nil { + return AliasParity{}, err + } + + parity := AliasParity{ + Tables: []AliasParityTable{}, + ProjectsChecked: len(projectIDs), + Ready: true, + } + for _, projectID := range projectIDs { + for _, table := range aliasOrphanEntityTables { + row, err := inspectAliasParityTable(ctx, store, projectID, table) + if err != nil { + return AliasParity{}, err + } + parity.Tables = append(parity.Tables, row) + parity.RawCount += row.RawCount + parity.AliasReachableCount += row.AliasReachableCount + parity.AliasedEntities += row.AliasedEntities + parity.OrphanDelta += row.OrphanDelta + parity.MultiAlias += row.MultiAlias + parity.DanglingAliases += row.DanglingAliases + if row.Inconsistent && row.Inconsistency != "" { + parity.Inconsistencies = append(parity.Inconsistencies, row.Inconsistency) + } + } + } + parity.TablesChecked = len(parity.Tables) + // Multi-alias is warning-only and does not gate Ready; only orphan/dangling + // damage and internal count inconsistency do. + if parity.OrphanDelta > 0 || parity.DanglingAliases > 0 || len(parity.Inconsistencies) > 0 { + parity.Ready = false + } + return parity, nil +} + +func listAliasParityProjectIDs(ctx context.Context, store *Store) ([]string, error) { + rows, err := store.db.QueryContext(ctx, `SELECT id FROM projects ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("list projects for alias parity: %w", err) + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scan project id for alias parity: %w", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate projects for alias parity: %w", err) + } + return ids, nil +} + +func inspectAliasParityTable(ctx context.Context, store *Store, projectID string, table aliasOrphanEntityTable) (AliasParityTable, error) { + result := AliasParityTable{ + ProjectID: projectID, + Kind: table.kind, + Table: table.table, + Namespace: table.namespace, + } + quotedTable := quoteSQLiteIdentifier(table.table) + + tx, err := store.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return result, fmt.Errorf("begin alias parity snapshot for %s: %w", table.table, err) + } + defer tx.Rollback() + + if err := tx.QueryRowContext(ctx, fmt.Sprintf( + `SELECT COUNT(*) FROM %s WHERE project_id = ?`, quotedTable, + ), projectID).Scan(&result.RawCount); err != nil { + return result, fmt.Errorf("count raw %s rows: %w", table.table, err) + } + + // One row per alias — the exact cardinality `loaf list` returns. + if err := tx.QueryRowContext(ctx, fmt.Sprintf(` +SELECT COUNT(*) +FROM aliases AS a +JOIN %s AS e ON e.project_id = a.project_id AND e.id = a.entity_id +WHERE a.project_id = ? AND a.entity_kind = ? AND a.namespace = ? +`, quotedTable), projectID, table.kind, table.namespace).Scan(&result.AliasReachableCount); err != nil { + return result, fmt.Errorf("count alias-reachable %s rows: %w", table.table, err) + } + + if err := tx.QueryRowContext(ctx, fmt.Sprintf(` +SELECT COUNT(*) +FROM %s AS e +WHERE e.project_id = ? + AND EXISTS ( + SELECT 1 FROM aliases AS a + WHERE a.project_id = e.project_id + AND a.entity_kind = ? + AND a.entity_id = e.id + AND a.namespace = ? + ) +`, quotedTable), projectID, table.kind, table.namespace).Scan(&result.AliasedEntities); err != nil { + return result, fmt.Errorf("count aliased %s entities: %w", table.table, err) + } + + if err := tx.QueryRowContext(ctx, fmt.Sprintf(` +SELECT COUNT(*) +FROM %s AS e +WHERE e.project_id = ? + AND NOT EXISTS ( + SELECT 1 FROM aliases AS a + WHERE a.project_id = e.project_id + AND a.entity_kind = ? + AND a.entity_id = e.id + AND a.namespace = ? + ) +`, quotedTable), projectID, table.kind, table.namespace).Scan(&result.OrphanDelta); err != nil { + return result, fmt.Errorf("count orphan %s rows: %w", table.table, err) + } + + // Dead aliases only — a forward reference the importer registered for an + // artifact that has no row yet is not divergence. See + // aliasOrphanDeadAliasPredicate: detector and repair share one definition. + if err := tx.QueryRowContext(ctx, fmt.Sprintf(` +SELECT COUNT(*) +FROM aliases AS a +WHERE a.project_id = ? + AND a.entity_kind = ? + AND a.namespace = ?`+aliasOrphanDeadAliasPredicate, quotedTable), projectID, table.kind, table.namespace).Scan(&result.DanglingAliases); err != nil { + return result, fmt.Errorf("count dangling %s aliases: %w", table.table, err) + } + + if err := tx.Commit(); err != nil { + return result, fmt.Errorf("commit alias parity snapshot for %s: %w", table.table, err) + } + + result.MultiAlias = result.AliasReachableCount - result.AliasedEntities + if result.OrphanDelta != result.RawCount-result.AliasedEntities { + result.Inconsistent = true + result.Inconsistency = fmt.Sprintf( + "alias parity internal inconsistency for %s project %s: orphan_delta=%d raw=%d aliased=%d", + table.table, projectID, result.OrphanDelta, result.RawCount, result.AliasedEntities, + ) + } + return result, nil +} + +func aliasParityDiagnostics(parity AliasParity) []Diagnostic { + var out []Diagnostic + if !parity.Ready { + out = append(out, aliasParityDivergenceDiagnostic(parity)) + } else if parity.MultiAlias == 0 { + out = append(out, aliasParityClearDiagnostic(parity)) + } + if parity.MultiAlias > 0 { + out = append(out, aliasParityMultiAliasDiagnostic(parity)) + } + return out +} + +func aliasParityDivergenceDiagnostic(parity AliasParity) Diagnostic { + divergent := make([]map[string]any, 0) + for _, table := range parity.Tables { + if table.OrphanDelta == 0 && table.DanglingAliases == 0 && !table.Inconsistent { + continue + } + row := map[string]any{ + "project_id": table.ProjectID, + "kind": table.Kind, + "table": table.Table, + "namespace": table.Namespace, + "raw_count": table.RawCount, + "alias_reachable_count": table.AliasReachableCount, + "aliased_entities": table.AliasedEntities, + "orphan_delta": table.OrphanDelta, + "multi_alias": table.MultiAlias, + "dangling_aliases": table.DanglingAliases, + } + if table.Inconsistent { + row["inconsistent"] = true + row["inconsistency"] = table.Inconsistency + } + divergent = append(divergent, row) + } + message := fmt.Sprintf( + "alias parity diverged (orphan_delta=%d, multi_alias=%d, dangling_aliases=%d); run: %s", + parity.OrphanDelta, + parity.MultiAlias, + parity.DanglingAliases, + AliasParityRepairCommand, + ) + if len(parity.Inconsistencies) > 0 { + message = fmt.Sprintf( + "alias parity diverged (orphan_delta=%d, multi_alias=%d, dangling_aliases=%d, inconsistencies=%d); run: %s", + parity.OrphanDelta, + parity.MultiAlias, + parity.DanglingAliases, + len(parity.Inconsistencies), + AliasParityRepairCommand, + ) + } + details := map[string]any{ + "raw_count": parity.RawCount, + "alias_reachable_count": parity.AliasReachableCount, + "aliased_entities": parity.AliasedEntities, + "orphan_delta": parity.OrphanDelta, + "multi_alias": parity.MultiAlias, + "dangling_aliases": parity.DanglingAliases, + "tables": divergent, + "preview_command": AliasParityRepairCommand, + } + if len(parity.Inconsistencies) > 0 { + details["inconsistencies"] = parity.Inconsistencies + } + return Diagnostic{ + Severity: "error", + Code: AliasParityDivergenceCode, + Category: RepairCategoryAliasIdentity, + Policy: DiagnosticPolicyInvalidLocalData, + Message: message, + Details: details, + } +} + +func aliasParityMultiAliasDiagnostic(parity AliasParity) Diagnostic { + divergent := make([]map[string]any, 0) + for _, table := range parity.Tables { + if table.MultiAlias == 0 { + continue + } + divergent = append(divergent, map[string]any{ + "project_id": table.ProjectID, + "kind": table.Kind, + "table": table.Table, + "namespace": table.Namespace, + "raw_count": table.RawCount, + "alias_reachable_count": table.AliasReachableCount, + "aliased_entities": table.AliasedEntities, + "orphan_delta": table.OrphanDelta, + "multi_alias": table.MultiAlias, + "dangling_aliases": table.DanglingAliases, + }) + } + return Diagnostic{ + Severity: "warn", + Code: AliasParityMultiAliasCode, + Category: RepairCategoryAliasIdentity, + Message: fmt.Sprintf( + "alias multi-alias present (multi_alias=%d); entities hold more than one alias; no automated repair", + parity.MultiAlias, + ), + Details: map[string]any{ + "raw_count": parity.RawCount, + "alias_reachable_count": parity.AliasReachableCount, + "aliased_entities": parity.AliasedEntities, + "orphan_delta": parity.OrphanDelta, + "multi_alias": parity.MultiAlias, + "dangling_aliases": parity.DanglingAliases, + "tables": divergent, + }, + } +} + +func aliasParityClearDiagnostic(parity AliasParity) Diagnostic { + return Diagnostic{ + Severity: "info", + Code: AliasParityClearCode, + Category: RepairCategoryAliasIdentity, + Message: fmt.Sprintf( + "alias parity clear: %d project(s), %d table check(s); raw_count=%d equals alias_reachable_count; multi_alias=0; dangling_aliases=0", + parity.ProjectsChecked, + parity.TablesChecked, + parity.RawCount, + ), + Details: map[string]any{ + "projects_checked": parity.ProjectsChecked, + "tables_checked": parity.TablesChecked, + "raw_count": parity.RawCount, + "alias_reachable_count": parity.AliasReachableCount, + "aliased_entities": parity.AliasedEntities, + "orphan_delta": parity.OrphanDelta, + "multi_alias": parity.MultiAlias, + "dangling_aliases": parity.DanglingAliases, + }, + } +} diff --git a/internal/state/alias_parity_test.go b/internal/state/alias_parity_test.go new file mode 100644 index 000000000..5f39eba05 --- /dev/null +++ b/internal/state/alias_parity_test.go @@ -0,0 +1,432 @@ +package state + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "os" + "strings" + "testing" + + "github.com/levifig/loaf/internal/project" +) + +func TestStateDoctorAliasParityCleanFixture(t *testing.T) { + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + resolver := PathResolver{StateHome: stateHome} + + seedTask(t, stateHome, root, projectID, "task:clean0000000000000001", "Clean Task", "todo", "2026-06-24T13:03:00Z", true, "TASK-CLEAN") + seedSpec(t, stateHome, root, projectID, "spec:clean0000000000000001", "Clean Spec", "active", "2026-06-24T13:03:00Z", true, "SPEC-CLEAN") + + status, err := InspectWithOptions(root, resolver, InspectOptions{AliasParity: true}) + if err != nil { + t.Fatalf("InspectWithOptions() error = %v", err) + } + if status.Mode != ModeSQLiteReady { + t.Fatalf("Mode = %q, want %q; diagnostics = %#v", status.Mode, ModeSQLiteReady, status.Diagnostics) + } + assertNoDiagnostic(t, status.Diagnostics, AliasParityDivergenceCode) + + diagnostic := findDiagnostic(t, status.Diagnostics, AliasParityClearCode) + if diagnostic.Severity != "info" || diagnostic.Category != RepairCategoryAliasIdentity { + t.Fatalf("alias parity clear diagnostic = %#v, want info/%s", diagnostic, RepairCategoryAliasIdentity) + } + wantTablesChecked := len(aliasOrphanEntityTables) + assertDiagnosticDetail(t, status.Diagnostics, AliasParityClearCode, "projects_checked", 1) + assertDiagnosticDetail(t, status.Diagnostics, AliasParityClearCode, "tables_checked", wantTablesChecked) + assertDiagnosticDetail(t, status.Diagnostics, AliasParityClearCode, "raw_count", 2) + assertDiagnosticDetail(t, status.Diagnostics, AliasParityClearCode, "alias_reachable_count", 2) + assertDiagnosticDetail(t, status.Diagnostics, AliasParityClearCode, "orphan_delta", 0) + assertDiagnosticDetail(t, status.Diagnostics, AliasParityClearCode, "dangling_aliases", 0) + if !strings.Contains(diagnostic.Message, "alias parity clear") { + t.Fatalf("clear message %q missing summary prefix", diagnostic.Message) + } + if !strings.Contains(diagnostic.Message, "dangling_aliases=0") { + t.Fatalf("clear message %q missing dangling_aliases=0", diagnostic.Message) + } + + store := openTestStore(t, root, stateHome) + defer store.Close() + parity, err := InspectAliasParity(context.Background(), store) + if err != nil { + t.Fatalf("InspectAliasParity() error = %v", err) + } + if !parity.Ready { + t.Fatalf("parity = %#v, want Ready=true", parity) + } + if parity.OrphanDelta != 0 || parity.DanglingAliases != 0 { + t.Fatalf("parity deltas = orphan=%d dangling=%d, want 0/0", parity.OrphanDelta, parity.DanglingAliases) + } + taskRow := findAliasParityTable(t, parity, projectID, "tasks") + if taskRow.RawCount != 1 || taskRow.AliasReachableCount != 1 || taskRow.OrphanDelta != 0 { + t.Fatalf("tasks parity = %#v, want raw=1 reachable=1 orphan=0", taskRow) + } +} + +func TestAliasParityStaysOffTheDefaultInspectPath(t *testing.T) { + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + resolver := PathResolver{StateHome: stateHome} + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-HOTPATH" + seedTask(t, stateHome, root, projectID, stableMigrationID("task", projectID, alias), "Hot Path Task", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, stableMigrationID("task", legacyID, alias), "Hot Path Task", "todo", "2026-06-13T10:00:00Z", false, "") + + status, err := Inspect(root, resolver) + if err != nil { + t.Fatalf("Inspect() error = %v", err) + } + if status.Mode != ModeSQLiteReady { + t.Fatalf("Mode = %q, want %q; diagnostics = %#v", status.Mode, ModeSQLiteReady, status.Diagnostics) + } + // Every list/read command calls Inspect; a global 27-project table scan does + // not belong on that path. + assertNoDiagnostic(t, status.Diagnostics, AliasParityDivergenceCode) + assertNoDiagnostic(t, status.Diagnostics, AliasParityClearCode) +} + +func TestAliasParityCountsAliasRowsNotAliasedEntities(t *testing.T) { + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + seedTask(t, stateHome, root, projectID, "task:multialias00000000001", "Two Aliases", "todo", "2026-06-24T13:03:00Z", true, "TASK-ONE") + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'task', 'task:multialias00000000001', 'task', 'TASK-TWO', ?, ?) +`, "alias:multialias0000000001", projectID, "2026-06-24T13:03:00Z", "2026-06-24T13:03:00Z") + + store := openTestStore(t, root, stateHome) + defer store.Close() + parity, err := InspectAliasParity(context.Background(), store) + if err != nil { + t.Fatalf("InspectAliasParity() error = %v", err) + } + tasks := findAliasParityTable(t, parity, projectID, "tasks") + // `loaf task list` INNER JOINs aliases, so it returns two rows for one task. + if tasks.AliasReachableCount != 2 || tasks.RawCount != 1 || tasks.MultiAlias != 1 { + t.Fatalf("tasks parity = %#v, want raw=1 reachable=2 multi_alias=1", tasks) + } + // Multi-alias is warning-only; Ready stays true with no orphan/dangling damage. + if !parity.Ready { + t.Fatalf("parity = %#v, want Ready=true while multi-alias is warning-only", parity) + } + + resolver := PathResolver{StateHome: stateHome} + status, err := InspectWithOptions(root, resolver, InspectOptions{AliasParity: true}) + if err != nil { + t.Fatalf("InspectWithOptions() error = %v", err) + } + if status.Mode != ModeSQLiteReady { + t.Fatalf("Mode = %q, want %q; diagnostics = %#v", status.Mode, ModeSQLiteReady, status.Diagnostics) + } + assertNoDiagnostic(t, status.Diagnostics, AliasParityDivergenceCode) + assertNoDiagnostic(t, status.Diagnostics, AliasParityClearCode) + diagnostic := findDiagnostic(t, status.Diagnostics, AliasParityMultiAliasCode) + if diagnostic.Severity != "warn" || diagnostic.Category != RepairCategoryAliasIdentity { + t.Fatalf("multi-alias diagnostic = %#v, want warn/%s", diagnostic, RepairCategoryAliasIdentity) + } + if diagnostic.Details["multi_alias"] != 1 { + t.Fatalf("multi_alias detail = %#v, want 1", diagnostic.Details["multi_alias"]) + } + // No repair action for multi-alias. + for _, action := range RepairPlanForStatus(Status{DatabasePath: status.DatabasePath, Diagnostics: status.Diagnostics}) { + if action.DiagnosticCode == AliasParityMultiAliasCode { + t.Fatalf("unexpected repair action for multi-alias: %#v", action) + } + } +} + +func TestStateDoctorAliasParityOrphanFinding(t *testing.T) { + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + resolver := PathResolver{StateHome: stateHome} + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-ORPHAN" + twinID := stableMigrationID("task", projectID, alias) + orphanID := stableMigrationID("task", legacyID, alias) + + seedTask(t, stateHome, root, projectID, twinID, "Twin Task", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "Twin Task", "todo", "2026-06-13T10:00:00Z", false, "") + + status, err := InspectWithOptions(root, resolver, InspectOptions{AliasParity: true}) + if err != nil { + t.Fatalf("InspectWithOptions() error = %v", err) + } + if status.Mode != ModeSQLiteReady { + t.Fatalf("Mode = %q, want usable sqlite-ready despite alias damage; diagnostics = %#v", status.Mode, status.Diagnostics) + } + + assertNoDiagnostic(t, status.Diagnostics, AliasParityClearCode) + diagnostic := findDiagnostic(t, status.Diagnostics, AliasParityDivergenceCode) + if diagnostic.Severity != "error" || diagnostic.Category != RepairCategoryAliasIdentity || diagnostic.Policy != DiagnosticPolicyInvalidLocalData { + t.Fatalf("alias parity diagnostic = %#v, want error/%s/%s", diagnostic, RepairCategoryAliasIdentity, DiagnosticPolicyInvalidLocalData) + } + if diagnostic.Details["orphan_delta"] != 1 { + t.Fatalf("orphan_delta = %#v, want 1", diagnostic.Details["orphan_delta"]) + } + if diagnostic.Details["dangling_aliases"] != 0 { + t.Fatalf("dangling_aliases = %#v, want 0", diagnostic.Details["dangling_aliases"]) + } + if diagnostic.Details["raw_count"] != 2 { + t.Fatalf("raw_count = %#v, want 2", diagnostic.Details["raw_count"]) + } + if diagnostic.Details["alias_reachable_count"] != 1 { + t.Fatalf("alias_reachable_count = %#v, want 1", diagnostic.Details["alias_reachable_count"]) + } + if !strings.Contains(diagnostic.Message, AliasParityRepairCommand) { + t.Fatalf("message %q missing repair command %q", diagnostic.Message, AliasParityRepairCommand) + } + if !strings.Contains(diagnostic.Message, "orphan_delta=1") { + t.Fatalf("message %q missing orphan_delta=1", diagnostic.Message) + } + + table := findAliasParityDetailTable(t, diagnostic, projectID, "tasks") + if table["raw_count"] != 2 || table["alias_reachable_count"] != 1 || table["orphan_delta"] != 1 || table["dangling_aliases"] != 0 { + t.Fatalf("tasks detail = %#v, want raw=2 reachable=1 orphan=1 dangling=0", table) + } + + action := findRepairAction(t, RepairPlanForStatus(Status{DatabasePath: status.DatabasePath, Diagnostics: status.Diagnostics}), "migrate-alias-orphans") + if action.DiagnosticCode != AliasParityDivergenceCode || action.Category != RepairCategoryAliasIdentity || action.Safe || action.Command != AliasParityRepairCommand { + t.Fatalf("repair action = %#v, want migrate-alias-orphans unsafe preview command", action) + } +} + +func TestStateDoctorAliasParityDanglingAliasFinding(t *testing.T) { + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + resolver := PathResolver{StateHome: stateHome} + + seedTask(t, stateHome, root, projectID, "task:kept000000000000000001", "Kept Task", "todo", "2026-06-24T13:03:00Z", true, "TASK-KEPT") + danglingAliasID := "alias:dangling000000000001" + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'task', 'task:missing0000000000001', 'task', 'TASK-MISSING', ?, ?) +`, danglingAliasID, projectID, "2026-06-24T13:03:00Z", "2026-06-24T13:03:00Z") + + status, err := InspectWithOptions(root, resolver, InspectOptions{AliasParity: true}) + if err != nil { + t.Fatalf("InspectWithOptions() error = %v", err) + } + if status.Mode != ModeSQLiteReady { + t.Fatalf("Mode = %q, want usable sqlite-ready; diagnostics = %#v", status.Mode, status.Diagnostics) + } + + assertNoDiagnostic(t, status.Diagnostics, AliasParityClearCode) + diagnostic := findDiagnostic(t, status.Diagnostics, AliasParityDivergenceCode) + if diagnostic.Details["dangling_aliases"] != 1 { + t.Fatalf("dangling_aliases = %#v, want 1", diagnostic.Details["dangling_aliases"]) + } + if diagnostic.Details["orphan_delta"] != 0 { + t.Fatalf("orphan_delta = %#v, want 0", diagnostic.Details["orphan_delta"]) + } + if !strings.Contains(diagnostic.Message, "dangling_aliases=1") { + t.Fatalf("message %q missing dangling_aliases=1", diagnostic.Message) + } + if !strings.Contains(diagnostic.Message, AliasParityRepairCommand) { + t.Fatalf("message %q missing repair command %q", diagnostic.Message, AliasParityRepairCommand) + } + + table := findAliasParityDetailTable(t, diagnostic, projectID, "tasks") + if table["raw_count"] != 1 || table["alias_reachable_count"] != 1 || table["orphan_delta"] != 0 || table["dangling_aliases"] != 1 { + t.Fatalf("tasks detail = %#v, want raw=1 reachable=1 orphan=0 dangling=1", table) + } + + action := findRepairAction(t, RepairPlanForStatus(Status{DatabasePath: status.DatabasePath, Diagnostics: status.Diagnostics}), "migrate-alias-orphans") + if action.Command != AliasParityRepairCommand { + t.Fatalf("repair command = %q, want %q", action.Command, AliasParityRepairCommand) + } +} + +func TestStateDoctorAliasParityDiagnosticPerformsNoWrites(t *testing.T) { + t.Run("orphan and dangling", func(t *testing.T) { + root, stateHome, projectID, path := seedAliasOrphanFixtureBase(t) + resolver := PathResolver{StateHome: stateHome} + legacyID := hex.EncodeToString(sha256Sum(path)) + alias := "TASK-NOWRITE" + twinID := stableMigrationID("task", projectID, alias) + orphanID := stableMigrationID("task", legacyID, alias) + + seedTask(t, stateHome, root, projectID, twinID, "No Write Twin", "todo", "2026-06-24T13:03:00Z", true, alias) + seedTask(t, stateHome, root, projectID, orphanID, "No Write Twin", "todo", "2026-06-13T10:00:00Z", false, "") + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'task', 'task:missing-nowrite000001', 'task', 'TASK-DANGLING-NW', ?, ?) +`, "alias:dangling-nowrite000001", projectID, "2026-06-24T13:03:00Z", "2026-06-24T13:03:00Z") + + assertAliasParityDiagnosticNoWrites(t, root, stateHome, resolver, AliasParityDivergenceCode) + if !entityExists(t, stateHome, root, "tasks", orphanID) { + t.Fatal("orphan row missing after InspectWithOptions") + } + if !entityExists(t, stateHome, root, "aliases", "alias:dangling-nowrite000001") { + t.Fatal("dangling alias missing after InspectWithOptions") + } + }) + + t.Run("multi-alias", func(t *testing.T) { + root, stateHome, projectID, _ := seedAliasOrphanFixtureBase(t) + resolver := PathResolver{StateHome: stateHome} + seedTask(t, stateHome, root, projectID, "task:multialias00000000001", "Two Aliases", "todo", "2026-06-24T13:03:00Z", true, "TASK-ONE") + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'task', 'task:multialias00000000001', 'task', 'TASK-TWO', ?, ?) +`, "alias:multialias0000000001", projectID, "2026-06-24T13:03:00Z", "2026-06-24T13:03:00Z") + + assertAliasParityDiagnosticNoWrites(t, root, stateHome, resolver, AliasParityMultiAliasCode) + if !entityExists(t, stateHome, root, "aliases", "alias:multialias0000000001") { + t.Fatal("second alias missing after InspectWithOptions") + } + }) +} + +func assertAliasParityDiagnosticNoWrites(t *testing.T, root project.Root, stateHome string, resolver PathResolver, wantCode string) { + t.Helper() + dbPath, err := resolver.DatabasePath(root) + if err != nil { + t.Fatalf("DatabasePath() error = %v", err) + } + // Checkpoint so file bytes are stable under WAL. + store := openTestStore(t, root, stateHome) + if _, err := store.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + store.Close() + t.Fatalf("wal_checkpoint: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + removeSQLiteSidecars(t, dbPath) + + before, err := os.ReadFile(dbPath) + if err != nil { + t.Fatalf("ReadFile(before) error = %v", err) + } + beforeHash := sha256.Sum256(before) + + status, err := InspectWithOptions(root, resolver, InspectOptions{AliasParity: true}) + if err != nil { + t.Fatalf("InspectWithOptions() error = %v", err) + } + assertDiagnostic(t, status.Diagnostics, wantCode) + + // Ensure any read-only connection is fully closed before re-hashing. + removeSQLiteSidecars(t, dbPath) + after, err := os.ReadFile(dbPath) + if err != nil { + t.Fatalf("ReadFile(after) error = %v", err) + } + afterHash := sha256.Sum256(after) + if !bytes.Equal(beforeHash[:], afterHash[:]) { + t.Fatalf("InspectWithOptions mutated database bytes: before=%x after=%x", beforeHash, afterHash) + } +} + +func findAliasParityTable(t *testing.T, parity AliasParity, projectID, table string) AliasParityTable { + t.Helper() + for _, row := range parity.Tables { + if row.ProjectID == projectID && row.Table == table { + return row + } + } + t.Fatalf("parity table %s for project %s not found in %#v", table, projectID, parity.Tables) + return AliasParityTable{} +} + +func findAliasParityDetailTable(t *testing.T, diagnostic Diagnostic, projectID, table string) map[string]any { + t.Helper() + raw, ok := diagnostic.Details["tables"] + if !ok { + t.Fatalf("diagnostic details missing tables: %#v", diagnostic.Details) + } + switch tables := raw.(type) { + case []map[string]any: + for _, row := range tables { + if row["project_id"] == projectID && row["table"] == table { + return row + } + } + case []any: + for _, item := range tables { + row, ok := item.(map[string]any) + if !ok { + continue + } + if row["project_id"] == projectID && row["table"] == table { + return row + } + } + default: + t.Fatalf("tables detail type %T = %#v", raw, raw) + } + t.Fatalf("table %s for project %s not found in %#v", table, projectID, raw) + return nil +} + +// A depends_on naming a task with no file is a forward reference: the importer +// registers its alias so the dependency renders by name, the doctor stays green, +// and the repair migration leaves it alone. Import → migrate → import converges. +// When the dependency leaves the markdown the alias goes dead, and only then is +// it collected. +func TestImportForwardReferenceAliasSurvivesUntilItsReferenceGoes(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAgentsFile(t, root.Path(), "tasks/TASK-001-example.md", `--- +id: TASK-001 +title: Example Task +status: todo +depends_on: [TASK-999] +--- +# Example Task +`) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("ApplyMarkdownMigration() error = %v", err) + } + assertAliasParityReady(t, first.DatabasePath, true) + + applied, err := ApplyAliasOrphanMigration(ctx, root, resolver, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("ApplyAliasOrphanMigration() error = %v", err) + } + if applied.Totals.AliasesDeleted != 0 { + t.Fatalf("aliases deleted = %d, want the forward reference left alone", applied.Totals.AliasesDeleted) + } + + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("re-import error = %v", err) + } + assertAliasParityReady(t, first.DatabasePath, true) + + // The reference leaves the markdown: the edge goes with it, and the alias it + // forward-declared becomes wreckage. + writeAgentsFile(t, root.Path(), "tasks/TASK-001-example.md", `--- +id: TASK-001 +title: Example Task +status: todo +--- +# Example Task +`) + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("third ApplyMarkdownMigration() error = %v", err) + } + assertAliasParityReady(t, first.DatabasePath, false) + + collected, err := ApplyAliasOrphanMigration(ctx, root, resolver, AliasOrphanApplyOptions{}) + if err != nil { + t.Fatalf("second ApplyAliasOrphanMigration() error = %v", err) + } + if collected.Totals.AliasesDeleted != 1 { + t.Fatalf("aliases deleted = %d, want the dead forward reference collected", collected.Totals.AliasesDeleted) + } + assertAliasParityReady(t, first.DatabasePath, true) +} + +func assertAliasParityReady(t *testing.T, databasePath string, want bool) { + t.Helper() + store := openStoreAt(t, databasePath) + defer store.Close() + parity, err := InspectAliasParity(context.Background(), store) + if err != nil { + t.Fatalf("InspectAliasParity() error = %v", err) + } + if parity.Ready != want { + t.Fatalf("alias parity ready = %t, want %t (%#v)", parity.Ready, want, parity) + } +} diff --git a/internal/state/artifact_body.go b/internal/state/artifact_body.go index ae0dd8abb..19c604b37 100644 --- a/internal/state/artifact_body.go +++ b/internal/state/artifact_body.go @@ -221,6 +221,19 @@ VALUES (?, ?, ?, ?, ?, ?) } func deleteArtifactSearchTx(ctx context.Context, tx *sql.Tx, row artifactSearchRow) error { + probe := artifactSearchIndexProbe(row.BodyKind) + var exists int + if err := tx.QueryRowContext(ctx, ` +SELECT EXISTS( + SELECT 1 FROM artifact_search + WHERE artifact_search MATCH ? AND rowid = ? +) +`, probe, row.RowID).Scan(&exists); err != nil { + return fmt.Errorf("probe artifact search row %d: %w", row.RowID, err) + } + if exists == 0 { + return nil + } if _, err := tx.ExecContext(ctx, ` INSERT INTO artifact_search(artifact_search, rowid, project_id, entity_kind, entity_id, body_kind, content) VALUES ('delete', ?, ?, ?, ?, ?, ?) @@ -230,6 +243,12 @@ VALUES ('delete', ?, ?, ?, ?, ?, ?) return nil } +func artifactSearchIndexProbe(bodyKind string) string { + kind := firstNonEmpty(strings.TrimSpace(bodyKind), ArtifactBodyKindMarkdown) + escaped := strings.ReplaceAll(kind, `"`, `""`) + return `body_kind:"` + escaped + `"` +} + func scanArtifactBody(row interface { Scan(dest ...any) error }) (ArtifactBody, error) { diff --git a/internal/state/journal_duplicate_migration.go b/internal/state/journal_duplicate_migration.go new file mode 100644 index 000000000..153c3d771 --- /dev/null +++ b/internal/state/journal_duplicate_migration.go @@ -0,0 +1,1132 @@ +package state + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/levifig/loaf/internal/project" +) + +// Journal-duplicates repair migration. +// +// Schema references to journal_entries.id (verified against internal/state/migrations/*.sql): +// +// - journal_search.journal_entry_id (0006_journal_search.sql; rebuilt in 0010_journal_first.sql) +// FTS5 derived index; not a foreign key. rowid mirrors journal_entries.rowid at insert time. +// - journal_origins.journal_entry_id (0011_journal_origins_and_deferrals.sql) — PRIMARY KEY, deliberately not an FK +// - journal_deferrals.journal_entry_id (0011) — NOT NULL UNIQUE, deliberately not an FK; +// repointed to the June-24 twin when free, otherwise deleted +// - intent_operations.journal_entry_id (0012_intents_and_explorations.sql) — optional projection ref; +// CHECK ties projection_version=1 to non-NULL journal_entry_id + spark_id; repoint-or-delete +// - journal_conversation_handles.journal_entry_id (0012) — NOT NULL, UNIQUE (journal_entry_id, handle_id), +// not an FK; repoint-or-delete +// - Polymorphic (entity_kind, entity_id) sites that may cite kind journal_entry. These are +// captured into the rollback manifest and deleted — never repointed at the surviving twin +// (same residue policy as alias-orphans). Covered by polymorphicEntityReferenceSweeps plus +// artifact_bodies/aliases: +// events, entity_tags, bundle_members, backend_mappings, +// exports (source_entity_kind / source_entity_id), +// relationships (from_* or to_*), +// artifact_bodies (+ artifact_search FTS mirror cleaned via deleteArtifactBodiesForEntityTx), +// aliases (alias-orphan dead-alias walk only covers seven non-journal entity tables) +// +// FTS strategy: targeted DELETE FROM journal_search WHERE journal_entry_id = ? inside the same +// apply transaction as the journal_entries deletion. A full RepairJournalSearch rebuild would work +// but is heavier and spans a second ceremony; targeted deletes keep apply/rollback symmetric — +// rollback restores the journal_entries row then rebuilds its FTS row via insertJournalSearchTx +// (the live write path), avoiding FTS rowid drift after re-insert. Restored artifact_bodies rows +// rebuild artifact_search the same way via restoreArtifactSearchTx. +// +// Window constants (june13OriginalImportWindow*, june24ReimportWindow*) and inTimestampWindow +// are shared with the alias-orphans migration — do not redefine them here. + +const ( + JournalDuplicateMigrationActionDryRun = "dry-run" + JournalDuplicateMigrationActionApply = "apply" + JournalDuplicateMigrationActionRollback = "rollback" + + journalDuplicateMigrationName = "journal-duplicates" + + journalDuplicateProofPair = "pair" + journalDuplicateProofUnproven = "unproven" + + journalDuplicateDispositionRetire = "retire" +) + +// JournalDuplicateMigrationResult is the preview/apply/rollback outcome for the +// journal-duplicates repair migration. Classification spans every project. +type JournalDuplicateMigrationResult struct { + ContractVersion int `json:"contract_version"` + DatabaseScope string `json:"database_scope"` + DatabasePath string `json:"database_path"` + ProjectID string `json:"project_id,omitempty"` + ProjectName string `json:"project_name,omitempty"` + ProjectCurrentPath string `json:"project_current_path,omitempty"` + Action string `json:"action"` + Applied bool `json:"applied"` + CopyRun bool `json:"copy_run"` + BackupPath string `json:"backup_path,omitempty"` + RollbackManifestPath string `json:"rollback_manifest_path,omitempty"` + Projects []JournalDuplicateProjectSummary `json:"projects"` + Totals JournalDuplicateCounts `json:"totals"` + Dispositions []JournalDuplicateDisposition `json:"dispositions,omitempty"` + OperatorFlags []string `json:"operator_flags,omitempty"` + Warnings []string `json:"warnings,omitempty"` + RowsRestored int `json:"rows_restored,omitempty"` +} + +// JournalDuplicateProjectSummary reports classification for one project. +type JournalDuplicateProjectSummary struct { + ProjectID string `json:"project_id"` + ProjectName string `json:"project_name,omitempty"` + ProjectCurrentPath string `json:"project_current_path,omitempty"` + Counts JournalDuplicateCounts `json:"counts"` + Classifications []JournalDuplicateRowClassify `json:"classifications,omitempty"` + Dispositions []JournalDuplicateDisposition `json:"dispositions,omitempty"` +} + +// JournalDuplicateCounts aggregates window and action counts. +type JournalDuplicateCounts struct { + June13Rows int `json:"june13_rows"` + June24Rows int `json:"june24_rows"` + Pairs int `json:"pairs"` + Retire int `json:"retire"` + Unproven int `json:"unproven"` + NamedDispositions int `json:"named_dispositions,omitempty"` + OperatorRetire int `json:"operator_retire,omitempty"` + EntriesRetired int `json:"entries_retired,omitempty"` +} + +// JournalDuplicateRowClassify is one journal entry's classification under the +// two-window natural-key pairing rules. +type JournalDuplicateRowClassify struct { + ProjectID string `json:"project_id"` + EntryID string `json:"entry_id"` + EntryType string `json:"entry_type"` + Scope string `json:"scope,omitempty"` + Message string `json:"message,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + Window string `json:"window,omitempty"` // "june13" or "june24" + Proof string `json:"proof"` + TwinID string `json:"twin_id,omitempty"` + Disposition string `json:"disposition,omitempty"` +} + +// JournalDuplicateDisposition is a planned action against a specific journal row. +type JournalDuplicateDisposition struct { + ProjectID string `json:"project_id"` + EntryID string `json:"entry_id"` + Action string `json:"action"` + Proof string `json:"proof,omitempty"` + TwinID string `json:"twin_id,omitempty"` + Flag string `json:"flag,omitempty"` + Note string `json:"note,omitempty"` +} + +// JournalDuplicateApplyOptions carries explicit per-row operator dispositions. +// --realias is not supported for this migration (journal rows carry no aliases). +type JournalDuplicateApplyOptions struct { + Retire []string // journal entry IDs to force-retire when classified unproven + Flags []string // verbatim flag strings for the manifest +} + +// JournalDuplicateRollbackManifest preserves every deleted/changed row for rollback. +type JournalDuplicateRollbackManifest struct { + ContractVersion int `json:"contract_version"` + Migration string `json:"migration"` + CreatedAt string `json:"created_at"` + DatabaseScope string `json:"database_scope"` + DatabasePath string `json:"database_path"` + OperatorFlags []string `json:"operator_flags,omitempty"` + OperatorDispositions []JournalDuplicateDisposition `json:"operator_dispositions,omitempty"` + Retirements []JournalDuplicateDisposition `json:"retirements,omitempty"` + DeletedRows []JournalDuplicateDeletedRow `json:"deleted_rows"` + Unlinks []JournalDuplicateUnlink `json:"unlinks,omitempty"` + Counts JournalDuplicateCounts `json:"counts"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// JournalDuplicateDeletedRow is one full row snapshot for rollback restore. +type JournalDuplicateDeletedRow struct { + Table string `json:"table"` + Columns []string `json:"columns"` + Values []any `json:"values"` + Order int `json:"order"` + Meta map[string]string `json:"meta,omitempty"` +} + +// JournalDuplicateUnlink records a soft-reference rewrite for rollback. +// NewID is empty when the row was deleted instead of repointed. +type JournalDuplicateUnlink struct { + Table string `json:"table"` + ProjectID string `json:"project_id"` + Column string `json:"column"` + KeyColumn string `json:"key_column,omitempty"` + RowID string `json:"row_id"` + PreviousID string `json:"previous_id"` + NewID string `json:"new_id,omitempty"` +} + +type journalDuplicateWindowRow struct { + id string + projectID string + entryType string + scope string + message string + createdAt string + window string +} + +// PreviewJournalDuplicateMigration classifies and simulates journal-duplicate repair on a copy. +func PreviewJournalDuplicateMigration(ctx context.Context, root project.Root, resolver PathResolver, options JournalDuplicateApplyOptions) (JournalDuplicateMigrationResult, error) { + status, err := requireJournalDuplicateMigrationStatus(root, resolver) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + source, err := OpenStoreReadOnly(status.DatabasePath) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + defer source.Close() + + tempDir, err := os.MkdirTemp("", "loaf-journal-duplicate-migration-*") + if err != nil { + return JournalDuplicateMigrationResult{}, fmt.Errorf("create journal-duplicate migration temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + copyPath := filepath.Join(tempDir, "state.sqlite") + if err := copySQLiteDatabase(ctx, source, copyPath, 0o600); err != nil { + return JournalDuplicateMigrationResult{}, err + } + copyStore, err := OpenStore(copyPath) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + defer copyStore.Close() + + result, manifest, err := planJournalDuplicateMigration(ctx, copyStore, journalDuplicateMigrationBaseResult(status, JournalDuplicateMigrationActionDryRun), options, journalDuplicateExecutedDispositions{}) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + result.OperatorFlags = append([]string{}, options.Flags...) + if err := applyJournalDuplicateMigrationManifest(ctx, copyStore, &manifest, nil); err != nil { + return JournalDuplicateMigrationResult{}, fmt.Errorf("simulate journal-duplicate migration: %w", err) + } + result.CopyRun = true + return result, nil +} + +// ApplyJournalDuplicateMigration backs up, writes a rollback manifest, and retires June-13 journal twins. +func ApplyJournalDuplicateMigration(ctx context.Context, root project.Root, resolver PathResolver, options JournalDuplicateApplyOptions) (JournalDuplicateMigrationResult, error) { + status, err := requireJournalDuplicateMigrationStatus(root, resolver) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + backup, err := Backup(ctx, root, resolver) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + store, err := openInitializedStore(root, resolver) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + defer store.Close() + + executed, err := readJournalDuplicateExecutedDispositions(filepath.Dir(backup.BackupPath)) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + result, manifest, err := planJournalDuplicateMigration(ctx, store, journalDuplicateMigrationBaseResult(status, JournalDuplicateMigrationActionApply), options, executed) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + result.BackupPath = backup.BackupPath + result.OperatorFlags = append([]string{}, options.Flags...) + if len(result.Warnings) > 0 { + return result, fmt.Errorf("journal-duplicate dispositions matched no rows: %s", strings.Join(result.Warnings, "; ")) + } + result.Applied = true + + manifestPath := "" + if err := applyJournalDuplicateMigrationManifest(ctx, store, &manifest, func(final JournalDuplicateRollbackManifest) error { + if !journalDuplicateManifestHasWork(final) { + return nil + } + path, err := writeJournalDuplicateRollbackManifest(final, filepath.Dir(backup.BackupPath), time.Now().UTC()) + if err != nil { + return err + } + manifestPath = path + return nil + }); err != nil { + if manifestPath != "" { + os.Remove(manifestPath) + } + result.Applied = false + return result, err + } + result.RollbackManifestPath = manifestPath + result.Totals.EntriesRetired = manifest.Counts.EntriesRetired + + verify, _, err := planJournalDuplicateMigration(ctx, store, journalDuplicateMigrationBaseResult(status, JournalDuplicateMigrationActionApply), JournalDuplicateApplyOptions{}, executed) + if err != nil { + return result, fmt.Errorf("post-apply verification: %w (backup %s, rollback manifest %s)", err, result.BackupPath, result.RollbackManifestPath) + } + if verify.Totals.Retire > 0 { + return result, fmt.Errorf("post-apply verification failed: %d retire-class journal duplicates remain (backup %s, rollback manifest %s)", verify.Totals.Retire, result.BackupPath, result.RollbackManifestPath) + } + return result, nil +} + +// RollbackJournalDuplicateMigration restores rows recorded in a journal-duplicate rollback manifest. +func RollbackJournalDuplicateMigration(ctx context.Context, root project.Root, resolver PathResolver, manifestPath string) (JournalDuplicateMigrationResult, error) { + if manifestPath == "" { + return JournalDuplicateMigrationResult{}, fmt.Errorf("journal-duplicate rollback requires a manifest path") + } + status, err := requireJournalDuplicateMigrationStatus(root, resolver) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + manifest, err := readJournalDuplicateRollbackManifest(manifestPath) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + backup, err := Backup(ctx, root, resolver) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + store, err := openInitializedStore(root, resolver) + if err != nil { + return JournalDuplicateMigrationResult{}, err + } + defer store.Close() + + result := journalDuplicateMigrationBaseResult(status, JournalDuplicateMigrationActionRollback) + result.Applied = true + result.BackupPath = backup.BackupPath + result.RollbackManifestPath = manifestPath + result.OperatorFlags = append([]string{}, manifest.OperatorFlags...) + if err := rollbackJournalDuplicateMigrationManifest(ctx, store, manifest, &result); err != nil { + return JournalDuplicateMigrationResult{}, err + } + return result, nil +} + +func journalDuplicateMigrationBaseResult(status Status, action string) JournalDuplicateMigrationResult { + return JournalDuplicateMigrationResult{ + ContractVersion: StateJSONContractVersion, + DatabaseScope: "global", + DatabasePath: status.DatabasePath, + ProjectID: status.ProjectID, + ProjectName: status.ProjectName, + ProjectCurrentPath: status.ProjectCurrentPath, + Action: action, + Projects: []JournalDuplicateProjectSummary{}, + } +} + +func requireJournalDuplicateMigrationStatus(root project.Root, resolver PathResolver) (Status, error) { + status, err := Inspect(root, resolver) + if err != nil { + return Status{}, err + } + switch status.Mode { + case ModeSQLiteReady: + return status, nil + case ModeMarkdownOnly: + return Status{}, fmt.Errorf("SQLite state database is not initialized; run `loaf state migrate markdown --apply` first") + case ModeInvalid: + return Status{}, fmt.Errorf("state database is invalid; run `loaf state doctor`") + default: + return Status{}, fmt.Errorf("state database is not ready; run `loaf state status`") + } +} + +func planJournalDuplicateMigration(ctx context.Context, store *Store, result JournalDuplicateMigrationResult, options JournalDuplicateApplyOptions, executed journalDuplicateExecutedDispositions) (JournalDuplicateMigrationResult, JournalDuplicateRollbackManifest, error) { + manifest := JournalDuplicateRollbackManifest{ + ContractVersion: StateJSONContractVersion, + Migration: journalDuplicateMigrationName, + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + DatabaseScope: result.DatabaseScope, + DatabasePath: result.DatabasePath, + OperatorFlags: append([]string{}, options.Flags...), + DeletedRows: []JournalDuplicateDeletedRow{}, + Metadata: map[string]string{ + "june13_window_start": june13OriginalImportWindowStart, + "june13_window_end": june13OriginalImportWindowEnd, + "june24_window_start": june24ReimportWindowStart, + "june24_window_end": june24ReimportWindowEnd, + }, + } + + retireSet := journalDuplicateOperatorRetireSet(options) + for _, id := range sortedKeys(retireSet) { + manifest.OperatorDispositions = append(manifest.OperatorDispositions, JournalDuplicateDisposition{ + EntryID: id, + Action: journalDuplicateDispositionRetire, + Flag: "--retire " + id, + }) + } + + projects, err := store.ListProjects(ctx) + if err != nil { + return result, manifest, err + } + + matched := map[string]struct{}{} + for _, project := range projects.Projects { + summary, err := classifyJournalDuplicatesForProject(ctx, store.db, project, retireSet) + if err != nil { + return result, manifest, err + } + result.Projects = append(result.Projects, summary) + result.Totals.June13Rows += summary.Counts.June13Rows + result.Totals.June24Rows += summary.Counts.June24Rows + result.Totals.Pairs += summary.Counts.Pairs + result.Totals.Retire += summary.Counts.Retire + result.Totals.Unproven += summary.Counts.Unproven + result.Totals.NamedDispositions += summary.Counts.NamedDispositions + result.Totals.OperatorRetire += summary.Counts.OperatorRetire + result.Dispositions = append(result.Dispositions, summary.Dispositions...) + for _, c := range summary.Classifications { + matched[c.EntryID] = struct{}{} + } + } + warnings, err := journalDuplicateUnmatchedDispositionWarnings(ctx, store.db, retireSet, matched, executed) + if err != nil { + return result, manifest, err + } + result.Warnings = append(result.Warnings, warnings...) + + manifest.Counts = JournalDuplicateCounts{ + June13Rows: result.Totals.June13Rows, + June24Rows: result.Totals.June24Rows, + Pairs: result.Totals.Pairs, + Retire: result.Totals.Retire, + Unproven: result.Totals.Unproven, + NamedDispositions: result.Totals.NamedDispositions, + OperatorRetire: result.Totals.OperatorRetire, + } + return result, manifest, nil +} + +func journalDuplicateOperatorRetireSet(options JournalDuplicateApplyOptions) map[string]struct{} { + retireSet := map[string]struct{}{} + for _, id := range options.Retire { + if id = strings.TrimSpace(id); id != "" { + retireSet[id] = struct{}{} + } + } + return retireSet +} + +func journalDuplicateUnmatchedDispositionWarnings(ctx context.Context, q interface { + QueryRowContext(context.Context, string, ...any) *sql.Row +}, retireSet map[string]struct{}, matched map[string]struct{}, executed journalDuplicateExecutedDispositions) ([]string, error) { + var warnings []string + for _, id := range sortedKeys(retireSet) { + if _, ok := matched[id]; ok { + continue + } + done, err := executed.retirementCarriedOut(ctx, q, id) + if err != nil { + return nil, err + } + if done { + continue + } + warnings = append(warnings, fmt.Sprintf("--retire %s matched no journal-duplicate row", id)) + } + return warnings, nil +} + +type journalDuplicateExecutedDispositions struct { + retired map[string]struct{} +} + +func readJournalDuplicateExecutedDispositions(dir string) (journalDuplicateExecutedDispositions, error) { + executed := journalDuplicateExecutedDispositions{retired: map[string]struct{}{}} + if dir == "" { + return executed, nil + } + paths, err := filepath.Glob(filepath.Join(dir, "journal-duplicate-rollback-*.json")) + if err != nil { + return executed, fmt.Errorf("list journal-duplicate rollback manifests: %w", err) + } + for _, path := range paths { + payload, err := os.ReadFile(path) + if err != nil { + return executed, fmt.Errorf("read journal-duplicate rollback manifest %s: %w", path, err) + } + var manifest struct { + Retirements []JournalDuplicateDisposition `json:"retirements"` + } + if err := json.Unmarshal(payload, &manifest); err != nil { + continue + } + for _, retirement := range manifest.Retirements { + executed.retired[retirement.EntryID] = struct{}{} + } + } + return executed, nil +} + +func (e journalDuplicateExecutedDispositions) retirementCarriedOut(ctx context.Context, q interface { + QueryRowContext(context.Context, string, ...any) *sql.Row +}, entryID string) (bool, error) { + if _, ok := e.retired[entryID]; !ok { + return false, nil + } + var found int + if err := q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM journal_entries WHERE id = ?)`, entryID).Scan(&found); err != nil { + return false, fmt.Errorf("look for journal entry %s: %w", entryID, err) + } + return found == 0, nil +} + +func classifyJournalDuplicatesForProject(ctx context.Context, q interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +}, project ProjectIdentity, retireSet map[string]struct{}) (JournalDuplicateProjectSummary, error) { + summary := JournalDuplicateProjectSummary{ + ProjectID: project.ID, + ProjectName: project.FriendlyName, + ProjectCurrentPath: project.CurrentPath, + Classifications: []JournalDuplicateRowClassify{}, + Dispositions: []JournalDuplicateDisposition{}, + } + + rows, err := q.QueryContext(ctx, ` +SELECT id, project_id, entry_type, COALESCE(scope, ''), message, created_at +FROM journal_entries +WHERE project_id = ? +ORDER BY created_at, id +`, project.ID) + if err != nil { + return summary, fmt.Errorf("list journal entries for project %s: %w", project.ID, err) + } + defer rows.Close() + + june13ByKey := map[string][]journalDuplicateWindowRow{} + june24ByKey := map[string][]journalDuplicateWindowRow{} + for rows.Next() { + var row journalDuplicateWindowRow + if err := rows.Scan(&row.id, &row.projectID, &row.entryType, &row.scope, &row.message, &row.createdAt); err != nil { + return summary, fmt.Errorf("scan journal entry: %w", err) + } + switch { + case inTimestampWindow(row.createdAt, june13OriginalImportWindowStart, june13OriginalImportWindowEnd): + row.window = "june13" + summary.Counts.June13Rows++ + key := journalDuplicateNaturalKey(row.entryType, row.scope, row.message) + june13ByKey[key] = append(june13ByKey[key], row) + case inTimestampWindow(row.createdAt, june24ReimportWindowStart, june24ReimportWindowEnd): + row.window = "june24" + summary.Counts.June24Rows++ + key := journalDuplicateNaturalKey(row.entryType, row.scope, row.message) + june24ByKey[key] = append(june24ByKey[key], row) + } + } + if err := rows.Err(); err != nil { + return summary, fmt.Errorf("list journal entries for project %s: %w", project.ID, err) + } + + keys := map[string]struct{}{} + for key := range june13ByKey { + if _, ok := june24ByKey[key]; ok { + keys[key] = struct{}{} + } + } + for key := range keys { + left := june13ByKey[key] + right := june24ByKey[key] + if len(left) == 1 && len(right) == 1 { + summary.Counts.Pairs++ + retire := left[0] + twin := right[0] + classify := JournalDuplicateRowClassify{ + ProjectID: project.ID, + EntryID: retire.id, + EntryType: retire.entryType, + Scope: retire.scope, + Message: retire.message, + CreatedAt: retire.createdAt, + Window: retire.window, + Proof: journalDuplicateProofPair, + TwinID: twin.id, + Disposition: journalDuplicateDispositionRetire, + } + summary.Classifications = append(summary.Classifications, classify) + summary.Counts.Retire++ + disp := JournalDuplicateDisposition{ + ProjectID: project.ID, + EntryID: retire.id, + Action: journalDuplicateDispositionRetire, + Proof: journalDuplicateProofPair, + TwinID: twin.id, + } + summary.Dispositions = append(summary.Dispositions, disp) + continue + } + // Multi-candidate on either side: refuse by default; --retire may force specific rows. + ambiguous := append(append([]journalDuplicateWindowRow{}, left...), right...) + sort.Slice(ambiguous, func(i, j int) bool { + if ambiguous[i].createdAt != ambiguous[j].createdAt { + return ambiguous[i].createdAt < ambiguous[j].createdAt + } + return ambiguous[i].id < ambiguous[j].id + }) + for _, row := range ambiguous { + classify := JournalDuplicateRowClassify{ + ProjectID: project.ID, + EntryID: row.id, + EntryType: row.entryType, + Scope: row.scope, + Message: row.message, + CreatedAt: row.createdAt, + Window: row.window, + Proof: journalDuplicateProofUnproven, + } + summary.Counts.Unproven++ + if _, force := retireSet[row.id]; force { + classify.Disposition = journalDuplicateDispositionRetire + summary.Counts.Retire++ + summary.Counts.OperatorRetire++ + summary.Counts.NamedDispositions++ + summary.Dispositions = append(summary.Dispositions, JournalDuplicateDisposition{ + ProjectID: project.ID, + EntryID: row.id, + Action: journalDuplicateDispositionRetire, + Proof: journalDuplicateProofUnproven, + Flag: "--retire " + row.id, + Note: "operator force-retire of unproven multi-candidate match", + }) + } + summary.Classifications = append(summary.Classifications, classify) + } + } + + sort.Slice(summary.Classifications, func(i, j int) bool { + return summary.Classifications[i].EntryID < summary.Classifications[j].EntryID + }) + sort.Slice(summary.Dispositions, func(i, j int) bool { + return summary.Dispositions[i].EntryID < summary.Dispositions[j].EntryID + }) + return summary, nil +} + +func journalDuplicateNaturalKey(entryType, scope, message string) string { + return entryType + "\x00" + scope + "\x00" + message +} + +func journalDuplicateManifestHasWork(manifest JournalDuplicateRollbackManifest) bool { + return len(manifest.DeletedRows) > 0 || len(manifest.Unlinks) > 0 || len(manifest.Retirements) > 0 +} + +func applyJournalDuplicateMigrationManifest(ctx context.Context, store *Store, manifest *JournalDuplicateRollbackManifest, beforeCommit func(JournalDuplicateRollbackManifest) error) error { + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin journal-duplicate migration: %w", err) + } + defer tx.Rollback() + + if _, err := tx.ExecContext(ctx, `PRAGMA defer_foreign_keys = ON`); err != nil { + return fmt.Errorf("defer foreign keys: %w", err) + } + + order := 0 + projects, err := listProjectsTx(ctx, tx, store.path) + if err != nil { + return err + } + + retireSet := map[string]struct{}{} + for _, d := range manifest.OperatorDispositions { + if d.Action == journalDuplicateDispositionRetire { + retireSet[d.EntryID] = struct{}{} + } + } + + // Fixed-point: retiring one multi-candidate row can leave a clean 1:1 pair + // that classification would auto-retire on a subsequent pass. Keep going + // until a pass plans no further retirements (operator --retire is only + // needed on the first pass; later passes use empty dispositions). + for pass := 0; pass < 32; pass++ { + passRetire := retireSet + if pass > 0 { + passRetire = map[string]struct{}{} + } + retiredThisPass := 0 + for _, project := range projects { + summary, err := classifyJournalDuplicatesForProject(ctx, tx, project, passRetire) + if err != nil { + return err + } + for _, c := range summary.Classifications { + if c.Disposition != journalDuplicateDispositionRetire { + continue + } + if err := retireJournalDuplicateTx(ctx, tx, project.ID, c, manifest, &order); err != nil { + return err + } + manifest.Retirements = append(manifest.Retirements, JournalDuplicateDisposition{ + ProjectID: project.ID, + EntryID: c.EntryID, + Action: journalDuplicateDispositionRetire, + Proof: c.Proof, + TwinID: c.TwinID, + }) + manifest.Counts.EntriesRetired++ + retiredThisPass++ + } + } + if retiredThisPass == 0 { + break + } + } + + if beforeCommit != nil { + if err := beforeCommit(*manifest); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit journal-duplicate migration: %w", err) + } + return nil +} + +func retireJournalDuplicateTx(ctx context.Context, tx *sql.Tx, projectID string, classify JournalDuplicateRowClassify, manifest *JournalDuplicateRollbackManifest, order *int) error { + entryID := classify.EntryID + twinID := classify.TwinID + + // FTS: targeted delete keeps the index consistent without a full rebuild. + if err := captureAndDeleteJournalDuplicateTx(ctx, tx, "journal_search", `WHERE journal_entry_id = ?`, []any{entryID}, manifest, order); err != nil { + return err + } + if err := captureAndDeleteJournalDuplicateTx(ctx, tx, "journal_origins", `WHERE journal_entry_id = ?`, []any{entryID}, manifest, order); err != nil { + return err + } + + // Soft refs that cannot be NULLed (NOT NULL columns / CHECK constraints). + // Repoint to the surviving June-24 twin when possible; otherwise delete. + if err := repointOrDeleteJournalSoftRefTx(ctx, tx, projectID, journalDuplicateSoftRef{ + table: "journal_deferrals", column: "journal_entry_id", keyColumn: "operation_key", uniqueColumn: true, + }, entryID, twinID, manifest, order); err != nil { + return err + } + if err := repointOrDeleteJournalSoftRefTx(ctx, tx, projectID, journalDuplicateSoftRef{ + table: "intent_operations", column: "journal_entry_id", keyColumn: "operation_key", uniqueColumn: false, + }, entryID, twinID, manifest, order); err != nil { + return err + } + if err := repointOrDeleteJournalConversationHandlesTx(ctx, tx, projectID, entryID, twinID, manifest, order); err != nil { + return err + } + + const journalEntityKind = "journal_entry" + if err := captureJournalDuplicateRowsTx(ctx, tx, "artifact_bodies", ` +SELECT * FROM artifact_bodies WHERE project_id = ? AND entity_kind = ? AND entity_id = ? +`, []any{projectID, journalEntityKind, entryID}, manifest, order, nil); err != nil { + return err + } + if _, _, err := deleteArtifactBodiesForEntityTx(ctx, tx, projectID, journalEntityKind, entryID); err != nil { + return err + } + for _, op := range polymorphicEntityReferenceSweeps(projectID, journalEntityKind, entryID) { + if err := captureAndDeleteJournalDuplicateTx(ctx, tx, op.table, op.where, op.args, manifest, order); err != nil { + return err + } + } + if err := captureAndDeleteJournalDuplicateTx(ctx, tx, "aliases", `WHERE project_id = ? AND entity_kind = ? AND entity_id = ?`, []any{projectID, journalEntityKind, entryID}, manifest, order); err != nil { + return err + } + + if err := captureAndDeleteJournalDuplicateTx(ctx, tx, "journal_entries", `WHERE project_id = ? AND id = ?`, []any{projectID, entryID}, manifest, order); err != nil { + return err + } + return nil +} + +type journalDuplicateSoftRef struct { + table string + column string + keyColumn string + uniqueColumn bool +} + +func repointOrDeleteJournalSoftRefTx(ctx context.Context, tx *sql.Tx, projectID string, ref journalDuplicateSoftRef, entryID, twinID string, manifest *JournalDuplicateRollbackManifest, order *int) error { + exists, err := sqliteTableExistsQ(ctx, tx, ref.table) + if err != nil { + return err + } + if !exists { + return nil + } + quotedTable := quoteSQLiteIdentifier(ref.table) + quotedColumn := quoteSQLiteIdentifier(ref.column) + quotedKey := quoteSQLiteIdentifier(ref.keyColumn) + + rows, err := tx.QueryContext(ctx, fmt.Sprintf(`SELECT %s FROM %s WHERE project_id = ? AND %s = ? ORDER BY %s`, quotedKey, quotedTable, quotedColumn, quotedKey), projectID, entryID) + if err != nil { + return fmt.Errorf("list %s.%s references: %w", ref.table, ref.column, err) + } + var keys []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + rows.Close() + return fmt.Errorf("scan %s.%s reference: %w", ref.table, ref.column, err) + } + keys = append(keys, key) + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("list %s.%s references: %w", ref.table, ref.column, err) + } + rows.Close() + + for _, key := range keys { + repoint := twinID != "" + if repoint && ref.uniqueColumn { + var taken int + if err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT EXISTS(SELECT 1 FROM %s WHERE %s = ?)`, quotedTable, quotedColumn), twinID).Scan(&taken); err != nil { + return fmt.Errorf("check %s.%s availability: %w", ref.table, ref.column, err) + } + repoint = taken == 0 + } + if repoint { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET %s = ? WHERE project_id = ? AND %s = ?`, quotedTable, quotedColumn, quotedKey), twinID, projectID, key); err != nil { + return fmt.Errorf("repoint %s.%s: %w", ref.table, ref.column, err) + } + manifest.Unlinks = append(manifest.Unlinks, JournalDuplicateUnlink{ + Table: ref.table, + ProjectID: projectID, + Column: ref.column, + KeyColumn: ref.keyColumn, + RowID: key, + PreviousID: entryID, + NewID: twinID, + }) + continue + } + if err := captureAndDeleteJournalDuplicateTx(ctx, tx, ref.table, fmt.Sprintf(`WHERE project_id = ? AND %s = ?`, quotedKey), []any{projectID, key}, manifest, order); err != nil { + return err + } + } + return nil +} + +func repointOrDeleteJournalConversationHandlesTx(ctx context.Context, tx *sql.Tx, projectID, entryID, twinID string, manifest *JournalDuplicateRollbackManifest, order *int) error { + exists, err := sqliteTableExistsQ(ctx, tx, "journal_conversation_handles") + if err != nil { + return err + } + if !exists { + return nil + } + rows, err := tx.QueryContext(ctx, ` +SELECT id, handle_id FROM journal_conversation_handles +WHERE project_id = ? AND journal_entry_id = ? +ORDER BY id +`, projectID, entryID) + if err != nil { + return fmt.Errorf("list journal_conversation_handles references: %w", err) + } + type handleRef struct { + id string + handleID string + } + var refs []handleRef + for rows.Next() { + var ref handleRef + if err := rows.Scan(&ref.id, &ref.handleID); err != nil { + rows.Close() + return fmt.Errorf("scan journal_conversation_handles reference: %w", err) + } + refs = append(refs, ref) + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("list journal_conversation_handles references: %w", err) + } + rows.Close() + + for _, ref := range refs { + repoint := twinID != "" + if repoint { + var taken int + if err := tx.QueryRowContext(ctx, ` +SELECT EXISTS(SELECT 1 FROM journal_conversation_handles WHERE journal_entry_id = ? AND handle_id = ?) +`, twinID, ref.handleID).Scan(&taken); err != nil { + return fmt.Errorf("check journal_conversation_handles availability: %w", err) + } + repoint = taken == 0 + } + if repoint { + if _, err := tx.ExecContext(ctx, ` +UPDATE journal_conversation_handles SET journal_entry_id = ? WHERE project_id = ? AND id = ? +`, twinID, projectID, ref.id); err != nil { + return fmt.Errorf("repoint journal_conversation_handles: %w", err) + } + manifest.Unlinks = append(manifest.Unlinks, JournalDuplicateUnlink{ + Table: "journal_conversation_handles", + ProjectID: projectID, + Column: "journal_entry_id", + KeyColumn: "id", + RowID: ref.id, + PreviousID: entryID, + NewID: twinID, + }) + continue + } + if err := captureAndDeleteJournalDuplicateTx(ctx, tx, "journal_conversation_handles", `WHERE project_id = ? AND id = ?`, []any{projectID, ref.id}, manifest, order); err != nil { + return err + } + } + return nil +} + +func captureAndDeleteJournalDuplicateTx(ctx context.Context, tx *sql.Tx, table string, where string, args []any, manifest *JournalDuplicateRollbackManifest, order *int) error { + exists, err := sqliteTableExistsQ(ctx, tx, table) + if err != nil { + return err + } + if !exists { + return nil + } + quoted := quoteSQLiteIdentifier(table) + if err := captureJournalDuplicateRowsTx(ctx, tx, table, fmt.Sprintf(`SELECT * FROM %s %s`, quoted, where), args, manifest, order, nil); err != nil { + return err + } + if _, err := execCountTx(ctx, tx, fmt.Sprintf(`DELETE FROM %s %s`, quoted, where), args...); err != nil { + return fmt.Errorf("delete %s rows: %w", table, err) + } + return nil +} + +func captureJournalDuplicateRowsTx(ctx context.Context, tx *sql.Tx, table string, query string, args []any, manifest *JournalDuplicateRollbackManifest, order *int, meta map[string]string) error { + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("capture %s rows: %w", table, err) + } + defer rows.Close() + scanned, err := scanRows(rows) + if err != nil { + return fmt.Errorf("scan %s rows for capture: %w", table, err) + } + for _, row := range scanned { + columns := make([]string, 0, len(row)) + for col := range row { + columns = append(columns, col) + } + sort.Strings(columns) + values := make([]any, len(columns)) + for i, col := range columns { + values[i] = row[col] + } + *order++ + entry := JournalDuplicateDeletedRow{ + Table: table, + Columns: columns, + Values: values, + Order: *order, + } + if len(meta) > 0 { + entry.Meta = meta + } + manifest.DeletedRows = append(manifest.DeletedRows, entry) + } + return nil +} + +func rollbackJournalDuplicateMigrationManifest(ctx context.Context, store *Store, manifest JournalDuplicateRollbackManifest, result *JournalDuplicateMigrationResult) error { + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin journal-duplicate rollback: %w", err) + } + defer tx.Rollback() + + if _, err := tx.ExecContext(ctx, `PRAGMA defer_foreign_keys = ON`); err != nil { + return fmt.Errorf("defer foreign keys: %w", err) + } + + // Restore deleted rows in reverse capture order. + rows := append([]JournalDuplicateDeletedRow{}, manifest.DeletedRows...) + sort.SliceStable(rows, func(i, j int) bool { return rows[i].Order > rows[j].Order }) + for _, row := range rows { + if row.Table == "journal_search" { + // FTS is rebuilt from restored journal_entries below so rowids stay aligned + // with the live insert path. + continue + } + if err := insertJournalDuplicateDeletedRowTx(ctx, tx, row); err != nil { + return err + } + result.RowsRestored++ + if row.Table == "journal_entries" { + if err := restoreJournalSearchForDeletedRowTx(ctx, tx, row); err != nil { + return err + } + } + if row.Table == "artifact_bodies" { + // FTS mirror is derived data: re-derive from the restored body rather than + // capturing artifact_search bytes in the rollback manifest. + if err := restoreArtifactSearchTx(ctx, tx, + journalDuplicateRowValueString(row, "project_id"), + journalDuplicateRowValueString(row, "entity_kind"), + journalDuplicateRowValueString(row, "entity_id"), + journalDuplicateRowValueString(row, "body_kind"), + journalDuplicateRowValueString(row, "content"), + ); err != nil { + return err + } + } + } + + // Undo repoints after row restores so PreviousID targets exist. + for _, unlink := range manifest.Unlinks { + keyColumn := firstNonEmpty(unlink.KeyColumn, "id") + if _, err := tx.ExecContext(ctx, fmt.Sprintf( + `UPDATE %s SET %s = ? WHERE project_id = ? AND %s = ?`, + quoteSQLiteIdentifier(unlink.Table), + quoteSQLiteIdentifier(unlink.Column), + quoteSQLiteIdentifier(keyColumn), + ), unlink.PreviousID, unlink.ProjectID, unlink.RowID); err != nil { + return fmt.Errorf("restore unlink %s.%s: %w", unlink.Table, unlink.Column, err) + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit journal-duplicate rollback: %w", err) + } + return nil +} + +func insertJournalDuplicateDeletedRowTx(ctx context.Context, tx *sql.Tx, row JournalDuplicateDeletedRow) error { + if len(row.Columns) == 0 || len(row.Columns) != len(row.Values) { + return fmt.Errorf("deleted row for %s has mismatched columns/values", row.Table) + } + quoted := make([]string, len(row.Columns)) + placeholders := make([]string, len(row.Columns)) + args := make([]any, len(row.Values)) + for i, col := range row.Columns { + quoted[i] = quoteSQLiteIdentifier(col) + placeholders[i] = "?" + args[i] = normalizeManifestValue(row.Values[i]) + } + query := fmt.Sprintf(`INSERT OR REPLACE INTO %s (%s) VALUES (%s)`, quoteSQLiteIdentifier(row.Table), strings.Join(quoted, ", "), strings.Join(placeholders, ", ")) + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("restore row into %s: %w", row.Table, err) + } + return nil +} + +func restoreJournalSearchForDeletedRowTx(ctx context.Context, tx *sql.Tx, row JournalDuplicateDeletedRow) error { + projectID := journalDuplicateRowValueString(row, "project_id") + entryID := journalDuplicateRowValueString(row, "id") + entryType := journalDuplicateRowValueString(row, "entry_type") + scope := journalDuplicateRowValueString(row, "scope") + message := journalDuplicateRowValueString(row, "message") + harness := journalDuplicateRowValueString(row, "harness_session_id") + if projectID == "" || entryID == "" { + return nil + } + // Drop any stale FTS row then re-insert via the live write path. + if _, err := tx.ExecContext(ctx, `DELETE FROM journal_search WHERE journal_entry_id = ?`, entryID); err != nil { + return fmt.Errorf("clear journal_search before restore for %s: %w", entryID, err) + } + return insertJournalSearchTx(ctx, tx, projectID, entryID, harness, entryType, scope, message) +} + +func journalDuplicateRowValueString(row JournalDuplicateDeletedRow, column string) string { + for i, col := range row.Columns { + if col != column { + continue + } + if i >= len(row.Values) || row.Values[i] == nil { + return "" + } + switch v := row.Values[i].(type) { + case string: + return v + case []byte: + return string(v) + default: + return fmt.Sprint(v) + } + } + return "" +} + +func writeJournalDuplicateRollbackManifest(manifest JournalDuplicateRollbackManifest, dir string, now time.Time) (string, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("create journal-duplicate rollback manifest directory: %w", err) + } + for i := 0; i < 100; i++ { + suffix := "" + if i > 0 { + suffix = fmt.Sprintf("-%02d", i) + } + path := filepath.Join(dir, fmt.Sprintf("journal-duplicate-rollback-%s%s.json", now.Format("20060102T150405Z"), suffix)) + if _, err := os.Stat(path); err == nil { + continue + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("stat journal-duplicate rollback manifest: %w", err) + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return "", fmt.Errorf("encode journal-duplicate rollback manifest: %w", err) + } + payload = append(payload, '\n') + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, os.ErrExist) { + continue + } + return "", fmt.Errorf("create journal-duplicate rollback manifest: %w", err) + } + if _, err := file.Write(payload); err != nil { + _ = file.Close() + _ = os.Remove(path) + return "", fmt.Errorf("write journal-duplicate rollback manifest: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + _ = os.Remove(path) + return "", fmt.Errorf("sync journal-duplicate rollback manifest: %w", err) + } + if err := file.Close(); err != nil { + _ = os.Remove(path) + return "", fmt.Errorf("close journal-duplicate rollback manifest: %w", err) + } + if err := syncDirectory(dir); err != nil { + _ = os.Remove(path) + return "", fmt.Errorf("sync journal-duplicate rollback manifest directory: %w", err) + } + return path, nil + } + return "", fmt.Errorf("create journal-duplicate rollback manifest: exhausted timestamp suffixes") +} + +func readJournalDuplicateRollbackManifest(path string) (JournalDuplicateRollbackManifest, error) { + payload, err := os.ReadFile(path) + if err != nil { + return JournalDuplicateRollbackManifest{}, fmt.Errorf("read journal-duplicate rollback manifest: %w", err) + } + var manifest JournalDuplicateRollbackManifest + if err := json.Unmarshal(payload, &manifest); err != nil { + return JournalDuplicateRollbackManifest{}, fmt.Errorf("decode journal-duplicate rollback manifest: %w", err) + } + if manifest.Migration != "" && manifest.Migration != journalDuplicateMigrationName { + return JournalDuplicateRollbackManifest{}, fmt.Errorf("rollback manifest migration %q is not %s", manifest.Migration, journalDuplicateMigrationName) + } + return manifest, nil +} diff --git a/internal/state/journal_duplicate_migration_test.go b/internal/state/journal_duplicate_migration_test.go new file mode 100644 index 000000000..81cc71b7c --- /dev/null +++ b/internal/state/journal_duplicate_migration_test.go @@ -0,0 +1,807 @@ +package state + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + "time" + + "github.com/levifig/loaf/internal/project" +) + +func TestJournalDuplicatePairingAndAmbiguity(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID := seedJournalDuplicateFixtureBase(t) + + // Clean 1:1 pair across the two import windows. + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-pair-june13", "decision", "auth", "chose token rotation", "2026-06-13T01:39:42Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-pair-june24", "decision", "auth", "chose token rotation", "2026-06-24T13:03:15Z") + + // Ambiguous: two June-13 candidates share a key with one June-24. + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-amb-j13a", "discover", "scope", "ambiguous message", "2026-06-13T01:40:00Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-amb-j13b", "discover", "scope", "ambiguous message", "2026-06-13T01:41:00Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-amb-j24", "discover", "scope", "ambiguous message", "2026-06-24T13:03:30Z") + + // Legitimate same-day repeat outside the reimport window — not a pair. + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-legit-a", "todo", "work", "same text later", "2026-06-24T15:00:00Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-legit-b", "todo", "work", "same text later", "2026-06-24T16:00:00Z") + + // June-13 only — no twin. + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-solo-j13", "spark", "idea", "solo original", "2026-06-13T01:42:00Z") + + preview, err := PreviewJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("PreviewJournalDuplicateMigration() error = %v", err) + } + if !preview.CopyRun || preview.Applied { + t.Fatalf("preview copy_run/applied = %t/%t, want true/false", preview.CopyRun, preview.Applied) + } + if preview.Totals.Pairs != 1 { + t.Fatalf("pairs = %d, want 1", preview.Totals.Pairs) + } + if preview.Totals.Retire != 1 { + t.Fatalf("retire = %d, want 1 (the clean pair only)", preview.Totals.Retire) + } + if preview.Totals.Unproven != 3 { + t.Fatalf("unproven = %d, want 3", preview.Totals.Unproven) + } + if preview.Totals.June13Rows < 4 || preview.Totals.June24Rows < 2 { + t.Fatalf("window totals = june13=%d june24=%d, want june13>=4 june24>=2", preview.Totals.June13Rows, preview.Totals.June24Rows) + } + + byID := map[string]JournalDuplicateRowClassify{} + for _, project := range preview.Projects { + for _, c := range project.Classifications { + byID[c.EntryID] = c + } + } + if got := byID["je-pair-june13"]; got.Proof != journalDuplicateProofPair || got.Disposition != journalDuplicateDispositionRetire || got.TwinID != "je-pair-june24" { + t.Fatalf("pair classify = %#v, want pair/retire twin=je-pair-june24", got) + } + if _, ok := byID["je-pair-june24"]; ok { + t.Fatal("surviving June-24 twin must not be classified as a retire candidate") + } + for _, id := range []string{"je-amb-j13a", "je-amb-j13b", "je-amb-j24"} { + if got := byID[id]; got.Proof != journalDuplicateProofUnproven || got.Disposition != "" { + t.Fatalf("ambiguous %s classify = %#v, want unproven with empty disposition", id, got) + } + } + for _, id := range []string{"je-legit-a", "je-legit-b", "je-solo-j13"} { + if _, ok := byID[id]; ok { + t.Fatalf("%s should not be classified", id) + } + } + + // Preview must not touch the live database. + if !journalEntryExists(t, stateHome, root, "je-pair-june13") { + t.Fatal("preview deleted pair June-13 row on live DB") + } +} + +func TestJournalDuplicateApplyRollbackIdempotencyAndFTS(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID := seedJournalDuplicateFixtureBase(t) + + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-pair-june13", "decision", "auth", "chose token rotation", "2026-06-13T01:39:42Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-pair-june24", "decision", "auth", "chose token rotation", "2026-06-24T13:03:15Z") + seedJournalDuplicateOrigin(t, stateHome, root, projectID, "je-pair-june13", "2026-06-13T01:39:42Z") + seedJournalDuplicateOrigin(t, stateHome, root, projectID, "je-pair-june24", "2026-06-24T13:03:15Z") + + // Soft-ref on the June-13 copy that should repoint to the twin. + // Seed the spark first so journal-provenance integrity stays ModeSQLiteReady. + seedJournalDuplicateSpark(t, stateHome, root, projectID, "spark-pair-j13", "fixture spark") + seedJournalDuplicateDeferral(t, stateHome, root, projectID, "op-pair", "je-pair-june13", "spark-pair-j13") + + beforeCount := journalEntryCount(t, stateHome, root) + beforeSearch := journalSearchCount(t, stateHome, root) + + applied, err := ApplyJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("ApplyJournalDuplicateMigration() error = %v", err) + } + if !applied.Applied || applied.BackupPath == "" || applied.RollbackManifestPath == "" { + t.Fatalf("apply result incomplete: %#v", applied) + } + if applied.Totals.EntriesRetired != 1 { + t.Fatalf("entries_retired = %d, want 1", applied.Totals.EntriesRetired) + } + if _, err := os.Stat(applied.BackupPath); err != nil { + t.Fatalf("stat backup: %v", err) + } + if _, err := os.Stat(applied.RollbackManifestPath); err != nil { + t.Fatalf("stat manifest: %v", err) + } + if journalEntryExists(t, stateHome, root, "je-pair-june13") { + t.Fatal("June-13 pair still present after apply") + } + if !journalEntryExists(t, stateHome, root, "je-pair-june24") { + t.Fatal("June-24 twin was retired; want preserved") + } + if got := journalEntryCount(t, stateHome, root); got != beforeCount-1 { + t.Fatalf("journal_entries count = %d, want %d", got, beforeCount-1) + } + if got := journalSearchCount(t, stateHome, root); got != beforeSearch-1 { + t.Fatalf("journal_search count = %d, want %d", got, beforeSearch-1) + } + assertJournalSearchParityReady(t, stateHome, root) + + // Soft-ref repointed to survivor. + if got := journalDeferralEntryID(t, stateHome, root, projectID, "op-pair"); got != "je-pair-june24" { + t.Fatalf("deferral journal_entry_id = %q, want je-pair-june24", got) + } + + // Zero cross-window twins remain. + secondPreview, err := PreviewJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("second preview error = %v", err) + } + if secondPreview.Totals.Pairs != 0 || secondPreview.Totals.Retire != 0 { + t.Fatalf("second preview totals = %#v, want zero pairs/retire", secondPreview.Totals) + } + + // Idempotent second apply. + secondApply, err := ApplyJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("second apply error = %v", err) + } + if secondApply.Totals.EntriesRetired != 0 { + t.Fatalf("second apply entries_retired = %d, want 0", secondApply.Totals.EntriesRetired) + } + + // Rollback restores the June-13 row, its origin, FTS, and reverses the repoint. + rolled, err := RollbackJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath) + if err != nil { + t.Fatalf("RollbackJournalDuplicateMigration() error = %v", err) + } + if !rolled.Applied || rolled.RowsRestored == 0 { + t.Fatalf("rollback result = %#v, want restored rows", rolled) + } + if !journalEntryExists(t, stateHome, root, "je-pair-june13") { + t.Fatal("June-13 pair not restored after rollback") + } + if got := journalEntryCount(t, stateHome, root); got != beforeCount { + t.Fatalf("journal_entries after rollback = %d, want %d", got, beforeCount) + } + assertJournalSearchParityReady(t, stateHome, root) + if got := journalDeferralEntryID(t, stateHome, root, projectID, "op-pair"); got != "je-pair-june13" { + t.Fatalf("deferral after rollback = %q, want je-pair-june13", got) + } +} + +func TestJournalDuplicateOperatorRetireUnproven(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID := seedJournalDuplicateFixtureBase(t) + + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-amb-j13a", "discover", "scope", "ambiguous message", "2026-06-13T01:40:00Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-amb-j13b", "discover", "scope", "ambiguous message", "2026-06-13T01:41:00Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, "je-amb-j24", "discover", "scope", "ambiguous message", "2026-06-24T13:03:30Z") + + // Without disposition, nothing retires. + preview, err := PreviewJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("preview error = %v", err) + } + if preview.Totals.Retire != 0 || preview.Totals.Unproven != 3 { + t.Fatalf("preview totals = %#v, want retire=0 unproven=3", preview.Totals) + } + + applied, err := ApplyJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{ + Retire: []string{"je-amb-j13a"}, + Flags: []string{"--retire", "je-amb-j13a"}, + }) + if err != nil { + t.Fatalf("apply with --retire error = %v", err) + } + // Fixed-point: retiring one multi-candidate leaves a clean 1:1 pair, which + // the same apply pass then auto-retires (j13b). The June-24 survivor remains. + if applied.Totals.EntriesRetired != 2 { + t.Fatalf("entries_retired = %d, want 2 (operator force + follow-on pair)", applied.Totals.EntriesRetired) + } + if journalEntryExists(t, stateHome, root, "je-amb-j13a") { + t.Fatal("operator-retired unproven row still present") + } + if journalEntryExists(t, stateHome, root, "je-amb-j13b") { + t.Fatal("follow-on June-13 pair twin still present after fixed-point pass") + } + if !journalEntryExists(t, stateHome, root, "je-amb-j24") { + t.Fatal("June-24 survivor was removed") + } + assertJournalSearchParityReady(t, stateHome, root) +} + +func TestJournalDuplicateUnmatchedRetireRefused(t *testing.T) { + ctx := context.Background() + root, stateHome, _ := seedJournalDuplicateFixtureBase(t) + + _, err := ApplyJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{ + Retire: []string{"je-does-not-exist"}, + Flags: []string{"--retire", "je-does-not-exist"}, + }) + if err == nil { + t.Fatal("apply with unmatched --retire error = nil, want refusal") + } +} + +func TestJournalDuplicatePolymorphicResidueSweep(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID := seedJournalDuplicateFixtureBase(t) + + const ( + june13ID = "je-poly-june13" + june24ID = "je-poly-june24" + bodyText = "journal poly residue body needlexyz" + ) + seedJournalDuplicateEntry(t, stateHome, root, projectID, june13ID, "decision", "auth", "chose poly rotation", "2026-06-13T01:39:42Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, june24ID, "decision", "auth", "chose poly rotation", "2026-06-24T13:03:15Z") + + now := "2026-06-13T10:00:00Z" + // Both relationship directions against the June-13 row. + mustExecOpen(t, stateHome, root, ` +INSERT INTO relationships (id, project_id, from_entity_kind, from_entity_id, to_entity_kind, to_entity_id, relationship_type, reason, created_at, updated_at) +VALUES (?, ?, 'journal_entry', ?, 'spark', 'spark-poly-target', 'promoted_to', 'from-j13', ?, ?) +`, "rel-from-j13", projectID, june13ID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO relationships (id, project_id, from_entity_kind, from_entity_id, to_entity_kind, to_entity_id, relationship_type, reason, created_at, updated_at) +VALUES (?, ?, 'spark', 'spark-poly-source', 'journal_entry', ?, 'sourced_from', 'to-j13', ?, ?) +`, "rel-to-j13", projectID, june13ID, now, now) + // Twin residue that must survive the sweep. + mustExecOpen(t, stateHome, root, ` +INSERT INTO relationships (id, project_id, from_entity_kind, from_entity_id, to_entity_kind, to_entity_id, relationship_type, reason, created_at, updated_at) +VALUES (?, ?, 'journal_entry', ?, 'spark', 'spark-poly-twin', 'promoted_to', 'from-j24', ?, ?) +`, "rel-from-j24", projectID, june24ID, now, now) + + mustExecOpen(t, stateHome, root, ` +INSERT INTO tags (id, project_id, name, created_at, updated_at) VALUES (?, ?, 'journal-poly-tag', ?, ?) +`, "tag-journal-poly", projectID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO entity_tags (id, project_id, tag_id, entity_kind, entity_id, created_at, updated_at) +VALUES (?, ?, ?, 'journal_entry', ?, ?, ?) +`, "etag-journal-poly", projectID, "tag-journal-poly", june13ID, now, now) + + mustExecOpen(t, stateHome, root, ` +INSERT INTO events (id, project_id, entity_kind, entity_id, event_type, from_status, to_status, note, created_at, updated_at) +VALUES (?, ?, 'journal_entry', ?, 'noted', NULL, NULL, 'j13 event', ?, ?) +`, "event-j13", projectID, june13ID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO events (id, project_id, entity_kind, entity_id, event_type, from_status, to_status, note, created_at, updated_at) +VALUES (?, ?, 'journal_entry', ?, 'noted', NULL, NULL, 'j24 event', ?, ?) +`, "event-j24", projectID, june24ID, now, now) + + store := openTestStore(t, root, stateHome) + if _, err := store.UpsertArtifactBody(ctx, projectID, "journal_entry", june13ID, ArtifactBodyKindMarkdown, bodyText, ""); err != nil { + store.Close() + t.Fatalf("UpsertArtifactBody for june13: %v", err) + } + store.Close() + + mustExecOpen(t, stateHome, root, ` +INSERT INTO aliases (id, project_id, entity_kind, entity_id, namespace, alias, created_at, updated_at) +VALUES (?, ?, 'journal_entry', ?, 'journal_entry', 'poly-j13-alias', ?, ?) +`, "alias-j13", projectID, june13ID, now, now) + + mustExecOpen(t, stateHome, root, ` +INSERT INTO bundles (id, project_id, slug, title, created_at, updated_at) +VALUES (?, ?, 'journal-poly-bundle', 'Journal poly bundle', ?, ?) +`, "bundle-journal-poly", projectID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO bundle_members (id, project_id, bundle_id, entity_kind, entity_id, created_at, updated_at) +VALUES (?, ?, ?, 'journal_entry', ?, ?, ?) +`, "bm-journal-poly", projectID, "bundle-journal-poly", june13ID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO backend_mappings (id, project_id, backend, entity_kind, entity_id, external_kind, external_id, sync_status, created_at, updated_at) +VALUES (?, ?, 'linear', 'journal_entry', ?, 'issue', 'EXT-J13', 'synced', ?, ?) +`, "bmap-journal-poly", projectID, june13ID, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO exports (id, project_id, export_kind, format, path, source_entity_kind, source_entity_id, generated_at, created_at, updated_at) +VALUES (?, ?, 'render', 'markdown', 'out-j13.md', 'journal_entry', ?, ?, ?, ?) +`, "export-j13", projectID, june13ID, now, now, now) + mustExecOpen(t, stateHome, root, ` +INSERT INTO exports (id, project_id, export_kind, format, path, source_entity_kind, source_entity_id, generated_at, created_at, updated_at) +VALUES (?, ?, 'render', 'markdown', 'out-j24.md', 'journal_entry', ?, ?, ?, ?) +`, "export-j24", projectID, june24ID, now, now, now) + + beforeJ13 := snapshotJournalPolymorphicResidue(t, stateHome, root, projectID, june13ID) + if len(beforeJ13) == 0 { + t.Fatal("expected polymorphic residue on June-13 row before apply") + } + beforeJ24 := snapshotJournalPolymorphicResidue(t, stateHome, root, projectID, june24ID) + if len(beforeJ24) == 0 { + t.Fatal("expected twin residue on June-24 row before apply") + } + beforeBodies := snapshotArtifactBodiesNullable(t, stateHome, root, projectID, "journal_entry", june13ID) + if len(beforeBodies) == 0 { + t.Fatal("expected artifact_bodies row for June-13 before apply") + } + beforeSearch := snapshotArtifactSearchIndex(t, stateHome, root, "needlexyz") + if len(beforeSearch) == 0 { + t.Fatal("expected artifact_search index membership for needlexyz before apply") + } + + applied, err := ApplyJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("ApplyJournalDuplicateMigration() error = %v", err) + } + if applied.Totals.EntriesRetired != 1 { + t.Fatalf("entries_retired = %d, want 1", applied.Totals.EntriesRetired) + } + assertJournalSearchParityReady(t, stateHome, root) + + if got := countJournalPolymorphicResidue(t, stateHome, root, projectID, june13ID); got != 0 { + t.Fatalf("polymorphic residue citing retired id after apply = %d, want 0", got) + } + afterJ24 := snapshotJournalPolymorphicResidue(t, stateHome, root, projectID, june24ID) + if !equalRowSnapshots(beforeJ24, afterJ24) { + t.Fatalf("twin polymorphic residue changed after apply\nbefore=%v\nafter=%v", beforeJ24, afterJ24) + } + + manifest, err := readJournalDuplicateRollbackManifest(applied.RollbackManifestPath) + if err != nil { + t.Fatalf("read rollback manifest: %v", err) + } + for id, row := range beforeJ13 { + found := false + for _, deleted := range manifest.DeletedRows { + if deleted.Table != row.table { + continue + } + if journalDuplicateRowValueString(deleted, "id") == id { + found = true + break + } + } + if !found { + t.Fatalf("manifest missing deleted_rows entry for %s id=%s", row.table, id) + } + } + + // FTS body gone with the entity. + if n := artifactSearchMatchCount(t, stateHome, root, "needlexyz"); n != 0 { + t.Fatalf("artifact_search hits after apply = %d, want 0", n) + } + + secondApply, err := ApplyJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("second apply error = %v", err) + } + if secondApply.Totals.EntriesRetired != 0 { + t.Fatalf("second apply entries_retired = %d, want 0", secondApply.Totals.EntriesRetired) + } + + rolled, err := RollbackJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath) + if err != nil { + t.Fatalf("RollbackJournalDuplicateMigration() error = %v", err) + } + if !rolled.Applied { + t.Fatalf("rollback result = %#v, want applied", rolled) + } + assertJournalSearchParityReady(t, stateHome, root) + + afterRollbackJ13 := snapshotJournalPolymorphicResidue(t, stateHome, root, projectID, june13ID) + if !equalRowSnapshots(beforeJ13, afterRollbackJ13) { + t.Fatalf("June-13 polymorphic residue not restored byte-identically\nbefore=%v\nafter=%v", beforeJ13, afterRollbackJ13) + } + afterRollbackJ24 := snapshotJournalPolymorphicResidue(t, stateHome, root, projectID, june24ID) + if !equalRowSnapshots(beforeJ24, afterRollbackJ24) { + t.Fatalf("June-24 twin residue changed after rollback\nbefore=%v\nafter=%v", beforeJ24, afterRollbackJ24) + } + afterBodies := snapshotArtifactBodiesNullable(t, stateHome, root, projectID, "journal_entry", june13ID) + if !equalArtifactBodySnaps(beforeBodies, afterBodies) { + t.Fatalf("artifact_bodies not restored byte-identically\nbefore=%v\nafter=%v", beforeBodies, afterBodies) + } + afterSearch := snapshotArtifactSearchIndex(t, stateHome, root, "needlexyz") + if !reflect.DeepEqual(beforeSearch, afterSearch) { + t.Fatalf("artifact_search index state not restored\nbefore=%v\nafter=%v", beforeSearch, afterSearch) + } + if n := artifactSearchMatchCount(t, stateHome, root, "needlexyz"); n != 1 { + t.Fatalf("artifact_search hits after rollback = %d, want 1", n) + } +} + +func TestJournalDuplicateApplyToleratesMissingArtifactSearchMirror(t *testing.T) { + ctx := context.Background() + root, stateHome, projectID := seedJournalDuplicateFixtureBase(t) + + const ( + june13ID = "je-desync-june13" + june24ID = "je-desync-june24" + bodyText = "journal desync body needledesync" + ) + seedJournalDuplicateEntry(t, stateHome, root, projectID, june13ID, "decision", "auth", "chose desync rotation", "2026-06-13T01:39:42Z") + seedJournalDuplicateEntry(t, stateHome, root, projectID, june24ID, "decision", "auth", "chose desync rotation", "2026-06-24T13:03:15Z") + + store := openTestStore(t, root, stateHome) + if _, err := store.UpsertArtifactBody(ctx, projectID, "journal_entry", june13ID, ArtifactBodyKindMarkdown, bodyText, ""); err != nil { + store.Close() + t.Fatalf("UpsertArtifactBody: %v", err) + } + var rowID int64 + var proj, kind, eid, bkind, content string + if err := store.db.QueryRow(` +SELECT rowid, project_id, entity_kind, entity_id, body_kind, content +FROM artifact_bodies +WHERE project_id = ? AND entity_kind = 'journal_entry' AND entity_id = ? AND body_kind = ? +`, projectID, june13ID, ArtifactBodyKindMarkdown).Scan(&rowID, &proj, &kind, &eid, &bkind, &content); err != nil { + store.Close() + t.Fatalf("read artifact body for desync setup: %v", err) + } + if _, err := store.db.Exec(` +INSERT INTO artifact_search(artifact_search, rowid, project_id, entity_kind, entity_id, body_kind, content) +VALUES('delete', ?, ?, ?, ?, ?, ?) +`, rowID, proj, kind, eid, bkind, content); err != nil { + store.Close() + t.Fatalf("remove artifact_search index entry: %v", err) + } + store.Close() + + if n := artifactSearchMatchCount(t, stateHome, root, "needledesync"); n != 0 { + t.Fatalf("artifact_search hits after desync setup = %d, want 0", n) + } + + applied, err := ApplyJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, JournalDuplicateApplyOptions{}) + if err != nil { + t.Fatalf("ApplyJournalDuplicateMigration() error = %v", err) + } + if applied.Totals.EntriesRetired != 1 { + t.Fatalf("entries_retired = %d, want 1", applied.Totals.EntriesRetired) + } + if journalEntryExists(t, stateHome, root, june13ID) { + t.Fatal("June-13 pair still present after apply") + } + + rolled, err := RollbackJournalDuplicateMigration(ctx, root, PathResolver{StateHome: stateHome}, applied.RollbackManifestPath) + if err != nil { + t.Fatalf("RollbackJournalDuplicateMigration() error = %v", err) + } + if !rolled.Applied { + t.Fatalf("rollback result = %#v, want applied", rolled) + } + if !journalEntryExists(t, stateHome, root, june13ID) { + t.Fatal("June-13 pair not restored after rollback") + } + if n := artifactSearchMatchCount(t, stateHome, root, "needledesync"); n != 1 { + t.Fatalf("artifact_search hits after rollback = %d, want 1", n) + } + assertArtifactSearchMatchReadable(t, stateHome, root, "needledesync", projectID, "journal_entry", june13ID) +} + +// --- fixture helpers --- + +func seedJournalDuplicateFixtureBase(t *testing.T) (project.Root, string, string) { + t.Helper() + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + dbPath := filepath.Join(stateHome, "loaf", "loaf.sqlite") + t.Setenv("LOAF_DB", dbPath) + status, err := Initialize(ctx, root, PathResolver{StateHome: stateHome}) + if err != nil { + t.Fatalf("Initialize() error = %v", err) + } + return root, stateHome, status.ProjectID +} + +func seedJournalDuplicateEntry(t *testing.T, stateHome string, root project.Root, projectID, id, entryType, scope, message, createdAt string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO journal_entries (id, project_id, entry_type, scope, message, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?) +`, id, projectID, entryType, scope, message, createdAt, createdAt) + store := openTestStore(t, root, stateHome) + defer store.Close() + tx, err := store.db.Begin() + if err != nil { + t.Fatalf("begin: %v", err) + } + defer tx.Rollback() + if err := insertJournalSearchTx(context.Background(), tx, projectID, id, "", entryType, scope, message); err != nil { + t.Fatalf("insert journal_search for %s: %v", id, err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit journal_search for %s: %v", id, err) + } +} + +func seedJournalDuplicateOrigin(t *testing.T, stateHome string, root project.Root, projectID, entryID, createdAt string) { + t.Helper() + mustExecOpen(t, stateHome, root, ` +INSERT INTO journal_origins ( + journal_entry_id, project_id, envelope_version, capture_mechanism, created_at +) VALUES (?, ?, 1, 'test-fixture', ?) +`, entryID, projectID, createdAt) +} + +func seedJournalDuplicateSpark(t *testing.T, stateHome string, root project.Root, projectID, sparkID, text string) { + t.Helper() + now := time.Now().UTC().Format(time.RFC3339Nano) + mustExecOpen(t, stateHome, root, ` +INSERT INTO sparks (id, project_id, text, status, source_id, created_at, updated_at) +VALUES (?, ?, ?, 'captured', NULL, ?, ?) +`, sparkID, projectID, text, now, now) +} + +func seedJournalDuplicateDeferral(t *testing.T, stateHome string, root project.Root, projectID, operationKey, entryID, sparkID string) { + t.Helper() + digest := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + now := time.Now().UTC().Format(time.RFC3339Nano) + mustExecOpen(t, stateHome, root, ` +INSERT INTO journal_deferrals (project_id, operation_key, journal_entry_id, spark_id, stored_digest, created_at) +VALUES (?, ?, ?, ?, ?, ?) +`, projectID, operationKey, entryID, sparkID, digest, now) +} + +func journalEntryExists(t *testing.T, stateHome string, root project.Root, id string) bool { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var count int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM journal_entries WHERE id = ?`, id).Scan(&count); err != nil { + t.Fatalf("count journal_entries %s: %v", id, err) + } + return count > 0 +} + +func journalEntryCount(t *testing.T, stateHome string, root project.Root) int { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var count int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM journal_entries`).Scan(&count); err != nil { + t.Fatalf("count journal_entries: %v", err) + } + return count +} + +func journalSearchCount(t *testing.T, stateHome string, root project.Root) int { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var count int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM journal_search`).Scan(&count); err != nil { + t.Fatalf("count journal_search: %v", err) + } + return count +} + +func journalDeferralEntryID(t *testing.T, stateHome string, root project.Root, projectID, operationKey string) string { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var id string + if err := store.db.QueryRow(`SELECT journal_entry_id FROM journal_deferrals WHERE project_id = ? AND operation_key = ?`, projectID, operationKey).Scan(&id); err != nil { + t.Fatalf("read journal_deferrals: %v", err) + } + return id +} + +func assertJournalSearchParityReady(t *testing.T, stateHome string, root project.Root) { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + parity, err := InspectJournalSearchParity(context.Background(), store) + if err != nil { + t.Fatalf("InspectJournalSearchParity() error = %v", err) + } + if !parity.Ready { + t.Fatalf("journal search parity not ready: %#v", parity) + } +} + +type journalPolyRowSnap struct { + table string + cols map[string]string +} + +func snapshotJournalPolymorphicResidue(t *testing.T, stateHome string, root project.Root, projectID, entryID string) map[string]journalPolyRowSnap { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + out := map[string]journalPolyRowSnap{} + queries := []struct { + table string + sql string + args []any + }{ + {"relationships", `SELECT * FROM relationships WHERE project_id = ? AND ((from_entity_kind = 'journal_entry' AND from_entity_id = ?) OR (to_entity_kind = 'journal_entry' AND to_entity_id = ?))`, []any{projectID, entryID, entryID}}, + {"entity_tags", `SELECT * FROM entity_tags WHERE project_id = ? AND entity_kind = 'journal_entry' AND entity_id = ?`, []any{projectID, entryID}}, + {"events", `SELECT * FROM events WHERE project_id = ? AND entity_kind = 'journal_entry' AND entity_id = ?`, []any{projectID, entryID}}, + {"bundle_members", `SELECT * FROM bundle_members WHERE project_id = ? AND entity_kind = 'journal_entry' AND entity_id = ?`, []any{projectID, entryID}}, + {"backend_mappings", `SELECT * FROM backend_mappings WHERE project_id = ? AND entity_kind = 'journal_entry' AND entity_id = ?`, []any{projectID, entryID}}, + {"exports", `SELECT * FROM exports WHERE project_id = ? AND source_entity_kind = 'journal_entry' AND source_entity_id = ?`, []any{projectID, entryID}}, + {"artifact_bodies", `SELECT * FROM artifact_bodies WHERE project_id = ? AND entity_kind = 'journal_entry' AND entity_id = ?`, []any{projectID, entryID}}, + {"aliases", `SELECT * FROM aliases WHERE project_id = ? AND entity_kind = 'journal_entry' AND entity_id = ?`, []any{projectID, entryID}}, + } + for _, q := range queries { + rows, err := store.db.Query(q.sql, q.args...) + if err != nil { + t.Fatalf("query %s: %v", q.table, err) + } + scanned, err := scanRows(rows) + rows.Close() + if err != nil { + t.Fatalf("scan %s: %v", q.table, err) + } + for _, row := range scanned { + cols := map[string]string{} + id := "" + for col, val := range row { + s := "" + if val != nil { + switch v := val.(type) { + case string: + s = v + case []byte: + s = string(v) + default: + s = fmt.Sprint(v) + } + } + cols[col] = s + if col == "id" { + id = s + } + } + if id == "" { + t.Fatalf("%s row missing id: %#v", q.table, row) + } + out[id] = journalPolyRowSnap{table: q.table, cols: cols} + } + } + return out +} + +func countJournalPolymorphicResidue(t *testing.T, stateHome string, root project.Root, projectID, entryID string) int { + t.Helper() + return len(snapshotJournalPolymorphicResidue(t, stateHome, root, projectID, entryID)) +} + +func equalRowSnapshots(a, b map[string]journalPolyRowSnap) bool { + if len(a) != len(b) { + return false + } + for id, left := range a { + right, ok := b[id] + if !ok || left.table != right.table || len(left.cols) != len(right.cols) { + return false + } + for col, val := range left.cols { + if right.cols[col] != val { + return false + } + } + } + return true +} + +func artifactSearchMatchCount(t *testing.T, stateHome string, root project.Root, term string) int { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var n int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM artifact_search WHERE artifact_search MATCH ?`, term).Scan(&n); err != nil { + t.Fatalf("artifact_search MATCH %q: %v", term, err) + } + return n +} + +func assertArtifactSearchMatchReadable(t *testing.T, stateHome string, root project.Root, term, projectID, entityKind, entityID string) { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + var gotProjectID, gotKind, gotEntityID, gotBodyKind, gotContent string + err := store.db.QueryRow(` +SELECT project_id, entity_kind, entity_id, body_kind, content +FROM artifact_search +WHERE artifact_search MATCH ? +`, term).Scan(&gotProjectID, &gotKind, &gotEntityID, &gotBodyKind, &gotContent) + if err != nil { + t.Fatalf("read artifact_search MATCH columns for %q: %v", term, err) + } + if gotProjectID != projectID || gotKind != entityKind || gotEntityID != entityID { + t.Fatalf("MATCH columns = project=%q kind=%q id=%q, want project=%q kind=%q id=%q", + gotProjectID, gotKind, gotEntityID, projectID, entityKind, entityID) + } + if gotBodyKind == "" || gotContent == "" { + t.Fatalf("MATCH body_kind/content empty: body_kind=%q content=%q", gotBodyKind, gotContent) + } +} + +type artifactBodyNullableSnap struct { + ID string + ProjectID string + EntityKind string + EntityID string + BodyKind string + Content string + ContentHash string + SourceID *string + CreatedAt string + UpdatedAt string +} + +// Restored rows get fresh rowids (artifact_bodies keys on a TEXT id), so index +// membership is compared by logical columns, never by rowid. +type artifactSearchMatchSnap struct { + ProjectID string + EntityKind string + EntityID string + BodyKind string + Content string +} + +func snapshotArtifactBodiesNullable(t *testing.T, stateHome string, root project.Root, projectID, entityKind, entityID string) []artifactBodyNullableSnap { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + rows, err := store.db.Query(` +SELECT id, project_id, entity_kind, entity_id, body_kind, content, content_hash, source_id, created_at, updated_at +FROM artifact_bodies +WHERE project_id = ? AND entity_kind = ? AND entity_id = ? +ORDER BY body_kind, id +`, projectID, entityKind, entityID) + if err != nil { + t.Fatalf("query artifact_bodies: %v", err) + } + defer rows.Close() + var out []artifactBodyNullableSnap + for rows.Next() { + var snap artifactBodyNullableSnap + var sourceID sql.NullString + if err := rows.Scan( + &snap.ID, &snap.ProjectID, &snap.EntityKind, &snap.EntityID, &snap.BodyKind, + &snap.Content, &snap.ContentHash, &sourceID, &snap.CreatedAt, &snap.UpdatedAt, + ); err != nil { + t.Fatalf("scan artifact_bodies: %v", err) + } + if sourceID.Valid { + s := sourceID.String + snap.SourceID = &s + } + out = append(out, snap) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate artifact_bodies: %v", err) + } + return out +} + +func equalArtifactBodySnaps(a, b []artifactBodyNullableSnap) bool { + return reflect.DeepEqual(a, b) +} + +func snapshotArtifactSearchIndex(t *testing.T, stateHome string, root project.Root, term string) []artifactSearchMatchSnap { + t.Helper() + store := openTestStore(t, root, stateHome) + defer store.Close() + rows, err := store.db.Query(` +SELECT project_id, entity_kind, entity_id, body_kind, content +FROM artifact_search +WHERE artifact_search MATCH ? +`, term) + if err != nil { + t.Fatalf("query artifact_search MATCH %q: %v", term, err) + } + defer rows.Close() + var snaps []artifactSearchMatchSnap + for rows.Next() { + var m artifactSearchMatchSnap + if err := rows.Scan(&m.ProjectID, &m.EntityKind, &m.EntityID, &m.BodyKind, &m.Content); err != nil { + t.Fatalf("scan artifact_search MATCH: %v", err) + } + snaps = append(snaps, m) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate artifact_search MATCH: %v", err) + } + sort.Slice(snaps, func(i, j int) bool { + if snaps[i].EntityID != snaps[j].EntityID { + return snaps[i].EntityID < snaps[j].EntityID + } + return snaps[i].BodyKind < snaps[j].BodyKind + }) + return snaps +} diff --git a/internal/state/journal_first_migration.go b/internal/state/journal_first_migration.go index dc4005290..0d2726f83 100644 --- a/internal/state/journal_first_migration.go +++ b/internal/state/journal_first_migration.go @@ -305,7 +305,7 @@ func classifyJournalFirstSchema10TargetWithPolicy(databasePath string, allowJour if !integrityValid { return false, nil } - if _, operationalValid, err := inspectOperationalInvariants(ctx, store); err != nil { + if _, operationalValid, err := inspectOperationalInvariants(ctx, store, InspectOptions{}); err != nil { return false, fmt.Errorf("classify schema-10 operational invariants: %w", err) } else if !operationalValid { return false, nil diff --git a/internal/state/markdown_import.go b/internal/state/markdown_import.go index 1263cf697..d2358f1e2 100644 --- a/internal/state/markdown_import.go +++ b/internal/state/markdown_import.go @@ -70,6 +70,17 @@ type markdownImporter struct { taskIndex map[string]taskIndexEntry specIndex map[string]specIndexEntry sparkAliases map[string]string + // writtenSparks holds the spark IDs this import pass already wrote, keyed by + // the (text, source) pair identity resolution matches on. A journal file may + // carry the same spark line twice; without this, the second line resolves + // onto the row the first line just minted and one of the two intake items + // disappears. + writtenSparks map[string][]string + // writtenJournalEntries holds journal entry IDs this import pass already + // wrote, keyed by the (type, scope, message) identity tuple. Repeated + // identical journal lines bind to successive existing rows rather than the + // first match twice. + writtenJournalEntries map[string][]string // report accumulates in-transaction provenance/status outcomes. // Shared pointer so value-receiver methods can mutate the same report. report *ImportReport @@ -157,14 +168,16 @@ func (s *Store) importMarkdown(ctx context.Context, root project.Root) (ImportRe defer tx.Rollback() importer := markdownImporter{ - tx: tx, - root: root, - projectID: projectID, - now: time.Now().UTC().Format(time.RFC3339), - taskIndex: loadTaskIndex(root.Path()), - specIndex: loadSpecIndex(root.Path()), - sparkAliases: map[string]string{}, - report: &report, + tx: tx, + root: root, + projectID: projectID, + now: time.Now().UTC().Format(time.RFC3339), + taskIndex: loadTaskIndex(root.Path()), + specIndex: loadSpecIndex(root.Path()), + sparkAliases: map[string]string{}, + writtenSparks: map[string][]string{}, + writtenJournalEntries: map[string][]string{}, + report: &report, } if err := importer.importAll(ctx); err != nil { return emptyImportReport(), err @@ -226,7 +239,10 @@ func (m markdownImporter) importSpecs(ctx context.Context, agentsPath string) er return err } alias := firstNonEmpty(artifact.Frontmatter["id"], specAliasFromPath(path), artifact.Stem) - id := stableMigrationID("spec", m.projectID, alias) + id, err := m.resolveImportedEntityID(ctx, "spec", "spec", alias, stableMigrationID("spec", m.projectID, alias)) + if err != nil { + return err + } meta := m.specIndex[alias] sourceID, err := m.upsertSource(ctx, artifact, "markdown") if err != nil { @@ -279,15 +295,20 @@ func (m markdownImporter) importTasks(ctx context.Context, agentsPath string) er specAlias := firstNonEmpty(meta.Spec, artifact.Frontmatter["spec"]) var specID any + var resolvedSpecID string if specAlias != "" { - resolvedSpecID, err := m.ensureSpecPlaceholder(ctx, specAlias) + var err error + resolvedSpecID, err = m.ensureSpecPlaceholder(ctx, specAlias) if err != nil { return err } specID = resolvedSpecID } - id := stableMigrationID("task", m.projectID, alias) + id, err := m.resolveImportedEntityID(ctx, "task", "task", alias, stableMigrationID("task", m.projectID, alias)) + if err != nil { + return err + } title := firstNonEmpty(meta.Title, artifact.Frontmatter["title"], artifact.Heading, alias) status, writeStatus, err := m.resolveImportStatus( ctx, "tasks", LifecycleEntityTask, id, @@ -311,13 +332,15 @@ func (m markdownImporter) importTasks(ctx context.Context, agentsPath string) er return err } if specAlias != "" { - toID := stableMigrationID("spec", m.projectID, specAlias) - if err := m.upsertRelationship(ctx, "task", id, "spec", toID, "implements", "imported from task metadata"); err != nil { + if err := m.upsertRelationship(ctx, "task", id, "spec", resolvedSpecID, "implements", "imported from task metadata"); err != nil { return err } } for _, dependency := range taskDependencies(meta, artifact.Frontmatter["depends_on"]) { - toID := stableMigrationID("task", m.projectID, dependency) + toID, err := m.resolveImportedEntityID(ctx, "task", "task", dependency, stableMigrationID("task", m.projectID, dependency)) + if err != nil { + return err + } if err := m.upsertAlias(ctx, "task", toID, "task", dependency); err != nil { return err } @@ -347,7 +370,10 @@ func (m markdownImporter) importSimpleMarkdown(ctx context.Context, agentsPath s return err } alias := firstNonEmpty(artifact.Frontmatter["id"], artifact.Stem) - id := stableMigrationID(kind, m.projectID, alias) + id, err := m.resolveImportedEntityID(ctx, kind, kind, alias, stableMigrationID(kind, m.projectID, alias)) + if err != nil { + return err + } sourceID, err := m.upsertSource(ctx, artifact, "markdown") if err != nil { return err @@ -392,7 +418,10 @@ func (m markdownImporter) importShapingDrafts(ctx context.Context, agentsPath st continue } alias := firstNonEmpty(artifact.Frontmatter["id"], artifact.Stem) - id := stableMigrationID("shaping_draft", m.projectID, alias) + id, err := m.resolveImportedEntityID(ctx, "shaping_draft", "shaping_draft", alias, stableMigrationID("shaping_draft", m.projectID, alias)) + if err != nil { + return err + } sourceID, err := m.upsertSource(ctx, artifact, "markdown") if err != nil { return err @@ -473,7 +502,10 @@ func (m markdownImporter) importReports(ctx context.Context, agentsPath string) return err } alias := firstNonEmpty(artifact.Frontmatter["id"], artifact.Stem) - id := stableMigrationID("report", m.projectID, alias) + id, err := m.resolveImportedEntityID(ctx, "report", "report", alias, stableMigrationID("report", m.projectID, alias)) + if err != nil { + return err + } sourceID, err := m.upsertSource(ctx, artifact, "markdown") if err != nil { return err @@ -532,7 +564,11 @@ func (m markdownImporter) importSessionJournal(ctx context.Context, artifact sou if entryType == "session" { continue } - entryID := stableMigrationID("journal", m.projectID, artifact.RelPath, fmt.Sprint(lineNumber+1)) + derivedEntryID := stableMigrationID("journal", m.projectID, artifact.RelPath, fmt.Sprint(lineNumber+1)) + entryID, err := m.resolveImportedJournalID(ctx, entryType, scope, message, derivedEntryID) + if err != nil { + return err + } imported, err := m.upsertJournalEntry(ctx, entryID, entryType, scope, message, observedBranch, observedWorktree, harnessSessionID) if err != nil { return err @@ -540,23 +576,37 @@ func (m markdownImporter) importSessionJournal(ctx context.Context, artifact sou if !imported { continue } + journalKey := journalIdentityKey(entryType, scope, message) + m.writtenJournalEntries[journalKey] = append(m.writtenJournalEntries[journalKey], entryID) if entryType == "spark" { - sparkID := stableMigrationID("spark", m.projectID, artifact.RelPath, fmt.Sprint(lineNumber+1)) + derivedSparkID := stableMigrationID("spark", m.projectID, artifact.RelPath, fmt.Sprint(lineNumber+1)) + slug := sparkSlugFromMessage(message) + if slug == "" { + slug = sparkSlugFallback(message) + } + sparkID, err := m.resolveImportedSparkID(ctx, slug, message, sourceID, derivedSparkID) + if err != nil { + return err + } if err := m.upsertSpark(ctx, sparkID, scope, message, sourceID); err != nil { return err } - slug := sparkSlugFromMessage(message) - if slug != "" { - alias := "SPARK-" + slug - if err := m.upsertAlias(ctx, "spark", sparkID, "spark", alias); err != nil { - return err - } - m.sparkAliases[slug] = sparkID + key := sparkIdentityKey(message, sourceID) + m.writtenSparks[key] = append(m.writtenSparks[key], sparkID) + alias, err := m.freeSparkAlias(ctx, sparkID, "SPARK-"+slug) + if err != nil { + return err + } + if err := m.upsertAlias(ctx, "spark", sparkID, "spark", alias); err != nil { + return err } + m.sparkAliases[slug] = sparkID if err := m.deleteImportedRelationships(ctx, "spark", sparkID); err != nil { return err } - if target, ok := m.capturedIdeaTarget(message); ok { + if target, ok, err := m.capturedIdeaTarget(ctx, message); err != nil { + return err + } else if ok { if err := m.upsertAlias(ctx, target.kind, target.id, target.kind, target.alias); err != nil { return err } @@ -583,7 +633,10 @@ type relationshipTarget struct { func (m markdownImporter) importArtifactRelationships(ctx context.Context, fromKind string, fromID string, artifact sourceArtifact) error { for _, field := range relationshipFrontmatterFields() { for _, value := range splitFrontmatterList(artifact.Frontmatter[field]) { - target, ok := m.resolveRelationshipTarget(value) + target, ok, err := m.resolveRelationshipTarget(ctx, value) + if err != nil { + return err + } if !ok { continue } @@ -612,7 +665,10 @@ func (m markdownImporter) importSparkResolution(ctx context.Context, message str targetText := strings.TrimSpace(rest) targetText = strings.TrimPrefix(targetText, "promoted to ") targetText = strings.TrimPrefix(targetText, "resolved by ") - target, ok := m.resolveRelationshipTarget(targetText) + target, ok, err := m.resolveRelationshipTarget(ctx, targetText) + if err != nil { + return err + } if !ok { return nil } @@ -626,23 +682,31 @@ func (m markdownImporter) importSparkResolution(ctx context.Context, message str return m.upsertRelationship(ctx, "spark", sparkID, target.kind, target.id, relationshipType, "imported from resolve(spark) journal entry") } -func (m markdownImporter) resolveRelationshipTarget(value string) (relationshipTarget, bool) { +func (m markdownImporter) resolveRelationshipTarget(ctx context.Context, value string) (relationshipTarget, bool, error) { if alias, ok := m.resolveDraftRelationshipTarget(value); ok { + id, err := m.resolveImportedEntityID(ctx, "shaping_draft", "shaping_draft", alias, stableMigrationID("shaping_draft", m.projectID, alias)) + if err != nil { + return relationshipTarget{}, false, err + } return relationshipTarget{ kind: "shaping_draft", - id: stableMigrationID("shaping_draft", m.projectID, alias), + id: id, alias: alias, - }, true + }, true, nil } alias, kind, ok := relationshipAliasAndKind(value) if !ok { - return relationshipTarget{}, false + return relationshipTarget{}, false, nil + } + id, err := m.resolveImportedEntityID(ctx, kind, kind, alias, stableMigrationID(kind, m.projectID, alias)) + if err != nil { + return relationshipTarget{}, false, err } return relationshipTarget{ kind: kind, - id: stableMigrationID(kind, m.projectID, alias), + id: id, alias: alias, - }, true + }, true, nil } func (m markdownImporter) resolveDraftRelationshipTarget(value string) (string, bool) { @@ -663,8 +727,11 @@ func (m markdownImporter) resolveDraftRelationshipTarget(value string) (string, } func (m markdownImporter) ensureSpecPlaceholder(ctx context.Context, alias string) (string, error) { - id := stableMigrationID("spec", m.projectID, alias) - _, err := m.tx.ExecContext(ctx, ` + id, err := m.resolveImportedEntityID(ctx, "spec", "spec", alias, stableMigrationID("spec", m.projectID, alias)) + if err != nil { + return "", err + } + _, err = m.tx.ExecContext(ctx, ` INSERT INTO specs (id, project_id, title, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING @@ -679,8 +746,11 @@ ON CONFLICT(id) DO NOTHING } func (m markdownImporter) upsertSource(ctx context.Context, artifact sourceArtifact, sourceKind string) (string, error) { - id := stableMigrationID("source", m.projectID, artifact.RelPath) - _, err := m.tx.ExecContext(ctx, ` + id, err := m.resolveSourceID(ctx, artifact.RelPath) + if err != nil { + return "", err + } + _, err = m.tx.ExecContext(ctx, ` INSERT INTO sources (id, project_id, source_kind, path, hash, imported_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET @@ -978,6 +1048,226 @@ ON CONFLICT(project_id, namespace, alias) DO UPDATE SET return nil } +func (m markdownImporter) entityRowExists(ctx context.Context, entityKind string, entityID string) (bool, error) { + table := registeredEntityTable(entityKind) + if table == "" { + return false, nil + } + var found int + query := fmt.Sprintf(`SELECT EXISTS(SELECT 1 FROM %s WHERE project_id = ? AND id = ?)`, quoteSQLiteIdentifier(table)) + if err := m.tx.QueryRowContext(ctx, query, m.projectID, entityID).Scan(&found); err != nil { + return false, fmt.Errorf("look for %s row %s: %w", entityKind, entityID, err) + } + return found == 1, nil +} + +func (m markdownImporter) resolveImportedEntityID(ctx context.Context, entityKind string, namespace string, alias string, derivedID string) (string, error) { + if strings.TrimSpace(alias) == "" { + return derivedID, nil + } + var entityID, kind string + err := m.tx.QueryRowContext(ctx, ` +SELECT entity_id, entity_kind +FROM aliases +WHERE project_id = ? AND namespace = ? AND alias = ? +`, m.projectID, namespace, alias).Scan(&entityID, &kind) + if errors.Is(err, sql.ErrNoRows) { + return derivedID, nil + } + if err != nil { + return "", fmt.Errorf("resolve %s alias %s: %w", entityKind, alias, err) + } + if kind != entityKind { + return derivedID, nil + } + return entityID, nil +} + +// sparkIdentityKey is the pair spark identity resolution matches on: the exact +// text of the journal line and the file it came from. +func sparkIdentityKey(message string, sourceID string) string { + return message + "\x00" + sourceID +} + +// journalIdentityKey is the tuple journal identity resolution matches on. +// created_at is the import instant and cannot participate. +func journalIdentityKey(entryType string, scope string, message string) string { + return entryType + "\x00" + scope + "\x00" + message +} + +// resolveImportedJournalID reuses a markdown-import journal row only when that +// row is the same natural entry: same type, scope, and message, with a +// journal_origins row recording source_event = markdown_import. Native +// loaf journal log rows are never hijacked. Rows already consumed earlier in +// this pass are excluded so repeated identical lines bind to successive +// existing rows; when none remain the derived ID is used for a new row. +func (m markdownImporter) resolveImportedJournalID(ctx context.Context, entryType string, scope string, message string, derivedID string) (string, error) { + query := ` +SELECT j.id +FROM journal_entries AS j +WHERE j.project_id = ? + AND j.entry_type = ? + AND j.scope IS ? + AND j.message = ? + AND EXISTS ( + SELECT 1 FROM journal_origins AS o + WHERE o.project_id = j.project_id + AND o.journal_entry_id = j.id + AND o.source_event = 'markdown_import' + )` + args := []any{m.projectID, entryType, emptyToNil(scope), message} + if written := m.writtenJournalEntries[journalIdentityKey(entryType, scope, message)]; len(written) > 0 { + fragment, bound := parameterizedNotInFragment("j.id", written) + query += "\n AND " + fragment + args = append(args, bound...) + } + query += "\nORDER BY j.id\nLIMIT 1\n" + var id string + err := m.tx.QueryRowContext(ctx, query, args...).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return derivedID, nil + } + if err != nil { + return "", fmt.Errorf("resolve journal entry for %s(%s): %w", entryType, scope, err) + } + return id, nil +} + +// resolveImportedSparkID reuses an alias-reachable spark only when that row is +// unmistakably this same journal line: same source file, same text, exactly one +// candidate, and not a row this same import pass already wrote. A spark alias is +// the message's first word, so two unrelated sparks routinely share one — +// resolving on the alias alone would make the second import overwrite the first +// spark's text — and identity is looked up by content rather than by the alias +// so a spark that had to take a disambiguated alias is still found after a +// rekey. Rows written earlier in this pass are excluded because a file may +// repeat a spark line verbatim: those are two intake items, and matching the +// second onto the first would silently drop one. +// +// Convergence trade: a single distinct spark line reuses its row across a rekey +// re-import. A verbatim-duplicated line is different — the first import mints +// two rows (and two aliases); after a rekey the source_id salt no longer matches +// either, uniqueness is no longer one candidate, and the re-import mints two +// more. Further re-imports then stabilize at four alias-reachable rows and do +// not reintroduce alias-orphan damage. Accepting that one-time fork is the +// trade for keeping the resolver's "exactly one candidate" gate, which is what +// prevents unrelated sparks that share a first word from collapsing into one. +func (m markdownImporter) resolveImportedSparkID(ctx context.Context, slug string, message string, sourceID string, derivedID string) (string, error) { + if slug == "" { + return derivedID, nil + } + query := ` +SELECT sparks.id +FROM sparks +WHERE sparks.project_id = ? + AND sparks.text = ? + AND sparks.source_id IS ? + AND EXISTS ( + SELECT 1 FROM aliases + WHERE aliases.project_id = sparks.project_id + AND aliases.namespace = 'spark' + AND aliases.entity_kind = 'spark' + AND aliases.entity_id = sparks.id + )` + args := []any{m.projectID, message, emptyToNil(sourceID)} + if written := m.writtenSparks[sparkIdentityKey(message, sourceID)]; len(written) > 0 { + fragment, bound := parameterizedNotInFragment("sparks.id", written) + query += "\n AND " + fragment + args = append(args, bound...) + } + query += "\nORDER BY sparks.id\nLIMIT 2\n" + rows, err := m.tx.QueryContext(ctx, query, args...) + if err != nil { + return "", fmt.Errorf("resolve spark for SPARK-%s: %w", slug, err) + } + defer rows.Close() + var candidates []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return "", fmt.Errorf("scan spark candidate for SPARK-%s: %w", slug, err) + } + candidates = append(candidates, id) + } + if err := rows.Err(); err != nil { + return "", fmt.Errorf("resolve spark for SPARK-%s: %w", slug, err) + } + if len(candidates) != 1 { + return derivedID, nil + } + return candidates[0], nil +} + +// freeSparkAlias returns the alias this spark may claim without evicting +// another. Re-pointing a claimed alias is the mechanic that forked identity in +// the first place, and a spark alias is only the message's first word, so +// collisions between unrelated sparks are routine rather than exceptional: the +// later spark takes a numbered alias instead of stealing the base one. An alias +// this spark already holds, and one whose entity row is gone, are both free. +func (m markdownImporter) freeSparkAlias(ctx context.Context, sparkID string, base string) (string, error) { + for attempt := 1; attempt <= 64; attempt++ { + candidate := base + if attempt > 1 { + candidate = fmt.Sprintf("%s-%d", base, attempt) + } + free, err := m.sparkAliasIsFree(ctx, sparkID, candidate) + if err != nil { + return "", err + } + if free { + return candidate, nil + } + } + candidate := base + "-" + strings.TrimPrefix(sparkID, "spark:")[:8] + free, err := m.sparkAliasIsFree(ctx, sparkID, candidate) + if err != nil { + return "", err + } + if !free { + return "", fmt.Errorf("no free spark alias for %s under %s", sparkID, base) + } + return candidate, nil +} + +func (m markdownImporter) sparkAliasIsFree(ctx context.Context, sparkID string, alias string) (bool, error) { + var holder string + err := m.tx.QueryRowContext(ctx, ` +SELECT entity_id FROM aliases WHERE project_id = ? AND namespace = 'spark' AND alias = ? +`, m.projectID, alias).Scan(&holder) + if errors.Is(err, sql.ErrNoRows) { + return true, nil + } + if err != nil { + return false, fmt.Errorf("read spark alias %s: %w", alias, err) + } + if holder == sparkID { + return true, nil + } + // An alias whose spark no longer exists is dangling, not claimed. + exists, err := m.entityRowExists(ctx, "spark", holder) + if err != nil { + return false, err + } + return !exists, nil +} + +func (m markdownImporter) resolveSourceID(ctx context.Context, relPath string) (string, error) { + var id string + err := m.tx.QueryRowContext(ctx, ` +SELECT id FROM sources +WHERE project_id = ? AND path = ? +ORDER BY created_at, id +LIMIT 1 +`, m.projectID, relPath).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return stableMigrationID("source", m.projectID, relPath), nil + } + if err != nil { + return "", fmt.Errorf("resolve source %s: %w", relPath, err) + } + return id, nil +} + type sourceArtifact struct { Path string RelPath string @@ -1244,6 +1534,14 @@ func sparkSlugFromMessage(message string) string { return normalizeSparkSlug(prefix) } +// sparkSlugFallback derives a non-empty slug from message content alone so a +// punctuation-only spark still receives an alias. Project ID and source ID must +// not participate: both shift across a rekey and would re-fork identity. +func sparkSlugFallback(message string) string { + sum := sha256.Sum256([]byte(message)) + return "spark-" + hex.EncodeToString(sum[:4]) +} + func normalizeSparkSlug(value string) string { value = strings.TrimSpace(strings.TrimPrefix(value, "SPARK-")) value = strings.Trim(value, `"'`) @@ -1253,16 +1551,19 @@ func normalizeSparkSlug(value string) string { return strings.Trim(value, "-_") } -func (m markdownImporter) capturedIdeaTarget(message string) (relationshipTarget, bool) { +func (m markdownImporter) capturedIdeaTarget(ctx context.Context, message string) (relationshipTarget, bool, error) { _, targetText, ok := strings.Cut(message, " captured to ") if !ok { - return relationshipTarget{}, false + return relationshipTarget{}, false, nil + } + target, ok, err := m.resolveRelationshipTarget(ctx, targetText) + if err != nil { + return relationshipTarget{}, false, err } - target, ok := m.resolveRelationshipTarget(targetText) if !ok || target.kind != "idea" { - return relationshipTarget{}, false + return relationshipTarget{}, false, nil } - return target, true + return target, true, nil } func leadingIDFromPath(path string, pattern string) string { diff --git a/internal/state/markdown_import_alias_first_test.go b/internal/state/markdown_import_alias_first_test.go new file mode 100644 index 000000000..52739caf3 --- /dev/null +++ b/internal/state/markdown_import_alias_first_test.go @@ -0,0 +1,805 @@ +package state + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + "time" +) + +func TestImportAliasFirstRekeyReimportStable(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAliasFirstImportFixture(t, root.Path()) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("first ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, first.DatabasePath) + defer store.Close() + + beforeEntities := countProjectEntities(t, store, first.ProjectID) + beforeSources := countTableWhere(t, store, `SELECT COUNT(*) FROM sources WHERE project_id = ?`, first.ProjectID) + beforeAliasMap := aliasEntityMap(t, store, first.ProjectID) + beforeEntityIDs := entityIDSet(t, store, first.ProjectID) + beforeSourceIDs := sourceIDSet(t, store, first.ProjectID) + + if beforeEntities["specs"] < 1 || beforeEntities["tasks"] < 1 || beforeEntities["ideas"] < 1 { + t.Fatalf("fixture import too thin: %#v", beforeEntities) + } + if beforeSources < 1 { + t.Fatal("expected imported sources") + } + if len(beforeAliasMap) < 1 { + t.Fatal("expected imported aliases") + } + + newProjectID := "proj_aliasfirst_rekey_00000001" + rekeyProjectLikeLegacy(t, store, first.ProjectID, newProjectID, root.Path()) + + second, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + if second.ProjectID != newProjectID { + t.Fatalf("ProjectID after rekey reimport = %q, want %q", second.ProjectID, newProjectID) + } + + afterEntities := countProjectEntities(t, store, newProjectID) + afterSources := countTableWhere(t, store, `SELECT COUNT(*) FROM sources WHERE project_id = ?`, newProjectID) + afterAliasMap := aliasEntityMap(t, store, newProjectID) + afterEntityIDs := entityIDSet(t, store, newProjectID) + afterSourceIDs := sourceIDSet(t, store, newProjectID) + orphans := countAliasOrphans(t, store, newProjectID) + + if !mapsEqual(beforeEntities, afterEntities) { + t.Fatalf("entity counts drifted after rekey re-import\nbefore=%#v\nafter=%#v", beforeEntities, afterEntities) + } + if afterSources != beforeSources { + t.Fatalf("sources = %d, want %d", afterSources, beforeSources) + } + if orphans != 0 { + t.Fatalf("alias orphans = %d, want 0", orphans) + } + if !mapsEqual(beforeAliasMap, afterAliasMap) { + t.Fatalf("alias→entity map drifted\nbefore=%v\nafter=%v", beforeAliasMap, afterAliasMap) + } + if !stringSetsEqual(beforeEntityIDs, afterEntityIDs) { + t.Fatalf("entity IDs changed\nbefore=%v\nafter=%v", sortedKeys(beforeEntityIDs), sortedKeys(afterEntityIDs)) + } + if !stringSetsEqual(beforeSourceIDs, afterSourceIDs) { + t.Fatalf("source IDs changed\nbefore=%v\nafter=%v", sortedKeys(beforeSourceIDs), sortedKeys(afterSourceIDs)) + } + + // Derived IDs under the new project ID must not have been minted as twins. + for namespace, entityID := range afterAliasMap { + parts := strings.SplitN(namespace, "\x00", 2) + if len(parts) != 2 { + continue + } + kind, alias := parts[0], parts[1] + twin := stableMigrationID(kind, newProjectID, alias) + if twin == entityID { + continue + } + var count int + if err := store.db.QueryRowContext(ctx, fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE project_id = ? AND id = ?`, entityTableForKind(kind)), newProjectID, twin).Scan(&count); err != nil { + t.Fatalf("count twin %s: %v", twin, err) + } + if count != 0 { + t.Fatalf("twin row minted for %s/%s: %s", kind, alias, twin) + } + } +} + +func TestImportAliasFirstIdempotentNoChurn(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAliasFirstImportFixture(t, root.Path()) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("first ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, first.DatabasePath) + defer store.Close() + + beforeDump := logicalBusinessDump(t, store) + beforeEntities := countProjectEntities(t, store, first.ProjectID) + beforeSources := countTableWhere(t, store, `SELECT COUNT(*) FROM sources WHERE project_id = ?`, first.ProjectID) + beforeAliasMap := aliasEntityMap(t, store, first.ProjectID) + + second, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + if second.ProjectID != first.ProjectID { + t.Fatalf("ProjectID changed: %q -> %q", first.ProjectID, second.ProjectID) + } + + afterDump := logicalBusinessDump(t, store) + if beforeDump != afterDump { + t.Fatalf("canonical business dump drifted on no-op re-import\nbefore:\n%s\nafter:\n%s", beforeDump, afterDump) + } + if !mapsEqual(beforeEntities, countProjectEntities(t, store, first.ProjectID)) { + t.Fatalf("entity counts drifted on no-op re-import") + } + if got := countTableWhere(t, store, `SELECT COUNT(*) FROM sources WHERE project_id = ?`, first.ProjectID); got != beforeSources { + t.Fatalf("sources = %d, want %d", got, beforeSources) + } + if !mapsEqual(beforeAliasMap, aliasEntityMap(t, store, first.ProjectID)) { + t.Fatalf("alias→entity map drifted on no-op re-import") + } + if orphans := countAliasOrphans(t, store, first.ProjectID); orphans != 0 { + t.Fatalf("alias orphans after idempotent re-import = %d, want 0", orphans) + } +} + +func TestImportAliasFirstDoesNotRepointLiveAlias(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAliasFirstImportFixture(t, root.Path()) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, first.DatabasePath) + defer store.Close() + + var liveEntityID string + if err := store.db.QueryRowContext(ctx, ` +SELECT entity_id FROM aliases +WHERE project_id = ? AND namespace = 'spec' AND alias = 'SPEC-001' +`, first.ProjectID).Scan(&liveEntityID); err != nil { + t.Fatalf("read SPEC-001 alias: %v", err) + } + + // Force project_id rewrite without changing entity IDs, then re-import. + newProjectID := "proj_aliasfirst_repoint_000001" + rekeyProjectLikeLegacy(t, store, first.ProjectID, newProjectID, root.Path()) + + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("re-import error = %v", err) + } + + var afterEntityID string + if err := store.db.QueryRowContext(ctx, ` +SELECT entity_id FROM aliases +WHERE project_id = ? AND namespace = 'spec' AND alias = 'SPEC-001' +`, newProjectID).Scan(&afterEntityID); err != nil { + t.Fatalf("read SPEC-001 alias after reimport: %v", err) + } + if afterEntityID != liveEntityID { + t.Fatalf("alias re-pointed: %q -> %q", liveEntityID, afterEntityID) + } + var rowCount int + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM specs WHERE project_id = ?`, newProjectID).Scan(&rowCount); err != nil { + t.Fatalf("count specs: %v", err) + } + if rowCount != 1 { + t.Fatalf("specs rows = %d, want 1 (no twin)", rowCount) + } +} + +func writeAliasFirstImportFixture(t *testing.T, root string) { + t.Helper() + writeAgentsFile(t, root, "specs/SPEC-001-example.md", `--- +id: SPEC-001 +title: Alias First Spec +status: implementing +--- +# Alias First Spec +`) + writeAgentsFile(t, root, "tasks/TASK-001-example.md", `--- +id: TASK-001 +title: Alias First Task +status: todo +--- +# Alias First Task +`) + writeAgentsFile(t, root, "ideas/20260528-alias-idea.md", `--- +id: idea-alias-first +title: Alias First Idea +--- +# Alias First Idea +`) + writeAgentsFile(t, root, "drafts/20260528-brainstorm-alias.md", `--- +id: brainstorm-alias-first +title: Alias First Brainstorm +--- +# Alias First Brainstorm +`) + writeAgentsFile(t, root, "reports/report-alias.md", `--- +id: report-alias-first +title: Alias First Report +status: final +--- +# Alias First Report +`) + writeAgentsFile(t, root, "sessions/20260528-session.md", `--- +branch: feature/alias-first +--- +[2026-05-28 10:00] spark(scope): aliasfirst-spark capture this +`) + writeAgentsFile(t, root, "TASKS.json", `{ + "tasks": { + "TASK-001": { + "title": "Alias First Task", + "spec": "SPEC-001", + "status": "todo", + "priority": "P1" + } + }, + "specs": { + "SPEC-001": { + "title": "Alias First Spec", + "status": "implementing" + } + } +} +`) +} + +// rekeyProjectLikeLegacy mirrors rekeyLegacyProjectTx: insert opaque project, +// rewrite project_id on projectScopedRekeyTables, swap projects row. +// Also moves artifact_bodies and journal_origins so FK delete of the old +// project row succeeds (those tables are populated by import but absent from +// projectScopedRekeyTables). +func rekeyProjectLikeLegacy(t *testing.T, store *Store, legacyID string, nextID string, currentPath string) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC().Format(time.RFC3339) + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin rekey: %v", err) + } + defer tx.Rollback() + + var createdAt string + var friendly sql.NullString + if err := tx.QueryRowContext(ctx, `SELECT created_at, friendly_name FROM projects WHERE id = ?`, legacyID).Scan(&createdAt, &friendly); err != nil { + t.Fatalf("read legacy project: %v", err) + } + friendlyName := "project" + if friendly.Valid && strings.TrimSpace(friendly.String) != "" { + friendlyName = friendly.String + } + if _, err := tx.ExecContext(ctx, ` +INSERT INTO projects (id, identity_hash, friendly_name, current_path, last_seen_at, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?) +`, nextID, nextID, friendlyName, currentPath, now, createdAt, now); err != nil { + t.Fatalf("insert rekeyed project: %v", err) + } + for _, table := range projectScopedRekeyTables() { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET project_id = ? WHERE project_id = ?`, table), nextID, legacyID); err != nil { + t.Fatalf("rekey %s: %v", table, err) + } + } + for _, table := range []string{"artifact_bodies", "journal_origins"} { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET project_id = ? WHERE project_id = ?`, table), nextID, legacyID); err != nil { + t.Fatalf("rekey %s: %v", table, err) + } + } + if _, err := tx.ExecContext(ctx, `DELETE FROM projects WHERE id = ?`, legacyID); err != nil { + t.Fatalf("delete legacy project: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit rekey: %v", err) + } +} + +func countProjectEntities(t *testing.T, store *Store, projectID string) map[string]int { + t.Helper() + tables := []string{"specs", "tasks", "ideas", "brainstorms", "reports", "sparks", "shaping_drafts"} + out := make(map[string]int, len(tables)) + for _, table := range tables { + out[table] = countTableWhere(t, store, fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE project_id = ?`, table), projectID) + } + return out +} + +func countTableWhere(t *testing.T, store *Store, query string, args ...any) int { + t.Helper() + var n int + if err := store.db.QueryRowContext(context.Background(), query, args...).Scan(&n); err != nil { + t.Fatalf("count query: %v", err) + } + return n +} + +func aliasEntityMap(t *testing.T, store *Store, projectID string) map[string]string { + t.Helper() + rows, err := store.db.QueryContext(context.Background(), ` +SELECT namespace, alias, entity_id +FROM aliases +WHERE project_id = ? +ORDER BY namespace, alias +`, projectID) + if err != nil { + t.Fatalf("query aliases: %v", err) + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var namespace, alias, entityID string + if err := rows.Scan(&namespace, &alias, &entityID); err != nil { + t.Fatalf("scan alias: %v", err) + } + out[namespace+"\x00"+alias] = entityID + } + if err := rows.Err(); err != nil { + t.Fatalf("aliases rows: %v", err) + } + return out +} + +func entityIDSet(t *testing.T, store *Store, projectID string) map[string]struct{} { + t.Helper() + out := map[string]struct{}{} + for _, table := range []string{"specs", "tasks", "ideas", "brainstorms", "reports", "sparks", "shaping_drafts"} { + rows, err := store.db.QueryContext(context.Background(), fmt.Sprintf(`SELECT id FROM %s WHERE project_id = ?`, table), projectID) + if err != nil { + t.Fatalf("query %s ids: %v", table, err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + t.Fatalf("scan %s id: %v", table, err) + } + out[table+":"+id] = struct{}{} + } + err = rows.Err() + rows.Close() + if err != nil { + t.Fatalf("%s rows: %v", table, err) + } + } + return out +} + +func sourceIDSet(t *testing.T, store *Store, projectID string) map[string]struct{} { + t.Helper() + rows, err := store.db.QueryContext(context.Background(), `SELECT id FROM sources WHERE project_id = ?`, projectID) + if err != nil { + t.Fatalf("query sources: %v", err) + } + defer rows.Close() + out := map[string]struct{}{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan source id: %v", err) + } + out[id] = struct{}{} + } + if err := rows.Err(); err != nil { + t.Fatalf("sources rows: %v", err) + } + return out +} + +func countAliasOrphans(t *testing.T, store *Store, projectID string) int { + t.Helper() + // Entity row with no matching alias for its kind/namespace. + total := 0 + for kind, table := range map[string]string{ + "spec": "specs", + "task": "tasks", + "idea": "ideas", + "brainstorm": "brainstorms", + "report": "reports", + "spark": "sparks", + } { + var n int + query := fmt.Sprintf(` +SELECT COUNT(*) +FROM %s AS e +WHERE e.project_id = ? + AND NOT EXISTS ( + SELECT 1 FROM aliases AS a + WHERE a.project_id = e.project_id + AND a.entity_kind = ? + AND a.entity_id = e.id + AND a.namespace = ? + ) +`, table) + if err := store.db.QueryRowContext(context.Background(), query, projectID, kind, kind).Scan(&n); err != nil { + t.Fatalf("count orphans %s: %v", table, err) + } + total += n + } + return total +} + +func entityTableForKind(kind string) string { + switch kind { + case "spec": + return "specs" + case "task": + return "tasks" + case "idea": + return "ideas" + case "brainstorm": + return "brainstorms" + case "report": + return "reports" + case "spark": + return "sparks" + case "shaping_draft": + return "shaping_drafts" + default: + return kind + "s" + } +} + +func mapsEqual[K comparable, V comparable](a, b map[K]V) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func stringSetsEqual(a, b map[string]struct{}) bool { + if len(a) != len(b) { + return false + } + for k := range a { + if _, ok := b[k]; !ok { + return false + } + } + return true +} + +// A spark's alias is the message's first word, so unrelated sparks collide on +// it routinely. Alias-first resolution must never let the second one overwrite +// the first one's text. +func TestImportAliasFirstKeepsCollidingSparksDistinct(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAgentsFile(t, root.Path(), "sessions/20260528-sparks.md", `--- +branch: feature/sparks +--- +[2026-05-28 10:00] spark(scope): dedupe the state tables one day +[2026-05-28 10:05] spark(scope): dedupe something entirely different +`) + + result, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, result.DatabasePath) + defer store.Close() + + texts := sparkTexts(t, store, result.ProjectID) + if len(texts) != 2 { + t.Fatalf("spark rows = %v, want both journal lines preserved", texts) + } + for _, want := range []string{"dedupe the state tables one day", "dedupe something entirely different"} { + if _, ok := texts[want]; !ok { + t.Fatalf("spark %q missing from %v", want, texts) + } + } + + // Both rows keep an alias of their own: the second spark takes a numbered + // alias instead of evicting the first, so neither is orphaned. + if orphans := countAliasOrphans(t, store, result.ProjectID); orphans != 0 { + t.Fatalf("alias orphans after a first import = %d, want 0", orphans) + } + aliases := aliasEntityMap(t, store, result.ProjectID) + for _, want := range []string{"spark\x00SPARK-dedupe", "spark\x00SPARK-dedupe-2"} { + if _, ok := aliases[want]; !ok { + t.Fatalf("alias %q missing from %v", want, aliases) + } + } + + // Re-import is still idempotent: no third row, no rewritten text, no churn + // in which spark holds which alias. + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + if again := sparkTexts(t, store, result.ProjectID); len(again) != 2 { + t.Fatalf("spark rows after re-import = %v, want 2", again) + } + if orphans := countAliasOrphans(t, store, result.ProjectID); orphans != 0 { + t.Fatalf("alias orphans after re-import = %d, want 0", orphans) + } + if !mapsEqual(aliases, aliasEntityMap(t, store, result.ProjectID)) { + t.Fatalf("alias→entity map drifted on re-import\nbefore=%v\nafter=%v", aliases, aliasEntityMap(t, store, result.ProjectID)) + } +} + +// A journal file may carry the same spark line twice. Those are two intake +// items: identity resolution must not match the second onto the row the first +// line just minted. +func TestImportAliasFirstKeepsRepeatedSparkLinesDistinct(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAgentsFile(t, root.Path(), "sessions/20260613-repeats.md", `--- +branch: feature/sparks +--- +[2026-06-13 10:00] spark(x): widget idea worth keeping +[2026-06-13 10:10] spark(x): widget idea worth keeping +`) + + result, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, result.DatabasePath) + defer store.Close() + + if got := sparkRowCount(t, store, result.ProjectID); got != 2 { + t.Fatalf("spark rows = %d, want one per journal line", got) + } + if orphans := countAliasOrphans(t, store, result.ProjectID); orphans != 0 { + t.Fatalf("alias orphans after a first import = %d, want 0", orphans) + } + aliases := aliasEntityMap(t, store, result.ProjectID) + for _, want := range []string{"spark\x00SPARK-widget", "spark\x00SPARK-widget-2"} { + if _, ok := aliases[want]; !ok { + t.Fatalf("alias %q missing from %v", want, aliases) + } + } + + // Re-import adds nothing and reshuffles nothing. + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + if got := sparkRowCount(t, store, result.ProjectID); got != 2 { + t.Fatalf("spark rows after re-import = %d, want 2", got) + } + if !mapsEqual(aliases, aliasEntityMap(t, store, result.ProjectID)) { + t.Fatalf("alias→entity map drifted on re-import\nbefore=%v\nafter=%v", aliases, aliasEntityMap(t, store, result.ProjectID)) + } +} + +// A colliding spark that had to take a numbered alias is still found by the +// rekey re-import: identity is looked up by content, not by the base alias. +func TestImportAliasFirstCollidingSparksSurviveRekeyReimport(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAgentsFile(t, root.Path(), "sessions/20260528-sparks.md", `--- +branch: feature/sparks +--- +[2026-05-28 10:00] spark(scope): dedupe the state tables one day +[2026-05-28 10:05] spark(scope): dedupe something entirely different +`) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("first ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, first.DatabasePath) + defer store.Close() + beforeIDs := entityIDSet(t, store, first.ProjectID) + + newProjectID := "proj_sparkcollision_00000001" + rekeyProjectLikeLegacy(t, store, first.ProjectID, newProjectID, root.Path()) + + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + if orphans := countAliasOrphans(t, store, newProjectID); orphans != 0 { + t.Fatalf("alias orphans after rekey re-import = %d, want 0", orphans) + } + if afterIDs := entityIDSet(t, store, newProjectID); !stringSetsEqual(beforeIDs, afterIDs) { + t.Fatalf("entity IDs changed across rekey re-import\nbefore=%v\nafter=%v", sortedKeys(beforeIDs), sortedKeys(afterIDs)) + } +} + +// The rekey that caused the damage must not fork spark identity either. +func TestImportAliasFirstSparkSurvivesRekeyReimport(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAgentsFile(t, root.Path(), "sessions/20260528-one-spark.md", `--- +branch: feature/sparks +--- +[2026-05-28 10:00] spark(scope): dedupe the state tables one day +`) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("first ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, first.DatabasePath) + defer store.Close() + beforeIDs := entityIDSet(t, store, first.ProjectID) + + newProjectID := "proj_sparkrekey_000000000001" + rekeyProjectLikeLegacy(t, store, first.ProjectID, newProjectID, root.Path()) + + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + if orphans := countAliasOrphans(t, store, newProjectID); orphans != 0 { + t.Fatalf("alias orphans after rekey re-import = %d, want 0", orphans) + } + if afterIDs := entityIDSet(t, store, newProjectID); !stringSetsEqual(beforeIDs, afterIDs) { + t.Fatalf("entity IDs changed across rekey re-import\nbefore=%v\nafter=%v", sortedKeys(beforeIDs), sortedKeys(afterIDs)) + } +} + +// Journal entry IDs must not re-derive from the live project ID after a rekey: +// resolve by natural identity and reuse the existing row. +func TestImportAliasFirstJournalSurvivesRekeyReimport(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAgentsFile(t, root.Path(), "sessions/20260528-journal.md", `--- +branch: feature/journal +--- +[2026-05-28 10:00] decision(scope): chose X because Y +[2026-05-28 10:05] discover(scope): learned Z from the field +[2026-05-28 10:10] decision(scope): chose X because Y +`) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("first ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, first.DatabasePath) + defer store.Close() + + beforeJournal := countTableWhere(t, store, `SELECT COUNT(*) FROM journal_entries WHERE project_id = ?`, first.ProjectID) + if beforeJournal != 3 { + t.Fatalf("journal rows after first import = %d, want 3", beforeJournal) + } + beforeIDs := journalIDSet(t, store, first.ProjectID) + + newProjectID := "proj_journalrekey_000000001" + rekeyProjectLikeLegacy(t, store, first.ProjectID, newProjectID, root.Path()) + + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + afterJournal := countTableWhere(t, store, `SELECT COUNT(*) FROM journal_entries WHERE project_id = ?`, newProjectID) + if afterJournal != beforeJournal { + t.Fatalf("journal rows after rekey re-import = %d, want %d (zero new rows)", afterJournal, beforeJournal) + } + if afterIDs := journalIDSet(t, store, newProjectID); !stringSetsEqual(beforeIDs, afterIDs) { + t.Fatalf("journal IDs changed across rekey re-import\nbefore=%v\nafter=%v", sortedKeys(beforeIDs), sortedKeys(afterIDs)) + } + + // Idempotent third pass still adds nothing. + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("third ApplyMarkdownMigration() error = %v", err) + } + if got := countTableWhere(t, store, `SELECT COUNT(*) FROM journal_entries WHERE project_id = ?`, newProjectID); got != beforeJournal { + t.Fatalf("journal rows after third import = %d, want %d", got, beforeJournal) + } +} + +// A punctuation-only spark still receives an alias; rekey re-import must not +// mint a twin. +func TestImportAliasFirstPunctuationOnlySparkGetsAlias(t *testing.T) { + ctx := context.Background() + root := projectRoot(t) + stateHome := t.TempDir() + resolver := PathResolver{StateHome: stateHome} + writeAgentsFile(t, root.Path(), "sessions/20260528-punct.md", `--- +branch: feature/sparks +--- +[2026-05-28 10:00] spark(scope): !!! +`) + + first, err := ApplyMarkdownMigration(ctx, root, resolver) + if err != nil { + t.Fatalf("first ApplyMarkdownMigration() error = %v", err) + } + store := openStoreAt(t, first.DatabasePath) + defer store.Close() + + if got := sparkRowCount(t, store, first.ProjectID); got != 1 { + t.Fatalf("spark rows = %d, want 1", got) + } + if orphans := countAliasOrphans(t, store, first.ProjectID); orphans != 0 { + t.Fatalf("alias orphans after first import = %d, want 0", orphans) + } + aliases := aliasEntityMap(t, store, first.ProjectID) + if len(aliases) == 0 { + t.Fatal("expected punctuation-only spark to hold an alias") + } + beforeIDs := entityIDSet(t, store, first.ProjectID) + + newProjectID := "proj_punctspark_00000000001" + rekeyProjectLikeLegacy(t, store, first.ProjectID, newProjectID, root.Path()) + + if _, err := ApplyMarkdownMigration(ctx, root, resolver); err != nil { + t.Fatalf("second ApplyMarkdownMigration() error = %v", err) + } + if got := sparkRowCount(t, store, newProjectID); got != 1 { + t.Fatalf("spark rows after rekey re-import = %d, want 1", got) + } + if orphans := countAliasOrphans(t, store, newProjectID); orphans != 0 { + t.Fatalf("alias orphans after rekey re-import = %d, want 0", orphans) + } + if afterIDs := entityIDSet(t, store, newProjectID); !stringSetsEqual(beforeIDs, afterIDs) { + t.Fatalf("entity IDs changed across rekey re-import\nbefore=%v\nafter=%v", sortedKeys(beforeIDs), sortedKeys(afterIDs)) + } + parity, err := InspectAliasParity(ctx, store) + if err != nil { + t.Fatalf("InspectAliasParity() error = %v", err) + } + if !parity.Ready { + t.Fatalf("parity after punctuation-only spark rekey re-import = %#v, want Ready", parity) + } +} + +func journalIDSet(t *testing.T, store *Store, projectID string) map[string]struct{} { + t.Helper() + rows, err := store.db.QueryContext(context.Background(), ` +SELECT id FROM journal_entries WHERE project_id = ? ORDER BY id +`, projectID) + if err != nil { + t.Fatalf("query journal ids: %v", err) + } + defer rows.Close() + out := map[string]struct{}{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan journal id: %v", err) + } + out[id] = struct{}{} + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate journal ids: %v", err) + } + return out +} + +func sparkRowCount(t *testing.T, store *Store, projectID string) int { + t.Helper() + var count int + if err := store.db.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM sparks WHERE project_id = ?`, projectID).Scan(&count); err != nil { + t.Fatalf("count sparks: %v", err) + } + return count +} + +func sparkTexts(t *testing.T, store *Store, projectID string) map[string]struct{} { + t.Helper() + rows, err := store.db.QueryContext(context.Background(), `SELECT text FROM sparks WHERE project_id = ?`, projectID) + if err != nil { + t.Fatalf("list sparks: %v", err) + } + defer rows.Close() + texts := map[string]struct{}{} + for rows.Next() { + var text string + if err := rows.Scan(&text); err != nil { + t.Fatalf("scan spark: %v", err) + } + texts[text] = struct{}{} + } + if err := rows.Err(); err != nil { + t.Fatalf("list sparks: %v", err) + } + return texts +} diff --git a/internal/state/schema_upgrade.go b/internal/state/schema_upgrade.go index 000805e2c..d0ddccca8 100644 --- a/internal/state/schema_upgrade.go +++ b/internal/state/schema_upgrade.go @@ -78,7 +78,7 @@ func requireCurrentSchemaForDerivedRepair(ctx context.Context, s *Store) error { if _, err := s.ValidateCurrentSchema(ctx); err != nil { return fmt.Errorf("state database is invalid: %w", err) } - if _, valid, err := inspectOperationalInvariants(ctx, s); err != nil { + if _, valid, err := inspectOperationalInvariants(ctx, s, InspectOptions{}); err != nil { return fmt.Errorf("state database is invalid: inspect operational invariants: %w", err) } else if !valid { return fmt.Errorf("state database is invalid: operational invariants failed") @@ -110,7 +110,7 @@ func (s *Store) RequireCurrentSchema(ctx context.Context) error { if _, err := s.ValidateCurrentSchema(ctx); err != nil { return fmt.Errorf("state database is invalid: %w", err) } - if _, valid, err := inspectOperationalInvariants(ctx, s); err != nil { + if _, valid, err := inspectOperationalInvariants(ctx, s, InspectOptions{}); err != nil { return fmt.Errorf("state database is invalid: inspect operational invariants: %w", err) } else if !valid { return fmt.Errorf("state database is invalid: operational invariants failed") @@ -382,7 +382,7 @@ func classifySchemaUpgradeSourceWithPolicy(ctx context.Context, path string, roo return schemaUpgradeSource{}, fmt.Errorf("state database is invalid; schema upgrade requires clean behind-schema state") } } - if _, valid, err := inspectOperationalInvariants(ctx, store); err != nil { + if _, valid, err := inspectOperationalInvariants(ctx, store, InspectOptions{}); err != nil { return schemaUpgradeSource{}, fmt.Errorf("classify schema upgrade operational invariants: %w", err) } else if !valid { return schemaUpgradeSource{}, fmt.Errorf("state database is invalid: operational invariants failed") diff --git a/internal/state/status.go b/internal/state/status.go index b974993ad..3eba23f3b 100644 --- a/internal/state/status.go +++ b/internal/state/status.go @@ -29,6 +29,7 @@ const ( RepairCategoryMarkdownImport = "markdown-import" RepairCategoryCompatibilityExport = "compatibility-export" RepairCategoryJournalSearch = "journal-search" + RepairCategoryAliasIdentity = "alias-identity" ) const ( @@ -84,8 +85,22 @@ type Status struct { RepairPlan []RepairAction `json:"repair_plan"` } +// InspectOptions selects diagnostics too expensive for the hot path. Every +// entry here scans whole tables across every project in the global database, so +// only surfaces that exist to diagnose — `loaf state doctor` — turn them on. +type InspectOptions struct { + // AliasParity compares raw entity counts with alias-reachable counts and + // looks for dangling aliases, per project and per entity table. + AliasParity bool +} + // Inspect returns the current state-runtime status without creating files. func Inspect(root project.Root, resolver PathResolver) (Status, error) { + return InspectWithOptions(root, resolver, InspectOptions{}) +} + +// InspectWithOptions is Inspect with the expensive diagnostics selectable. +func InspectWithOptions(root project.Root, resolver PathResolver, options InspectOptions) (Status, error) { databasePath, err := resolver.DatabasePath(root) if err != nil { return Status{}, err @@ -160,7 +175,7 @@ func Inspect(root project.Root, resolver PathResolver) (Status, error) { status.Mode = ModeInvalid return status, nil } - invariantDiagnostics, invariantValid, err := inspectOperationalInvariants(context.Background(), store) + invariantDiagnostics, invariantValid, err := inspectOperationalInvariants(context.Background(), store, options) if err != nil { status.Diagnostics = append(status.Diagnostics, Diagnostic{ Severity: "error", @@ -390,6 +405,16 @@ func RepairPlanForStatus(status Status) []RepairAction { Path: status.DatabasePath, Safe: false, }) + case AliasParityDivergenceCode: + actions = appendRepairAction(actions, RepairAction{ + Code: "migrate-alias-orphans", + DiagnosticCode: diagnostic.Code, + Category: RepairCategoryAliasIdentity, + Description: "Preview and then apply alias-orphan migration to retire twin rows and delete dangling aliases.", + Command: AliasParityRepairCommand, + Path: status.DatabasePath, + Safe: false, + }) case "local-markdown-not-imported": actions = appendRepairAction(actions, RepairAction{ Code: "migrate-current-project-markdown", @@ -483,7 +508,7 @@ func inspectSchemaMigrations(ctx context.Context, store *Store, version int) ([] return diagnostics, valid } -func inspectOperationalInvariants(ctx context.Context, store *Store) ([]Diagnostic, bool, error) { +func inspectOperationalInvariants(ctx context.Context, store *Store, options InspectOptions) ([]Diagnostic, bool, error) { diagnostics := []Diagnostic{} valid := true @@ -548,6 +573,17 @@ func inspectOperationalInvariants(ctx context.Context, store *Store) ([]Diagnost }) } + if options.AliasParity { + aliasParity, err := InspectAliasParity(ctx, store) + if err != nil { + return nil, false, err + } + // Always emit a diagnostic: info all-clear when Ready (and no multi-alias), + // error when orphan/dangling diverged, warn for multi-alias. Mode stays + // ready either way — identity damage is detectable, not invalidating. + diagnostics = append(diagnostics, aliasParityDiagnostics(aliasParity)...) + } + journalProvenance, err := InspectJournalProvenanceIntegrity(ctx, store) if err != nil { return nil, false, err diff --git a/internal/state/storage_home_migration.go b/internal/state/storage_home_migration.go index 9a118374f..d5c38406b 100644 --- a/internal/state/storage_home_migration.go +++ b/internal/state/storage_home_migration.go @@ -871,7 +871,7 @@ func verifyStorageHomeSource(ctx context.Context, store *Store) error { return fmt.Errorf("legacy state SQLite integrity checks failed") } if version >= 9 { - if _, valid, err := inspectOperationalInvariants(ctx, store); err != nil { + if _, valid, err := inspectOperationalInvariants(ctx, store, InspectOptions{}); err != nil { return err } else if !valid { return fmt.Errorf("legacy state operational invariants failed") @@ -1106,7 +1106,7 @@ func verifyStorageHomeDestination(ctx context.Context, root project.Root, databa } else if !valid { return ProjectIdentity{}, fmt.Errorf("SQLite integrity checks failed") } - if _, valid, err := inspectOperationalInvariants(ctx, store); err != nil { + if _, valid, err := inspectOperationalInvariants(ctx, store, InspectOptions{}); err != nil { return ProjectIdentity{}, err } else if !valid { return ProjectIdentity{}, fmt.Errorf("operational invariants failed") diff --git a/plugins/loaf/skills/loaf-reference/SKILL.md b/plugins/loaf/skills/loaf-reference/SKILL.md index 79e686acc..968e36f1a 100644 --- a/plugins/loaf/skills/loaf-reference/SKILL.md +++ b/plugins/loaf/skills/loaf-reference/SKILL.md @@ -69,7 +69,7 @@ Names and one-line purposes only. Run `loaf --help` for options, argum | `loaf docs` | Manage docs/ indexing | index | | `loaf change` | Shape-first Change artifacts: git-canonical work context under docs/changes/ | init, check, list | | `loaf render` | Maintain committed durable Markdown renders | sweep | -| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | +| `loaf state` | Manage native SQLite state | path, status, init, doctor, repair legacy-project-database, repair relationship-origin, repair journal-search, migrate markdown, migrate storage-home, migrate schema, migrate deferrals, migrate lifecycle-statuses, migrate alias-orphans, migrate journal-duplicates, migrate journal-first, backup, backup verify, backup restore, restore-ephemerals, verify-ephemerals, export, export all, export triage, export spec, export release-readiness | | `loaf journal` | Record and read the project-scoped journal (the durable record across all conversations) | log, recent, search, show, context, export, defer | | `loaf project` | Manage durable project identity | list, show, identity, rename, move, delete | | `loaf migrate` | Run native migration workflows | markdown, storage-home, schema, lifecycle-statuses, journal-first, worktree-storage |