From 132406f23e2a8a0f82783a36db803961e97ab1ac Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 20 Jul 2026 19:32:53 +0300 Subject: [PATCH 1/5] Replace Drive case storage with deterministic SQLite --- .agents/skills/dialogue-lab-closeout/SKILL.md | 22 +- .../dialogue-lab-closeout/agents/openai.yaml | 2 +- .agents/skills/dialogue-lab-followup/SKILL.md | 25 +- .../dialogue-lab-followup/agents/openai.yaml | 2 +- .agents/skills/dialogue-lab-intake/SKILL.md | 26 +- .../dialogue-lab-intake/agents/openai.yaml | 2 +- .agents/skills/dialogue-lab-posting/SKILL.md | 20 +- .../dialogue-lab-posting/agents/openai.yaml | 2 +- .../dialogue-lab-strategy-review/SKILL.md | 19 +- .../agents/openai.yaml | 2 +- .codex/config.toml.example | 20 +- .gitignore | 11 + AGENTS.md | 12 +- CONTRIBUTING.md | 6 +- README.md | 194 ++---- config/drive-files.toml | 38 -- config/storage.toml | 12 + docs/cli-payloads.md | 107 +++ docs/drive-connector-boundary.md | 19 - docs/evidence-base.md | 5 + docs/migration-receipt.json.example | 31 +- docs/reply-strategy-guide.md | 5 + docs/rollback.md | 26 +- docs/rollout.md | 36 +- evals/README.md | 10 +- evals/cases/synthetic-cases.json | 8 +- pyproject.toml | 4 +- src/dialogue_lab/__init__.py | 2 +- src/dialogue_lab/cli.py | 449 +++++++++---- src/dialogue_lab/drive_models.py | 22 - src/dialogue_lab/drive_protocol.py | 81 --- src/dialogue_lab/enums.py | 1 - src/dialogue_lab/errors.py | 6 + src/dialogue_lab/identifiers.py | 5 - src/dialogue_lab/lifecycle.py | 14 +- src/dialogue_lab/manual_version.py | 40 -- src/dialogue_lab/migration_receipt.py | 14 +- src/dialogue_lab/models.py | 75 +-- src/dialogue_lab/pending_sync.py | 66 -- src/dialogue_lab/readback.py | 8 +- src/dialogue_lab/schema.py | 139 ++-- src/dialogue_lab/source_consistency.py | 48 +- src/dialogue_lab/storage.py | 617 ++++++++++++++++++ src/dialogue_lab/validation.py | 27 +- tests/helpers.py | 2 +- tests/test_cli_commands.py | 276 ++++++-- tests/test_identity_identifiers.py | 20 - tests/test_migration_cli.py | 52 ++ tests/test_pending_receipt_cli.py | 81 --- tests/test_schema_source_write.py | 142 ---- tests/test_sqlite_storage.py | 199 ++++++ uv.lock | 2 +- 52 files changed, 1848 insertions(+), 1206 deletions(-) delete mode 100644 config/drive-files.toml create mode 100644 config/storage.toml create mode 100644 docs/cli-payloads.md delete mode 100644 docs/drive-connector-boundary.md create mode 100644 docs/evidence-base.md create mode 100644 docs/reply-strategy-guide.md delete mode 100644 src/dialogue_lab/drive_models.py delete mode 100644 src/dialogue_lab/drive_protocol.py delete mode 100644 src/dialogue_lab/manual_version.py delete mode 100644 src/dialogue_lab/pending_sync.py create mode 100644 src/dialogue_lab/storage.py create mode 100644 tests/test_migration_cli.py delete mode 100644 tests/test_pending_receipt_cli.py delete mode 100644 tests/test_schema_source_write.py create mode 100644 tests/test_sqlite_storage.py diff --git a/.agents/skills/dialogue-lab-closeout/SKILL.md b/.agents/skills/dialogue-lab-closeout/SKILL.md index 8470ee9..cbcf69b 100644 --- a/.agents/skills/dialogue-lab-closeout/SKILL.md +++ b/.agents/skills/dialogue-lab-closeout/SKILL.md @@ -1,30 +1,28 @@ --- name: dialogue-lab-closeout -description: Close an Israel Facebook Dialogue Lab case after explicit user instruction, reported abandonment, no-response closure, or a clearly ended exchange. Use to validate closure, classify only observable outcomes, score the highest outcome, and record one controlled next test. +description: Close an Israel Facebook Dialogue Lab case from observable evidence and record the result through one approval-gated SQLite transaction. --- # Dialogue Lab Closeout ## Required inputs -- Target Case ID and the allowed closure condition. +- Target Case ID and allowed closure condition. - Any final public turn, reaction, or user report needed to establish the observable ending. ## Workflow -1. Read the complete live Manual, verify compatibility, record its revision, and validate the live Case Log schema. -2. Load the complete Case and Turn graph. Run `dialogue-lab validate-parent-graph`; resolve any required missing public turn before closure. -3. Verify privacy. Record observable chronology only. Never infer persuasion from silence, deletion, blocking, a reaction, or disappearance. -4. Choose the Manual-allowed closed status, Outcome Class, and highest outcome score reached. Record concise Outcome Notes, What Worked, What Failed, and exactly one Next Test. -5. Run `dialogue-lab validate-case` and `dialogue-lab validate-transition` on the planned result. -6. Re-read relevant rows, Pending Sync, source revision state, and schema. Run source-consistency before the explicit-approval write. -7. Read every written field back with `dialogue-lab verify-readback`. On failure, show PENDING SYNC and do not claim closure was recorded. +1. Run `dialogue-lab doctor` and load the complete Case and Turn graph once with `case-show`. +2. Resolve any missing public Turn through `$dialogue-lab-followup` before closure. +3. Verify privacy and record observable chronology only. Never infer persuasion from silence, deletion, blocking, a reaction, or disappearance. +4. Choose the closed status, Outcome Class, highest outcome score reached, concise Outcome Notes, What Worked, What Failed, and exactly one Next Test. +5. Prepare one closeout payload. After explicit approval, run exactly one `dialogue-lab case-close --case-id --approved` transaction and report its compact receipt. ## Safety -- Do not close a case solely because no new message is visible unless the user requests or reports no-response closure. -- Do not alter canonical documents other than approved Case Log fields. Never post to Facebook. +- Do not close solely because no new message is visible unless the user requests or reports no-response closure. +- Never access or write `General responses`, and never post to Facebook. ## Output -Return a compact closeout receipt containing Case ID, final status, Outcome Score, Outcome Class, observable basis, What Worked, What Failed, Next Test, Manual revision, source-consistency result, and read-back result. +Return Case ID, final status, Outcome Score, Outcome Class, observable basis, What Worked, What Failed, Next Test, and read-back status. diff --git a/.agents/skills/dialogue-lab-closeout/agents/openai.yaml b/.agents/skills/dialogue-lab-closeout/agents/openai.yaml index 781fea9..178a5bc 100644 --- a/.agents/skills/dialogue-lab-closeout/agents/openai.yaml +++ b/.agents/skills/dialogue-lab-closeout/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Dialogue Lab Closeout" short_description: "Close a case from observable outcomes" - default_prompt: "Use -lab-closeout to close this Dialogue Lab case." + default_prompt: "Use $dialogue-lab-closeout to close this Dialogue Lab case." diff --git a/.agents/skills/dialogue-lab-followup/SKILL.md b/.agents/skills/dialogue-lab-followup/SKILL.md index 59aed3a..0fc044b 100644 --- a/.agents/skills/dialogue-lab-followup/SKILL.md +++ b/.agents/skills/dialogue-lab-followup/SKILL.md @@ -1,6 +1,6 @@ --- name: dialogue-lab-followup -description: Process a new public Facebook comment, reply, reaction, screenshot, text, or permalink in an existing Israel Facebook Dialogue Lab case. Use to extend the turn graph, resolve parentage with explicit confidence, update Active Exchange state, and recommend the next reply. +description: Process supplied public content in an existing Israel Facebook Dialogue Lab case, resolve parentage, and prepare one approval-gated SQLite follow-up transaction. --- # Dialogue Lab Follow-up @@ -8,26 +8,19 @@ description: Process a new public Facebook comment, reply, reaction, screenshot, ## Required inputs - Existing Case ID or enough Post ID + Root Comment ID data to resolve it. -- Exact new public turn or reaction, supplied URL, observed time, and any visible parent context. +- Exact new public turn or reaction, supplied URL, observed time, and visible parent context. ## Workflow -1. Read the complete live Manual and record its compatible version and revision state. Read the live Case Log schema and run `dialogue-lab schema-check`. -2. Parse supplied URLs with `dialogue-lab parse-url`, preserve each exact URL, resolve canonical identity, and load the existing Case plus every relevant Turn. -3. Assign or reuse only `P1`, `P2`, … or `USER`. Identify the direct parent from visible context or user confirmation; record Parent Confidence. Ask only when ambiguity materially changes the reply. -4. Build the candidate incoming Turn, then run `dialogue-lab validate-turn` and `dialogue-lab validate-parent-graph`. Multiple children may share a parent. Show a compact Thread Map when the graph is branched. -5. Load the live Strategy Guide and Evidence Base as needed, verify material claims, and answer the operative claim without treating private planning as public context. -6. Produce the standard reply output. Ignore incidental abuse; use `no_engagement` for credible threats, doxxing, calls for violence, spam, or no testable claim. - -## Canonical write gate - -- Reads are automatic. Any Case/Turn append or update requires explicit approval and a Manual-allowed lifecycle action. -- Before writing, re-read relevant rows, Pending Sync, source states, and schema; run source-consistency and case-local Turn ID allocation on the fresh rows. -- After the runtime connector call, read the Case and Turn back and run `dialogue-lab verify-readback`. -- On write or verification failure, show a complete PENDING SYNC record and do not claim Drive was updated. Never post to Facebook or write protected documents. +1. Run `dialogue-lab doctor`, resolve the Case with `case-find` when needed, then load it once with `case-show`. +2. Parse supplied URLs with `parse-url`, preserve each exact URL, and confirm the Case identity. +3. Assign or reuse only `P1`, `P2`, ... or `USER`. Resolve the direct parent from visible context or user confirmation and retain Parent Confidence; ask only when ambiguity changes the reply. +4. Build one incoming Turn payload. The `case-followup` transaction allocates the Turn ID, validates the complete parent graph, appends the Turn, updates Case state, commits, and reads back. +5. Read repository strategy and evidence Markdown only as needed, verify material claims, and answer the operative claim without treating private planning as public context. +6. Return the standard output. After explicit approval, run exactly one `dialogue-lab case-followup --case-id --approved` transaction and report its compact receipt. ## Output Return, in order: `Case ID / Status`, optional `Thread Map`, `Claim Map`, `Recommended Reply`, `Shorter Version`, `Why This Approach`, `Fact Check`, `Conversational Potential`, `Log Tags`, and `Next Action`. -Use the Manual's exact Next Action line and the Facebook thread's language. +Use the Facebook thread's language. Ignore incidental abuse; use `no_engagement` for credible threats, doxxing, calls for violence, spam, or no testable claim. diff --git a/.agents/skills/dialogue-lab-followup/agents/openai.yaml b/.agents/skills/dialogue-lab-followup/agents/openai.yaml index db158f3..201bc3f 100644 --- a/.agents/skills/dialogue-lab-followup/agents/openai.yaml +++ b/.agents/skills/dialogue-lab-followup/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Dialogue Lab Follow-up" short_description: "Process a new turn in an existing case" - default_prompt: "Use -lab-followup to process this new public turn." + default_prompt: "Use $dialogue-lab-followup to process this new public turn." diff --git a/.agents/skills/dialogue-lab-intake/SKILL.md b/.agents/skills/dialogue-lab-intake/SKILL.md index c305175..81b4d2e 100644 --- a/.agents/skills/dialogue-lab-intake/SKILL.md +++ b/.agents/skills/dialogue-lab-intake/SKILL.md @@ -1,6 +1,6 @@ --- name: dialogue-lab-intake -description: Start or identify an Israel Facebook Dialogue Lab case from a public post, root comment, screenshot, text, or Facebook URL. Use for new-case intake, duplicate-case detection, claim mapping, fact checking, and a first reply recommendation; canonical Drive writes remain approval-gated. +description: Start or identify an Israel Facebook Dialogue Lab case from supplied public content, perform deterministic duplicate detection, and prepare one approval-gated SQLite intake transaction. --- # Dialogue Lab Intake @@ -12,24 +12,16 @@ description: Start or identify an Israel Facebook Dialogue Lab case from a publi ## Workflow -1. Read `config/drive-files.toml`. Through the Google Drive plugin, read the complete live Operating Manual and record its version plus revision or modified state. Run `dialogue-lab manual-version` and stop write preparation if compatibility fails. -2. Read Case Log metadata, Cases/Turns headers, Data Dictionary enums/validations, and relevant current rows. Run `dialogue-lab schema-check` against the observed schema. Preserve the live schema. -3. Parse every supplied Facebook URL with `dialogue-lab parse-url`. Preserve the exact URL; never infer a parent solely from `reply_comment_id`. -4. Build `facebook::` with `dialogue-lab case-key`. Search Cases for that identity before allocating. If found, hand off to `$dialogue-lab-followup`. -5. Extract the post, target and preceding public turns, first recorded comment, current case-local participant reference, material claims, hidden assumption, hostility, evidence confidence, privacy issues, and supplied URLs. Store no names or profile links. -6. Load the live Strategy Guide and Evidence Base only as needed. Verify material current, historical, legal, military, statistical, and media-authenticity claims with current sources; separate fact, assessment, allegation, legal status, and moral judgment. -7. Produce the standard output below. Do not write exploratory or intermediate drafts. - -## Canonical write gate - -- Read automatically; write only after explicit user approval at a Manual-allowed lifecycle point. -- Immediately before a write, re-read relevant rows, unresolved Pending Sync, source revision state, and schema. Run the identifier, record, transition, graph, and source-consistency CLI checks. -- Send only a typed Case Log append/update request through the runtime connector. Never write `General responses`, the Strategy Guide, or the Evidence Base here. -- Read the exact row back and run `dialogue-lab verify-readback`. A connector success without a matching read-back is failure. -- On failure, show a complete PENDING SYNC record and block new Case ID allocation. Never post to Facebook. +1. Run `dialogue-lab doctor` once for the configured database. +2. Parse every supplied Facebook URL with `dialogue-lab parse-url`; preserve exact URLs and never infer a parent solely from `reply_comment_id`. +3. Resolve `Post ID + Root Comment ID` with `dialogue-lab case-find`. If found, hand off to `$dialogue-lab-followup` without allocating or writing. +4. Extract the public context, case-local participant references, material claims, hidden assumption, hostility, evidence confidence, privacy state, and supplied URLs. Store no names or profile links. +5. Read repository strategy and evidence Markdown only as needed. Verify material current claims with authoritative sources and distinguish fact, assessment, allegation, legal status, and moral judgment. +6. Prepare one `case-intake` JSON payload containing the Case and initial public Turns. Do not persist exploratory drafts. +7. Return the standard output. After explicit approval, run exactly one `dialogue-lab case-intake --date --approved` transaction and report its compact receipt. ## Output Return, in order: `Case ID / Status`, `Claim Map`, `Recommended Reply`, `Shorter Version`, `Why This Approach`, `Fact Check`, `Conversational Potential`, `Log Tags`, and `Next Action`. -Use the Manual's exact Next Action line. The public reply must use the thread language, focus on one pivotal point, sound natural, remain useful to silent readers, offer a face-saving path where warranted, and never fabricate evidence. +The public reply must use the thread language, focus on one pivotal point, sound natural, remain useful to silent readers, offer a face-saving path where warranted, and never fabricate evidence. diff --git a/.agents/skills/dialogue-lab-intake/agents/openai.yaml b/.agents/skills/dialogue-lab-intake/agents/openai.yaml index 75cd5a4..266b1bb 100644 --- a/.agents/skills/dialogue-lab-intake/agents/openai.yaml +++ b/.agents/skills/dialogue-lab-intake/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Dialogue Lab Intake" short_description: "Start or identify a Dialogue Lab case" - default_prompt: "Use -lab-intake to analyze this Facebook root-comment discussion." + default_prompt: "Use $dialogue-lab-intake to analyze this Facebook root-comment discussion." diff --git a/.agents/skills/dialogue-lab-posting/SKILL.md b/.agents/skills/dialogue-lab-posting/SKILL.md index d130e8d..cf33586 100644 --- a/.agents/skills/dialogue-lab-posting/SKILL.md +++ b/.agents/skills/dialogue-lab-posting/SKILL.md @@ -1,6 +1,6 @@ --- name: dialogue-lab-posting -description: Record a user's explicit confirmation that a Dialogue Lab reply was posted to Facebook. Use when the user says a reply was posted and supplies or confirms the exact published wording, permalink, identifiers, or posting time. This skill records state; it never publishes to Facebook. +description: Record a user's explicit confirmation that a Dialogue Lab reply was posted, using one deterministic SQLite transaction; this skill never publishes to Facebook. --- # Dialogue Lab Posting Confirmation @@ -9,25 +9,21 @@ description: Record a user's explicit confirmation that a Dialogue Lab reply was - Target Case ID and parent Turn ID. - Exact wording actually published, even when it differs from the recommendation. -- Supplied permalink, IDs, and posting time when available. +- Supplied permalink, identifiers, posting time, and optional Draft Turn ID. ## Workflow -1. Read the complete live Manual, verify compatibility, and record its revision state. Read and validate the live Case Log schema. -2. Resolve the case and parent turn. Parse any supplied URL with `dialogue-lab parse-url`; preserve it exactly and do not derive the immediate parent solely from `reply_comment_id`. -3. Find an approved Draft row representing the posted reply. Update that row when appropriate; otherwise prepare a new Outgoing Posted Turn. Preserve exact published wording without rewriting it. -4. Run `dialogue-lab validate-turn`, `dialogue-lab validate-transition`, and `dialogue-lab validate-parent-graph`. Update Case timestamps and status as the Manual requires. -5. Re-read relevant rows, unresolved Pending Sync, source revisions, and schema. Run `dialogue-lab source-consistency` before the approval-gated connector write. -6. Read every written field back and run `dialogue-lab verify-readback`. -7. If writing or read-back fails, show a complete PENDING SYNC record and state that Drive was not verified. +1. Run `dialogue-lab doctor` and load the Case once with `case-show`. +2. Parse any supplied URL with `parse-url`; preserve it exactly and do not derive the immediate parent solely from `reply_comment_id`. +3. Prepare one Outgoing Posted Turn payload containing the exact published wording. Include `draft_turn_id` only when an existing Outgoing Draft must be marked Replaced. +4. Run exactly one `dialogue-lab case-record-posting --case-id --approved` transaction. It allocates the Turn ID, validates the graph and lifecycle, writes atomically, and reads back the committed state. ## Safety - Never publish, edit, or delete Facebook content. - Never assume the posted text equals an earlier draft. -- Never write `General responses`, the Strategy Guide, or the Evidence Base. -- Never treat connector acknowledgement as successful posting confirmation without matching Case Log read-back. +- Never access or write `General responses`. ## Output -Return a compact posting receipt containing Case ID, Turn ID, state, exact-text verification, URL/ID preservation status, Manual version and revision, source-consistency result, and read-back result. +Return only the compact command receipt plus exact-text and permalink preservation status. diff --git a/.agents/skills/dialogue-lab-posting/agents/openai.yaml b/.agents/skills/dialogue-lab-posting/agents/openai.yaml index bae0e5a..aab162a 100644 --- a/.agents/skills/dialogue-lab-posting/agents/openai.yaml +++ b/.agents/skills/dialogue-lab-posting/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Dialogue Lab Posting" short_description: "Record an explicitly confirmed posted reply" - default_prompt: "Use -lab-posting to record my exact published reply." + default_prompt: "Use $dialogue-lab-posting to record my exact published reply." diff --git a/.agents/skills/dialogue-lab-strategy-review/SKILL.md b/.agents/skills/dialogue-lab-strategy-review/SKILL.md index 5b291b8..5b3b472 100644 --- a/.agents/skills/dialogue-lab-strategy-review/SKILL.md +++ b/.agents/skills/dialogue-lab-strategy-review/SKILL.md @@ -1,6 +1,6 @@ --- name: dialogue-lab-strategy-review -description: Review strategy evidence across closed Israel Facebook Dialogue Lab cases. Use for comparable-case analysis after at least three cases, formal review at twenty closed cases, confounder-aware pattern assessment, and proposed Reply Strategy Guide changes that must not be applied without separate explicit approval. +description: Review deterministic SQLite evidence across closed Israel Facebook Dialogue Lab cases and propose, but never apply, repository strategy-guide changes. --- # Dialogue Lab Strategy Review @@ -12,19 +12,16 @@ description: Review strategy evidence across closed Israel Facebook Dialogue Lab ## Workflow -1. Read the complete live Manual, record its compatible version and revision, validate the live Case Log schema, and read the complete live Reply Strategy Guide. -2. Load only closed Cases and their relevant Turns. Reject fewer than three reasonably comparable cases. Label reviews below twenty total closed cases as preliminary. +1. Run `dialogue-lab doctor`, read `docs/reply-strategy-guide.md`, and load all closed Cases and Turns in one `dialogue-lab strategy-dataset` call. +2. Reject fewer than three reasonably comparable cases. Label reviews below twenty total closed cases as preliminary. 3. Compare topic, hostility, evidence confidence, reply length, thread position, and strategy. Separate commenter outcomes from silent-reader signals. -4. Report sample sizes, contradictory evidence, missing data, and confounders. Do not infer causation from simple correlation or treat one case as a reusable finding. -5. Distinguish supported findings, provisional hypotheses, and inconclusive results. Keep case-specific lessons in the Case Log. -6. Draft the smallest exact Strategy Guide change only when the Manual's evidence threshold is satisfied. Do not apply it in this run. +4. Report sample sizes, contradictory evidence, missing data, and confounders. Do not infer causation from correlation or treat one case as a reusable finding. +5. Distinguish supported findings, provisional hypotheses, and inconclusive results. Keep case-specific lessons in SQLite. +6. Draft the smallest exact Strategy Guide change only when evidence supports it. Do not edit the guide in this run; a separate explicit approval and reviewed repository change are required. -## Write policy +## Safety -- Case Log and source reads may run automatically. -- Never update the Strategy Guide without a separate explicit user approval for the exact proposed wording and a fresh source-consistency check. -- Never write the Evidence Base or `General responses`, and never post to Facebook. -- If a separately approved later write occurs, require revision recheck and connector read-back. +- Never access or write `General responses`, and never post to Facebook. ## Output diff --git a/.agents/skills/dialogue-lab-strategy-review/agents/openai.yaml b/.agents/skills/dialogue-lab-strategy-review/agents/openai.yaml index 18e8b4c..b57a221 100644 --- a/.agents/skills/dialogue-lab-strategy-review/agents/openai.yaml +++ b/.agents/skills/dialogue-lab-strategy-review/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Dialogue Lab Strategy Review" short_description: "Review strategy evidence across closed cases" - default_prompt: "Use -lab-strategy-review to review comparable closed cases." + default_prompt: "Use $dialogue-lab-strategy-review to review comparable closed cases." diff --git a/.codex/config.toml.example b/.codex/config.toml.example index 1bfe152..b1f5e2a 100644 --- a/.codex/config.toml.example +++ b/.codex/config.toml.example @@ -1,17 +1,7 @@ -# Safe example only; this file does not establish a live connection. +# Safe local example. The canonical database path is supplied through the +# DIALOGUE_LAB_DB environment variable and must resolve outside the Git repo. [dialogue_lab] -drive_config = "config/drive-files.toml" -connector = "google-drive-plugin" -reads = "automatic" +storage_config = "config/storage.toml" +database_environment_variable = "DIALOGUE_LAB_DB" writes = "explicit-approval" -delete = false -move = false -rename = false -share = false -single_writer = true - -# Configure the Google Drive plugin in the Codex environment. Keep OAuth and -# environment-specific connector settings outside the repository. -[mcp.google_drive] -enabled = true -environment_specific_connection = "CONFIGURE_IN_CODEX" +transactional = true diff --git a/.gitignore b/.gitignore index b4474c9..7489d9e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,14 @@ htmlcov/ *.env .env* !.env.example +*.sqlite3 +*.sqlite3-wal +*.sqlite3-shm +*.sqlite3-journal +*.db +*.db-wal +*.db-shm +*.db-journal +data/ +backups/ +exports/ diff --git a/AGENTS.md b/AGENTS.md index 53b9da1..4316127 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,14 +10,14 @@ ## HasbaraTops -- Before case work, read the live Operating Manual through the configured Drive ID, verify that its version is supported, and record the revision state used. -- Treat Google Drive as canonical and repository files as non-canonical implementation; never use or commit repository exports as substitutes for live canonical sources. -- Use `config/drive-files.toml`, preserve the live Case Log schema, and check `Post ID + Root Comment ID` for duplicates before allocating a Case ID. +- Before case work, run `dialogue-lab doctor` against the configured SQLite database and stop if local documents, schema, or database integrity fail validation. +- Treat repository Markdown as canonical governance, strategy, and evidence content and the configured SQLite database as canonical Case and Turn state; do not use Google Drive or MCP for Dialogue Lab work. +- Use `config/storage.toml`; preserve the SQLite schema version and enforce unique `Post ID + Root Comment ID` identity before Case ID allocation. - Invoke the matching Dialogue Lab skill for intake, follow-up, posting confirmation, closeout, or strategy review. - Never edit, move, rename, replace, delete, import, or summarize `General responses` without the user's explicit instruction for that exact action. -- Before a canonical write, verify that relevant source state has not materially changed; require explicit approval, read the record back, and compare every expected field. -- After a failed required write, follow the Manual's Recovery procedure and block new Case ID allocation while any `PENDING SYNC` is unresolved. -- Never open, inspect, click, scroll, or otherwise interact with Facebook unless the user explicitly asks. Never post to Facebook autonomously. Never store credentials, OAuth material, secrets, or canonical document bodies in Git. +- Perform canonical writes only through explicit-approval `dialogue-lab` commands that use SQLite transactions and committed read-back verification. +- After a failed canonical write, verify rollback and database integrity; block further writes while either remains unresolved. +- Never open, inspect, click, scroll, or otherwise interact with Facebook unless the user explicitly asks. Never post to Facebook autonomously. Never commit SQLite databases, exports, backups, credentials, or secrets. ### Repo checkout and worktrees: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0127de0..ee1d002 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,8 @@ # Contributing Open an issue before substantial changes. Keep changes narrowly scoped, preserve -the Google Drive canonical-source boundary, and never commit credentials, -canonical document bodies, Case Log exports, or public Facebook content. +the local SQLite transaction boundary, and never commit databases, exports, +backups, credentials, secrets, or public Facebook content. Before proposing a change, run: @@ -10,7 +10,7 @@ Before proposing a change, run: uv run pytest uv run ruff check . uv run mypy -uv run dialogue-lab doctor +uv run dialogue-lab --database doctor ``` Contributions are accepted only under the repository's proprietary license. diff --git a/README.md b/README.md index 67e8ce9..b69e48b 100644 --- a/README.md +++ b/README.md @@ -1,164 +1,100 @@ -# Israel Facebook Dialogue Lab — Codex Runtime +# Israel Facebook Dialogue Lab -This repository is the non-canonical execution, validation, and testing layer for the Israel Facebook Dialogue Lab. Google Drive remains the canonical source for workflow rules, case state, strategy findings, and factual material. +This repository is the canonical governance and execution layer for the Dialogue Lab. Repository Markdown owns governance, strategy, and reusable evidence. One SQLite database outside Git owns Case and Turn state. + +The runtime performs no Google Drive or MCP operations. ## Architecture ```text -Codex app / CLI / IDE - ├─ AGENTS.md bootstrap - ├─ .agents/skills workflow instructions - ├─ dialogue-lab deterministic CLI - ├─ synthetic tests and evals - └─ runtime-managed Google Drive connector boundary - │ - ▼ -Google Drive — canonical source of truth - ├─ 00 — Dialogue Lab Operating Manual - ├─ 01 — Dialogue Lab Case Log - ├─ 02 — Reply Strategy Guide - └─ 03 — Israel Claims and Evidence Base +AGENTS.md canonical governance +docs/reply-strategy-guide.md canonical cross-case strategy +docs/evidence-base.md canonical reusable evidence +external SQLite database canonical Cases and Turns +dialogue-lab CLI only canonical storage boundary +.agents/skills/ model-driven analysis and reply workflows ``` -`General responses` is user-owned, protected, and non-canonical. The repository contains only its non-secret Drive ID and a deny-by-default policy. Canonical document bodies, Case Log exports, public Facebook text, backups, and credentials must not be committed. - -The skills control model-driven workflows. Python controls identifiers, URL parsing, schemas, enums, lifecycle invariants, parent graphs, Pending Sync, source consistency, read-back comparison, and migration receipts. The Google Drive plugin is the external data boundary; it is runtime-managed and is not imported into Python. +`General responses` is protected and outside this workflow. The CLI exposes no operation for it. -## Installation +## Deterministic boundary -Python 3.12 or newer is supported. +Python and SQLite own URL parsing, identifiers, duplicate identity, schema constraints, lifecycle transitions, parent graphs, transactional writes, committed read-back, open-case summaries, strategy datasets, backups, and migration receipts. -```powershell -python -m venv .venv -.venv\Scripts\Activate.ps1 -python -m pip install uv -uv sync --extra dev --frozen -uv run dialogue-lab --version -``` +The model owns interpretation, materially ambiguous parentage, reply drafting, fact-check judgment, and strategy analysis. Each canonical workflow ends in at most one high-level write command. -Run the checks: +## Installation ```powershell -uv run pytest -uv run ruff check . -uv run mypy +uv sync --frozen --extra dev +$env:DIALOGUE_LAB_DB = '\dialogue-lab.sqlite3' +uv run dialogue-lab db-init --approved uv run dialogue-lab doctor ``` -Refresh the lock only after intentionally changing `pyproject.toml`: +`DIALOGUE_LAB_DB` must resolve outside the Git repository. `--database ` may override it and must appear before the subcommand. -```powershell -uv lock --upgrade -uv sync --extra dev --frozen -``` +Database initialization, imports, backups, and Case or Turn mutations require `--approved`. Read commands do not mutate state. -Optional local checks, when installed: +## High-level commands -```powershell -pip-audit -gitleaks git --no-banner +```text +dialogue-lab doctor +dialogue-lab db-init --approved +dialogue-lab db-status +dialogue-lab db-backup --destination --approved +dialogue-lab db-import --approved + +dialogue-lab case-find --post-id --root-comment-id +dialogue-lab case-show --case-id +dialogue-lab case-list-open +dialogue-lab strategy-dataset + +dialogue-lab case-intake --date --approved +dialogue-lab case-followup --case-id --approved +dialogue-lab case-record-posting --case-id --approved +dialogue-lab case-close --case-id --approved ``` -## Codex setup - -Codex reads the root `AGENTS.md` for repository bootstrap rules and discovers skills under `.agents/skills/`. Connect the Google Drive plugin in Codex, then copy `.codex/config.toml.example` to the environment-specific Codex configuration surface. The example contains no credentials and does not establish a connection by itself. - -Read operations may run automatically. Case Log appends and updates require explicit user approval, a fresh source-consistency check, and targeted read-back. Delete, move, rename, sharing, autonomous Facebook posting, arbitrary Drive writes, and automatic edits to canonical documents are disabled. - -## Live source startup - -Before every case operation: - -1. Read the complete live Operating Manual and record its Drive revision or modified state. -2. Run `dialogue-lab manual-version` and stop canonical write preparation on incompatibility. -3. Read current Case Log metadata, exact headers, Data Dictionary values/validations, and relevant rows. -4. Run `dialogue-lab schema-check` against a connector-derived schema JSON document. -5. Record the Case Log modified state and schema signature. -6. Load the Strategy Guide and Evidence Base only when the operation needs them; record their revision states. -7. Record the repository Git commit. - -Never use a repository copy instead of a live canonical read. - -## Operations - -### New case intake +The high-level write commands allocate identifiers, validate all affected records, write inside an immediate SQLite transaction, commit, reopen, and compare the committed records. Errors produce compact JSON on stderr and a nonzero exit code. -Invoke `$dialogue-lab-intake`. Parse every supplied URL, build `Post ID + Root Comment ID`, and search for duplicates before allocation. Immediately before allocation, re-read current Case IDs and unresolved Pending Sync records. An exploratory reply is not logged; only an explicitly approved or explicitly saved unposted reply may become an Outgoing Draft turn. +See [CLI payload contracts](docs/cli-payloads.md) for exact JSON shapes. -### Duplicate intake +## Dialogue workflows -When the identity already exists, do not allocate a new Case ID. Resolve the existing case and continue with `$dialogue-lab-followup`. +- Intake: `$dialogue-lab-intake` +- Follow-up: `$dialogue-lab-followup` +- Posting confirmation: `$dialogue-lab-posting` +- Closeout: `$dialogue-lab-closeout` +- Strategy review: `$dialogue-lab-strategy-review` -### Follow-up and branches +Skills use read commands automatically. They may pass `--approved` only after the user approves the exact canonical write. No skill publishes to Facebook. -Invoke `$dialogue-lab-followup`. Append the exact incoming public turn, assign a case-local participant reference, resolve its direct parent, retain Parent Confidence, and validate the full parent graph. Multiple children may share a parent. Show a Thread Map for a branched Active Exchange. +## Migration -### Posting confirmation +1. Prepare a UTF-8 JSON snapshot with `cases` and `turns` arrays using the exact payload contract. +2. Initialize an empty outside-Git database. +3. Create a verified empty backup. +4. Run `db-import` once after explicit approval. +5. Run `doctor`, compare counts and representative records, create a populated backup, and record `docs/migration-receipt.json.example` with actual values. -Invoke `$dialogue-lab-posting` only after the user confirms publication. Record the exact wording actually posted, even when it differs from an earlier recommendation. Preserve the supplied permalink and identifiers on the matching turn. The skill records state; it never posts to Facebook. +Import is atomic and only accepts an empty database. Duplicate Case IDs, duplicate `Post ID + Root Comment ID`, invalid enums, invalid URLs, missing parents, cycles, and foreign-key violations stop the transaction. -### Closure +## Safety -Invoke `$dialogue-lab-closeout` only under a Manual-allowed closure condition. Record observable outcomes, the highest outcome score, What Worked, What Failed, and one Next Test. Do not infer persuasion from silence, deletion, blocking, a reaction, or disappearance. +- Never commit SQLite databases, journals, exports, backups, credentials, secrets, or public Facebook text. +- Never access Facebook unless explicitly requested; never post autonomously. +- Never overwrite a backup. Rollback restores a verified backup to a new path. +- Keep one canonical writer. -### Strategy review +See [controlled rollout](docs/rollout.md) and [rollback](docs/rollback.md). -Invoke `$dialogue-lab-strategy-review` only on closed cases. Require at least three reasonably comparable cases and treat the first review at twenty closed cases as the first formal review. Report samples and confounders; do not infer causality from simple correlation. Propose Strategy Guide wording but never apply it without separate explicit approval. +## Development -### Pending Sync recovery - -After a failed required write or mismatched read-back, construct a complete PENDING SYNC record in the interaction and do not claim that Drive changed. Reconcile or explicitly resolve it before allocating another Case ID. - -### Source-consistency recovery - -Record source states at operation start and compare them immediately before writing. If a relevant source changed, reload it, revalidate the operation, and regenerate any affected write. If compatibility cannot be established, stop the write and report the Manual and repository versions. - -## CLI - -```text -dialogue-lab doctor -dialogue-lab parse-url -dialogue-lab manual-version -dialogue-lab schema-check -dialogue-lab source-consistency -dialogue-lab case-key --post-id --root-comment-id -dialogue-lab next-case-id --date YYYY-MM-DD --existing -dialogue-lab next-turn-id --case-id --existing -dialogue-lab validate-case -dialogue-lab validate-turn -dialogue-lab validate-transition -dialogue-lab validate-parent-graph -dialogue-lab pending-sync-check -dialogue-lab verify-readback --expected --actual -dialogue-lab migration-receipt +```powershell +uv run pytest +uv run ruff check . +uv run mypy ``` -Commands emit JSON and use non-zero exit status for validation failures. They do not print secrets and expose no Facebook publishing command. - -## Connector boundary - -`DriveProtocol` specifies semantic reads, Case Log writes, read-back, and revision-state operations. The live Google Drive plugin executes outside Python. Codex must translate a validated typed request into the narrow connector call and translate the response back into typed values. A local adapter is not presented as a live connector. - -The initial writer is single-process and non-transactional. Immediately before allocating an ID, re-read rows, check duplicates and Pending Sync, calculate the next ID, verify revision state, write after approval, read back, and compare. An optional future Apps Script or MCP gateway may make allocation atomic; see [connector boundary](docs/drive-connector-boundary.md). - -## Version upgrades - -- Manual version changes: read the new Manual, run compatibility checks, review behavior deltas, update repository compatibility only after evidence, and name the exact synchronized version. -- Case Log header changes: stop writes, inspect the live Data Dictionary and formulas, update header resolution and the schema signature, then rerun targeted tests. -- Enum changes: update enums, schema validation, lifecycle rules, fixtures, and CLI validation together. -- New canonical files: add only non-secret IDs and source-state handling; do not export content into Git. -- Repository compatibility changes: update `project.repository_version`, tests, and rollout receipt expectations in one change. - -## Rollout and rollback - -Follow [the controlled rollout](docs/rollout.md) and [rollback procedure](docs/rollback.md). The previous ChatGPT Project must become read-only reference after cutover; never leave two active writers. A receipt template is at [docs/migration-receipt.json.example](docs/migration-receipt.json.example). - -Do not create cutover tags, named Drive versions, or offline backups during repository construction. Those are production state changes and require the documented gate and explicit approval. - -## Limitations - -- Initial single-writer mode does not provide atomic Google Sheets allocation. -- Python does not call the runtime-managed Google Drive connector directly. -- Every canonical write needs explicit human approval and connector read-back. -- No code publishes to Facebook. -- Parallel writers remain disabled until a narrow atomic gateway exists. +Tests use temporary SQLite databases and synthetic public text only. diff --git a/config/drive-files.toml b/config/drive-files.toml deleted file mode 100644 index 8c57e18..0000000 --- a/config/drive-files.toml +++ /dev/null @@ -1,38 +0,0 @@ -[project] -name = "Israel Facebook Dialogue Lab" -timezone = "Asia/Jerusalem" -repository_version = "0.1.0" - -[folder] -id = "15swU8SH2E1w_eNDDPvRd4j4S8i-M0kEh" - -[operating_manual] -id = "13F4MoD0oar-aamkrLgWVEC66bPPIXQSHkyA-zK3gzaA" -minimum_supported_version = "2.7" -supported_major_version = 2 - -[case_log] -id = "1BysryBcXiA0W5xN_J_kdWj0wuf8RHweSkc6w1c3Tz8k" -schema_signature = "777cb5a318d78c4f95fdec8ac559bbfa230648ae98ad26049f26b7c337cf5c0e" - -[case_log.sheets] -cases = "Cases" -turns = "Turns" -data_dictionary = "Data Dictionary" -strategy_taxonomy = "Strategy Taxonomy" -dashboard = "Dashboard" - -[case_log.headers] -cases = ["Case ID", "Case Title", "Created At", "Updated At", "Status", "Topic", "Post Text", "Post URL", "Post ID", "Root Comment ID", "Source Links", "Privacy Checked", "Outcome Score 0–5", "Outcome Class", "Outcome Notes", "User Rating 1–5", "What Worked", "What Failed", "Next Test", "Closed At"] -turns = ["Case ID", "Turn ID", "Parent Turn ID", "Parent Confidence", "Participant Ref", "Direction", "Kind", "State", "Exact Text", "Post ID", "Root Comment ID", "Reply Comment ID", "Exact URL", "URL Supplied At", "Observed At", "Notes"] - -[reply_strategy_guide] -id = "1-IcaaAxzvPIDNYpO0gVpoI-bgWPOlKZz6-GhXsxJNpw" - -[evidence_base] -id = "1ULMsy7XncP_bXK2TPNJQHOPLcNg2t9eV9XeHMgoPxU0" - -[general_responses] -id = "1Q10V7szfmcoDLme0y3KkDAn_WdYvXVGvwQS0yI-SiGU" -writable = false -canonical = false diff --git a/config/storage.toml b/config/storage.toml new file mode 100644 index 0000000..7357138 --- /dev/null +++ b/config/storage.toml @@ -0,0 +1,12 @@ +[project] +name = "Israel Facebook Dialogue Lab" +timezone = "Asia/Jerusalem" + +[database] +schema_version = 1 +path_environment_variable = "DIALOGUE_LAB_DB" + +[documents] +governance = "AGENTS.md" +strategy_guide = "docs/reply-strategy-guide.md" +evidence_base = "docs/evidence-base.md" diff --git a/docs/cli-payloads.md b/docs/cli-payloads.md new file mode 100644 index 0000000..f037106 --- /dev/null +++ b/docs/cli-payloads.md @@ -0,0 +1,107 @@ +# CLI payload contracts + +All payloads are UTF-8 JSON. Field names are lowercase `snake_case`. Unknown fields and missing required values are rejected before mutation. + +## Database import + +`db-import` accepts one object: + +```json +{ + "cases": [ + { + "case_id": "CASE-20260720-001", + "case_title": "Short title", + "created_at": "2026-07-20 10:00", + "updated_at": "2026-07-20 10:00", + "status": "Posted", + "topic": "Topic", + "post_text": "Exact public post text", + "post_url": "https://www.facebook.com/example/posts/123?comment_id=456", + "post_id": "123", + "root_comment_id": "456", + "source_links": [], + "privacy_checked": true, + "outcome_score": null, + "outcome_class": null, + "outcome_notes": "", + "user_rating": null, + "what_worked": "", + "what_failed": "", + "next_test": "", + "closed_at": "" + } + ], + "turns": [] +} +``` + +Every imported Case and Turn requires its allocated identifier. `source_links` is an array. Open Cases require an exact Facebook comment or reply permalink. + +## Case intake + +`case-intake` accepts a Case without `case_id` and zero or more Turns without `case_id` or `turn_id`: + +```json +{ + "case": { + "case_title": "Short title", + "created_at": "2026-07-20 10:00", + "updated_at": "2026-07-20 10:00", + "status": "Posted", + "topic": "Topic", + "post_text": "Exact public post text", + "post_url": "https://www.facebook.com/example/posts/123?comment_id=456", + "post_id": "123", + "root_comment_id": "456", + "source_links": [], + "privacy_checked": true + }, + "turns": [ + { + "parent_turn_id": null, + "parent_confidence": null, + "participant_ref": "P1", + "direction": "Incoming", + "kind": "Comment", + "state": "Received", + "exact_text": "Exact public comment", + "reply_comment_id": null, + "exact_url": "https://www.facebook.com/example/posts/123?comment_id=456", + "url_supplied_at": "2026-07-20 10:00", + "observed_at": "2026-07-20 10:00", + "notes": "" + } + ] +} +``` + +The command derives Case and Turn identifiers and forces each Turn identity to match the Case. + +## Follow-up and posting + +`case-followup` and `case-record-posting` accept one Turn without `case_id`, `turn_id`, `post_id`, or `root_comment_id`. Required Turn fields match the intake example. + +A posting payload must use `direction: "Outgoing"` and `state: "Posted"`. It may include `draft_turn_id` to mark one existing Outgoing Draft as Replaced in the same transaction. + +## Closeout + +`case-close` accepts: + +```json +{ + "status": "Closed - Substantive", + "updated_at": "2026-07-21 10:00", + "outcome_score": 3, + "outcome_class": "Substantive Engagement", + "outcome_notes": "Observable outcome only", + "user_rating": null, + "what_worked": "Concise observation", + "what_failed": "Concise observation", + "next_test": "One controlled test", + "closed_at": "2026-07-21 10:00", + "reason": "explicit closeout" +} +``` + +Only closure fields are updated. The Case identity and public context remain unchanged. diff --git a/docs/drive-connector-boundary.md b/docs/drive-connector-boundary.md deleted file mode 100644 index be1e207..0000000 --- a/docs/drive-connector-boundary.md +++ /dev/null @@ -1,19 +0,0 @@ -# Google Drive connector boundary - -The Codex runtime owns Google Drive authentication and connector invocation. Python owns deterministic request validation and response verification. Do not place OAuth credentials, cookies, service-account keys, or provider internals in this repository. - -## Required semantic operations - -Reads: `read_operating_manual`, `read_strategy_guide`, `read_evidence_base`, `read_case_log_schema`, `read_case_rows`, `read_turn_rows`, `find_case_by_identity`, `find_unresolved_pending_sync`, and `read_file_revision_state`. - -Approval-gated Case Log writes: `append_case`, `update_case`, `append_turn`, and `update_turn`. - -Verification reads: `read_back_case` and `read_back_turn`. - -The runtime must validate a `DriveWriteRequest`, recheck source state, execute one narrow connector call, perform a targeted read-back, and run deterministic comparison. Connector acknowledgement is not success. - -## Optional atomic gateway - -A future Apps Script or MCP gateway may expose only `findCaseByIdentity`, `allocateCase`, `appendCase`, `appendTurn`, `updateTurn`, `recordPostedReply`, `updateCaseStatus`, `closeCase`, and `readCanonicalDocument`. - -It must provide atomic identifier allocation, exact schema validation, source-revision checks, single-purpose methods, read-back responses, and actionable errors. It must expose no generic spreadsheet writer, `General responses` access, credential output, or Facebook publishing. diff --git a/docs/evidence-base.md b/docs/evidence-base.md new file mode 100644 index 0000000..d0d867a --- /dev/null +++ b/docs/evidence-base.md @@ -0,0 +1,5 @@ +# Evidence Base + +No reusable evidence entries have been imported into the local canonical repository. + +Case-specific fact checks and source links belong in SQLite. Add reusable evidence here only through a reviewed commit. diff --git a/docs/migration-receipt.json.example b/docs/migration-receipt.json.example index d6675f6..1a36a73 100644 --- a/docs/migration-receipt.json.example +++ b/docs/migration-receipt.json.example @@ -1,24 +1,19 @@ { - "case_log_modified_state": "YYYY-MM-DDTHH:MM:SSZ", - "case_log_schema_signature": "64-character SHA-256", - "codex_environment": "Codex app", "cutover_timestamp": "YYYY-MM-DDTHH:MM:SS+03:00", - "drive_connection_status": "read-only validated; approved write not yet exercised", + "timezone": "Asia/Jerusalem", + "database_schema_version": 1, + "database_integrity": "ok", + "database_backup": "external verified backup path", + "imported_case_count": 0, + "imported_turn_count": 0, + "repository_commit": "Git commit", + "test_results": "pytest, Ruff, and mypy passed", "known_limitations": [ - "single-writer mode", - "no atomic Sheets allocation" + "single local writer" ], - "manual_revision_state": "Drive revision or modified state", - "manual_version": "2.7", - "previous_writer_disabled": false, - "repository_commit": "Git commit", - "repository_tag": "pre-codex-cutover", "rollback_instructions": [ - "Disable Codex writes", - "Reconcile Pending Sync", - "Enable exactly one writer" - ], - "test_results": "pytest, Ruff, and mypy passed", - "timezone": "Asia/Jerusalem", - "writer_enabled": false + "Stop canonical writes", + "Restore the verified SQLite backup", + "Run dialogue-lab doctor" + ] } diff --git a/docs/reply-strategy-guide.md b/docs/reply-strategy-guide.md new file mode 100644 index 0000000..69cc13b --- /dev/null +++ b/docs/reply-strategy-guide.md @@ -0,0 +1,5 @@ +# Reply Strategy Guide + +No cross-case strategy changes have been approved in the local canonical repository. + +Case-specific observations belong in SQLite. Proposed changes require a separate explicit approval and a reviewed commit. diff --git a/docs/rollback.md b/docs/rollback.md index f964660..411e0cb 100644 --- a/docs/rollback.md +++ b/docs/rollback.md @@ -2,24 +2,16 @@ ## Pre-cutover checkpoint -1. Record the exact Manual version and Drive revision or modified state. -2. Record the Case Log schema signature and modified state. -3. Record the repository commit and validated check results. -4. After explicit approval, create local tag `pre-codex-cutover`. -5. Request named Drive versions where the live surface supports them. -6. After explicit approval, export offline backups of the four canonical files to an encrypted location outside the repository. Never include `General responses` unless explicitly requested. -7. Generate and verify a migration receipt. - -After verified cutover, create local tag `codex-v1.0` only with explicit approval. +1. Record the repository commit, schema version, schema signature, database integrity, and row counts. +2. After explicit approval, create a consistent SQLite backup outside Git and verify it with `db-status`. +3. Generate and verify a migration receipt. ## Rollback -1. Obtain explicit user authorization. -2. Disable Codex writes and verify no operation is in progress. -3. Reconcile every Pending Sync record. -4. Restore or select the recorded Drive checkpoint when necessary. -5. Verify Manual revision, Case Log schema, Case Log modified state, and required records. -6. Re-enable exactly one writer. -7. Generate a rollback migration receipt. +1. Obtain explicit user authorization and stop canonical writes. +2. Preserve the failed database for diagnosis; do not overwrite it. +3. Restore the verified backup to a new outside-Git path. +4. Point `DIALOGUE_LAB_DB` to the restored database and run `dialogue-lab doctor`. +5. Verify schema, integrity, Case count, Turn count, and the affected records before resuming one writer. -Never leave Codex and the previous runtime writable at the same time. +Never copy, restore, or overwrite a canonical database without explicit approval. diff --git a/docs/rollout.md b/docs/rollout.md index acc7ff5..8c0c680 100644 --- a/docs/rollout.md +++ b/docs/rollout.md @@ -1,37 +1,31 @@ # Controlled rollout -## Stage 1 — Local deterministic validation +## Stage 1 - Local deterministic validation -Run the frozen dependency sync, unit tests, Ruff, mypy, CLI smoke checks, synthetic eval validation, and lock reproducibility checks. Use synthetic fixtures only. +Run unit tests, Ruff, mypy, CLI smoke checks, synthetic eval validation, and lock reproducibility checks. Use temporary SQLite databases only. -## Stage 2 — Read-only Drive validation +## Stage 2 - Canonical content preparation -Connect the Google Drive plugin. Read all four canonical sources, verify Manual compatibility, verify exact Case Log schema and formulas, confirm `General responses` classification, and record source revision states. Perform no writes. +Review repository governance, strategy, and evidence Markdown. Keep `General responses` outside the workflow unless the user explicitly names an exact action for it. -## Stage 3 — Controlled write test +## Stage 3 - Empty database validation -Only after explicit approval and only when the Manual permits it, perform one safe production Case Log write and read every field back. Do not create disposable test pollution. Reconcile the record only through an authorized canonical lifecycle action. +After explicit approval, initialize a database outside Git, run `dialogue-lab doctor`, create a verified backup, and record its schema signature and integrity result. -## Stage 4 — Shadow use +## Stage 4 - Deterministic import -Run Codex alongside the previous workflow while exactly one system remains writable. Compare identity, turns, exact URLs, statuses, parent graphs, factual classifications, and reply outputs. Fix the producing implementation rather than patching isolated outputs. Do not duplicate canonical public text in shadow logs. +Prepare one JSON snapshot containing Cases and Turns. Run `db-import` once after explicit approval. The import must be atomic, preserve exact public text and URLs, reject duplicate identity, and pass committed count and integrity checks. -Use at least ten representative shadow cases unless fewer cases exist or the user explicitly approves less. Include new intake, duplicate intake, a branch, ambiguous parentage, posting confirmation, closure, hostility, a legal claim, and a failed-write simulation. +## Stage 5 - Shadow validation -## Stage 5 — Cutover gate +Compare imported identities, turns, exact URLs, statuses, parent graphs, classifications, and representative high-level command results against the source snapshot. Fix the producing implementation instead of patching individual rows. -Require zero duplicate-identity mismatches, URL corruption incidents, parent-graph failures, unverified writes, protected-file write attempts, and unresolved Pending Sync records. Require all deterministic tests, lint, and type checks to pass, successful read-only Drive validation, successful approved read-back write validation when exercised, the minimum shadow sample, and explicit user approval. +## Stage 6 - Cutover gate -## Stage 6 — Cutover +Require zero duplicate identities, URL corruption incidents, graph failures, unverified writes, schema mismatches, or integrity failures. Require checks passed, a verified backup, a complete migration receipt, and explicit user approval. -Make Codex the only canonical writer, freeze the previous ChatGPT Project in reference mode, record the migration receipt, create the approved repository tag, and remain in single-writer mode. Re-enabling the old writer is an explicit rollback decision. +## Stage 7 - Cutover -## Stage 7 — Optional private GitHub and CI +Set `DIALOGUE_LAB_DB` to the verified external database and use only high-level `dialogue-lab` commands for canonical Case and Turn operations. Keep one writer. -Only after explicit approval, create a private remote, push validated history, require pull-request checks, and enable secret/dependency scanning. CI must exclude canonical exports, backups, and credentials and must never write to production Drive. - -## Previous runtime modes - -Reference mode may read historical conversations but may not allocate IDs, append turns, update lifecycle state, or write canonical files. - -Rollback mode requires explicit authorization and the procedure in `docs/rollback.md`. Exactly one writer must remain enabled. +Rollback requires explicit authorization and `docs/rollback.md`. diff --git a/evals/README.md b/evals/README.md index b16c75e..ff6857e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,8 +1,8 @@ # Synthetic evals -`cases/synthetic-cases.json` contains no real Case Log rows, participant names, or Facebook text. +`cases/synthetic-cases.json` contains no real Case or Turn rows, participant names, or Facebook text. -- Deterministic assertions: URL identity, identifier allocation, lifecycle, graph, schema, Pending Sync, read-back, and source-consistency expectations are executable tests. -- Qualitative reply review: claim alignment, one pivotal point, natural thread language, legal precision, face-saving correction, and silent-reader usefulness require human or model review under the live Manual. -- Drive integration checks: connector reads, revision states, schema inspection, approval-gated writes, and read-back must run against the live sources and must never be inferred from synthetic results. -- Human approval: canonical writes, Strategy Guide edits, cutover, backups, tags, remote publication, and rollback require explicit approval. +- Deterministic assertions cover URL identity, identifier allocation, lifecycle, graph validation, SQLite schema and integrity, approval gates, committed read-back, and rollback behavior. +- Qualitative reply review covers claim alignment, one pivotal point, natural thread language, legal precision, face-saving correction, and silent-reader usefulness. +- Storage validation runs only against temporary local SQLite databases and never contacts an external service. +- Canonical writes, repository strategy edits, cutover, backups, tags, remote publication, and rollback require explicit approval. diff --git a/evals/cases/synthetic-cases.json b/evals/cases/synthetic-cases.json index a7e9b4d..7499756 100644 --- a/evals/cases/synthetic-cases.json +++ b/evals/cases/synthetic-cases.json @@ -12,8 +12,8 @@ {"id": "edited-posting-wording", "kind": "posting", "expected": "exact published wording preserved"}, {"id": "substantive-closeout", "kind": "closeout", "expected": "observable outcome recorded"}, {"id": "no-response-closeout", "kind": "closeout", "expected": "no persuasion inferred"}, - {"id": "failed-drive-write", "kind": "recovery", "expected": "Pending Sync blocks allocation"}, - {"id": "schema-incompatibility", "kind": "schema", "expected": "canonical writes blocked"}, - {"id": "manual-changed-mid-operation", "kind": "source-consistency", "expected": "reload required"}, - {"id": "unsupported-manual-major", "kind": "compatibility", "expected": "fail safely"} + {"id": "sqlite-write-rollback", "kind": "recovery", "expected": "transaction rolls back and integrity remains valid"}, + {"id": "schema-version-mismatch", "kind": "schema", "expected": "canonical writes blocked"}, + {"id": "missing-local-document", "kind": "readiness", "expected": "doctor fails safely"}, + {"id": "unapproved-sqlite-write", "kind": "write-safety", "expected": "write rejected before database mutation"} ] diff --git a/pyproject.toml b/pyproject.toml index d87faf5..a45d608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "israel-facebook-dialogue-lab" -version = "0.1.0" -description = "Non-canonical Codex execution and validation layer for the Israel Facebook Dialogue Lab" +version = "0.2.0" +description = "Deterministic local execution and SQLite storage for the Israel Facebook Dialogue Lab" readme = "README.md" requires-python = ">=3.12" license = { text = "Proprietary" } diff --git a/src/dialogue_lab/__init__.py b/src/dialogue_lab/__init__.py index 6603f92..e41be7c 100644 --- a/src/dialogue_lab/__init__.py +++ b/src/dialogue_lab/__init__.py @@ -1,3 +1,3 @@ """Deterministic execution layer for the Israel Facebook Dialogue Lab.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/dialogue_lab/cli.py b/src/dialogue_lab/cli.py index cbb5bd6..a967965 100644 --- a/src/dialogue_lab/cli.py +++ b/src/dialogue_lab/cli.py @@ -4,9 +4,12 @@ import argparse import json +import os +import sqlite3 import sys import tomllib from collections.abc import Mapping +from dataclasses import replace from datetime import date from pathlib import Path from typing import Any @@ -14,24 +17,17 @@ from . import __version__ from .case_identity import make_case_identity from .enums import CaseStatus, OutcomeClass, ParentConfidence, TurnDirection, TurnKind, TurnState -from .errors import DialogueLabError +from .errors import DialogueLabError, StorageError, WriteSafetyError from .facebook_url import parse_facebook_url from .identifiers import next_case_id, next_turn_id -from .lifecycle import validate_transition -from .manual_version import require_supported_manual +from .lifecycle import CLOSED_STATUSES, validate_posted_turn, validate_transition from .migration_receipt import migration_receipt_from_mapping, render_migration_receipt -from .models import ( - CaseRecord, - LifecycleTransition, - PendingSyncRecord, - TurnRecord, - to_jsonable, -) +from .models import CaseRecord, LifecycleTransition, TurnRecord, to_jsonable from .parent_graph import validate_parent_graph -from .pending_sync import require_no_pending_sync from .readback import verify_readback -from .schema import schema_signature, validate_schema -from .source_consistency import check_source_consistency +from .schema import CASE_FIELDS, SCHEMA_VERSION, TURN_FIELDS +from .source_consistency import file_sha256 +from .storage import SQLiteStore from .validation import validate_case, validate_turn @@ -57,7 +53,30 @@ def _optional_string(value: object) -> str | None: return None if value is None else str(value) +def _optional_int(value: object) -> int | None: + return None if value is None else int(str(value)) + + +def _boolean(value: object, label: str) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str) and value.lower() in {"true", "yes"}: + return True + if isinstance(value, str) and value.lower() in {"false", "no"}: + return False + raise DialogueLabError(f"{label} must be a boolean") + + +def _reject_unknown( + payload: Mapping[str, Any], allowed: set[str], label: str +) -> None: + unknown = sorted(set(payload) - allowed) + if unknown: + raise DialogueLabError(f"unknown {label} fields: {', '.join(unknown)}") + + def _case_record(payload: Mapping[str, Any]) -> CaseRecord: + _reject_unknown(payload, set(CASE_FIELDS), "Case") links = payload.get("source_links", []) return CaseRecord( case_id=str(payload.get("case_id", "")), @@ -71,7 +90,7 @@ def _case_record(payload: Mapping[str, Any]) -> CaseRecord: post_id=str(payload.get("post_id", "")), root_comment_id=str(payload.get("root_comment_id", "")), source_links=tuple(str(item) for item in _list(links, "source_links")), - privacy_checked=bool(payload.get("privacy_checked", False)), + privacy_checked=_boolean(payload.get("privacy_checked", False), "privacy_checked"), outcome_score=_optional_int(payload.get("outcome_score")), outcome_class=( OutcomeClass(str(payload["outcome_class"])) @@ -87,11 +106,8 @@ def _case_record(payload: Mapping[str, Any]) -> CaseRecord: ) -def _optional_int(value: object) -> int | None: - return None if value is None else int(str(value)) - - def _turn_record(payload: Mapping[str, Any]) -> TurnRecord: + _reject_unknown(payload, set(TURN_FIELDS), "Turn") return TurnRecord( case_id=str(payload.get("case_id", "")), turn_id=str(payload.get("turn_id", "")), @@ -116,86 +132,122 @@ def _turn_record(payload: Mapping[str, Any]) -> TurnRecord: ) -def _pending_record(payload: Mapping[str, Any]) -> PendingSyncRecord: - return PendingSyncRecord( - operation=str(payload.get("operation", "")), - target_file=str(payload.get("target_file", "")), - target_sheet=str(payload.get("target_sheet", "")), - case_id=str(payload.get("case_id", "")), - turn_id=_optional_string(payload.get("turn_id")), - expected_values=_mapping(payload.get("expected_values", {}), "expected_values"), - failure=str(payload.get("failure", "")), - last_verified_state=_mapping( - payload.get("last_verified_state", {}), "last_verified_state" - ), - manual_version=str(payload.get("manual_version", "")), - source_revision_state={ - str(key): str(value) - for key, value in _mapping( - payload.get("source_revision_state", {}), "source_revision_state" - ).items() - }, - required_reconciliation_action=str(payload.get("required_reconciliation_action", "")), - created_at=str(payload.get("created_at", "")), - resolved=bool(payload.get("resolved", False)), +def _print(value: object) -> None: + print( + json.dumps( + to_jsonable(value), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) ) -def _print(value: object) -> None: - print(json.dumps(to_jsonable(value), ensure_ascii=False, indent=2, sort_keys=True)) +def _repo_root(start: Path) -> Path: + current = start.resolve() + for candidate in (current, *current.parents): + if (candidate / ".git").exists(): + return candidate + raise StorageError("run the command from the HasbaraTops repository") + + +def _outside_repository(path: Path) -> Path: + resolved = path.resolve() + root = _repo_root(Path.cwd()) + try: + resolved.relative_to(root) + except ValueError: + return resolved + raise WriteSafetyError("canonical databases, exports, and backups must stay outside Git") -def _doctor() -> dict[str, object]: - root = Path.cwd() - config_path = root / "config" / "drive-files.toml" +def _storage_config(root: Path) -> dict[str, Any]: + config_path = root / "config" / "storage.toml" + if not config_path.is_file(): + raise StorageError(f"storage config does not exist: {config_path}") + with config_path.open("rb") as stream: + return tomllib.load(stream) + + +def _database_path(args: argparse.Namespace) -> Path: + config = _storage_config(_repo_root(Path.cwd())) + environment_variable = str(config["database"]["path_environment_variable"]) + configured = args.database or os.environ.get(environment_variable) + if not configured: + raise StorageError(f"set {environment_variable} or pass --database") + return _outside_repository(Path(str(configured))) + + +def _store(args: argparse.Namespace) -> SQLiteStore: + return SQLiteStore(_database_path(args)) + + +def _doctor(args: argparse.Namespace) -> dict[str, object]: + root = _repo_root(Path.cwd()) + config_path = root / "config" / "storage.toml" + config = _storage_config(root) + documents = config["documents"] checks: dict[str, bool] = { - "git_repository": (root / ".git").exists(), - "drive_config": config_path.exists(), - "agents_bootstrap": (root / "AGENTS.md").exists(), + "storage_config": config_path.is_file(), + "agents": (root / str(documents["governance"])).is_file(), + "strategy_guide": (root / str(documents["strategy_guide"])).is_file(), + "evidence_base": (root / str(documents["evidence_base"])).is_file(), + } + configured_version = int(config["database"]["schema_version"]) + checks["configured_schema_version"] = configured_version == SCHEMA_VERSION + document_revisions = { + name: file_sha256(root / str(path)) + for name, path in documents.items() + if (root / str(path)).is_file() } - configured_signature = "" - if config_path.exists(): - with config_path.open("rb") as stream: - config = tomllib.load(stream) - configured_signature = str(config["case_log"]["schema_signature"]) - checks["schema_signature"] = configured_signature == schema_signature().value + status = _store(args).status() + checks["database_integrity"] = status["integrity"] == "ok" return { "ok": all(checks.values()), "checks": checks, - "repository_schema_signature": schema_signature().value, - "configured_schema_signature": configured_signature, - "drive_connection": "runtime-managed; not exercised by the Python doctor", - "writer_policy": "single writer; explicit approval; read-back required", + "database": status, + "document_revisions": document_revisions, + "configured_schema_version": configured_version, + "writer_policy": "transactional; explicit approval; committed read-back", } +def _add_database_write_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--approved", action="store_true") + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="dialogue-lab", - description="Deterministic validation for the Israel Facebook Dialogue Lab", + description="Deterministic local operations for the Israel Facebook Dialogue Lab", ) parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + parser.add_argument("--database", help="SQLite path; defaults to DIALOGUE_LAB_DB") sub = parser.add_subparsers(dest="command", required=True) - sub.add_parser("doctor", help="check local repository/configuration readiness") + sub.add_parser("doctor", help="verify local docs, configuration, and database integrity") + init = sub.add_parser("db-init", help="initialize or verify the canonical SQLite database") + _add_database_write_arguments(init) + sub.add_parser("db-status", help="report schema, integrity, and row counts") + backup = sub.add_parser("db-backup", help="create a consistent non-overwriting backup") + backup.add_argument("--destination", required=True) + _add_database_write_arguments(backup) + import_parser = sub.add_parser("db-import", help="atomically import cases and turns JSON") + import_parser.add_argument("json_file") + _add_database_write_arguments(import_parser) + parse = sub.add_parser("parse-url", help="parse a Facebook URL without rewriting it") parse.add_argument("url") - manual = sub.add_parser("manual-version", help="parse and validate Manual text or a file") - manual.add_argument("text_or_file") - schema = sub.add_parser("schema-check", help="validate an observed schema JSON file") - schema.add_argument("schema_json") - consistency = sub.add_parser("source-consistency", help="compare source revision states") - consistency.add_argument("json_file") case_key = sub.add_parser("case-key", help="generate normalized canonical case identity") case_key.add_argument("--post-id", required=True) case_key.add_argument("--root-comment-id", required=True) - case_id = sub.add_parser("next-case-id", help="allocate a date-local Case ID") + case_id = sub.add_parser("next-case-id", help="calculate a date-local Case ID") case_id.add_argument("--date", required=True) case_id.add_argument("--existing", required=True) - case_id.add_argument("--pending-sync") - turn_id = sub.add_parser("next-turn-id", help="allocate a case-local Turn ID") + turn_id = sub.add_parser("next-turn-id", help="calculate a case-local Turn ID") turn_id.add_argument("--case-id", required=True) turn_id.add_argument("--existing", required=True) + validate_case_parser = sub.add_parser("validate-case", help="validate a Case record") validate_case_parser.add_argument("json_file") validate_turn_parser = sub.add_parser("validate-turn", help="validate a Turn record") @@ -204,13 +256,36 @@ def _build_parser() -> argparse.ArgumentParser: transition.add_argument("json_file") graph = sub.add_parser("validate-parent-graph", help="validate a case-local parent graph") graph.add_argument("json_file") - pending = sub.add_parser("pending-sync-check", help="fail while PENDING SYNC is unresolved") - pending.add_argument("json_file") readback = sub.add_parser("verify-readback", help="compare expected and actual fields") readback.add_argument("--expected", required=True) readback.add_argument("--actual", required=True) receipt = sub.add_parser("migration-receipt", help="validate and render a migration receipt") receipt.add_argument("json_file") + + find = sub.add_parser("case-find", help="resolve one Case by canonical identity") + find.add_argument("--post-id", required=True) + find.add_argument("--root-comment-id", required=True) + show = sub.add_parser("case-show", help="return one Case and its complete Turn graph") + show.add_argument("--case-id", required=True) + sub.add_parser("case-list-open", help="return exact open-case permalink summaries") + sub.add_parser("strategy-dataset", help="return all closed Cases and their Turns") + + intake = sub.add_parser("case-intake", help="create one Case and initial Turns atomically") + intake.add_argument("json_file") + intake.add_argument("--date", required=True) + _add_database_write_arguments(intake) + followup = sub.add_parser("case-followup", help="record one incoming Turn atomically") + followup.add_argument("--case-id", required=True) + followup.add_argument("json_file") + _add_database_write_arguments(followup) + posting = sub.add_parser("case-record-posting", help="record a confirmed posted reply") + posting.add_argument("--case-id", required=True) + posting.add_argument("json_file") + _add_database_write_arguments(posting) + close = sub.add_parser("case-close", help="close one Case atomically") + close.add_argument("--case-id", required=True) + close.add_argument("json_file") + _add_database_write_arguments(close) return parser @@ -227,52 +302,195 @@ def _ids_from_json(value: object, key: str, case_id: str | None = None) -> list[ return output +def _turn_for_case( + payload: Mapping[str, Any], case: CaseRecord, turn_id: str +) -> TurnRecord: + values = dict(payload) + values.pop("draft_turn_id", None) + values.update( + { + "case_id": case.case_id, + "turn_id": turn_id, + "post_id": case.post_id, + "root_comment_id": case.root_comment_id, + } + ) + return _turn_record(values) + + +def _run_database_command(args: argparse.Namespace, command: str) -> object: + store = _store(args) + if command == "db-init": + return store.initialize(approved=bool(args.approved)) + if command == "db-status": + return store.status() + if command == "db-backup": + destination = _outside_repository(Path(str(args.destination))) + return store.backup(destination, approved=bool(args.approved)) + if command == "db-import": + payload = _mapping(_load_json(str(args.json_file)), "database import") + _reject_unknown(payload, {"cases", "turns"}, "database import") + cases = [ + _case_record(_mapping(item, "Case import row")) + for item in _list(payload.get("cases")) + ] + turns = [ + _turn_record(_mapping(item, "Turn import row")) + for item in _list(payload.get("turns")) + ] + return store.import_records(cases, turns, approved=bool(args.approved)) + if command == "case-find": + case = store.find_case(str(args.post_id), str(args.root_comment_id)) + result: dict[str, object] = {"found": False} + if case is not None: + result = {"found": True, "case_id": case.case_id, "status": case.status.value} + return result + if command == "case-show": + case_id = str(args.case_id) + return {"case": store.get_case(case_id), "turns": store.get_turns(case_id)} + if command == "case-list-open": + return store.list_open_summaries() + if command == "strategy-dataset": + return store.strategy_dataset() + if command == "case-intake": + payload = _mapping(_load_json(str(args.json_file)), "case intake") + _reject_unknown(payload, {"case", "turns"}, "case intake") + case_payload = _mapping(payload.get("case"), "case") + existing = store.find_case( + str(case_payload.get("post_id", "")), str(case_payload.get("root_comment_id", "")) + ) + if existing is not None: + return { + "created": False, + "duplicate": True, + "case_id": existing.case_id, + "status": existing.status.value, + } + case_id = next_case_id(store.case_ids(), on_date=date.fromisoformat(str(args.date))) + case = _case_record({**case_payload, "case_id": case_id}) + initial_turns: list[TurnRecord] = [] + for item in _list(payload.get("turns", []), "turns"): + turn_id = next_turn_id(turn.turn_id for turn in initial_turns) + initial_turns.append( + _turn_for_case(_mapping(item, "initial Turn"), case, turn_id) + ) + return store.create_case(case, initial_turns, approved=bool(args.approved)) + if command in {"case-followup", "case-record-posting"}: + case_id = str(args.case_id) + case = store.get_case(case_id) + if case.status in CLOSED_STATUSES: + raise DialogueLabError(f"case is closed: {case_id}") + payload = _mapping(_load_json(str(args.json_file)), "Turn payload") + _reject_unknown(payload, {*TURN_FIELDS, "draft_turn_id"}, "Turn") + turn_id = next_turn_id(store.turn_ids(case_id)) + turn = _turn_for_case(payload, case, turn_id) + if command == "case-followup": + if turn.direction is not TurnDirection.INCOMING: + raise DialogueLabError("case-followup requires an Incoming Turn") + target_status = CaseStatus.ACTIVE_EXCHANGE + reason = "incoming public turn" + replace_draft_id = None + else: + validate_posted_turn(turn) + target_status = CaseStatus.POSTED if case.status is CaseStatus.DRAFT else case.status + reason = "posting confirmed" + replace_draft_id = _optional_string(payload.get("draft_turn_id")) + return store.add_turn( + turn, + target_status=target_status, + updated_at=turn.observed_at, + reason=reason, + replace_draft_id=replace_draft_id, + approved=bool(args.approved), + ) + if command == "case-close": + case = store.get_case(str(args.case_id)) + payload = _mapping(_load_json(str(args.json_file)), "case close") + close_fields = { + "status", + "updated_at", + "outcome_score", + "outcome_class", + "outcome_notes", + "user_rating", + "what_worked", + "what_failed", + "next_test", + "closed_at", + "reason", + } + _reject_unknown(payload, close_fields, "case close") + status = CaseStatus(str(payload.get("status", ""))) + if status not in CLOSED_STATUSES: + raise DialogueLabError("case-close requires a Closed status") + required_text = ( + "updated_at", + "outcome_notes", + "what_worked", + "what_failed", + "next_test", + "closed_at", + ) + missing = [name for name in required_text if not str(payload.get(name, "")).strip()] + if missing or payload.get("outcome_score") is None or payload.get("outcome_class") is None: + raise DialogueLabError("case-close payload lacks required outcome fields") + updated = replace( + case, + updated_at=str(payload["updated_at"]), + status=status, + outcome_score=int(str(payload["outcome_score"])), + outcome_class=OutcomeClass(str(payload["outcome_class"])), + outcome_notes=str(payload["outcome_notes"]), + user_rating=_optional_int(payload.get("user_rating")), + what_worked=str(payload["what_worked"]), + what_failed=str(payload["what_failed"]), + next_test=str(payload["next_test"]), + closed_at=str(payload["closed_at"]), + ) + return store.close_case( + updated, + reason=str(payload.get("reason", "explicit closeout")), + approved=bool(args.approved), + ) + raise DialogueLabError(f"unknown database command: {command}") + + def _run(args: argparse.Namespace) -> object: command = str(args.command) if command == "doctor": - result = _doctor() + result = _doctor(args) if not result["ok"]: raise DialogueLabError("local doctor checks failed") return result + database_commands = { + "db-init", + "db-status", + "db-backup", + "db-import", + "case-find", + "case-show", + "case-list-open", + "strategy-dataset", + "case-intake", + "case-followup", + "case-record-posting", + "case-close", + } + if command in database_commands: + return _run_database_command(args, command) if command == "parse-url": return parse_facebook_url(str(args.url)) - if command == "manual-version": - source = Path(str(args.text_or_file)) - text = source.read_text(encoding="utf-8") if source.is_file() else str(args.text_or_file) - version, warnings = require_supported_manual(text) - return {"version": str(version), "warnings": warnings} - if command == "schema-check": - signature = validate_schema(_mapping(_load_json(str(args.schema_json)))) - return {"compatible": True, "schema_signature": signature.value} - if command == "source-consistency": - payload = _mapping(_load_json(str(args.json_file))) - start = {str(k): str(v) for k, v in _mapping(payload.get("operation_start", {})).items()} - current = {str(k): str(v) for k, v in _mapping(payload.get("current", {})).items()} - material_value = payload.get("material_sources") - material = ( - [str(item) for item in _list(material_value, "material_sources")] - if material_value is not None - else None - ) - return check_source_consistency(start, current, material_sources=material) if command == "case-key": return {"case_key": make_case_identity(args.post_id, args.root_comment_id).key} if command == "next-case-id": existing = _ids_from_json(_load_json(str(args.existing)), "case_id") - pending_records: list[PendingSyncRecord] = [] - if args.pending_sync: - pending_records = [ - _pending_record(_mapping(item)) - for item in _list(_load_json(str(args.pending_sync))) - ] - allocated = next_case_id( - existing, - on_date=date.fromisoformat(str(args.date)), - pending_sync=pending_records, - ) - return {"case_id": allocated} + return { + "case_id": next_case_id(existing, on_date=date.fromisoformat(str(args.date))) + } if command == "next-turn-id": - existing = _ids_from_json(_load_json(str(args.existing)), "turn_id", str(args.case_id)) + existing = _ids_from_json( + _load_json(str(args.existing)), "turn_id", str(args.case_id) + ) return {"case_id": args.case_id, "turn_id": next_turn_id(existing)} if command == "validate-case": validate_case(_case_record(_mapping(_load_json(str(args.json_file))))) @@ -294,12 +512,6 @@ def _run(args: argparse.Namespace) -> object: turns = [_turn_record(_mapping(item)) for item in _list(_load_json(str(args.json_file)))] validate_parent_graph(turns) return {"valid": True, "turn_count": len(turns)} - if command == "pending-sync-check": - records = [ - _pending_record(_mapping(item)) for item in _list(_load_json(str(args.json_file))) - ] - require_no_pending_sync(records) - return {"clear": True} if command == "verify-readback": expected = _mapping(_load_json(str(args.expected))) actual = _mapping(_load_json(str(args.actual))) @@ -321,9 +533,20 @@ def main(argv: list[str] | None = None) -> int: try: _print(_run(args)) return 0 - except (DialogueLabError, OSError, ValueError, KeyError) as error: - category = getattr(error, "category", "input_error") - print(json.dumps({"ok": False, "error": str(error), "category": category}), file=sys.stderr) + except (DialogueLabError, OSError, ValueError, KeyError, sqlite3.Error) as error: + category = ( + "storage_error" + if isinstance(error, sqlite3.Error) + else getattr(error, "category", "input_error") + ) + print( + json.dumps( + {"ok": False, "error": str(error), "category": category}, + separators=(",", ":"), + sort_keys=True, + ), + file=sys.stderr, + ) return 2 diff --git a/src/dialogue_lab/drive_models.py b/src/dialogue_lab/drive_models.py deleted file mode 100644 index 02e2e31..0000000 --- a/src/dialogue_lab/drive_models.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Typed envelopes for the runtime-managed Google Drive connector boundary.""" - -from collections.abc import Mapping -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ConnectorRequest: - action: str - file_id: str - sheet: str | None - payload: Mapping[str, object] - source_revision_state: Mapping[str, str] - - -@dataclass(frozen=True) -class ConnectorResponse: - connector_reported_success: bool - record_identity: str | None - values: Mapping[str, object] - revision_state: Mapping[str, str] - error_category: str | None = None diff --git a/src/dialogue_lab/drive_protocol.py b/src/dialogue_lab/drive_protocol.py deleted file mode 100644 index 8bf93a8..0000000 --- a/src/dialogue_lab/drive_protocol.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Provider-independent Drive protocol and write-policy boundary. - -The Codex Google Drive plugin is runtime-managed and is not imported by Python. -Implementations must execute connector calls outside this package, then pass typed -responses back through deterministic validation and read-back comparison. -""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from typing import Protocol - -from .drive_models import ConnectorRequest -from .errors import WriteSafetyError -from .models import DriveWriteRequest, SourceRevisionState - -CASE_LOG_ID = "1BysryBcXiA0W5xN_J_kdWj0wuf8RHweSkc6w1c3Tz8k" -GENERAL_RESPONSES_ID = "1Q10V7szfmcoDLme0y3KkDAn_WdYvXVGvwQS0yI-SiGU" -STRATEGY_GUIDE_ID = "1-IcaaAxzvPIDNYpO0gVpoI-bgWPOlKZz6-GhXsxJNpw" -EVIDENCE_BASE_ID = "1ULMsy7XncP_bXK2TPNJQHOPLcNg2t9eV9XeHMgoPxU0" - - -class DriveProtocol(Protocol): - """Semantic operations required by Dialogue Lab workflows.""" - - def read_operating_manual(self) -> tuple[str, SourceRevisionState]: ... - def read_strategy_guide(self) -> tuple[str, SourceRevisionState]: ... - def read_evidence_base(self) -> tuple[str, SourceRevisionState]: ... - def read_case_log_schema(self) -> Mapping[str, object]: ... - def read_case_rows(self) -> Sequence[Mapping[str, object]]: ... - def read_turn_rows(self, case_id: str | None = None) -> Sequence[Mapping[str, object]]: ... - def find_case_by_identity( - self, post_id: str, root_comment_id: str - ) -> Mapping[str, object] | None: ... - def find_unresolved_pending_sync(self) -> Sequence[Mapping[str, object]]: ... - def append_case(self, request: DriveWriteRequest) -> Mapping[str, object]: ... - def update_case(self, request: DriveWriteRequest) -> Mapping[str, object]: ... - def append_turn(self, request: DriveWriteRequest) -> Mapping[str, object]: ... - def update_turn(self, request: DriveWriteRequest) -> Mapping[str, object]: ... - def read_back_case(self, case_id: str) -> Mapping[str, object]: ... - def read_back_turn(self, case_id: str, turn_id: str) -> Mapping[str, object]: ... - def read_file_revision_state(self, file_id: str) -> SourceRevisionState: ... - - -def validate_drive_write_request(request: DriveWriteRequest) -> None: - """Enforce the initial least-privilege, explicit-approval write policy.""" - if not request.explicitly_approved: - raise WriteSafetyError("canonical writes require explicit user approval") - if request.operation not in {"append_case", "update_case", "append_turn", "update_turn"}: - raise WriteSafetyError(f"disabled Drive operation: {request.operation}") - if request.target_file_id == GENERAL_RESPONSES_ID: - if ( - not request.protected_override_instruction - or not request.protected_override_instruction.strip() - ): - raise WriteSafetyError("General responses is protected and read-only") - raise WriteSafetyError( - "General responses cannot be written through the Case Log connector boundary" - ) - if request.target_file_id in {STRATEGY_GUIDE_ID, EVIDENCE_BASE_ID}: - raise WriteSafetyError("Strategy Guide and Evidence Base writes are never automatic") - if request.target_file_id != CASE_LOG_ID: - raise WriteSafetyError("write target is outside the configured Case Log") - if request.target_sheet not in {"Cases", "Turns"}: - raise WriteSafetyError("canonical Case Log writes are limited to Cases and Turns") - if not request.record_identity.strip() or not request.values: - raise WriteSafetyError("write request requires record identity and values") - - -def connector_request( - request: DriveWriteRequest, source_revision_state: Mapping[str, str] -) -> ConnectorRequest: - """Build a transport-neutral envelope after deterministic policy validation.""" - validate_drive_write_request(request) - return ConnectorRequest( - action=request.operation, - file_id=request.target_file_id, - sheet=request.target_sheet, - payload={"record_identity": request.record_identity, "values": dict(request.values)}, - source_revision_state=dict(source_revision_state), - ) diff --git a/src/dialogue_lab/enums.py b/src/dialogue_lab/enums.py index e8e2baa..db5969f 100644 --- a/src/dialogue_lab/enums.py +++ b/src/dialogue_lab/enums.py @@ -17,7 +17,6 @@ class CaseStatus(StringEnum): CLOSED_CLAIM_NARROWED = "Closed - Claim Narrowed" CLOSED_CORRECTION = "Closed - Correction" CLOSED_ABANDONED = "Closed - Abandoned" - PENDING_SYNC = "Pending Sync" class TurnDirection(StringEnum): diff --git a/src/dialogue_lab/errors.py b/src/dialogue_lab/errors.py index b7e875d..0daef14 100644 --- a/src/dialogue_lab/errors.py +++ b/src/dialogue_lab/errors.py @@ -35,3 +35,9 @@ class WriteSafetyError(DialogueLabError): """Raised when a requested canonical write violates safety policy.""" category = "write_safety_error" + + +class StorageError(DialogueLabError): + """Raised when local canonical storage is missing, incompatible, or corrupt.""" + + category = "storage_error" diff --git a/src/dialogue_lab/identifiers.py b/src/dialogue_lab/identifiers.py index 55d1391..03a9bc9 100644 --- a/src/dialogue_lab/identifiers.py +++ b/src/dialogue_lab/identifiers.py @@ -6,7 +6,6 @@ from zoneinfo import ZoneInfo from .errors import IdentifierError -from .models import PendingSyncRecord CASE_ID_RE = re.compile(r"^CASE-(\d{8})-(\d{3})$") TURN_ID_RE = re.compile(r"^T(\d{3})$") @@ -27,12 +26,8 @@ def next_case_id( existing_ids: Iterable[str], *, on_date: date | None = None, - pending_sync: Iterable[PendingSyncRecord] = (), ) -> str: """Allocate the next Jerusalem-date Case ID from freshly supplied rows.""" - unresolved = [record for record in pending_sync if not record.resolved] - if unresolved: - raise IdentifierError("new Case ID allocation blocked by unresolved PENDING SYNC") values = list(existing_ids) _require_unique(values, "Case IDs") parsed: list[tuple[str, int]] = [] diff --git a/src/dialogue_lab/lifecycle.py b/src/dialogue_lab/lifecycle.py index 585dd59..f6967be 100644 --- a/src/dialogue_lab/lifecycle.py +++ b/src/dialogue_lab/lifecycle.py @@ -14,15 +14,9 @@ } ALLOWED_TRANSITIONS: dict[CaseStatus, set[CaseStatus]] = { - CaseStatus.DRAFT: {CaseStatus.POSTED, CaseStatus.PENDING_SYNC}, - CaseStatus.POSTED: {CaseStatus.ACTIVE_EXCHANGE, CaseStatus.PENDING_SYNC, *CLOSED_STATUSES}, - CaseStatus.ACTIVE_EXCHANGE: {CaseStatus.PENDING_SYNC, *CLOSED_STATUSES}, - CaseStatus.PENDING_SYNC: { - CaseStatus.DRAFT, - CaseStatus.POSTED, - CaseStatus.ACTIVE_EXCHANGE, - *CLOSED_STATUSES, - }, + CaseStatus.DRAFT: {CaseStatus.POSTED}, + CaseStatus.POSTED: {CaseStatus.ACTIVE_EXCHANGE, *CLOSED_STATUSES}, + CaseStatus.ACTIVE_EXCHANGE: {*CLOSED_STATUSES}, **{status: set() for status in CLOSED_STATUSES}, } @@ -56,7 +50,7 @@ def validate_posted_turn(turn: TurnRecord) -> None: def validate_closure_evidence(*, claimed_persuasion: bool, evidence: set[str]) -> None: - """Reject outcome inference from signals the Manual says are non-probative.""" + """Reject outcome inference from non-probative signals.""" non_probative = {"silence", "deletion", "blocking", "reaction", "disappearance"} if claimed_persuasion and evidence and evidence <= non_probative: raise LifecycleError("persuasion cannot be inferred from the supplied closure signals") diff --git a/src/dialogue_lab/manual_version.py b/src/dialogue_lab/manual_version.py deleted file mode 100644 index 8ea66a1..0000000 --- a/src/dialogue_lab/manual_version.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Operating Manual version parsing and compatibility checks.""" - -import re - -from .errors import CompatibilityError -from .models import ManualVersion - -VERSION_RE = re.compile(r"\bVersion\s+(\d+)\.(\d+)\b", re.IGNORECASE) - - -def parse_manual_version(text: str) -> ManualVersion: - """Parse the first explicit Manual version marker from live text.""" - match = VERSION_RE.search(text) - if match is None: - raise CompatibilityError("Operating Manual version marker was not found") - return ManualVersion(int(match.group(1)), int(match.group(2)), match.group(0)) - - -def require_supported_manual( - text: str, - *, - minimum: str = "2.7", - supported_major: int = 2, -) -> tuple[ManualVersion, tuple[str, ...]]: - """Return the live version and compatible-minor warnings or fail safely.""" - version = parse_manual_version(text) - minimum_version = parse_manual_version(f"Version {minimum}") - if version.major != supported_major: - raise CompatibilityError( - f"unsupported Manual major version {version.major}; " - f"repository supports {supported_major}" - ) - if version < minimum_version: - raise CompatibilityError( - f"Manual {version} is older than minimum supported version {minimum_version}" - ) - warnings: tuple[str, ...] = () - if version.minor > minimum_version.minor: - warnings = (f"Manual {version} is newer; revalidate repository compatibility",) - return version, warnings diff --git a/src/dialogue_lab/migration_receipt.py b/src/dialogue_lab/migration_receipt.py index 4268b8a..162014e 100644 --- a/src/dialogue_lab/migration_receipt.py +++ b/src/dialogue_lab/migration_receipt.py @@ -18,16 +18,12 @@ def migration_receipt_from_mapping(payload: Mapping[str, object]) -> MigrationRe return MigrationReceipt( cutover_timestamp=str(payload["cutover_timestamp"]), timezone=str(payload["timezone"]), - manual_version=str(payload["manual_version"]), - manual_revision_state=str(payload["manual_revision_state"]), - case_log_schema_signature=str(payload["case_log_schema_signature"]), - case_log_modified_state=str(payload["case_log_modified_state"]), + database_schema_version=int(str(payload["database_schema_version"])), + database_integrity=str(payload["database_integrity"]), + database_backup=str(payload["database_backup"]), + imported_case_count=int(str(payload["imported_case_count"])), + imported_turn_count=int(str(payload["imported_turn_count"])), repository_commit=str(payload["repository_commit"]), - repository_tag=str(payload["repository_tag"]), - codex_environment=str(payload["codex_environment"]), - drive_connection_status=str(payload["drive_connection_status"]), - writer_enabled=bool(payload["writer_enabled"]), - previous_writer_disabled=bool(payload["previous_writer_disabled"]), test_results=str(payload["test_results"]), known_limitations=tuple(str(item) for item in _list(payload["known_limitations"])), rollback_instructions=tuple(str(item) for item in _list(payload["rollback_instructions"])), diff --git a/src/dialogue_lab/models.py b/src/dialogue_lab/models.py index 591c2c5..a55629f 100644 --- a/src/dialogue_lab/models.py +++ b/src/dialogue_lab/models.py @@ -1,9 +1,9 @@ -"""Typed, provider-independent domain records.""" +"""Typed provider-independent domain records.""" from __future__ import annotations from collections.abc import Mapping -from dataclasses import asdict, dataclass, field +from dataclasses import asdict, dataclass from enum import Enum from typing import Any @@ -52,16 +52,6 @@ class ParsedFacebookURL: errors: tuple[str, ...] = () -@dataclass(frozen=True, order=True) -class ManualVersion: - major: int - minor: int - source_text: str = field(compare=False, default="") - - def __str__(self) -> str: - return f"{self.major}.{self.minor}" - - @dataclass(frozen=True) class SchemaSignature: value: str @@ -123,35 +113,7 @@ class LifecycleTransition: @dataclass(frozen=True) -class PendingSyncRecord: - operation: str - target_file: str - target_sheet: str - case_id: str - turn_id: str | None - expected_values: Mapping[str, object] - failure: str - last_verified_state: Mapping[str, object] - manual_version: str - source_revision_state: Mapping[str, str] - required_reconciliation_action: str - created_at: str - resolved: bool = False - - -@dataclass(frozen=True) -class DriveWriteRequest: - operation: str - target_file_id: str - target_sheet: str - record_identity: str - values: Mapping[str, object] - explicitly_approved: bool - protected_override_instruction: str | None = None - - -@dataclass(frozen=True) -class DriveWriteVerification: +class WriteVerification: succeeded: bool expected: Mapping[str, object] actual: Mapping[str, object] @@ -167,37 +129,16 @@ class ThreadMapEntry: state: TurnState -@dataclass(frozen=True) -class SourceRevisionState: - source: str - file_id: str - revision_or_modified_time: str - - -@dataclass(frozen=True) -class SourceConsistencyResult: - consistent: bool - operation_start: Mapping[str, str] - current: Mapping[str, str] - changed_sources: tuple[str, ...] - requires_reload: bool - blocking_reasons: tuple[str, ...] - - @dataclass(frozen=True) class MigrationReceipt: cutover_timestamp: str timezone: str - manual_version: str - manual_revision_state: str - case_log_schema_signature: str - case_log_modified_state: str + database_schema_version: int + database_integrity: str + database_backup: str + imported_case_count: int + imported_turn_count: int repository_commit: str - repository_tag: str - codex_environment: str - drive_connection_status: str - writer_enabled: bool - previous_writer_disabled: bool test_results: str known_limitations: tuple[str, ...] rollback_instructions: tuple[str, ...] diff --git a/src/dialogue_lab/pending_sync.py b/src/dialogue_lab/pending_sync.py deleted file mode 100644 index d9add1c..0000000 --- a/src/dialogue_lab/pending_sync.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Recovery records and allocation blocking after failed required writes.""" - -from collections.abc import Iterable, Mapping -from dataclasses import replace - -from .errors import WriteSafetyError -from .identifiers import now_jerusalem -from .models import PendingSyncRecord - - -def create_pending_sync( - *, - operation: str, - target_file: str, - target_sheet: str, - case_id: str, - turn_id: str | None, - expected_values: Mapping[str, object], - failure: str, - last_verified_state: Mapping[str, object], - manual_version: str, - source_revision_state: Mapping[str, str], - required_reconciliation_action: str, -) -> PendingSyncRecord: - required_text = { - "operation": operation, - "target_file": target_file, - "target_sheet": target_sheet, - "case_id": case_id, - "failure": failure, - "manual_version": manual_version, - "required_reconciliation_action": required_reconciliation_action, - } - missing = [name for name, value in required_text.items() if not value.strip()] - if missing: - raise WriteSafetyError(f"incomplete PENDING SYNC record: {', '.join(missing)}") - return PendingSyncRecord( - operation=operation, - target_file=target_file, - target_sheet=target_sheet, - case_id=case_id, - turn_id=turn_id, - expected_values=dict(expected_values), - failure=failure, - last_verified_state=dict(last_verified_state), - manual_version=manual_version, - source_revision_state=dict(source_revision_state), - required_reconciliation_action=required_reconciliation_action, - created_at=now_jerusalem().isoformat(timespec="seconds"), - ) - - -def unresolved_pending_sync(records: Iterable[PendingSyncRecord]) -> tuple[PendingSyncRecord, ...]: - return tuple(record for record in records if not record.resolved) - - -def require_no_pending_sync(records: Iterable[PendingSyncRecord]) -> None: - unresolved = unresolved_pending_sync(records) - if unresolved: - raise WriteSafetyError( - f"operation blocked by {len(unresolved)} unresolved PENDING SYNC record(s)" - ) - - -def mark_reconciled(record: PendingSyncRecord) -> PendingSyncRecord: - return replace(record, resolved=True) diff --git a/src/dialogue_lab/readback.py b/src/dialogue_lab/readback.py index 7cd8c8a..c081b2c 100644 --- a/src/dialogue_lab/readback.py +++ b/src/dialogue_lab/readback.py @@ -2,20 +2,20 @@ from collections.abc import Mapping -from .models import DriveWriteVerification +from .models import WriteVerification def verify_readback( expected: Mapping[str, object], actual: Mapping[str, object] -) -> DriveWriteVerification: - """Compare every expected field exactly; connector success alone is irrelevant.""" +) -> WriteVerification: + """Compare every expected field exactly.""" mismatches: list[str] = [] for key, expected_value in expected.items(): if key not in actual: mismatches.append(f"missing field: {key}") elif actual[key] != expected_value: mismatches.append(f"value mismatch: {key}") - return DriveWriteVerification( + return WriteVerification( succeeded=not mismatches, expected=dict(expected), actual=dict(actual), diff --git a/src/dialogue_lab/schema.py b/src/dialogue_lab/schema.py index 2263353..f4fca81 100644 --- a/src/dialogue_lab/schema.py +++ b/src/dialogue_lab/schema.py @@ -1,11 +1,10 @@ -"""Live Case Log schema compatibility and deterministic signatures.""" +"""Canonical SQLite schema metadata and deterministic signatures.""" from __future__ import annotations import hashlib import json -from collections.abc import Mapping, Sequence -from typing import Any +from collections.abc import Mapping from .enums import ( CaseStatus, @@ -15,56 +14,73 @@ TurnKind, TurnState, ) -from .errors import CompatibilityError from .models import SchemaSignature -REQUIRED_SHEETS = ( - "Cases", - "Turns", - "Data Dictionary", - "Strategy Taxonomy", - "Dashboard", +SCHEMA_VERSION = 1 + +CASE_FIELDS = ( + "case_id", + "case_title", + "created_at", + "updated_at", + "status", + "topic", + "post_text", + "post_url", + "post_id", + "root_comment_id", + "source_links", + "privacy_checked", + "outcome_score", + "outcome_class", + "outcome_notes", + "user_rating", + "what_worked", + "what_failed", + "next_test", + "closed_at", ) -CASE_HEADERS = ( - "Case ID", "Case Title", "Created At", "Updated At", "Status", "Topic", - "Post Text", "Post URL", "Post ID", "Root Comment ID", "Source Links", - "Privacy Checked", "Outcome Score 0–5", "Outcome Class", "Outcome Notes", - "User Rating 1–5", "What Worked", "What Failed", "Next Test", "Closed At", -) - -TURN_HEADERS = ( - "Case ID", "Turn ID", "Parent Turn ID", "Parent Confidence", "Participant Ref", - "Direction", "Kind", "State", "Exact Text", "Post ID", "Root Comment ID", - "Reply Comment ID", "Exact URL", "URL Supplied At", "Observed At", "Notes", +TURN_FIELDS = ( + "case_id", + "turn_id", + "parent_turn_id", + "parent_confidence", + "participant_ref", + "direction", + "kind", + "state", + "exact_text", + "post_id", + "root_comment_id", + "reply_comment_id", + "exact_url", + "url_supplied_at", + "observed_at", + "notes", ) EXPECTED_ENUMS: dict[str, tuple[str, ...]] = { - "Cases.Status": tuple(item.value for item in CaseStatus), - "Cases.Privacy Checked": ("Yes", "No"), - "Cases.Outcome Class": tuple(item.value for item in OutcomeClass), - "Turns.Parent Confidence": tuple(item.value for item in ParentConfidence), - "Turns.Direction": tuple(item.value for item in TurnDirection), - "Turns.Kind": tuple(item.value for item in TurnKind), - "Turns.State": tuple(item.value for item in TurnState), + "cases.status": tuple(item.value for item in CaseStatus), + "cases.outcome_class": tuple(item.value for item in OutcomeClass), + "turns.parent_confidence": tuple(item.value for item in ParentConfidence), + "turns.direction": tuple(item.value for item in TurnDirection), + "turns.kind": tuple(item.value for item in TurnKind), + "turns.state": tuple(item.value for item in TurnState), } -ESSENTIAL_DASHBOARD_FORMULAS = ( - '=COUNTA(Cases!A2:A)', - '=COUNTA(Turns!B2:B)', - '=COUNTIF(Cases!E2:E,"Draft")', - '=COUNTIF(Cases!E2:E,"Posted")', - '=COUNTIF(Cases!E2:E,"Active Exchange")', - '=COUNTIF(Cases!E2:E,"Closed*")', -) - def expected_schema_payload() -> dict[str, object]: return { - "sheets": list(REQUIRED_SHEETS), - "headers": {"Cases": list(CASE_HEADERS), "Turns": list(TURN_HEADERS)}, + "schema_version": SCHEMA_VERSION, + "tables": {"cases": list(CASE_FIELDS), "turns": list(TURN_FIELDS)}, "enums": {key: list(values) for key, values in EXPECTED_ENUMS.items()}, - "essential_dashboard_formulas": list(ESSENTIAL_DASHBOARD_FORMULAS), + "constraints": [ + "cases.case_id primary key", + "cases(post_id, root_comment_id) unique", + "turns(case_id, turn_id) primary key", + "turns.case_id references cases.case_id", + ], } @@ -76,46 +92,3 @@ def schema_signature(payload: Mapping[str, object] | None = None) -> SchemaSigna separators=(",", ":"), ).encode("utf-8") return SchemaSignature(hashlib.sha256(canonical).hexdigest()) - - -def _as_strings(value: Any, label: str) -> tuple[str, ...]: - if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): - raise CompatibilityError(f"{label} must be a list") - return tuple(str(item) for item in value) - - -def validate_schema(observed: Mapping[str, Any]) -> SchemaSignature: - sheets = observed.get("sheets") - if not isinstance(sheets, Mapping): - raise CompatibilityError("schema JSON requires a sheets object") - if set(sheets) != set(REQUIRED_SHEETS): - raise CompatibilityError( - f"sheet mismatch: expected {list(REQUIRED_SHEETS)}, observed {sorted(sheets)}" - ) - expected_headers = {"Cases": CASE_HEADERS, "Turns": TURN_HEADERS} - for sheet, expected in expected_headers.items(): - actual = _as_strings(sheets[sheet], f"{sheet} headers") - if actual != expected: - raise CompatibilityError( - f"{sheet} header mismatch: expected {list(expected)}, observed {list(actual)}" - ) - enums = observed.get("enums") - if not isinstance(enums, Mapping): - raise CompatibilityError("schema JSON requires an enums object") - for field, expected in EXPECTED_ENUMS.items(): - if field not in enums: - raise CompatibilityError(f"missing enum definition: {field}") - actual = _as_strings(enums[field], field) - if actual != expected: - raise CompatibilityError( - f"enum mismatch for {field}: expected {list(expected)}, observed {list(actual)}" - ) - formulas = observed.get("dashboard_formulas") - if formulas is not None: - actual_formulas = set(_as_strings(formulas, "dashboard_formulas")) - missing = [ - formula for formula in ESSENTIAL_DASHBOARD_FORMULAS if formula not in actual_formulas - ] - if missing: - raise CompatibilityError(f"missing essential Dashboard formulas: {missing}") - return schema_signature() diff --git a/src/dialogue_lab/source_consistency.py b/src/dialogue_lab/source_consistency.py index cf3393b..2aad6df 100644 --- a/src/dialogue_lab/source_consistency.py +++ b/src/dialogue_lab/source_consistency.py @@ -1,41 +1,13 @@ -"""Guard one operation against stale or mixed canonical-source state.""" +"""Deterministic revision identifiers for repository canonical documents.""" -from collections.abc import Iterable, Mapping +import hashlib +from pathlib import Path -from .models import SourceConsistencyResult - -def check_source_consistency( - operation_start: Mapping[str, str], - current: Mapping[str, str], - *, - material_sources: Iterable[str] | None = None, -) -> SourceConsistencyResult: - """Compare revision states and block when a relevant source changed or vanished.""" - material = set(material_sources) if material_sources is not None else set(operation_start) - changed: list[str] = [] - reasons: list[str] = [] - for source, start_state in operation_start.items(): - current_state = current.get(source) - if current_state is None: - changed.append(source) - if source in material: - reasons.append(f"{source} revision state is unavailable") - elif current_state != start_state: - changed.append(source) - if source in material: - reasons.append(f"{source} changed during the operation") - for source in current.keys() - operation_start.keys(): - changed.append(source) - if source in material: - reasons.append(f"{source} was introduced during the operation") - changed_tuple = tuple(sorted(set(changed))) - requires_reload = any(source in material for source in changed_tuple) - return SourceConsistencyResult( - consistent=not reasons, - operation_start=dict(operation_start), - current=dict(current), - changed_sources=changed_tuple, - requires_reload=requires_reload, - blocking_reasons=tuple(reasons), - ) +def file_sha256(path: Path) -> str: + """Return a stable content revision without printing or copying document bodies.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(65536), b""): + digest.update(block) + return digest.hexdigest() diff --git a/src/dialogue_lab/storage.py b/src/dialogue_lab/storage.py new file mode 100644 index 0000000..4dda3f9 --- /dev/null +++ b/src/dialogue_lab/storage.py @@ -0,0 +1,617 @@ +"""Transactional SQLite storage for canonical Dialogue Lab case state. + +All mutations require an explicit approval flag, run inside an immediate +transaction, and are read back after commit. The caller supplies the database +path; this module never discovers credentials, contacts a network service, or +writes outside that path and explicitly requested backup destinations. +""" + +from __future__ import annotations + +import json +import sqlite3 +from collections import defaultdict +from collections.abc import Iterable, Mapping +from dataclasses import replace +from pathlib import Path + +from .enums import ( + CaseStatus, + OutcomeClass, + ParentConfidence, + TurnDirection, + TurnKind, + TurnState, +) +from .errors import StorageError, WriteSafetyError +from .facebook_url import parse_facebook_url +from .lifecycle import CLOSED_STATUSES, validate_transition +from .models import CaseRecord, LifecycleTransition, TurnRecord, to_jsonable +from .parent_graph import validate_parent_graph +from .readback import verify_readback +from .schema import CASE_FIELDS, SCHEMA_VERSION, TURN_FIELDS, schema_signature +from .validation import validate_case, validate_turn + + +def _sql_enum(values: Iterable[str]) -> str: + return ", ".join("'" + value.replace("'", "''") + "'" for value in values) + + +_STATUS_SQL = _sql_enum(item.value for item in CaseStatus) +_OUTCOME_SQL = _sql_enum(item.value for item in OutcomeClass) +_CONFIDENCE_SQL = _sql_enum(item.value for item in ParentConfidence) +_DIRECTION_SQL = _sql_enum(item.value for item in TurnDirection) +_KIND_SQL = _sql_enum(item.value for item in TurnKind) +_STATE_SQL = _sql_enum(item.value for item in TurnState) +_SCHEMA_SIGNATURE = schema_signature().value + +_SCHEMA_SQL = f""" +CREATE TABLE IF NOT EXISTS storage_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS cases ( + case_id TEXT PRIMARY KEY, + case_title TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ({_STATUS_SQL})), + topic TEXT NOT NULL, + post_text TEXT NOT NULL, + post_url TEXT NOT NULL, + post_id TEXT NOT NULL, + root_comment_id TEXT NOT NULL, + source_links TEXT NOT NULL DEFAULT '[]', + privacy_checked INTEGER NOT NULL CHECK (privacy_checked IN (0, 1)), + outcome_score INTEGER CHECK (outcome_score BETWEEN 0 AND 5), + outcome_class TEXT CHECK (outcome_class IS NULL OR outcome_class IN ({_OUTCOME_SQL})), + outcome_notes TEXT NOT NULL DEFAULT '', + user_rating INTEGER CHECK (user_rating BETWEEN 1 AND 5), + what_worked TEXT NOT NULL DEFAULT '', + what_failed TEXT NOT NULL DEFAULT '', + next_test TEXT NOT NULL DEFAULT '', + closed_at TEXT NOT NULL DEFAULT '', + UNIQUE (post_id, root_comment_id) +); + +CREATE TABLE IF NOT EXISTS turns ( + case_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + parent_turn_id TEXT, + parent_confidence TEXT CHECK ( + parent_confidence IS NULL OR parent_confidence IN ({_CONFIDENCE_SQL}) + ), + participant_ref TEXT NOT NULL, + direction TEXT NOT NULL CHECK (direction IN ({_DIRECTION_SQL})), + kind TEXT NOT NULL CHECK (kind IN ({_KIND_SQL})), + state TEXT NOT NULL CHECK (state IN ({_STATE_SQL})), + exact_text TEXT NOT NULL, + post_id TEXT NOT NULL, + root_comment_id TEXT NOT NULL, + reply_comment_id TEXT, + exact_url TEXT, + url_supplied_at TEXT, + observed_at TEXT NOT NULL, + notes TEXT NOT NULL DEFAULT '', + PRIMARY KEY (case_id, turn_id), + FOREIGN KEY (case_id) REFERENCES cases(case_id) ON DELETE RESTRICT, + FOREIGN KEY (case_id, parent_turn_id) REFERENCES turns(case_id, turn_id) + DEFERRABLE INITIALLY DEFERRED +); + +CREATE INDEX IF NOT EXISTS turns_case_observed_idx + ON turns(case_id, observed_at, turn_id); +INSERT INTO storage_metadata(key, value) + VALUES ('schema_signature', '{_SCHEMA_SIGNATURE}') + ON CONFLICT(key) DO NOTHING; +PRAGMA user_version = {SCHEMA_VERSION}; +""" + + +def _require_approval(approved: bool) -> None: + if not approved: + raise WriteSafetyError("canonical SQLite writes require explicit approval") + + +def _case_values(record: CaseRecord) -> dict[str, object]: + return { + "case_id": record.case_id, + "case_title": record.case_title, + "created_at": record.created_at, + "updated_at": record.updated_at, + "status": record.status.value, + "topic": record.topic, + "post_text": record.post_text, + "post_url": record.post_url, + "post_id": record.post_id, + "root_comment_id": record.root_comment_id, + "source_links": json.dumps(list(record.source_links), ensure_ascii=False), + "privacy_checked": int(record.privacy_checked), + "outcome_score": record.outcome_score, + "outcome_class": record.outcome_class.value if record.outcome_class else None, + "outcome_notes": record.outcome_notes, + "user_rating": record.user_rating, + "what_worked": record.what_worked, + "what_failed": record.what_failed, + "next_test": record.next_test, + "closed_at": record.closed_at, + } + + +def _turn_values(record: TurnRecord) -> dict[str, object]: + return { + "case_id": record.case_id, + "turn_id": record.turn_id, + "parent_turn_id": record.parent_turn_id, + "parent_confidence": ( + record.parent_confidence.value if record.parent_confidence else None + ), + "participant_ref": record.participant_ref, + "direction": record.direction.value, + "kind": record.kind.value, + "state": record.state.value, + "exact_text": record.exact_text, + "post_id": record.post_id, + "root_comment_id": record.root_comment_id, + "reply_comment_id": record.reply_comment_id, + "exact_url": record.exact_url, + "url_supplied_at": record.url_supplied_at, + "observed_at": record.observed_at, + "notes": record.notes, + } + + +def _case_from_row(row: sqlite3.Row) -> CaseRecord: + raw_links = json.loads(str(row["source_links"])) + if not isinstance(raw_links, list): + raise StorageError(f"invalid source_links JSON for {row['case_id']}") + return CaseRecord( + case_id=str(row["case_id"]), + case_title=str(row["case_title"]), + created_at=str(row["created_at"]), + updated_at=str(row["updated_at"]), + status=CaseStatus(str(row["status"])), + topic=str(row["topic"]), + post_text=str(row["post_text"]), + post_url=str(row["post_url"]), + post_id=str(row["post_id"]), + root_comment_id=str(row["root_comment_id"]), + source_links=tuple(str(item) for item in raw_links), + privacy_checked=bool(row["privacy_checked"]), + outcome_score=(int(row["outcome_score"]) if row["outcome_score"] is not None else None), + outcome_class=( + OutcomeClass(str(row["outcome_class"])) + if row["outcome_class"] is not None + else None + ), + outcome_notes=str(row["outcome_notes"]), + user_rating=(int(row["user_rating"]) if row["user_rating"] is not None else None), + what_worked=str(row["what_worked"]), + what_failed=str(row["what_failed"]), + next_test=str(row["next_test"]), + closed_at=str(row["closed_at"]), + ) + + +def _turn_from_row(row: sqlite3.Row) -> TurnRecord: + return TurnRecord( + case_id=str(row["case_id"]), + turn_id=str(row["turn_id"]), + parent_turn_id=(str(row["parent_turn_id"]) if row["parent_turn_id"] else None), + parent_confidence=( + ParentConfidence(str(row["parent_confidence"])) + if row["parent_confidence"] is not None + else None + ), + participant_ref=str(row["participant_ref"]), + direction=TurnDirection(str(row["direction"])), + kind=TurnKind(str(row["kind"])), + state=TurnState(str(row["state"])), + exact_text=str(row["exact_text"]), + post_id=str(row["post_id"]), + root_comment_id=str(row["root_comment_id"]), + reply_comment_id=( + str(row["reply_comment_id"]) if row["reply_comment_id"] is not None else None + ), + exact_url=str(row["exact_url"]) if row["exact_url"] is not None else None, + url_supplied_at=( + str(row["url_supplied_at"]) if row["url_supplied_at"] is not None else None + ), + observed_at=str(row["observed_at"]), + notes=str(row["notes"]), + ) + + +class SQLiteStore: + """Narrow canonical storage API; no generic SQL execution is exposed.""" + + def __init__(self, path: Path) -> None: + self.path = path.resolve() + + def _connect( + self, *, require_exists: bool = True, writable: bool = False + ) -> sqlite3.Connection: + if require_exists and not self.path.is_file(): + raise StorageError(f"database does not exist: {self.path}") + target = self.path if writable else f"{self.path.as_uri()}?mode=ro" + connection = sqlite3.connect( + target, + timeout=5.0, + isolation_level=None, + uri=not writable, + ) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA busy_timeout = 5000") + return connection + + def initialize(self, *, approved: bool) -> Mapping[str, object]: + """Create or verify the schema without replacing an existing database.""" + _require_approval(approved) + self.path.parent.mkdir(parents=True, exist_ok=True) + with self._connect(require_exists=False, writable=True) as connection: + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + if version not in {0, SCHEMA_VERSION}: + raise StorageError( + f"unsupported database schema version: {version}; expected {SCHEMA_VERSION}" + ) + if version == 0: + tables = connection.execute( + """SELECT name FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%'""" + ).fetchall() + if tables: + raise StorageError("refusing to initialize a non-empty unversioned database") + connection.execute("PRAGMA journal_mode = WAL") + connection.executescript(_SCHEMA_SQL) + return self.status() + + def status(self) -> Mapping[str, object]: + with self._connect() as connection: + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + if version != SCHEMA_VERSION: + raise StorageError( + f"database schema version mismatch: {version}; expected {SCHEMA_VERSION}" + ) + signature_row = connection.execute( + "SELECT value FROM storage_metadata WHERE key = 'schema_signature'" + ).fetchone() + actual_signature = str(signature_row["value"]) if signature_row else "" + if actual_signature != _SCHEMA_SIGNATURE: + raise StorageError("database schema signature mismatch") + case_columns = tuple( + str(row["name"]) for row in connection.execute("PRAGMA table_info(cases)") + ) + turn_columns = tuple( + str(row["name"]) for row in connection.execute("PRAGMA table_info(turns)") + ) + if case_columns != CASE_FIELDS or turn_columns != TURN_FIELDS: + raise StorageError("database table columns do not match the canonical schema") + integrity = str(connection.execute("PRAGMA integrity_check").fetchone()[0]) + foreign_keys = connection.execute("PRAGMA foreign_key_check").fetchall() + case_count = int(connection.execute("SELECT COUNT(*) FROM cases").fetchone()[0]) + turn_count = int(connection.execute("SELECT COUNT(*) FROM turns").fetchone()[0]) + if integrity != "ok" or foreign_keys: + raise StorageError("database integrity check failed") + return { + "ok": True, + "schema_version": version, + "schema_signature": actual_signature, + "integrity": integrity, + "case_count": case_count, + "turn_count": turn_count, + } + + def backup(self, destination: Path, *, approved: bool) -> Mapping[str, object]: + """Create a verified backup, never overwrite, and remove partial output on failure.""" + _require_approval(approved) + target = destination.resolve() + if target.exists(): + raise StorageError(f"backup destination already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + try: + with self._connect() as source, sqlite3.connect(target) as backup: + source.backup(backup) + status = SQLiteStore(target).status() + except Exception: + target.unlink(missing_ok=True) + raise + return {"ok": True, "backup": str(target), "database": status} + + def case_ids(self) -> list[str]: + with self._connect() as connection: + rows = connection.execute("SELECT case_id FROM cases ORDER BY case_id").fetchall() + return [str(row["case_id"]) for row in rows] + + def turn_ids(self, case_id: str) -> list[str]: + with self._connect() as connection: + rows = connection.execute( + "SELECT turn_id FROM turns WHERE case_id = ? ORDER BY turn_id", (case_id,) + ).fetchall() + return [str(row["turn_id"]) for row in rows] + + def _get_case(self, connection: sqlite3.Connection, case_id: str) -> CaseRecord: + row = connection.execute("SELECT * FROM cases WHERE case_id = ?", (case_id,)).fetchone() + if row is None: + raise StorageError(f"case not found: {case_id}") + return _case_from_row(row) + + def _get_turns(self, connection: sqlite3.Connection, case_id: str) -> list[TurnRecord]: + rows = connection.execute( + "SELECT * FROM turns WHERE case_id = ? ORDER BY turn_id", (case_id,) + ).fetchall() + return [_turn_from_row(row) for row in rows] + + def get_case(self, case_id: str) -> CaseRecord: + with self._connect() as connection: + return self._get_case(connection, case_id) + + def get_turns(self, case_id: str) -> list[TurnRecord]: + with self._connect() as connection: + self._get_case(connection, case_id) + return self._get_turns(connection, case_id) + + def find_case(self, post_id: str, root_comment_id: str) -> CaseRecord | None: + with self._connect() as connection: + row = connection.execute( + "SELECT * FROM cases WHERE post_id = ? AND root_comment_id = ?", + (post_id, root_comment_id), + ).fetchone() + return _case_from_row(row) if row is not None else None + + def list_open_summaries(self) -> list[dict[str, str]]: + with self._connect() as connection: + rows = connection.execute("SELECT * FROM cases ORDER BY case_id").fetchall() + output: list[dict[str, str]] = [] + for row in rows: + case = _case_from_row(row) + if case.status in CLOSED_STATUSES: + continue + parsed = parse_facebook_url(case.post_url) + if parsed.root_comment_id is None and parsed.reply_comment_id is None: + raise StorageError( + f"open case lacks a comment or reply permalink: {case.case_id}" + ) + output.append( + { + "case_id": case.case_id, + "status": case.status.value, + "exact_permalink": parsed.original_url, + } + ) + return output + + def strategy_dataset(self) -> Mapping[str, object]: + with self._connect() as connection: + cases = [ + _case_from_row(row) + for row in connection.execute("SELECT * FROM cases ORDER BY case_id") + if CaseStatus(str(row["status"])) in CLOSED_STATUSES + ] + turns = { + case.case_id: self._get_turns(connection, case.case_id) for case in cases + } + return {"cases": cases, "turns": turns} + + @staticmethod + def _insert_case(connection: sqlite3.Connection, record: CaseRecord) -> None: + connection.execute( + """INSERT INTO cases VALUES ( + :case_id, :case_title, :created_at, :updated_at, :status, :topic, + :post_text, :post_url, :post_id, :root_comment_id, :source_links, + :privacy_checked, :outcome_score, :outcome_class, :outcome_notes, + :user_rating, :what_worked, :what_failed, :next_test, :closed_at + )""", + _case_values(record), + ) + + @staticmethod + def _insert_turn(connection: sqlite3.Connection, record: TurnRecord) -> None: + connection.execute( + """INSERT INTO turns VALUES ( + :case_id, :turn_id, :parent_turn_id, :parent_confidence, + :participant_ref, :direction, :kind, :state, :exact_text, :post_id, + :root_comment_id, :reply_comment_id, :exact_url, :url_supplied_at, + :observed_at, :notes + )""", + _turn_values(record), + ) + + @staticmethod + def _validate_bundle(cases: list[CaseRecord], turns: list[TurnRecord]) -> None: + by_case = {case.case_id: case for case in cases} + if len(by_case) != len(cases): + raise StorageError("duplicate Case IDs in write payload") + grouped: dict[str, list[TurnRecord]] = defaultdict(list) + for case in cases: + validate_case(case) + for turn in turns: + validate_turn(turn) + parent_case = by_case.get(turn.case_id) + if parent_case is None: + raise StorageError(f"turn references missing Case: {turn.case_id}") + if (turn.post_id, turn.root_comment_id) != ( + parent_case.post_id, + parent_case.root_comment_id, + ): + raise StorageError(f"turn identity conflicts with Case: {turn.turn_id}") + grouped[turn.case_id].append(turn) + for case_turns in grouped.values(): + validate_parent_graph(case_turns) + + def import_records( + self, cases: list[CaseRecord], turns: list[TurnRecord], *, approved: bool + ) -> Mapping[str, object]: + """Atomically import a validated snapshot into an empty database.""" + _require_approval(approved) + self._validate_bundle(cases, turns) + try: + with self._connect(writable=True) as connection: + connection.execute("BEGIN IMMEDIATE") + existing = int(connection.execute("SELECT COUNT(*) FROM cases").fetchone()[0]) + existing += int(connection.execute("SELECT COUNT(*) FROM turns").fetchone()[0]) + if existing: + raise StorageError("database import requires empty cases and turns tables") + for case in cases: + self._insert_case(connection, case) + for turn in turns: + self._insert_turn(connection, turn) + connection.commit() + except sqlite3.Error as error: + raise StorageError(f"database import failed: {error}") from error + status = self.status() + if status["case_count"] != len(cases) or status["turn_count"] != len(turns): + raise StorageError("database import read-back count mismatch") + for case in cases: + if self.get_case(case.case_id) != case: + raise StorageError(f"database import Case read-back mismatch: {case.case_id}") + expected_turns: dict[str, list[TurnRecord]] = defaultdict(list) + for turn in turns: + expected_turns[turn.case_id].append(turn) + for case_id, case_turns in expected_turns.items(): + if self.get_turns(case_id) != sorted(case_turns, key=lambda item: item.turn_id): + raise StorageError(f"database import Turn read-back mismatch: {case_id}") + return status + + def create_case( + self, case: CaseRecord, turns: list[TurnRecord], *, approved: bool + ) -> Mapping[str, object]: + """Atomically create one Case and its initial public-turn graph.""" + _require_approval(approved) + self._validate_bundle([case], turns) + try: + with self._connect(writable=True) as connection: + connection.execute("BEGIN IMMEDIATE") + self._insert_case(connection, case) + for turn in turns: + self._insert_turn(connection, turn) + connection.commit() + except sqlite3.Error as error: + raise StorageError(f"case creation failed: {error}") from error + actual_case = self.get_case(case.case_id) + actual_turns = self.get_turns(case.case_id) + if actual_case != case or actual_turns != sorted(turns, key=lambda item: item.turn_id): + raise StorageError("case creation read-back mismatch") + return {"created": True, "case_id": case.case_id, "turn_count": len(turns)} + + def add_turn( + self, + turn: TurnRecord, + *, + target_status: CaseStatus, + updated_at: str, + reason: str, + replace_draft_id: str | None, + approved: bool, + ) -> Mapping[str, object]: + """Atomically append one turn, optionally retire a Draft, and update Case state.""" + _require_approval(approved) + validate_turn(turn) + try: + with self._connect(writable=True) as connection: + connection.execute("BEGIN IMMEDIATE") + case = self._get_case(connection, turn.case_id) + if (turn.post_id, turn.root_comment_id) != ( + case.post_id, + case.root_comment_id, + ): + raise StorageError(f"turn identity conflicts with Case: {turn.turn_id}") + validate_transition(LifecycleTransition(case.status, target_status, reason)) + existing_turns = self._get_turns(connection, turn.case_id) + if replace_draft_id is not None: + draft = next( + (item for item in existing_turns if item.turn_id == replace_draft_id), + None, + ) + if ( + draft is None + or draft.direction is not TurnDirection.OUTGOING + or draft.state is not TurnState.DRAFT + ): + raise StorageError( + f"replacement target is not an Outgoing Draft: {replace_draft_id}" + ) + connection.execute( + "UPDATE turns SET state = ? WHERE case_id = ? AND turn_id = ?", + (TurnState.REPLACED.value, turn.case_id, replace_draft_id), + ) + existing_turns = [ + item + if item.turn_id != replace_draft_id + else replace(item, state=TurnState.REPLACED) + for item in existing_turns + ] + validate_parent_graph([*existing_turns, turn]) + self._insert_turn(connection, turn) + connection.execute( + "UPDATE cases SET status = ?, updated_at = ? WHERE case_id = ?", + (target_status.value, updated_at, turn.case_id), + ) + connection.commit() + except sqlite3.Error as error: + raise StorageError(f"turn write failed: {error}") from error + actual_turns = self.get_turns(turn.case_id) + actual = next((item for item in actual_turns if item.turn_id == turn.turn_id), None) + actual_case = self.get_case(turn.case_id) + if ( + actual != turn + or actual_case.status is not target_status + or actual_case.updated_at != updated_at + ): + raise StorageError("turn write read-back mismatch") + if replace_draft_id is not None: + replaced = next( + (item for item in actual_turns if item.turn_id == replace_draft_id), None + ) + if replaced is None or replaced.state is not TurnState.REPLACED: + raise StorageError("Draft replacement read-back mismatch") + return { + "case_id": turn.case_id, + "turn_id": turn.turn_id, + "status": actual_case.status.value, + "readback": "verified", + } + + def close_case( + self, updated: CaseRecord, *, reason: str, approved: bool + ) -> Mapping[str, object]: + """Atomically write only closure fields and verify the committed Case.""" + _require_approval(approved) + validate_case(updated) + try: + with self._connect(writable=True) as connection: + connection.execute("BEGIN IMMEDIATE") + current = self._get_case(connection, updated.case_id) + validate_transition(LifecycleTransition(current.status, updated.status, reason)) + connection.execute( + """UPDATE cases SET + updated_at = :updated_at, + status = :status, + outcome_score = :outcome_score, + outcome_class = :outcome_class, + outcome_notes = :outcome_notes, + user_rating = :user_rating, + what_worked = :what_worked, + what_failed = :what_failed, + next_test = :next_test, + closed_at = :closed_at + WHERE case_id = :case_id""", + _case_values(updated), + ) + connection.commit() + except sqlite3.Error as error: + raise StorageError(f"case close failed: {error}") from error + actual = self.get_case(updated.case_id) + expected = to_jsonable(updated) + observed = to_jsonable(actual) + if not isinstance(expected, Mapping) or not isinstance(observed, Mapping): + raise StorageError("case close read-back serialization failed") + verification = verify_readback(expected, observed) + if not verification.succeeded: + raise StorageError("case close read-back mismatch") + return { + "case_id": actual.case_id, + "status": actual.status.value, + "readback": "verified", + } diff --git a/src/dialogue_lab/validation.py b/src/dialogue_lab/validation.py index 2d07ce3..8c03b38 100644 --- a/src/dialogue_lab/validation.py +++ b/src/dialogue_lab/validation.py @@ -2,10 +2,11 @@ import re -from .enums import CaseStatus, TurnDirection, TurnState +from .enums import TurnDirection, TurnState from .errors import DialogueLabError from .facebook_url import parse_facebook_url from .identifiers import CASE_ID_RE, TURN_ID_RE +from .lifecycle import CLOSED_STATUSES from .models import CaseRecord, TurnRecord PARTICIPANT_RE = re.compile(r"^(?:USER|P[1-9]\d*)$") @@ -27,14 +28,28 @@ def validate_case(record: CaseRecord) -> None: missing = [name for name, value in required.items() if not value.strip()] if missing: raise DialogueLabError(f"missing required Case fields: {', '.join(missing)}") + if not record.privacy_checked: + raise DialogueLabError("Privacy Checked must be true") if record.outcome_score is not None and not 0 <= record.outcome_score <= 5: - raise DialogueLabError("Outcome Score 0–5 must be between 0 and 5") + raise DialogueLabError("Outcome Score 0-5 must be between 0 and 5") if record.user_rating is not None and not 1 <= record.user_rating <= 5: - raise DialogueLabError("User Rating 1–5 must be between 1 and 5") - if record.status.value.startswith("Closed -") and not record.closed_at: + raise DialogueLabError("User Rating 1-5 must be between 1 and 5") + if record.status in CLOSED_STATUSES and not record.closed_at: raise DialogueLabError("Closed At is required for closed cases") - if record.status is CaseStatus.PENDING_SYNC and record.closed_at: - raise DialogueLabError("Pending Sync cannot be marked closed") + + parsed = parse_facebook_url(record.post_url) + if not parsed.is_facebook_url: + raise DialogueLabError("Post URL must be a Facebook URL") + if parsed.post_id is not None and parsed.post_id != record.post_id: + raise DialogueLabError("Post URL Post ID conflicts with Case Post ID") + if parsed.root_comment_id is not None and parsed.root_comment_id != record.root_comment_id: + raise DialogueLabError("Post URL Root Comment ID conflicts with Case Root Comment ID") + if ( + record.status not in CLOSED_STATUSES + and parsed.root_comment_id is None + and parsed.reply_comment_id is None + ): + raise DialogueLabError("open Case Post URL requires a comment or reply identifier") def validate_turn(record: TurnRecord) -> None: diff --git a/tests/helpers.py b/tests/helpers.py index c39a3ef..998f913 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -19,7 +19,7 @@ def make_case(**overrides: object) -> CaseRecord: "status": CaseStatus.POSTED, "topic": "Synthetic topic", "post_text": "Synthetic public post text.", - "post_url": "https://www.facebook.com/example/posts/123", + "post_url": "https://www.facebook.com/example/posts/123?comment_id=456", "post_id": "123", "root_comment_id": "456", "privacy_checked": True, diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 7b063ce..4bb070f 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -4,9 +4,8 @@ import pytest from dialogue_lab.cli import main -from dialogue_lab.enums import CaseStatus, TurnDirection, TurnState +from dialogue_lab.enums import CaseStatus, OutcomeClass, TurnDirection, TurnState from dialogue_lab.models import to_jsonable -from dialogue_lab.schema import expected_schema_payload from tests.helpers import make_case, make_reply, make_turn @@ -15,74 +14,207 @@ def _write(path: Path, value: object) -> Path: return path -def _run_ok(capsys: pytest.CaptureFixture[str], *args: str) -> dict[str, object]: +def _run_ok(capsys: pytest.CaptureFixture[str], *args: str) -> object: assert main(list(args)) == 0 - result = json.loads(capsys.readouterr().out) - assert isinstance(result, dict) - return result + return json.loads(capsys.readouterr().out) -def test_all_required_cli_commands_smoke( +def _as_dict(value: object) -> dict[str, object]: + assert isinstance(value, dict) + return value + + +def test_transactional_case_cli_lifecycle( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - assert _run_ok(capsys, "doctor")["ok"] is True - manual = tmp_path / "manual.txt" - manual.write_text("Version 2.7 — synthetic", encoding="utf-8") - assert _run_ok(capsys, "manual-version", str(manual))["version"] == "2.7" - - expected = expected_schema_payload() - headers = expected["headers"] - assert isinstance(headers, dict) - observed = { - "sheets": { - "Cases": headers["Cases"], - "Turns": headers["Turns"], - "Data Dictionary": [], - "Strategy Taxonomy": [], - "Dashboard": [], - }, - "enums": expected["enums"], - "dashboard_formulas": expected["essential_dashboard_formulas"], + database = tmp_path / "canonical.sqlite3" + db = ("--database", str(database)) + initialized = _as_dict(_run_ok(capsys, *db, "db-init", "--approved")) + assert initialized["integrity"] == "ok" + assert _as_dict(_run_ok(capsys, *db, "doctor"))["ok"] is True + + case_payload = _as_dict(to_jsonable(make_case())) + case_payload.pop("case_id") + turn_payload = _as_dict(to_jsonable(make_turn())) + intake_path = _write( + tmp_path / "intake.json", + {"case": case_payload, "turns": [turn_payload]}, + ) + intake = _as_dict( + _run_ok( + capsys, + *db, + "case-intake", + str(intake_path), + "--date", + "2026-07-17", + "--approved", + ) + ) + assert intake == { + "created": True, + "case_id": "CASE-20260717-001", + "turn_count": 1, } - schema_path = _write(tmp_path / "schema.json", observed) - assert _run_ok(capsys, "schema-check", str(schema_path))["compatible"] is True - consistency_path = _write( - tmp_path / "consistency.json", - {"operation_start": {"manual": "r1"}, "current": {"manual": "r1"}}, + duplicate = _as_dict( + _run_ok( + capsys, + *db, + "case-intake", + str(intake_path), + "--date", + "2026-07-17", + "--approved", + ) + ) + assert duplicate["duplicate"] is True + assert duplicate["case_id"] == "CASE-20260717-001" + + open_cases = _run_ok(capsys, *db, "case-list-open") + assert open_cases == [ + { + "case_id": "CASE-20260717-001", + "status": "Posted", + "exact_permalink": case_payload["post_url"], + } + ] + found = _as_dict( + _run_ok( + capsys, + *db, + "case-find", + "--post-id", + "123", + "--root-comment-id", + "456", + ) + ) + assert found["case_id"] == "CASE-20260717-001" + + followup = make_reply( + "ignored", + "T001", + observed_at="2026-07-17 11:00", + exact_text="Synthetic follow-up.", + ) + followup_path = _write(tmp_path / "followup.json", followup) + followup_receipt = _as_dict( + _run_ok( + capsys, + *db, + "case-followup", + "--case-id", + "CASE-20260717-001", + str(followup_path), + "--approved", + ) + ) + assert followup_receipt["turn_id"] == "T002" + assert followup_receipt["status"] == "Active Exchange" + + posted = make_reply( + "ignored", + "T002", + participant_ref="USER", + direction=TurnDirection.OUTGOING, + state=TurnState.POSTED, + exact_text="Exact published reply.", + observed_at="2026-07-17 12:00", + ) + posting_path = _write(tmp_path / "posting.json", posted) + posting_receipt = _as_dict( + _run_ok( + capsys, + *db, + "case-record-posting", + "--case-id", + "CASE-20260717-001", + str(posting_path), + "--approved", + ) ) - assert _run_ok(capsys, "source-consistency", str(consistency_path))["consistent"] is True - assert _run_ok( - capsys, "case-key", "--post-id", "123", "--root-comment-id", "456" + assert posting_receipt["turn_id"] == "T003" + assert posting_receipt["readback"] == "verified" + + close_path = _write( + tmp_path / "close.json", + { + "status": CaseStatus.CLOSED_SUBSTANTIVE, + "updated_at": "2026-07-18 10:00", + "outcome_score": 3, + "outcome_class": OutcomeClass.SUBSTANTIVE_ENGAGEMENT, + "outcome_notes": "Observable exchange of reasons.", + "what_worked": "Narrow question.", + "what_failed": "Long setup.", + "next_test": "Lead with the narrow question.", + "closed_at": "2026-07-18 10:00", + "reason": "explicit closeout", + }, + ) + close_receipt = _as_dict( + _run_ok( + capsys, + *db, + "case-close", + "--case-id", + "CASE-20260717-001", + str(close_path), + "--approved", + ) + ) + assert close_receipt["status"] == "Closed - Substantive" + assert _run_ok(capsys, *db, "case-list-open") == [] + dataset = _as_dict(_run_ok(capsys, *db, "strategy-dataset")) + assert len(dataset["cases"]) == 1 # type: ignore[arg-type] + assert len(dataset["turns"]["CASE-20260717-001"]) == 3 # type: ignore[index] + + +def test_pure_validation_commands_and_migration_receipt( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert _as_dict( + _run_ok( + capsys, + "case-key", + "--post-id", + "123", + "--root-comment-id", + "456", + ) )["case_key"] == "facebook:123:456" ids_path = _write(tmp_path / "case-ids.json", ["CASE-20260717-001"]) - assert _run_ok( - capsys, - "next-case-id", - "--date", - "2026-07-17", - "--existing", - str(ids_path), + assert _as_dict( + _run_ok( + capsys, + "next-case-id", + "--date", + "2026-07-17", + "--existing", + str(ids_path), + ) )["case_id"] == "CASE-20260717-002" turns_path = _write( tmp_path / "turn-ids.json", [{"case_id": "CASE-20260717-001", "turn_id": "T001"}], ) - assert _run_ok( - capsys, - "next-turn-id", - "--case-id", - "CASE-20260717-001", - "--existing", - str(turns_path), + assert _as_dict( + _run_ok( + capsys, + "next-turn-id", + "--case-id", + "CASE-20260717-001", + "--existing", + str(turns_path), + ) )["turn_id"] == "T002" case_path = _write(tmp_path / "case.json", make_case()) - assert _run_ok(capsys, "validate-case", str(case_path))["valid"] is True + assert _as_dict(_run_ok(capsys, "validate-case", str(case_path)))["valid"] is True turn = make_turn() turn_path = _write(tmp_path / "turn.json", turn) - assert _run_ok(capsys, "validate-turn", str(turn_path))["valid"] is True + assert _as_dict(_run_ok(capsys, "validate-turn", str(turn_path)))["valid"] is True transition_path = _write( tmp_path / "transition.json", { @@ -91,7 +223,9 @@ def test_all_required_cli_commands_smoke( "reason": "new turn", }, ) - assert _run_ok(capsys, "validate-transition", str(transition_path))["valid"] is True + assert _as_dict( + _run_ok(capsys, "validate-transition", str(transition_path)) + )["valid"] is True graph_path = _write( tmp_path / "graph.json", [ @@ -105,21 +239,33 @@ def test_all_required_cli_commands_smoke( ), ], ) - assert _run_ok(capsys, "validate-parent-graph", str(graph_path))["turn_count"] == 2 - pending_path = _write(tmp_path / "pending.json", []) - assert _run_ok(capsys, "pending-sync-check", str(pending_path))["clear"] is True - - expected_path = _write(tmp_path / "expected.json", {"Turn ID": "T001"}) - actual_path = _write(tmp_path / "actual.json", {"Turn ID": "T001"}) - assert _run_ok( - capsys, - "verify-readback", - "--expected", - str(expected_path), - "--actual", - str(actual_path), + assert _as_dict( + _run_ok(capsys, "validate-parent-graph", str(graph_path)) + )["turn_count"] == 2 + + expected_path = _write(tmp_path / "expected.json", {"turn_id": "T001"}) + actual_path = _write(tmp_path / "actual.json", {"turn_id": "T001"}) + assert _as_dict( + _run_ok( + capsys, + "verify-readback", + "--expected", + str(expected_path), + "--actual", + str(actual_path), + ) )["succeeded"] is True receipt_path = Path(__file__).parents[1] / "docs" / "migration-receipt.json.example" - receipt = _run_ok(capsys, "migration-receipt", str(receipt_path)) - assert receipt["manual_version"] == "2.7" + receipt = _as_dict(_run_ok(capsys, "migration-receipt", str(receipt_path))) + assert receipt["database_schema_version"] == 1 + + +def test_write_command_rejects_missing_approval( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + database = tmp_path / "canonical.sqlite3" + assert main(["--database", str(database), "db-init"]) == 2 + error = json.loads(capsys.readouterr().err) + assert error["category"] == "write_safety_error" + assert not database.exists() diff --git a/tests/test_identity_identifiers.py b/tests/test_identity_identifiers.py index de986a3..fe2cd6f 100644 --- a/tests/test_identity_identifiers.py +++ b/tests/test_identity_identifiers.py @@ -5,7 +5,6 @@ from dialogue_lab.case_identity import find_duplicate_case, make_case_identity from dialogue_lab.errors import DialogueLabError, IdentifierError from dialogue_lab.identifiers import next_case_id, next_turn_id -from dialogue_lab.models import PendingSyncRecord from tests.helpers import make_case @@ -50,25 +49,6 @@ def test_malformed_and_duplicate_case_ids_are_rejected() -> None: ) -def test_unresolved_pending_sync_blocks_case_allocation() -> None: - record = PendingSyncRecord( - operation="append_case", - target_file="case-log", - target_sheet="Cases", - case_id="CASE-20260717-001", - turn_id=None, - expected_values={"Case ID": "CASE-20260717-001"}, - failure="connector unavailable", - last_verified_state={}, - manual_version="2.7", - source_revision_state={"manual": "r1"}, - required_reconciliation_action="retry append and read back", - created_at="2026-07-17T10:00:00+03:00", - ) - with pytest.raises(IdentifierError, match="PENDING SYNC"): - next_case_id([], on_date=date(2026, 7, 17), pending_sync=[record]) - - def test_turn_ids_are_case_local_and_sequential() -> None: assert next_turn_id([]) == "T001" assert next_turn_id(["T001", "T003"]) == "T004" diff --git a/tests/test_migration_cli.py b/tests/test_migration_cli.py new file mode 100644 index 0000000..adfc93a --- /dev/null +++ b/tests/test_migration_cli.py @@ -0,0 +1,52 @@ +import json +from pathlib import Path + +import pytest + +from dialogue_lab.cli import main +from dialogue_lab.migration_receipt import migration_receipt_from_mapping +from dialogue_lab.models import to_jsonable + + +def test_migration_receipt_requires_database_evidence_and_git_commit() -> None: + payload: dict[str, object] = { + "cutover_timestamp": "2026-07-20T10:00:00+03:00", + "timezone": "Asia/Jerusalem", + "database_schema_version": 1, + "database_integrity": "ok", + "database_backup": "external/snapshot.sqlite3", + "imported_case_count": 1, + "imported_turn_count": 2, + "repository_commit": "deadbeef", + "test_results": "passed", + "known_limitations": ["single local writer"], + "rollback_instructions": ["restore the verified SQLite backup"], + } + receipt = migration_receipt_from_mapping(payload) + serialized = to_jsonable(receipt) + assert serialized["database_schema_version"] == 1 + assert serialized["repository_commit"] == "deadbeef" + + +def test_cli_parse_url_and_version_emit_success(capsys: pytest.CaptureFixture[str]) -> None: + assert main(["parse-url", "https://facebook.com/reel/123?comment_id=456"]) == 0 + output = json.loads(capsys.readouterr().out) + assert output["post_id"] == "123" + with pytest.raises(SystemExit) as exit_info: + main(["--version"]) + assert exit_info.value.code == 0 + + +def test_no_python_module_exposes_facebook_publishing_behavior() -> None: + source_root = Path(__file__).parents[1] / "src" / "dialogue_lab" + text = "\n".join(path.read_text(encoding="utf-8") for path in source_root.glob("*.py")) + assert "post_to_facebook" not in text + assert "publish_to_facebook" not in text + + +def test_synthetic_eval_catalog_covers_local_storage_scenarios() -> None: + path = Path(__file__).parents[1] / "evals" / "cases" / "synthetic-cases.json" + cases = json.loads(path.read_text(encoding="utf-8")) + ids = {case["id"] for case in cases} + assert len(cases) == 17 + assert {"new-case-intake", "sqlite-write-rollback", "schema-version-mismatch"} <= ids diff --git a/tests/test_pending_receipt_cli.py b/tests/test_pending_receipt_cli.py deleted file mode 100644 index 9fb366a..0000000 --- a/tests/test_pending_receipt_cli.py +++ /dev/null @@ -1,81 +0,0 @@ -import json -from pathlib import Path - -import pytest - -from dialogue_lab.cli import main -from dialogue_lab.errors import WriteSafetyError -from dialogue_lab.migration_receipt import migration_receipt_from_mapping -from dialogue_lab.models import to_jsonable -from dialogue_lab.pending_sync import ( - create_pending_sync, - mark_reconciled, - require_no_pending_sync, -) - - -def test_failed_write_creates_complete_pending_sync_and_blocks_until_reconciled() -> None: - record = create_pending_sync( - operation="append_turn", - target_file="case-log-id", - target_sheet="Turns", - case_id="CASE-20260717-001", - turn_id="T001", - expected_values={"Turn ID": "T001"}, - failure="read-back mismatch", - last_verified_state={"row": "absent"}, - manual_version="2.7", - source_revision_state={"manual": "r1", "case_log": "m1"}, - required_reconciliation_action="retry append and verify", - ) - with pytest.raises(WriteSafetyError, match="unresolved"): - require_no_pending_sync([record]) - require_no_pending_sync([mark_reconciled(record)]) - - -def test_migration_receipt_requires_manual_version_and_git_commit() -> None: - payload: dict[str, object] = { - "cutover_timestamp": "2026-07-17T10:00:00+03:00", - "timezone": "Asia/Jerusalem", - "manual_version": "2.7", - "manual_revision_state": "r1", - "case_log_schema_signature": "a" * 64, - "case_log_modified_state": "m1", - "repository_commit": "deadbeef", - "repository_tag": "pre-codex-cutover", - "codex_environment": "Codex app", - "drive_connection_status": "read-only verified", - "writer_enabled": False, - "previous_writer_disabled": False, - "test_results": "passed", - "known_limitations": ["single writer"], - "rollback_instructions": ["enable exactly one writer"], - } - receipt = migration_receipt_from_mapping(payload) - serialized = to_jsonable(receipt) - assert serialized["manual_version"] == "2.7" - assert serialized["repository_commit"] == "deadbeef" - - -def test_cli_parse_url_and_version_emit_success(capsys: pytest.CaptureFixture[str]) -> None: - assert main(["parse-url", "https://facebook.com/reel/123?comment_id=456"]) == 0 - output = json.loads(capsys.readouterr().out) - assert output["post_id"] == "123" - with pytest.raises(SystemExit) as exit_info: - main(["--version"]) - assert exit_info.value.code == 0 - - -def test_no_python_module_exposes_facebook_publishing_behavior() -> None: - source_root = Path(__file__).parents[1] / "src" / "dialogue_lab" - text = "\n".join(path.read_text(encoding="utf-8") for path in source_root.glob("*.py")) - assert "post_to_facebook" not in text - assert "publish_to_facebook" not in text - - -def test_synthetic_eval_catalog_covers_required_scenarios() -> None: - path = Path(__file__).parents[1] / "evals" / "cases" / "synthetic-cases.json" - cases = json.loads(path.read_text(encoding="utf-8")) - ids = {case["id"] for case in cases} - assert len(cases) == 17 - assert {"new-case-intake", "failed-drive-write", "unsupported-manual-major"} <= ids diff --git a/tests/test_schema_source_write.py b/tests/test_schema_source_write.py deleted file mode 100644 index 0e799a0..0000000 --- a/tests/test_schema_source_write.py +++ /dev/null @@ -1,142 +0,0 @@ -import copy - -import pytest - -from dialogue_lab.drive_protocol import CASE_LOG_ID, GENERAL_RESPONSES_ID, connector_request -from dialogue_lab.errors import CompatibilityError, WriteSafetyError -from dialogue_lab.manual_version import require_supported_manual -from dialogue_lab.models import DriveWriteRequest -from dialogue_lab.readback import verify_readback -from dialogue_lab.schema import expected_schema_payload, schema_signature, validate_schema -from dialogue_lab.source_consistency import check_source_consistency - - -def _observed_schema() -> dict[str, object]: - expected = expected_schema_payload() - headers = expected["headers"] - assert isinstance(headers, dict) - return { - "sheets": { - "Cases": headers["Cases"], - "Turns": headers["Turns"], - "Data Dictionary": [], - "Strategy Taxonomy": [], - "Dashboard": [], - }, - "enums": expected["enums"], - "dashboard_formulas": expected["essential_dashboard_formulas"], - } - - -def test_manual_version_27_is_supported() -> None: - version, warnings = require_supported_manual("Version 2.7 — synthetic") - assert str(version) == "2.7" - assert warnings == () - - -def test_manual_below_minimum_and_unknown_major_fail() -> None: - with pytest.raises(CompatibilityError, match="older"): - require_supported_manual("Version 2.6") - with pytest.raises(CompatibilityError, match="major"): - require_supported_manual("Version 3.0") - - -def test_newer_minor_warns_without_claiming_sync() -> None: - version, warnings = require_supported_manual("Version 2.8") - assert str(version) == "2.8" - assert warnings and "revalidate" in warnings[0] - - -def test_live_shape_schema_signature_is_deterministic() -> None: - first = validate_schema(_observed_schema()) - second = schema_signature() - assert first == second - assert len(first.value) == 64 - - -def test_header_enum_and_formula_mismatch_block_writes() -> None: - header_bad = _observed_schema() - assert isinstance(header_bad["sheets"], dict) - header_bad["sheets"]["Cases"] = ["Wrong"] - with pytest.raises(CompatibilityError, match="header mismatch"): - validate_schema(header_bad) - - enum_bad = _observed_schema() - assert isinstance(enum_bad["enums"], dict) - enum_bad["enums"]["Turns.State"] = ["Received"] - with pytest.raises(CompatibilityError, match="enum mismatch"): - validate_schema(enum_bad) - - formula_bad = _observed_schema() - formula_bad["dashboard_formulas"] = [] - with pytest.raises(CompatibilityError, match="Dashboard formulas"): - validate_schema(formula_bad) - - -def test_changed_manual_or_schema_requires_revalidation() -> None: - start = {"manual": "r1", "case_log_schema": "s1", "strategy": "r1"} - manual_changed = check_source_consistency(start, {**start, "manual": "r2"}) - assert not manual_changed.consistent and manual_changed.requires_reload - schema_changed = check_source_consistency(start, {**start, "case_log_schema": "s2"}) - assert not schema_changed.consistent - - -def test_non_material_change_can_remain_consistent() -> None: - start = {"manual": "r1", "strategy": "r1"} - result = check_source_consistency( - start, {"manual": "r1", "strategy": "r2"}, material_sources={"manual"} - ) - assert result.consistent - assert result.changed_sources == ("strategy",) - - -def test_readback_requires_exact_text_and_every_expected_field() -> None: - expected = {"Turn ID": "T001", "Exact Text": "exact", "State": "Posted"} - assert verify_readback(expected, copy.deepcopy(expected)).succeeded - mismatch = verify_readback(expected, {"Turn ID": "T001", "Exact Text": "changed"}) - assert not mismatch.succeeded - assert "value mismatch: Exact Text" in mismatch.mismatches - assert "missing field: State" in mismatch.mismatches - - -def test_connector_success_without_matching_readback_is_still_failure() -> None: - connector_response = {"success": True} - result = verify_readback({"Case ID": "CASE-20260717-001"}, connector_response) - assert not result.succeeded - - -def test_write_policy_rejects_unapproved_protected_and_arbitrary_targets() -> None: - unapproved = DriveWriteRequest( - operation="append_case", - target_file_id=CASE_LOG_ID, - target_sheet="Cases", - record_identity="CASE-20260717-001", - values={"Case ID": "CASE-20260717-001"}, - explicitly_approved=False, - ) - with pytest.raises(WriteSafetyError, match="approval"): - connector_request(unapproved, {"manual": "r1"}) - protected = DriveWriteRequest( - operation="append_case", - target_file_id=GENERAL_RESPONSES_ID, - target_sheet="Cases", - record_identity="CASE-20260717-001", - values={"Case ID": "CASE-20260717-001"}, - explicitly_approved=True, - ) - with pytest.raises(WriteSafetyError, match="protected"): - connector_request(protected, {"manual": "r1"}) - - -def test_valid_case_log_write_builds_typed_connector_envelope() -> None: - request = DriveWriteRequest( - operation="append_turn", - target_file_id=CASE_LOG_ID, - target_sheet="Turns", - record_identity="CASE-20260717-001:T001", - values={"Turn ID": "T001"}, - explicitly_approved=True, - ) - envelope = connector_request(request, {"manual": "r1", "case_log": "m1"}) - assert envelope.action == "append_turn" - assert envelope.source_revision_state["manual"] == "r1" diff --git a/tests/test_sqlite_storage.py b/tests/test_sqlite_storage.py new file mode 100644 index 0000000..5553754 --- /dev/null +++ b/tests/test_sqlite_storage.py @@ -0,0 +1,199 @@ +import sqlite3 +from dataclasses import replace +from pathlib import Path + +import pytest + +from dialogue_lab.enums import CaseStatus, OutcomeClass, TurnDirection, TurnState +from dialogue_lab.errors import DialogueLabError, StorageError, WriteSafetyError +from dialogue_lab.storage import SQLiteStore +from dialogue_lab.validation import validate_case +from tests.helpers import make_case, make_reply, make_turn + + +def test_database_initialization_requires_approval_and_reports_integrity( + tmp_path: Path, +) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + with pytest.raises(WriteSafetyError, match="approval"): + store.initialize(approved=False) + + status = store.initialize(approved=True) + assert status["integrity"] == "ok" + assert status["schema_version"] == 1 + assert status["case_count"] == 0 + + +def test_case_creation_is_atomic_unique_and_read_back(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case() + turn = make_turn() + + receipt = store.create_case(case, [turn], approved=True) + assert receipt == { + "created": True, + "case_id": "CASE-20260717-001", + "turn_count": 1, + } + assert store.get_case(case.case_id) == case + assert store.get_turns(case.case_id) == [turn] + + duplicate = make_case(case_id="CASE-20260717-002") + with pytest.raises(StorageError, match="UNIQUE"): + store.create_case(duplicate, [], approved=True) + assert store.status()["case_count"] == 1 + + +def test_failed_import_rolls_back_every_row(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + first = make_case(case_id="CASE-20260717-001") + duplicate_identity = make_case(case_id="CASE-20260717-002") + + with pytest.raises(StorageError, match="UNIQUE"): + store.import_records([first, duplicate_identity], [], approved=True) + + assert store.status()["case_count"] == 0 + + +def test_schema_version_mismatch_blocks_reads(tmp_path: Path) -> None: + path = tmp_path / "dialogue-lab.sqlite3" + store = SQLiteStore(path) + store.initialize(approved=True) + with sqlite3.connect(path) as connection: + connection.execute("PRAGMA user_version = 99") + + with pytest.raises(StorageError, match="schema version mismatch"): + store.status() + + +def test_initialization_refuses_an_unrelated_database(tmp_path: Path) -> None: + path = tmp_path / "unrelated.sqlite3" + with sqlite3.connect(path) as connection: + connection.execute("CREATE TABLE unrelated(value TEXT)") + + with pytest.raises(StorageError, match="non-empty unversioned"): + SQLiteStore(path).initialize(approved=True) + + +def test_followup_updates_status_and_verifies_committed_turn(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case() + first = make_turn() + store.create_case(case, [first], approved=True) + followup = make_reply( + "T002", + "T001", + observed_at="2026-07-17 11:00", + ) + + receipt = store.add_turn( + followup, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at=followup.observed_at, + reason="incoming public turn", + replace_draft_id=None, + approved=True, + ) + + assert receipt["readback"] == "verified" + assert store.get_case(case.case_id).status is CaseStatus.ACTIVE_EXCHANGE + assert store.get_turns(case.case_id)[-1] == followup + + +def test_posting_retires_named_draft_in_same_transaction(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case(status=CaseStatus.ACTIVE_EXCHANGE) + first = make_turn() + draft = make_reply( + "T002", + "T001", + participant_ref="USER", + direction=TurnDirection.OUTGOING, + state=TurnState.DRAFT, + ) + store.create_case(case, [first, draft], approved=True) + posted = make_reply( + "T003", + "T001", + participant_ref="USER", + direction=TurnDirection.OUTGOING, + state=TurnState.POSTED, + exact_text="Exact published wording.", + observed_at="2026-07-17 12:00", + ) + + store.add_turn( + posted, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at=posted.observed_at, + reason="posting confirmed", + replace_draft_id="T002", + approved=True, + ) + + turns = store.get_turns(case.case_id) + assert turns[1].state is TurnState.REPLACED + assert turns[2] == posted + + +def test_close_case_writes_only_validated_outcome_and_reads_back(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case(status=CaseStatus.ACTIVE_EXCHANGE) + store.create_case(case, [make_turn()], approved=True) + closed = replace( + case, + status=CaseStatus.CLOSED_SUBSTANTIVE, + updated_at="2026-07-18 10:00", + outcome_score=3, + outcome_class=OutcomeClass.SUBSTANTIVE_ENGAGEMENT, + outcome_notes="Observable exchange of reasons.", + what_worked="Narrow question.", + what_failed="Long setup.", + next_test="Lead with the narrow question.", + closed_at="2026-07-18 10:00", + ) + + receipt = store.close_case(closed, reason="explicit closeout", approved=True) + + assert receipt["readback"] == "verified" + assert store.get_case(case.case_id) == closed + + +def test_open_case_requires_and_returns_exact_comment_permalink(tmp_path: Path) -> None: + invalid = make_case(post_url="https://www.facebook.com/example/posts/123") + with pytest.raises(DialogueLabError, match="comment or reply"): + validate_case(invalid) + + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case( + post_url=( + "https://www.facebook.com/example/posts/123?comment_id=456" + "&__cft__[0]=tracking#comment" + ) + ) + store.create_case(case, [], approved=True) + assert store.list_open_summaries() == [ + { + "case_id": case.case_id, + "status": "Posted", + "exact_permalink": case.post_url, + } + ] + + +def test_backup_is_consistent_and_never_overwrites(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + store.create_case(make_case(), [], approved=True) + destination = tmp_path / "backups" / "snapshot.sqlite3" + + store.backup(destination, approved=True) + assert SQLiteStore(destination).status()["case_count"] == 1 + with pytest.raises(StorageError, match="already exists"): + store.backup(destination, approved=True) diff --git a/uv.lock b/uv.lock index e719d9f..b829692 100644 --- a/uv.lock +++ b/uv.lock @@ -22,7 +22,7 @@ wheels = [ [[package]] name = "israel-facebook-dialogue-lab" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "tzdata" }, From c2b8a6dcdcce9741a66645198a7cabcb71109f11 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 22 Jul 2026 04:33:22 +0300 Subject: [PATCH 2/5] Redesign Dialogue Lab case and turn identity --- .agents/skills/dialogue-lab-closeout/SKILL.md | 4 +- .agents/skills/dialogue-lab-followup/SKILL.md | 15 +- .agents/skills/dialogue-lab-intake/SKILL.md | 15 +- .../dialogue-lab-intake/agents/openai.yaml | 2 +- .agents/skills/dialogue-lab-posting/SKILL.md | 6 +- .../dialogue-lab-strategy-review/SKILL.md | 2 +- AGENTS.history.json | 88 ++ AGENTS.md | 8 +- README.md | 27 +- docs/cli-payloads.md | 30 +- docs/migration-receipt.json.example | 18 +- docs/rollback.md | 10 +- docs/rollout.md | 8 +- evals/README.md | 2 +- evals/cases/synthetic-cases.json | 20 +- src/dialogue_lab/case_identity.py | 27 +- src/dialogue_lab/cli.py | 128 ++- src/dialogue_lab/identifiers.py | 35 +- src/dialogue_lab/migration_receipt.py | 23 +- src/dialogue_lab/models.py | 42 +- src/dialogue_lab/schema.py | 6 +- src/dialogue_lab/storage.py | 963 ++++++++++++++++-- src/dialogue_lab/validation.py | 17 +- tests/helpers.py | 4 +- tests/test_cli_commands.py | 147 ++- tests/test_identity_identifiers.py | 53 +- tests/test_migration_cli.py | 26 +- tests/test_sqlite_storage.py | 448 +++++++- 28 files changed, 1822 insertions(+), 352 deletions(-) create mode 100644 AGENTS.history.json diff --git a/.agents/skills/dialogue-lab-closeout/SKILL.md b/.agents/skills/dialogue-lab-closeout/SKILL.md index cbcf69b..f0140c0 100644 --- a/.agents/skills/dialogue-lab-closeout/SKILL.md +++ b/.agents/skills/dialogue-lab-closeout/SKILL.md @@ -12,11 +12,11 @@ description: Close an Israel Facebook Dialogue Lab case from observable evidence ## Workflow -1. Run `dialogue-lab doctor` and load the complete Case and Turn graph once with `case-show`. +1. Load the complete Case and Turn graph once with `case-show`. 2. Resolve any missing public Turn through `$dialogue-lab-followup` before closure. 3. Verify privacy and record observable chronology only. Never infer persuasion from silence, deletion, blocking, a reaction, or disappearance. 4. Choose the closed status, Outcome Class, highest outcome score reached, concise Outcome Notes, What Worked, What Failed, and exactly one Next Test. -5. Prepare one closeout payload. After explicit approval, run exactly one `dialogue-lab case-close --case-id --approved` transaction and report its compact receipt. +5. Prepare one closeout payload. After explicit approval, run `dialogue-lab check`; if it passes, run exactly one `dialogue-lab case-close --case-id --approved` transaction and report its compact receipt. ## Safety diff --git a/.agents/skills/dialogue-lab-followup/SKILL.md b/.agents/skills/dialogue-lab-followup/SKILL.md index 0fc044b..aca248e 100644 --- a/.agents/skills/dialogue-lab-followup/SKILL.md +++ b/.agents/skills/dialogue-lab-followup/SKILL.md @@ -7,17 +7,18 @@ description: Process supplied public content in an existing Israel Facebook Dial ## Required inputs -- Existing Case ID or enough Post ID + Root Comment ID data to resolve it. +- Existing Case ID or enough Post ID + Root Comment ID and branch context to select one candidate. - Exact new public turn or reaction, supplied URL, observed time, and visible parent context. ## Workflow -1. Run `dialogue-lab doctor`, resolve the Case with `case-find` when needed, then load it once with `case-show`. -2. Parse supplied URLs with `parse-url`, preserve each exact URL, and confirm the Case identity. -3. Assign or reuse only `P1`, `P2`, ... or `USER`. Resolve the direct parent from visible context or user confirmation and retain Parent Confidence; ask only when ambiguity changes the reply. -4. Build one incoming Turn payload. The `case-followup` transaction allocates the Turn ID, validates the complete parent graph, appends the Turn, updates Case state, commits, and reads back. -5. Read repository strategy and evidence Markdown only as needed, verify material claims, and answer the operative claim without treating private planning as public context. -6. Return the standard output. After explicit approval, run exactly one `dialogue-lab case-followup --case-id --approved` transaction and report its compact receipt. +1. Treat an explicit Case ID as definitive; otherwise use root-based `case-find` only for candidate discovery and select the Case from branch context, asking when multiple candidates remain materially ambiguous. Load the selected Case once with `case-show`. +2. Parse supplied URLs with `parse-url`, preserve each exact URL, and confirm that the selected Case contains the tracked branch. Never identify a Case from its latest reply. +3. When sibling branches already stored in one Case must be tracked independently, prepare one `case-split-branch` command with an outside-Git backup and exact new title/topic. After explicit approval, run `dialogue-lab check`; if it passes, run the split and use its committed Case/Turn mapping. +4. Assign or reuse only `P1`, `P2`, ... or `USER`. Resolve the direct parent from visible context or user confirmation and retain Parent Confidence; ask only when ambiguity changes the reply. +5. Build one incoming Turn payload. A supplied `reply_comment_id` is its strongest duplicate identity; without one, the transaction uses Case ID + Parent Turn ID (including a null root) + Direction + Exact Text. The transaction returns the existing Turn on a duplicate or allocates a Turn ID, validates the complete parent graph, appends, commits, and reads back. +6. Read repository strategy and evidence Markdown only as needed, verify material claims, and answer the operative claim without treating private planning as public context. +7. Return the standard output. After explicit approval, run `dialogue-lab check`; if it passes, run exactly one `dialogue-lab case-followup --case-id --approved` transaction and report its compact receipt. ## Output diff --git a/.agents/skills/dialogue-lab-intake/SKILL.md b/.agents/skills/dialogue-lab-intake/SKILL.md index 81b4d2e..127ca91 100644 --- a/.agents/skills/dialogue-lab-intake/SKILL.md +++ b/.agents/skills/dialogue-lab-intake/SKILL.md @@ -1,6 +1,6 @@ --- name: dialogue-lab-intake -description: Start or identify an Israel Facebook Dialogue Lab case from supplied public content, perform deterministic duplicate detection, and prepare one approval-gated SQLite intake transaction. +description: Start or identify an Israel Facebook Dialogue Lab case from supplied public content, resolve Case-ID or root candidates, and prepare one approval-gated SQLite intake transaction. --- # Dialogue Lab Intake @@ -12,13 +12,12 @@ description: Start or identify an Israel Facebook Dialogue Lab case from supplie ## Workflow -1. Run `dialogue-lab doctor` once for the configured database. -2. Parse every supplied Facebook URL with `dialogue-lab parse-url`; preserve exact URLs and never infer a parent solely from `reply_comment_id`. -3. Resolve `Post ID + Root Comment ID` with `dialogue-lab case-find`. If found, hand off to `$dialogue-lab-followup` without allocating or writing. -4. Extract the public context, case-local participant references, material claims, hidden assumption, hostility, evidence confidence, privacy state, and supplied URLs. Store no names or profile links. -5. Read repository strategy and evidence Markdown only as needed. Verify material current claims with authoritative sources and distinguish fact, assessment, allegation, legal status, and moral judgment. -6. Prepare one `case-intake` JSON payload containing the Case and initial public Turns. Do not persist exploratory drafts. -7. Return the standard output. After explicit approval, run exactly one `dialogue-lab case-intake --date --approved` transaction and report its compact receipt. +1. Parse every supplied Facebook URL with `dialogue-lab parse-url`; preserve exact URLs and never infer a parent solely from `reply_comment_id`. +2. Use an explicitly supplied Case ID as definitive. Otherwise run `dialogue-lab case-find` by `Post ID + Root Comment ID`, treat every result as a candidate, and compare its Turn graph with the supplied reply branch. Hand off an identified existing Case to `$dialogue-lab-followup`; an unmatched branch may become a new Case even when the Facebook root already has candidates. +3. Extract the public context, case-local participant references, material claims, hidden assumption, hostility, evidence confidence, privacy state, and supplied URLs. Store no names or profile links. +4. Read repository strategy and evidence Markdown only as needed. Verify material current claims with authoritative sources and distinguish fact, assessment, allegation, legal status, and moral judgment. +5. Prepare one `case-intake` JSON payload containing the Case and initial public Turns. Do not persist exploratory drafts. +6. Return the standard output. After explicit approval, run `dialogue-lab check`; if it passes, run exactly one `dialogue-lab case-intake --approved` transaction and report its compact receipt. ## Output diff --git a/.agents/skills/dialogue-lab-intake/agents/openai.yaml b/.agents/skills/dialogue-lab-intake/agents/openai.yaml index 266b1bb..13bada5 100644 --- a/.agents/skills/dialogue-lab-intake/agents/openai.yaml +++ b/.agents/skills/dialogue-lab-intake/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Dialogue Lab Intake" short_description: "Start or identify a Dialogue Lab case" - default_prompt: "Use $dialogue-lab-intake to analyze this Facebook root-comment discussion." + default_prompt: "Use $dialogue-lab-intake to analyze this Facebook reply branch." diff --git a/.agents/skills/dialogue-lab-posting/SKILL.md b/.agents/skills/dialogue-lab-posting/SKILL.md index cf33586..50c57a7 100644 --- a/.agents/skills/dialogue-lab-posting/SKILL.md +++ b/.agents/skills/dialogue-lab-posting/SKILL.md @@ -13,10 +13,10 @@ description: Record a user's explicit confirmation that a Dialogue Lab reply was ## Workflow -1. Run `dialogue-lab doctor` and load the Case once with `case-show`. -2. Parse any supplied URL with `parse-url`; preserve it exactly and do not derive the immediate parent solely from `reply_comment_id`. +1. Load the Case once with `case-show`. +2. Parse any supplied URL with `parse-url`; preserve it exactly. Use supplied `reply_comment_id` as the strongest duplicate identity, but resolve the immediate parent only from visible context or user confirmation. 3. Prepare one Outgoing Posted Turn payload containing the exact published wording. Include `draft_turn_id` only when an existing Outgoing Draft must be marked Replaced. -4. Run exactly one `dialogue-lab case-record-posting --case-id --approved` transaction. It allocates the Turn ID, validates the graph and lifecycle, writes atomically, and reads back the committed state. +4. After explicit approval, run `dialogue-lab check`; if it passes, run exactly one `dialogue-lab case-record-posting --case-id --approved` transaction. Without `reply_comment_id`, it deduplicates by Case ID + Parent Turn ID (including a null root) + Direction + Exact Text. It otherwise allocates the Turn ID, validates the graph and lifecycle, writes atomically, and reads back the committed state. ## Safety diff --git a/.agents/skills/dialogue-lab-strategy-review/SKILL.md b/.agents/skills/dialogue-lab-strategy-review/SKILL.md index 5b3b472..72a753d 100644 --- a/.agents/skills/dialogue-lab-strategy-review/SKILL.md +++ b/.agents/skills/dialogue-lab-strategy-review/SKILL.md @@ -12,7 +12,7 @@ description: Review deterministic SQLite evidence across closed Israel Facebook ## Workflow -1. Run `dialogue-lab doctor`, read `docs/reply-strategy-guide.md`, and load all closed Cases and Turns in one `dialogue-lab strategy-dataset` call. +1. Read `docs/reply-strategy-guide.md` and load all closed Cases and Turns in one `dialogue-lab strategy-dataset` call. 2. Reject fewer than three reasonably comparable cases. Label reviews below twenty total closed cases as preliminary. 3. Compare topic, hostility, evidence confidence, reply length, thread position, and strategy. Separate commenter outcomes from silent-reader signals. 4. Report sample sizes, contradictory evidence, missing data, and confounders. Do not infer causation from correlation or treat one case as a reusable finding. diff --git a/AGENTS.history.json b/AGENTS.history.json new file mode 100644 index 0000000..4fd6d60 --- /dev/null +++ b/AGENTS.history.json @@ -0,0 +1,88 @@ +{ + "version": 2, + "entries": [ + { + "id": "2026-07-22-case-turn-identity-redesign", + "date": "2026-07-22", + "rules": [ + "HASBARA-IDENTITY-01", + "HASBARA-IDENTITY-02" + ], + "relations": { + "requires": [ + "configured SQLite database is canonical Case and Turn state", + "canonical writes use explicit-approval transactional commands with read-back" + ] + }, + "reason": "Replace mutable Facebook-root and date-scoped identity with stable Case and Turn keys.", + "preserved": "SQLite schema version 1, case-local Turn IDs and parent graph, explicit write approval, transactional rollback, committed read-back, and Facebook-access restrictions.", + "introduced": "Globally sequential Case IDs, multiple Cases per Facebook root, root candidate lookup, and deterministic Turn duplicate identities.", + "validation": "The candidate permits separate same-root reply branches, keeps explicit Case-ID selection definitive, covers null-parent root Turns, and excludes latest-reply identity.", + "original_sha256": "c11a2f51ac39e4ff2cac7dee48f581533c256b898a556c866e85e05ebb5b20e5", + "rules_sha256": "64e705236213c82f520f93a1f4a40cd1746cdc629c5c0def29a5ef86219009f5", + "global_rules_sha256": "8ef798b3330613534c7b03ca75aeb7fb9413a4c6774e2f7388d258a09140ed42" + }, + { + "id": "2026-07-22-canonical-markdown-check", + "date": "2026-07-22", + "rules": [ + "HASBARA-CHECK-01", + "HASBARA-CANONICAL-01" + ], + "relations": { + "requires": [ + "HASBARA-IDENTITY-01", + "explicit-approval transactional SQLite writes" + ] + }, + "reason": "Give the mandatory readiness command a plain operational name and keep its canonical Markdown scope explicit.", + "preserved": "The pre-case readiness gate, SQLite schema and integrity checks, canonical Case and Turn state, and external-write restrictions.", + "introduced": "The check command validates canonical repository Markdown and configured SQLite before case work.", + "validation": "The old command name has no current reference or alias, while the readiness gate continues to hash configured repository Markdown before checking SQLite.", + "original_sha256": "64e705236213c82f520f93a1f4a40cd1746cdc629c5c0def29a5ef86219009f5", + "rules_sha256": "9f172e07852ddd349ccf7bf65ca115cc5ba8a0d05a62e629aba547ab56310830", + "global_rules_sha256": "8ef798b3330613534c7b03ca75aeb7fb9413a4c6774e2f7388d258a09140ed42" + }, + { + "id": "2026-07-22-readiness-and-open-case-presentation", + "date": "2026-07-22", + "rules": [ + "HASBARA-CHECK-01", + "HASBARA-OPEN-CASES-01" + ], + "relations": { + "requires": [ + "HASBARA-CANONICAL-01", + "HASBARA-IDENTITY-01" + ] + }, + "reason": "Avoid redundant readiness work on ordinary reads and prevent root permalinks from being presented as latest-comment links.", + "preserved": "Readiness validation before canonical writes and after failures, exact supplied URL preservation, and the rule that mutable latest-Turn state is never identity.", + "introduced": "Read-only queries may reuse fresh evidence, and open-Case presentation reports the latest public Turn permalink or an explicit missing-link state without root fallback.", + "validation": "The open-Case producer selects public Turns only, exposes the selected Turn ID, reports missing URLs explicitly, and skills place readiness checks immediately before approved writes rather than routine reads.", + "original_sha256": "9f172e07852ddd349ccf7bf65ca115cc5ba8a0d05a62e629aba547ab56310830", + "rules_sha256": "a13eb777209419f0fd81ee29bde8c2c3091041ec8e737d3e1970529f7c18c0ca", + "global_rules_sha256": "8ef798b3330613534c7b03ca75aeb7fb9413a4c6774e2f7388d258a09140ed42" + }, + { + "id": "2026-07-22-readiness-evidence-reuse", + "date": "2026-07-22", + "rules": [ + "HASBARA-CHECK-01" + ], + "relations": { + "requires": [ + "HASBARA-CANONICAL-01", + "HASBARA-OPEN-CASES-01" + ] + }, + "reason": "Prevent redundant readiness checks during ordinary read-only queries when fresh sufficient evidence already exists.", + "preserved": "Mandatory readiness checks before canonical writes, after failures, and whenever database state is uncertain.", + "introduced": "Ordinary read-only queries must reuse fresh sufficient readiness evidence rather than optionally rerunning the check.", + "validation": "The rule retains every write and uncertainty gate while making the previously optional read-only reuse behavior mandatory.", + "original_sha256": "a13eb777209419f0fd81ee29bde8c2c3091041ec8e737d3e1970529f7c18c0ca", + "rules_sha256": "0c66144eed98d6320a6d0ccbad65e84257a28f7856e28e3bc0e50b4b763f6497", + "global_rules_sha256": "8ef798b3330613534c7b03ca75aeb7fb9413a4c6774e2f7388d258a09140ed42" + } + ] +} diff --git a/AGENTS.md b/AGENTS.md index 4316127..a901737 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,9 +10,11 @@ ## HasbaraTops -- Before case work, run `dialogue-lab doctor` against the configured SQLite database and stop if local documents, schema, or database integrity fail validation. -- Treat repository Markdown as canonical governance, strategy, and evidence content and the configured SQLite database as canonical Case and Turn state; do not use Google Drive or MCP for Dialogue Lab work. -- Use `config/storage.toml`; preserve the SQLite schema version and enforce unique `Post ID + Root Comment ID` identity before Case ID allocation. +- [HASBARA-CHECK-01] Run `dialogue-lab check` before a canonical write, after a failed write or readiness check, or when database state is uncertain; ordinary read-only queries must reuse fresh sufficient evidence. +- [HASBARA-CANONICAL-01] Treat repository Markdown as canonical governance, strategy, and evidence content and the configured SQLite database as canonical Case and Turn state; do not use MCP for Dialogue Lab work. +- [HASBARA-IDENTITY-01] Use `config/storage.toml`; preserve the SQLite schema version; treat Case ID as definitive, allow multiple Cases per `Post ID + Root Comment ID`, and treat root lookup as candidate discovery only. +- [HASBARA-IDENTITY-02] Deduplicate Turns by supplied `reply_comment_id`; when absent, use Case ID + Parent Turn ID (including null roots) + Direction + Exact Text; never use mutable latest-reply state as identity. +- [HASBARA-OPEN-CASES-01] When presenting open Cases, use each Case's latest public Turn supplied exact URL; never substitute the Case root URL, and mark a missing link explicitly. - Invoke the matching Dialogue Lab skill for intake, follow-up, posting confirmation, closeout, or strategy review. - Never edit, move, rename, replace, delete, import, or summarize `General responses` without the user's explicit instruction for that exact action. - Perform canonical writes only through explicit-approval `dialogue-lab` commands that use SQLite transactions and committed read-back verification. diff --git a/README.md b/README.md index b69e48b..0a88e10 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository is the canonical governance and execution layer for the Dialogue Lab. Repository Markdown owns governance, strategy, and reusable evidence. One SQLite database outside Git owns Case and Turn state. -The runtime performs no Google Drive or MCP operations. +The runtime reads repository Markdown and the configured SQLite database only. ## Architecture @@ -29,7 +29,7 @@ The model owns interpretation, materially ambiguous parentage, reply drafting, f uv sync --frozen --extra dev $env:DIALOGUE_LAB_DB = '\dialogue-lab.sqlite3' uv run dialogue-lab db-init --approved -uv run dialogue-lab doctor +uv run dialogue-lab check ``` `DIALOGUE_LAB_DB` must resolve outside the Git repository. `--database ` may override it and must appear before the subcommand. @@ -39,18 +39,21 @@ Database initialization, imports, backups, and Case or Turn mutations require `- ## High-level commands ```text -dialogue-lab doctor +dialogue-lab check dialogue-lab db-init --approved dialogue-lab db-status dialogue-lab db-backup --destination --approved dialogue-lab db-import --approved +dialogue-lab db-migrate-identity --backup-destination --approved +dialogue-lab case-find --case-id dialogue-lab case-find --post-id --root-comment-id dialogue-lab case-show --case-id +dialogue-lab case-split-branch --case-id --branch-root-turn-id --new-case-title --new-topic <topic> --backup-destination <outside-repo-path> --approved dialogue-lab case-list-open dialogue-lab strategy-dataset -dialogue-lab case-intake <payload.json> --date <YYYY-MM-DD> --approved +dialogue-lab case-intake <payload.json> --approved dialogue-lab case-followup --case-id <id> <payload.json> --approved dialogue-lab case-record-posting --case-id <id> <payload.json> --approved dialogue-lab case-close --case-id <id> <payload.json> --approved @@ -60,6 +63,14 @@ The high-level write commands allocate identifiers, validate all affected record See [CLI payload contracts](docs/cli-payloads.md) for exact JSON shapes. +## Identity model + +`case_id` is the definitive Case key. Case IDs use `Case-NNN` and are allocated from one global sequence; dates and Facebook identifiers are not part of the key. Separate reply branches may therefore be represented by multiple Cases with the same `post_id` and `root_comment_id`. `case-find --case-id` resolves one Case, while a root-based lookup returns every matching candidate for explicit selection. + +A non-null `reply_comment_id` from a supplied permalink is globally unique across Turns. Without that value, the deterministic identity is `case_id + parent_turn_id + direction + exact_text`; `parent_turn_id` is null for a root Turn. Mutable ordering or a “latest reply” is never identity. + +`case-list-open` returns one row per Case with the latest public Turn's supplied exact URL. It never substitutes the Case root URL; when the latest Turn has no supplied URL, it returns `permalink_status: "missing"` and a null permalink. This ordering is presentation only and never identity. When sibling branches in one Case must be tracked independently, `case-split-branch` keeps the selected branch in a newly allocated Case, copies its shared ancestor path with fresh case-local Turn IDs, creates a verified backup, commits transactionally, and reads both graphs back. It refuses to copy a shared ancestor carrying a globally unique `reply_comment_id`. + ## Dialogue workflows - Intake: `$dialogue-lab-intake` @@ -70,15 +81,17 @@ See [CLI payload contracts](docs/cli-payloads.md) for exact JSON shapes. Skills use read commands automatically. They may pass `--approved` only after the user approves the exact canonical write. No skill publishes to Facebook. -## Migration +## Import and identity migration 1. Prepare a UTF-8 JSON snapshot with `cases` and `turns` arrays using the exact payload contract. 2. Initialize an empty outside-Git database. 3. Create a verified empty backup. 4. Run `db-import` once after explicit approval. -5. Run `doctor`, compare counts and representative records, create a populated backup, and record `docs/migration-receipt.json.example` with actual values. +5. Run `check`, compare counts and representative records, create a populated backup, and record `docs/migration-receipt.json.example` with actual values. + +Import is atomic and only accepts an empty database. Duplicate Case IDs, duplicate Turn identities, invalid enums, invalid URLs, missing parents, cycles, and foreign-key violations stop the transaction. Repeated `Post ID + Root Comment ID` values are allowed. -Import is atomic and only accepts an empty database. Duplicate Case IDs, duplicate `Post ID + Root Comment ID`, invalid enums, invalid URLs, missing parents, cycles, and foreign-key violations stop the transaction. +For an existing schema-version-1 database, `db-migrate-identity` first creates and verifies the approved outside-Git backup, then deterministically renumbers Cases by `(created_at, full existing case_id)` and rewrites every Case/Turn reference in one transaction. The full old ID is only a stable tie-breaker; its date and suffix are never parsed as identity. The command preserves schema version 1, commits, reopens the database, verifies the complete mapping and integrity, and emits a migration receipt. Any failure rolls back and blocks further writes until rollback and integrity are verified. ## Safety diff --git a/docs/cli-payloads.md b/docs/cli-payloads.md index f037106..0ca3d12 100644 --- a/docs/cli-payloads.md +++ b/docs/cli-payloads.md @@ -10,7 +10,7 @@ All payloads are UTF-8 JSON. Field names are lowercase `snake_case`. Unknown fie { "cases": [ { - "case_id": "CASE-20260720-001", + "case_id": "Case-004", "case_title": "Short title", "created_at": "2026-07-20 10:00", "updated_at": "2026-07-20 10:00", @@ -36,7 +36,13 @@ All payloads are UTF-8 JSON. Field names are lowercase `snake_case`. Unknown fie } ``` -Every imported Case and Turn requires its allocated identifier. `source_links` is an array. Open Cases require an exact Facebook comment or reply permalink. +Every imported Case and Turn requires its allocated identifier. Case IDs must use `Case-NNN` from the global sequence. `source_links` is an array. Open Cases require an exact Facebook comment or reply permalink. + +## Case lookup + +`case-find --case-id Case-004` uses `case_id` as the definitive key and returns exactly that Case when it exists. `case-find --post-id 123 --root-comment-id 456` returns a `candidates` list because multiple Cases may intentionally track separate reply branches under one Facebook root. A root match never silently selects or reuses a Case. + +`case-list-open` reports `last_turn_id`, `last_comment_permalink`, and `permalink_status` for each open Case. The permalink is the latest public Turn's supplied exact URL. A missing URL is reported as null with `permalink_status: "missing"`; the Case root URL is never substituted. Latest-Turn ordering is a presentation choice, not Case or Turn identity. ## Case intake @@ -76,7 +82,9 @@ Every imported Case and Turn requires its allocated identifier. `source_links` i } ``` -The command derives Case and Turn identifiers and forces each Turn identity to match the Case. +The command allocates the next globally unique sequential Case ID and derives Case-local Turn IDs. It does not treat `post_id + root_comment_id` as Case identity, so another Case may be created for a separate branch under the same root. + +Turn duplicate detection first uses a supplied permalink's non-null `reply_comment_id`, which is globally unique across Turns. Otherwise it uses the exact tuple `case_id + parent_turn_id + direction + exact_text`. A root Turn participates in the fallback with `parent_turn_id: null`. Mutable state, timestamps, ordering, and the latest reply do not determine identity. ## Follow-up and posting @@ -84,6 +92,14 @@ The command derives Case and Turn identifiers and forces each Turn identity to m A posting payload must use `direction: "Outgoing"` and `state: "Posted"`. It may include `draft_turn_id` to mark one existing Outgoing Draft as Replaced in the same transaction. +## Branch split + +```text +dialogue-lab case-split-branch --case-id <id> --branch-root-turn-id <turn-id> --new-case-title <title> --new-topic <topic> --backup-destination <outside-repo-path> --approved +``` + +The branch root must be a non-root Turn with another branch remaining in the source Case. The command allocates the next global Case ID, copies the shared ancestor path with fresh case-local Turn IDs, moves the selected branch and all descendants, preserves exact public text and URLs, and verifies the backup and both committed graphs. It stops when a copied shared ancestor has `reply_comment_id`, because that identifier is globally unique. + ## Closeout `case-close` accepts: @@ -105,3 +121,11 @@ A posting payload must use `direction: "Outgoing"` and `state: "Posted"`. It may ``` Only closure fields are updated. The Case identity and public context remain unchanged. + +## Identity migration + +```text +dialogue-lab db-migrate-identity --backup-destination <outside-repo-path> --approved +``` + +This command is the only supported path for renumbering an existing canonical database. It requires explicit approval, creates and verifies a non-overwriting backup, preserves schema version 1, renumbers Cases in stable creation/allocation order, updates every Turn and graph reference transactionally, and verifies the committed mapping and integrity before success. Its JSON receipt reports the backup, unchanged schema version, migrated counts, committed read-back, and integrity result. A failed migration rolls back and blocks further writes until rollback and integrity are verified. diff --git a/docs/migration-receipt.json.example b/docs/migration-receipt.json.example index 1a36a73..cc71b43 100644 --- a/docs/migration-receipt.json.example +++ b/docs/migration-receipt.json.example @@ -1,11 +1,20 @@ { + "operation": "db-migrate-identity", "cutover_timestamp": "YYYY-MM-DDTHH:MM:SS+03:00", "timezone": "Asia/Jerusalem", - "database_schema_version": 1, + "database_schema_version_before": 1, + "database_schema_version_after": 1, "database_integrity": "ok", "database_backup": "external verified backup path", - "imported_case_count": 0, - "imported_turn_count": 0, + "migrated_case_count": 1, + "verified_turn_count": 12, + "first_case_id": "Case-001", + "last_case_id": "Case-001", + "case_id_map": { + "legacy Case ID": "Case-001" + }, + "backup_verified": true, + "committed_read_back": "verified", "repository_commit": "Git commit", "test_results": "pytest, Ruff, and mypy passed", "known_limitations": [ @@ -14,6 +23,7 @@ "rollback_instructions": [ "Stop canonical writes", "Restore the verified SQLite backup", - "Run dialogue-lab doctor" + "Use the recorded pre-migration repository commit", + "Run dialogue-lab check against the restored database" ] } diff --git a/docs/rollback.md b/docs/rollback.md index 411e0cb..58c4cfe 100644 --- a/docs/rollback.md +++ b/docs/rollback.md @@ -2,16 +2,16 @@ ## Pre-cutover checkpoint -1. Record the repository commit, schema version, schema signature, database integrity, and row counts. -2. After explicit approval, create a consistent SQLite backup outside Git and verify it with `db-status`. -3. Generate and verify a migration receipt. +1. Record the repository commit, schema version, schema signature, database integrity, row counts, and current Case-ID sequence. +2. Run the explicitly approved migration command with a new outside-Git backup destination; it must verify the legacy schema, locked source snapshot, and backup before writing. +3. Preserve its migration receipt with the pre-migration repository commit. ## Rollback 1. Obtain explicit user authorization and stop canonical writes. 2. Preserve the failed database for diagnosis; do not overwrite it. 3. Restore the verified backup to a new outside-Git path. -4. Point `DIALOGUE_LAB_DB` to the restored database and run `dialogue-lab doctor`. -5. Verify schema, integrity, Case count, Turn count, and the affected records before resuming one writer. +4. Use the recorded pre-migration repository commit, point `DIALOGUE_LAB_DB` to the restored database, and run `dialogue-lab check`. +5. Verify schema version 1, integrity, Case count, Turn count, Case/Turn references, and the affected records before resuming one writer. Never copy, restore, or overwrite a canonical database without explicit approval. diff --git a/docs/rollout.md b/docs/rollout.md index 8c0c680..409f621 100644 --- a/docs/rollout.md +++ b/docs/rollout.md @@ -10,15 +10,15 @@ Review repository governance, strategy, and evidence Markdown. Keep `General res ## Stage 3 - Empty database validation -After explicit approval, initialize a database outside Git, run `dialogue-lab doctor`, create a verified backup, and record its schema signature and integrity result. +After explicit approval, initialize a database outside Git, run `dialogue-lab check`, create a verified backup, and record its schema signature and integrity result. -## Stage 4 - Deterministic import +## Stage 4 - Deterministic import or identity migration -Prepare one JSON snapshot containing Cases and Turns. Run `db-import` once after explicit approval. The import must be atomic, preserve exact public text and URLs, reject duplicate identity, and pass committed count and integrity checks. +For an empty database, prepare one JSON snapshot containing Cases and Turns and run `db-import` once after explicit approval. For an existing schema-version-1 database, run `db-migrate-identity` with an explicitly approved outside-Git backup destination. The operation must be atomic, preserve exact public text, URLs, Turn graphs, and schema version 1, assign `Case-NNN` in stable creation/allocation order, and pass committed mapping, count, and integrity checks. ## Stage 5 - Shadow validation -Compare imported identities, turns, exact URLs, statuses, parent graphs, classifications, and representative high-level command results against the source snapshot. Fix the producing implementation instead of patching individual rows. +Compare Case IDs, Turns, exact URLs, statuses, parent graphs, classifications, and representative high-level command results against the source state. Verify explicit Case lookup, multi-candidate root lookup, and both Turn duplicate paths. Fix the producing implementation instead of patching individual rows. ## Stage 6 - Cutover gate diff --git a/evals/README.md b/evals/README.md index ff6857e..f387784 100644 --- a/evals/README.md +++ b/evals/README.md @@ -2,7 +2,7 @@ `cases/synthetic-cases.json` contains no real Case or Turn rows, participant names, or Facebook text. -- Deterministic assertions cover URL identity, identifier allocation, lifecycle, graph validation, SQLite schema and integrity, approval gates, committed read-back, and rollback behavior. +- Deterministic assertions cover global Case-ID allocation, definitive Case lookup, multi-candidate root lookup, Turn duplicate identity, lifecycle, graph validation, schema-version-1 integrity, approved migration backups, committed read-back, and rollback behavior. - Qualitative reply review covers claim alignment, one pivotal point, natural thread language, legal precision, face-saving correction, and silent-reader usefulness. - Storage validation runs only against temporary local SQLite databases and never contacts an external service. - Canonical writes, repository strategy edits, cutover, backups, tags, remote publication, and rollback require explicit approval. diff --git a/evals/cases/synthetic-cases.json b/evals/cases/synthetic-cases.json index 7499756..b64f69b 100644 --- a/evals/cases/synthetic-cases.json +++ b/evals/cases/synthetic-cases.json @@ -1,9 +1,17 @@ [ - {"id": "new-case-intake", "kind": "intake", "expected": "new identity and reply output"}, - {"id": "duplicate-case-intake", "kind": "duplicate", "expected": "existing case reused"}, - {"id": "same-post-new-root", "kind": "identity", "expected": "different case identity"}, + {"id": "new-case-intake", "kind": "intake", "expected": "next global Case-NNN allocated"}, + {"id": "explicit-case-lookup", "kind": "identity", "expected": "case_id resolves exactly one Case"}, + {"id": "same-root-separate-cases", "kind": "identity", "expected": "both Cases accepted with different Case-NNN identities"}, + {"id": "same-root-lookup", "kind": "identity", "expected": "all matching Cases returned as candidates without implicit selection"}, + {"id": "open-case-latest-permalink", "kind": "readiness", "expected": "latest public Turn exact URL returned or missing reported without root fallback"}, + {"id": "reply-comment-turn-duplicate", "kind": "duplicate", "expected": "same supplied reply_comment_id rejected across Cases"}, + {"id": "fallback-turn-duplicate", "kind": "duplicate", "expected": "same case_id, parent_turn_id, direction, and exact_text rejected"}, + {"id": "root-turn-fallback-duplicate", "kind": "duplicate", "expected": "null parent_turn_id participates in the exact fallback tuple"}, + {"id": "sibling-turns-distinct-text", "kind": "identity", "expected": "siblings with different exact_text accepted"}, + {"id": "mutable-latest-turn", "kind": "identity", "expected": "latest reply ordering never used as identity"}, {"id": "existing-case-followup", "kind": "followup", "expected": "case-local turn appended"}, {"id": "branched-thread", "kind": "graph", "expected": "multiple children accepted and map shown"}, + {"id": "split-branch-case", "kind": "graph", "expected": "selected branch moves to next Case-NNN with copied ancestors and verified backup"}, {"id": "ambiguous-parent", "kind": "graph", "expected": "confidence retained and clarification requested when material"}, {"id": "hostile-concrete-claim", "kind": "reply", "expected": "insult ignored and operative claim answered"}, {"id": "pure-abuse", "kind": "reply", "expected": "no-engagement considered"}, @@ -13,7 +21,11 @@ {"id": "substantive-closeout", "kind": "closeout", "expected": "observable outcome recorded"}, {"id": "no-response-closeout", "kind": "closeout", "expected": "no persuasion inferred"}, {"id": "sqlite-write-rollback", "kind": "recovery", "expected": "transaction rolls back and integrity remains valid"}, + {"id": "identity-migration-stable-order", "kind": "migration", "expected": "existing Cases deterministically renumbered Case-001 onward in stable creation order"}, + {"id": "identity-migration-schema-version", "kind": "migration", "expected": "schema version remains 1"}, + {"id": "identity-migration-backup", "kind": "migration", "expected": "approved verified backup precedes the transactional write"}, + {"id": "identity-migration-readback", "kind": "migration", "expected": "committed Case and Turn references match the receipt"}, {"id": "schema-version-mismatch", "kind": "schema", "expected": "canonical writes blocked"}, - {"id": "missing-local-document", "kind": "readiness", "expected": "doctor fails safely"}, + {"id": "missing-local-document", "kind": "readiness", "expected": "check fails safely"}, {"id": "unapproved-sqlite-write", "kind": "write-safety", "expected": "write rejected before database mutation"} ] diff --git a/src/dialogue_lab/case_identity.py b/src/dialogue_lab/case_identity.py index 6f19d47..edbb21e 100644 --- a/src/dialogue_lab/case_identity.py +++ b/src/dialogue_lab/case_identity.py @@ -1,20 +1,21 @@ -"""Canonical case identity helpers.""" +"""Definitive Case-ID lookup and secondary Facebook-root discovery.""" from collections.abc import Iterable -from .errors import IdentifierError -from .models import CaseIdentity, CaseRecord +from .models import CaseRecord -def make_case_identity(post_id: str, root_comment_id: str) -> CaseIdentity: - return CaseIdentity(post_id, root_comment_id) +def find_case(case_id: str, cases: Iterable[CaseRecord]) -> CaseRecord | None: + """Resolve one Case by its definitive identifier.""" + return next((case for case in cases if case.case_id == case_id), None) -def find_duplicate_case( - identity: CaseIdentity, cases: Iterable[CaseRecord] -) -> CaseRecord | None: - matches = [case for case in cases if case.identity == identity] - if len(matches) > 1: - ids = ", ".join(case.case_id for case in matches) - raise IdentifierError(f"duplicate canonical identity already exists in cases: {ids}") - return matches[0] if matches else None +def find_case_candidates( + post_id: str, root_comment_id: str, cases: Iterable[CaseRecord] +) -> list[CaseRecord]: + """Return all Cases on a root; callers must select by definitive Case ID.""" + return [ + case + for case in cases + if case.post_id == post_id and case.root_comment_id == root_comment_id + ] diff --git a/src/dialogue_lab/cli.py b/src/dialogue_lab/cli.py index a967965..0f1e1a1 100644 --- a/src/dialogue_lab/cli.py +++ b/src/dialogue_lab/cli.py @@ -10,12 +10,10 @@ import tomllib from collections.abc import Mapping from dataclasses import replace -from datetime import date from pathlib import Path from typing import Any from . import __version__ -from .case_identity import make_case_identity from .enums import CaseStatus, OutcomeClass, ParentConfidence, TurnDirection, TurnKind, TurnState from .errors import DialogueLabError, StorageError, WriteSafetyError from .facebook_url import parse_facebook_url @@ -182,7 +180,7 @@ def _store(args: argparse.Namespace) -> SQLiteStore: return SQLiteStore(_database_path(args)) -def _doctor(args: argparse.Namespace) -> dict[str, object]: +def _check(args: argparse.Namespace) -> dict[str, object]: root = _repo_root(Path.cwd()) config_path = root / "config" / "storage.toml" config = _storage_config(root) @@ -225,24 +223,27 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--database", help="SQLite path; defaults to DIALOGUE_LAB_DB") sub = parser.add_subparsers(dest="command", required=True) - sub.add_parser("doctor", help="verify local docs, configuration, and database integrity") + sub.add_parser( + "check", help="verify canonical Markdown, configuration, and database integrity" + ) init = sub.add_parser("db-init", help="initialize or verify the canonical SQLite database") _add_database_write_arguments(init) sub.add_parser("db-status", help="report schema, integrity, and row counts") backup = sub.add_parser("db-backup", help="create a consistent non-overwriting backup") backup.add_argument("--destination", required=True) _add_database_write_arguments(backup) + identity_migration = sub.add_parser( + "db-migrate-identity", help="migrate Case and Turn identity transactionally" + ) + identity_migration.add_argument("--backup-destination", required=True) + _add_database_write_arguments(identity_migration) import_parser = sub.add_parser("db-import", help="atomically import cases and turns JSON") import_parser.add_argument("json_file") _add_database_write_arguments(import_parser) parse = sub.add_parser("parse-url", help="parse a Facebook URL without rewriting it") parse.add_argument("url") - case_key = sub.add_parser("case-key", help="generate normalized canonical case identity") - case_key.add_argument("--post-id", required=True) - case_key.add_argument("--root-comment-id", required=True) - case_id = sub.add_parser("next-case-id", help="calculate a date-local Case ID") - case_id.add_argument("--date", required=True) + case_id = sub.add_parser("next-case-id", help="calculate the next global Case ID") case_id.add_argument("--existing", required=True) turn_id = sub.add_parser("next-turn-id", help="calculate a case-local Turn ID") turn_id.add_argument("--case-id", required=True) @@ -262,17 +263,28 @@ def _build_parser() -> argparse.ArgumentParser: receipt = sub.add_parser("migration-receipt", help="validate and render a migration receipt") receipt.add_argument("json_file") - find = sub.add_parser("case-find", help="resolve one Case by canonical identity") - find.add_argument("--post-id", required=True) - find.add_argument("--root-comment-id", required=True) + find = sub.add_parser( + "case-find", help="resolve by definitive Case ID or discover root candidates" + ) + find.add_argument("--case-id") + find.add_argument("--post-id") + find.add_argument("--root-comment-id") show = sub.add_parser("case-show", help="return one Case and its complete Turn graph") show.add_argument("--case-id", required=True) - sub.add_parser("case-list-open", help="return exact open-case permalink summaries") + split = sub.add_parser( + "case-split-branch", help="move one reply branch into the next global Case" + ) + split.add_argument("--case-id", required=True) + split.add_argument("--branch-root-turn-id", required=True) + split.add_argument("--new-case-title", required=True) + split.add_argument("--new-topic", required=True) + split.add_argument("--backup-destination", required=True) + _add_database_write_arguments(split) + sub.add_parser("case-list-open", help="return latest-Turn links for open Cases") sub.add_parser("strategy-dataset", help="return all closed Cases and their Turns") intake = sub.add_parser("case-intake", help="create one Case and initial Turns atomically") intake.add_argument("json_file") - intake.add_argument("--date", required=True) _add_database_write_arguments(intake) followup = sub.add_parser("case-followup", help="record one incoming Turn atomically") followup.add_argument("--case-id", required=True) @@ -315,6 +327,16 @@ def _turn_for_case( "root_comment_id": case.root_comment_id, } ) + exact_url = _optional_string(values.get("exact_url")) + if exact_url is not None: + parsed = parse_facebook_url(exact_url) + if parsed.reply_comment_id is not None: + supplied = _optional_string(values.get("reply_comment_id")) + if supplied is not None and supplied != parsed.reply_comment_id: + raise DialogueLabError( + "reply_comment_id conflicts with the supplied Exact URL" + ) + values["reply_comment_id"] = parsed.reply_comment_id return _turn_record(values) @@ -327,6 +349,9 @@ def _run_database_command(args: argparse.Namespace, command: str) -> object: if command == "db-backup": destination = _outside_repository(Path(str(args.destination))) return store.backup(destination, approved=bool(args.approved)) + if command == "db-migrate-identity": + destination = _outside_repository(Path(str(args.backup_destination))) + return store.migrate_identity(destination, approved=bool(args.approved)) if command == "db-import": payload = _mapping(_load_json(str(args.json_file)), "database import") _reject_unknown(payload, {"cases", "turns"}, "database import") @@ -340,14 +365,42 @@ def _run_database_command(args: argparse.Namespace, command: str) -> object: ] return store.import_records(cases, turns, approved=bool(args.approved)) if command == "case-find": - case = store.find_case(str(args.post_id), str(args.root_comment_id)) - result: dict[str, object] = {"found": False} - if case is not None: - result = {"found": True, "case_id": case.case_id, "status": case.status.value} - return result + case_id = _optional_string(args.case_id) + post_id = _optional_string(args.post_id) + root_comment_id = _optional_string(args.root_comment_id) + if case_id is not None: + if post_id is not None or root_comment_id is not None: + raise DialogueLabError( + "case-find accepts either --case-id or a Post ID + Root Comment ID pair" + ) + case = store.get_case(case_id) + return {"found": True, "case_id": case.case_id, "status": case.status.value} + if post_id is None or root_comment_id is None: + raise DialogueLabError( + "case-find requires --case-id or both --post-id and --root-comment-id" + ) + candidates = store.find_cases(post_id, root_comment_id) + return { + "found": bool(candidates), + "candidate_count": len(candidates), + "candidates": [ + {"case_id": case.case_id, "status": case.status.value} + for case in candidates + ], + } if command == "case-show": case_id = str(args.case_id) return {"case": store.get_case(case_id), "turns": store.get_turns(case_id)} + if command == "case-split-branch": + destination = _outside_repository(Path(str(args.backup_destination))) + return store.split_case_branch( + str(args.case_id), + str(args.branch_root_turn_id), + new_case_title=str(args.new_case_title), + new_topic=str(args.new_topic), + backup_destination=destination, + approved=bool(args.approved), + ) if command == "case-list-open": return store.list_open_summaries() if command == "strategy-dataset": @@ -356,25 +409,15 @@ def _run_database_command(args: argparse.Namespace, command: str) -> object: payload = _mapping(_load_json(str(args.json_file)), "case intake") _reject_unknown(payload, {"case", "turns"}, "case intake") case_payload = _mapping(payload.get("case"), "case") - existing = store.find_case( - str(case_payload.get("post_id", "")), str(case_payload.get("root_comment_id", "")) - ) - if existing is not None: - return { - "created": False, - "duplicate": True, - "case_id": existing.case_id, - "status": existing.status.value, - } - case_id = next_case_id(store.case_ids(), on_date=date.fromisoformat(str(args.date))) - case = _case_record({**case_payload, "case_id": case_id}) + case = _case_record({**case_payload, "case_id": ""}) initial_turns: list[TurnRecord] = [] for item in _list(payload.get("turns", []), "turns"): - turn_id = next_turn_id(turn.turn_id for turn in initial_turns) initial_turns.append( - _turn_for_case(_mapping(item, "initial Turn"), case, turn_id) + _turn_for_case(_mapping(item, "initial Turn"), case, "") ) - return store.create_case(case, initial_turns, approved=bool(args.approved)) + return store.create_case( + case, initial_turns, approved=bool(args.approved), allocate_ids=True + ) if command in {"case-followup", "case-record-posting"}: case_id = str(args.case_id) case = store.get_case(case_id) @@ -382,8 +425,7 @@ def _run_database_command(args: argparse.Namespace, command: str) -> object: raise DialogueLabError(f"case is closed: {case_id}") payload = _mapping(_load_json(str(args.json_file)), "Turn payload") _reject_unknown(payload, {*TURN_FIELDS, "draft_turn_id"}, "Turn") - turn_id = next_turn_id(store.turn_ids(case_id)) - turn = _turn_for_case(payload, case, turn_id) + turn = _turn_for_case(payload, case, "") if command == "case-followup": if turn.direction is not TurnDirection.INCOMING: raise DialogueLabError("case-followup requires an Incoming Turn") @@ -457,18 +499,20 @@ def _run_database_command(args: argparse.Namespace, command: str) -> object: def _run(args: argparse.Namespace) -> object: command = str(args.command) - if command == "doctor": - result = _doctor(args) + if command == "check": + result = _check(args) if not result["ok"]: - raise DialogueLabError("local doctor checks failed") + raise DialogueLabError("local readiness checks failed") return result database_commands = { "db-init", "db-status", "db-backup", + "db-migrate-identity", "db-import", "case-find", "case-show", + "case-split-branch", "case-list-open", "strategy-dataset", "case-intake", @@ -480,13 +524,9 @@ def _run(args: argparse.Namespace) -> object: return _run_database_command(args, command) if command == "parse-url": return parse_facebook_url(str(args.url)) - if command == "case-key": - return {"case_key": make_case_identity(args.post_id, args.root_comment_id).key} if command == "next-case-id": existing = _ids_from_json(_load_json(str(args.existing)), "case_id") - return { - "case_id": next_case_id(existing, on_date=date.fromisoformat(str(args.date))) - } + return {"case_id": next_case_id(existing)} if command == "next-turn-id": existing = _ids_from_json( _load_json(str(args.existing)), "turn_id", str(args.case_id) diff --git a/src/dialogue_lab/identifiers.py b/src/dialogue_lab/identifiers.py index 03a9bc9..f65aa3a 100644 --- a/src/dialogue_lab/identifiers.py +++ b/src/dialogue_lab/identifiers.py @@ -1,13 +1,13 @@ -"""Case-local and date-local identifier allocation.""" +"""Global Case and case-local Turn identifier allocation.""" import re from collections.abc import Iterable -from datetime import date, datetime +from datetime import datetime from zoneinfo import ZoneInfo from .errors import IdentifierError -CASE_ID_RE = re.compile(r"^CASE-(\d{8})-(\d{3})$") +CASE_ID_RE = re.compile(r"^Case-(\d{3,})$") TURN_ID_RE = re.compile(r"^T(\d{3})$") JERUSALEM = ZoneInfo("Asia/Jerusalem") @@ -22,25 +22,28 @@ def _require_unique(values: list[str], label: str) -> None: raise IdentifierError(f"duplicate {label}: {', '.join(duplicates)}") +def case_id_number(value: str) -> int: + """Validate a canonical Case ID and return its positive sequence number.""" + match = CASE_ID_RE.fullmatch(value) + if match is None: + raise IdentifierError(f"malformed Case ID: {value}") + number = int(match.group(1)) + if number < 1 or value != f"Case-{number:03d}": + raise IdentifierError(f"malformed Case ID: {value}") + return number + + def next_case_id( existing_ids: Iterable[str], - *, - on_date: date | None = None, ) -> str: - """Allocate the next Jerusalem-date Case ID from freshly supplied rows.""" + """Allocate the next globally sequential Case ID from freshly supplied rows.""" values = list(existing_ids) _require_unique(values, "Case IDs") - parsed: list[tuple[str, int]] = [] + numbers: list[int] = [] for value in values: - match = CASE_ID_RE.fullmatch(value) - if match is None: - raise IdentifierError(f"malformed Case ID: {value}") - parsed.append((match.group(1), int(match.group(2)))) - target = (on_date or now_jerusalem().date()).strftime("%Y%m%d") - sequence = max((number for day, number in parsed if day == target), default=0) + 1 - if sequence > 999: - raise IdentifierError(f"Case ID sequence exhausted for {target}") - return f"CASE-{target}-{sequence:03d}" + numbers.append(case_id_number(value)) + sequence = max(numbers, default=0) + 1 + return f"Case-{sequence:03d}" def next_turn_id(existing_ids: Iterable[str]) -> str: diff --git a/src/dialogue_lab/migration_receipt.py b/src/dialogue_lab/migration_receipt.py index 162014e..3952c36 100644 --- a/src/dialogue_lab/migration_receipt.py +++ b/src/dialogue_lab/migration_receipt.py @@ -15,14 +15,31 @@ def migration_receipt_from_mapping(payload: Mapping[str, object]) -> MigrationRe raise DialogueLabError( f"migration receipt fields mismatch; missing={missing}, extra={extra}" ) + case_id_map = payload["case_id_map"] + if not isinstance(case_id_map, dict): + raise DialogueLabError("migration receipt case_id_map must be a JSON object") + backup_verified = payload["backup_verified"] + if not isinstance(backup_verified, bool): + raise DialogueLabError("migration receipt backup_verified must be a boolean") return MigrationReceipt( + operation=str(payload["operation"]), cutover_timestamp=str(payload["cutover_timestamp"]), timezone=str(payload["timezone"]), - database_schema_version=int(str(payload["database_schema_version"])), + database_schema_version_before=int( + str(payload["database_schema_version_before"]) + ), + database_schema_version_after=int( + str(payload["database_schema_version_after"]) + ), database_integrity=str(payload["database_integrity"]), database_backup=str(payload["database_backup"]), - imported_case_count=int(str(payload["imported_case_count"])), - imported_turn_count=int(str(payload["imported_turn_count"])), + migrated_case_count=int(str(payload["migrated_case_count"])), + verified_turn_count=int(str(payload["verified_turn_count"])), + first_case_id=str(payload["first_case_id"]), + last_case_id=str(payload["last_case_id"]), + case_id_map={str(key): str(value) for key, value in case_id_map.items()}, + backup_verified=backup_verified, + committed_read_back=str(payload["committed_read_back"]), repository_commit=str(payload["repository_commit"]), test_results=str(payload["test_results"]), known_limitations=tuple(str(item) for item in _list(payload["known_limitations"])), diff --git a/src/dialogue_lab/models.py b/src/dialogue_lab/models.py index a55629f..3b3bfb7 100644 --- a/src/dialogue_lab/models.py +++ b/src/dialogue_lab/models.py @@ -15,30 +15,6 @@ TurnKind, TurnState, ) -from .errors import DialogueLabError - - -def _required(value: str, name: str) -> str: - cleaned = value.strip() - if not cleaned: - raise DialogueLabError(f"{name} is required") - return cleaned - - -@dataclass(frozen=True) -class CaseIdentity: - post_id: str - root_comment_id: str - - def __post_init__(self) -> None: - object.__setattr__(self, "post_id", _required(self.post_id, "post_id")) - object.__setattr__( - self, "root_comment_id", _required(self.root_comment_id, "root_comment_id") - ) - - @property - def key(self) -> str: - return f"facebook:{self.post_id}:{self.root_comment_id}" @dataclass(frozen=True) @@ -80,11 +56,6 @@ class CaseRecord: next_test: str = "" closed_at: str = "" - @property - def identity(self) -> CaseIdentity: - return CaseIdentity(self.post_id, self.root_comment_id) - - @dataclass(frozen=True) class TurnRecord: case_id: str @@ -131,13 +102,20 @@ class ThreadMapEntry: @dataclass(frozen=True) class MigrationReceipt: + operation: str cutover_timestamp: str timezone: str - database_schema_version: int + database_schema_version_before: int + database_schema_version_after: int database_integrity: str database_backup: str - imported_case_count: int - imported_turn_count: int + migrated_case_count: int + verified_turn_count: int + first_case_id: str + last_case_id: str + case_id_map: Mapping[str, str] + backup_verified: bool + committed_read_back: str repository_commit: str test_results: str known_limitations: tuple[str, ...] diff --git a/src/dialogue_lab/schema.py b/src/dialogue_lab/schema.py index f4fca81..f23222b 100644 --- a/src/dialogue_lab/schema.py +++ b/src/dialogue_lab/schema.py @@ -77,9 +77,13 @@ def expected_schema_payload() -> dict[str, object]: "enums": {key: list(values) for key, values in EXPECTED_ENUMS.items()}, "constraints": [ "cases.case_id primary key", - "cases(post_id, root_comment_id) unique", + "cases.case_id canonical positive Case-NNN check", + "cases(post_id, root_comment_id) candidate index", "turns(case_id, turn_id) primary key", "turns.case_id references cases.case_id", + "turns.reply_comment_id globally unique when present", + "turns(case_id, coalesced parent_turn_id, direction, exact_text) unique " + "when reply_comment_id absent", ], } diff --git a/src/dialogue_lab/storage.py b/src/dialogue_lab/storage.py index 4dda3f9..2ff94fd 100644 --- a/src/dialogue_lab/storage.py +++ b/src/dialogue_lab/storage.py @@ -24,7 +24,7 @@ TurnState, ) from .errors import StorageError, WriteSafetyError -from .facebook_url import parse_facebook_url +from .identifiers import next_case_id, next_turn_id, now_jerusalem from .lifecycle import CLOSED_STATUSES, validate_transition from .models import CaseRecord, LifecycleTransition, TurnRecord, to_jsonable from .parent_graph import validate_parent_graph @@ -45,14 +45,33 @@ def _sql_enum(values: Iterable[str]) -> str: _STATE_SQL = _sql_enum(item.value for item in TurnState) _SCHEMA_SIGNATURE = schema_signature().value -_SCHEMA_SQL = f""" -CREATE TABLE IF NOT EXISTS storage_metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); +_LEGACY_IDENTITY_SCHEMA_SIGNATURE = schema_signature( + { + "schema_version": SCHEMA_VERSION, + "tables": {"cases": list(CASE_FIELDS), "turns": list(TURN_FIELDS)}, + "enums": { + "cases.status": [item.value for item in CaseStatus], + "cases.outcome_class": [item.value for item in OutcomeClass], + "turns.parent_confidence": [item.value for item in ParentConfidence], + "turns.direction": [item.value for item in TurnDirection], + "turns.kind": [item.value for item in TurnKind], + "turns.state": [item.value for item in TurnState], + }, + "constraints": [ + "cases.case_id primary key", + "cases(post_id, root_comment_id) unique", + "turns(case_id, turn_id) primary key", + "turns.case_id references cases.case_id", + ], + } +).value -CREATE TABLE IF NOT EXISTS cases ( - case_id TEXT PRIMARY KEY, +_CASES_SQL = f""" +CREATE TABLE {{if_not_exists}}{{table}} ( + case_id TEXT PRIMARY KEY CHECK ( + case_id = printf('Case-%03d', CAST(substr(case_id, 6) AS INTEGER)) + AND CAST(substr(case_id, 6) AS INTEGER) > 0 + ), case_title TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, @@ -71,11 +90,12 @@ def _sql_enum(values: Iterable[str]) -> str: what_worked TEXT NOT NULL DEFAULT '', what_failed TEXT NOT NULL DEFAULT '', next_test TEXT NOT NULL DEFAULT '', - closed_at TEXT NOT NULL DEFAULT '', - UNIQUE (post_id, root_comment_id) -); + closed_at TEXT NOT NULL DEFAULT '' +) +""" -CREATE TABLE IF NOT EXISTS turns ( +_TURNS_SQL = f""" +CREATE TABLE {{if_not_exists}}{{table}} ( case_id TEXT NOT NULL, turn_id TEXT NOT NULL, parent_turn_id TEXT, @@ -95,25 +115,133 @@ def _sql_enum(values: Iterable[str]) -> str: observed_at TEXT NOT NULL, notes TEXT NOT NULL DEFAULT '', PRIMARY KEY (case_id, turn_id), - FOREIGN KEY (case_id) REFERENCES cases(case_id) ON DELETE RESTRICT, - FOREIGN KEY (case_id, parent_turn_id) REFERENCES turns(case_id, turn_id) + FOREIGN KEY (case_id) REFERENCES {{cases_table}}(case_id) ON DELETE RESTRICT, + FOREIGN KEY (case_id, parent_turn_id) REFERENCES {{table}}(case_id, turn_id) DEFERRABLE INITIALLY DEFERRED +) +""" + +_SCHEMA_SQL = f""" +CREATE TABLE IF NOT EXISTS storage_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL ); +{_CASES_SQL.format(if_not_exists="IF NOT EXISTS ", table="cases")}; +{_TURNS_SQL.format(if_not_exists="IF NOT EXISTS ", table="turns", cases_table="cases")}; + CREATE INDEX IF NOT EXISTS turns_case_observed_idx ON turns(case_id, observed_at, turn_id); +CREATE INDEX IF NOT EXISTS cases_root_candidates_idx + ON cases(post_id, root_comment_id, created_at, case_id); +CREATE UNIQUE INDEX IF NOT EXISTS turns_reply_comment_id_uq + ON turns(reply_comment_id) WHERE reply_comment_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS turns_fallback_identity_uq + ON turns(case_id, COALESCE(parent_turn_id, ''), direction, exact_text) + WHERE reply_comment_id IS NULL; INSERT INTO storage_metadata(key, value) VALUES ('schema_signature', '{_SCHEMA_SIGNATURE}') ON CONFLICT(key) DO NOTHING; PRAGMA user_version = {SCHEMA_VERSION}; """ +_EXPECTED_INDEX_SQL = { + "turns_case_observed_idx": """CREATE INDEX turns_case_observed_idx + ON turns(case_id, observed_at, turn_id)""", + "cases_root_candidates_idx": """CREATE INDEX cases_root_candidates_idx + ON cases(post_id, root_comment_id, created_at, case_id)""", + "turns_reply_comment_id_uq": """CREATE UNIQUE INDEX turns_reply_comment_id_uq + ON turns(reply_comment_id) WHERE reply_comment_id IS NOT NULL""", + "turns_fallback_identity_uq": """CREATE UNIQUE INDEX turns_fallback_identity_uq + ON turns(case_id, COALESCE(parent_turn_id, ''), direction, exact_text) + WHERE reply_comment_id IS NULL""", +} + + +def _normalized_sql(value: str) -> str: + return " ".join(value.lower().replace("if not exists ", "").split()) + def _require_approval(approved: bool) -> None: if not approved: raise WriteSafetyError("canonical SQLite writes require explicit approval") +def _rows( + connection: sqlite3.Connection, table: str, fields: tuple[str, ...], order_by: str +) -> list[dict[str, object]]: + selected = ", ".join(fields) + return [ + {field: row[field] for field in fields} + for row in connection.execute( + f"SELECT {selected} FROM {table} ORDER BY {order_by}" # noqa: S608 + ) + ] + + +def _snapshot(connection: sqlite3.Connection) -> dict[str, list[dict[str, object]]]: + return { + "cases": _rows(connection, "cases", CASE_FIELDS, "created_at, case_id"), + "turns": _rows(connection, "turns", TURN_FIELDS, "case_id, turn_id"), + } + + +def _verify_database_shape( + connection: sqlite3.Connection, *, expected_signature: str +) -> dict[str, object]: + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + if version != SCHEMA_VERSION: + raise StorageError( + f"database schema version mismatch: {version}; expected {SCHEMA_VERSION}" + ) + signature_row = connection.execute( + "SELECT value FROM storage_metadata WHERE key = 'schema_signature'" + ).fetchone() + actual_signature = str(signature_row["value"]) if signature_row else "" + if actual_signature != expected_signature: + raise StorageError("database schema signature mismatch") + case_columns = tuple( + str(row["name"]) for row in connection.execute("PRAGMA table_info(cases)") + ) + turn_columns = tuple( + str(row["name"]) for row in connection.execute("PRAGMA table_info(turns)") + ) + if case_columns != CASE_FIELDS or turn_columns != TURN_FIELDS: + raise StorageError("database table columns do not match the canonical schema") + integrity = str(connection.execute("PRAGMA integrity_check").fetchone()[0]) + foreign_keys = connection.execute("PRAGMA foreign_key_check").fetchall() + if integrity != "ok" or foreign_keys: + raise StorageError("database integrity check failed") + return { + "schema_version": version, + "schema_signature": actual_signature, + "integrity": integrity, + "case_count": int(connection.execute("SELECT COUNT(*) FROM cases").fetchone()[0]), + "turn_count": int(connection.execute("SELECT COUNT(*) FROM turns").fetchone()[0]), + } + + +def _verify_legacy_identity_shape(connection: sqlite3.Connection) -> dict[str, object]: + state = _verify_database_shape( + connection, expected_signature=_LEGACY_IDENTITY_SCHEMA_SIGNATURE + ) + root_unique = False + for row in connection.execute("PRAGMA index_list(cases)"): + if int(row["unique"]) != 1: + continue + columns = tuple( + str(item["name"]) + for item in connection.execute( + f"PRAGMA index_info('{str(row['name'])}')" + ) + ) + if columns == ("post_id", "root_comment_id"): + root_unique = True + if not root_unique: + raise StorageError("legacy database lacks its root identity UNIQUE constraint") + return state + + def _case_values(record: CaseRecord) -> dict[str, object]: return { "case_id": record.case_id, @@ -223,6 +351,97 @@ def _turn_from_row(row: sqlite3.Row) -> TurnRecord: ) +def _find_turn_duplicate( + connection: sqlite3.Connection, turn: TurnRecord +) -> TurnRecord | None: + if turn.reply_comment_id is not None: + row = connection.execute( + "SELECT * FROM turns WHERE reply_comment_id = ?", + (turn.reply_comment_id,), + ).fetchone() + else: + row = connection.execute( + """SELECT * FROM turns + WHERE case_id = ? AND parent_turn_id IS ? AND direction = ? + AND exact_text = ? AND reply_comment_id IS NULL""", + ( + turn.case_id, + turn.parent_turn_id, + turn.direction.value, + turn.exact_text, + ), + ).fetchone() + return _turn_from_row(row) if row is not None else None + + +def _duplicate_turn_receipt( + duplicate: TurnRecord, requested: TurnRecord +) -> Mapping[str, object]: + if duplicate.case_id != requested.case_id: + raise StorageError( + "reply_comment_id belongs to a different Case: " + f"{duplicate.case_id}/{duplicate.turn_id}" + ) + return { + "created": False, + "duplicate": True, + "duplicate_identity": ( + "reply_comment_id" if requested.reply_comment_id is not None else "fallback" + ), + "case_id": duplicate.case_id, + "turn_id": duplicate.turn_id, + } + + +def _branch_partition( + turns: list[TurnRecord], branch_root_turn_id: str +) -> tuple[list[TurnRecord], list[TurnRecord], list[TurnRecord]]: + """Return copied ancestors, moved branch Turns, and source remainder. + + Shared ancestors must not carry globally unique reply identifiers because + the split intentionally copies them into the new Case. The branch root must + be a non-root Turn, and another branch must remain in the source Case. + """ + validate_parent_graph(turns) + by_id = {turn.turn_id: turn for turn in turns} + branch_root = by_id.get(branch_root_turn_id) + if branch_root is None: + raise StorageError(f"branch root Turn not found: {branch_root_turn_id}") + if branch_root.parent_turn_id is None: + raise StorageError("branch split requires a non-root Turn") + + descendant_ids = {branch_root_turn_id} + changed = True + while changed: + changed = False + for turn in turns: + if turn.parent_turn_id in descendant_ids and turn.turn_id not in descendant_ids: + descendant_ids.add(turn.turn_id) + changed = True + + ancestor_ids: list[str] = [] + parent_id: str | None = branch_root.parent_turn_id + while parent_id is not None: + ancestor = by_id[parent_id] + ancestor_ids.append(parent_id) + parent_id = ancestor.parent_turn_id + ancestor_ids.reverse() + ancestor_id_set = set(ancestor_ids) + other_branch_ids = set(by_id) - descendant_ids - ancestor_id_set + if not other_branch_ids: + raise StorageError("branch split would leave no separate branch in the source Case") + + ancestors = [turn for turn in turns if turn.turn_id in ancestor_id_set] + if any(turn.reply_comment_id is not None for turn in ancestors): + raise StorageError( + "shared branch ancestors with reply_comment_id cannot be copied across Cases" + ) + moved = [turn for turn in turns if turn.turn_id in descendant_ids] + remaining = [turn for turn in turns if turn.turn_id not in descendant_ids] + validate_parent_graph(remaining) + return ancestors, moved, remaining + + class SQLiteStore: """Narrow canonical storage API; no generic SQL execution is exposed.""" @@ -263,44 +482,61 @@ def initialize(self, *, approved: bool) -> Mapping[str, object]: ).fetchall() if tables: raise StorageError("refusing to initialize a non-empty unversioned database") - connection.execute("PRAGMA journal_mode = WAL") - connection.executescript(_SCHEMA_SQL) + connection.execute("PRAGMA journal_mode = WAL") + connection.executescript(_SCHEMA_SQL) return self.status() def status(self) -> Mapping[str, object]: with self._connect() as connection: - version = int(connection.execute("PRAGMA user_version").fetchone()[0]) - if version != SCHEMA_VERSION: - raise StorageError( - f"database schema version mismatch: {version}; expected {SCHEMA_VERSION}" + state = _verify_database_shape( + connection, expected_signature=_SCHEMA_SIGNATURE + ) + named_indexes = { + str(row["name"]): _normalized_sql(str(row["sql"])) + for row in connection.execute( + """SELECT name, sql FROM sqlite_master + WHERE type = 'index' AND name NOT LIKE 'sqlite_autoindex%'""" ) - signature_row = connection.execute( - "SELECT value FROM storage_metadata WHERE key = 'schema_signature'" + } + expected_indexes = { + name: _normalized_sql(sql) for name, sql in _EXPECTED_INDEX_SQL.items() + } + if named_indexes != expected_indexes: + raise StorageError("database indexes do not match the canonical schema") + cases_sql_row = connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'cases'" ).fetchone() - actual_signature = str(signature_row["value"]) if signature_row else "" - if actual_signature != _SCHEMA_SIGNATURE: - raise StorageError("database schema signature mismatch") - case_columns = tuple( - str(row["name"]) for row in connection.execute("PRAGMA table_info(cases)") + cases_sql = _normalized_sql(str(cases_sql_row["sql"])) + if "case_id = printf('case-%03d'" not in cases_sql: + raise StorageError("cases table lacks the canonical Case ID constraint") + legacy_unique = [ + row + for row in connection.execute("PRAGMA index_list(cases)") + if str(row["origin"]) == "u" + ] + if legacy_unique: + raise StorageError("cases table retains a legacy UNIQUE constraint") + foreign_keys = sorted( + ( + str(row["table"]), + str(row["from"]), + str(row["to"]), + str(row["on_delete"]), + ) + for row in connection.execute("PRAGMA foreign_key_list(turns)") ) - turn_columns = tuple( - str(row["name"]) for row in connection.execute("PRAGMA table_info(turns)") + expected_foreign_keys = sorted( + [ + ("cases", "case_id", "case_id", "RESTRICT"), + ("turns", "case_id", "case_id", "NO ACTION"), + ("turns", "parent_turn_id", "turn_id", "NO ACTION"), + ] ) - if case_columns != CASE_FIELDS or turn_columns != TURN_FIELDS: - raise StorageError("database table columns do not match the canonical schema") - integrity = str(connection.execute("PRAGMA integrity_check").fetchone()[0]) - foreign_keys = connection.execute("PRAGMA foreign_key_check").fetchall() - case_count = int(connection.execute("SELECT COUNT(*) FROM cases").fetchone()[0]) - turn_count = int(connection.execute("SELECT COUNT(*) FROM turns").fetchone()[0]) - if integrity != "ok" or foreign_keys: - raise StorageError("database integrity check failed") + if foreign_keys != expected_foreign_keys: + raise StorageError("database foreign keys do not match the canonical schema") return { "ok": True, - "schema_version": version, - "schema_signature": actual_signature, - "integrity": integrity, - "case_count": case_count, - "turn_count": turn_count, + **state, } def backup(self, destination: Path, *, approved: bool) -> Mapping[str, object]: @@ -319,9 +555,201 @@ def backup(self, destination: Path, *, approved: bool) -> Mapping[str, object]: raise return {"ok": True, "backup": str(target), "database": status} + def migrate_identity( + self, backup_destination: Path, *, approved: bool + ) -> Mapping[str, object]: + """Replace legacy Case/root identity constraints without changing schema version. + + Cases are renumbered by ``(created_at, existing case_id)``. The full + existing identifier is only a deterministic tie-breaker; no embedded + date or date-local suffix is interpreted. + """ + _require_approval(approved) + target = backup_destination.resolve() + if target == self.path: + raise StorageError("backup destination must differ from the canonical database") + with self._connect() as preflight: + signature_row = preflight.execute( + "SELECT value FROM storage_metadata WHERE key = 'schema_signature'" + ).fetchone() + signature = str(signature_row["value"]) if signature_row else "" + if signature == _SCHEMA_SIGNATURE: + self.status() + raise StorageError("identity migration is already applied") + _verify_legacy_identity_shape(preflight) + if target.exists(): + raise StorageError(f"backup destination already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + + source_snapshot: dict[str, list[dict[str, object]]] | None = None + backup_verified = False + try: + with self._connect(writable=True) as connection: + _verify_legacy_identity_shape(connection) + source_snapshot = _snapshot(connection) + with sqlite3.connect(target) as backup: + connection.backup(backup) + with sqlite3.connect(target) as backup_read: + backup_read.row_factory = sqlite3.Row + _verify_legacy_identity_shape(backup_read) + if _snapshot(backup_read) != source_snapshot: + raise StorageError("identity migration backup read-back mismatch") + backup_verified = True + + connection.execute("BEGIN IMMEDIATE") + if _snapshot(connection) != source_snapshot: + raise StorageError("database changed while identity migration was starting") + case_rows = source_snapshot["cases"] + turn_rows = source_snapshot["turns"] + case_id_map = { + str(row["case_id"]): f"Case-{index:03d}" + for index, row in enumerate(case_rows, start=1) + } + + connection.execute( + _CASES_SQL.format( + if_not_exists="", table="cases_identity_new" + ) + ) + connection.execute( + _TURNS_SQL.format( + if_not_exists="", + table="turns_identity_new", + cases_table="cases_identity_new", + ) + ) + case_insert = ", ".join("?" for _ in CASE_FIELDS) + turn_insert = ", ".join("?" for _ in TURN_FIELDS) + for row in case_rows: + values = dict(row) + values["case_id"] = case_id_map[str(row["case_id"])] + connection.execute( + f"INSERT INTO cases_identity_new VALUES ({case_insert})", + tuple(values[field] for field in CASE_FIELDS), + ) + for row in turn_rows: + values = dict(row) + values["case_id"] = case_id_map[str(row["case_id"])] + connection.execute( + f"INSERT INTO turns_identity_new VALUES ({turn_insert})", + tuple(values[field] for field in TURN_FIELDS), + ) + + connection.execute("DROP TABLE turns") + connection.execute("DROP TABLE cases") + connection.execute("ALTER TABLE cases_identity_new RENAME TO cases") + connection.execute("ALTER TABLE turns_identity_new RENAME TO turns") + connection.execute( + """CREATE INDEX turns_case_observed_idx + ON turns(case_id, observed_at, turn_id)""" + ) + connection.execute( + """CREATE INDEX cases_root_candidates_idx + ON cases(post_id, root_comment_id, created_at, case_id)""" + ) + connection.execute( + """CREATE UNIQUE INDEX turns_reply_comment_id_uq + ON turns(reply_comment_id) WHERE reply_comment_id IS NOT NULL""" + ) + connection.execute( + """CREATE UNIQUE INDEX turns_fallback_identity_uq + ON turns(case_id, COALESCE(parent_turn_id, ''), direction, exact_text) + WHERE reply_comment_id IS NULL""" + ) + connection.execute( + "UPDATE storage_metadata SET value = ? WHERE key = 'schema_signature'", + (_SCHEMA_SIGNATURE,), + ) + connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + + expected_cases = [] + for row in case_rows: + expected = dict(row) + expected["case_id"] = case_id_map[str(row["case_id"])] + expected_cases.append(expected) + expected_cases.sort( + key=lambda row: (str(row["created_at"]), str(row["case_id"])) + ) + expected_turns = [] + for row in turn_rows: + expected = dict(row) + expected["case_id"] = case_id_map[str(row["case_id"])] + expected_turns.append(expected) + expected_turns.sort( + key=lambda row: (str(row["case_id"]), str(row["turn_id"])) + ) + expected_snapshot = {"cases": expected_cases, "turns": expected_turns} + _verify_database_shape(connection, expected_signature=_SCHEMA_SIGNATURE) + if _snapshot(connection) != expected_snapshot: + raise StorageError("identity migration transactional read-back mismatch") + connection.commit() + except Exception as error: + recovery_errors: list[str] = [] + if target.exists() and backup_verified: + try: + with sqlite3.connect(target) as backup_read: + backup_read.row_factory = sqlite3.Row + _verify_legacy_identity_shape(backup_read) + if source_snapshot is not None and _snapshot( + backup_read + ) != source_snapshot: + raise StorageError("identity migration backup verification failed") + except Exception as backup_error: + recovery_errors.append(str(backup_error)) + elif target.exists(): + try: + target.unlink() + except OSError as cleanup_error: + recovery_errors.append( + f"partial backup cleanup failed: {cleanup_error}" + ) + try: + with self._connect() as rolled_back: + _verify_legacy_identity_shape(rolled_back) + if source_snapshot is not None and _snapshot( + rolled_back + ) != source_snapshot: + raise StorageError("identity migration rollback verification failed") + except Exception as rollback_error: + recovery_errors.append(str(rollback_error)) + if recovery_errors: + raise StorageError( + "identity migration failed and recovery verification failed: " + + "; ".join(recovery_errors) + ) from error + backup_state = "retained" if backup_verified else "removed" + raise StorageError( + "identity migration failed; rollback verified; " + f"backup {backup_state}: {error}" + ) from error + + status = self.status() + with self._connect() as committed: + if _snapshot(committed) != expected_snapshot: + raise StorageError("identity migration committed read-back mismatch") + return { + "operation": "db-migrate-identity", + "cutover_timestamp": now_jerusalem().isoformat(), + "timezone": "Asia/Jerusalem", + "database_schema_version_before": SCHEMA_VERSION, + "database_schema_version_after": status["schema_version"], + "database_integrity": status["integrity"], + "database_backup": str(target), + "migrated_case_count": status["case_count"], + "verified_turn_count": status["turn_count"], + "first_case_id": next(iter(case_id_map.values()), ""), + "last_case_id": next(reversed(case_id_map.values()), ""), + "case_id_map": case_id_map, + "backup_verified": True, + "committed_read_back": "verified", + } + def case_ids(self) -> list[str]: with self._connect() as connection: - rows = connection.execute("SELECT case_id FROM cases ORDER BY case_id").fetchall() + rows = connection.execute( + "SELECT case_id FROM cases " + "ORDER BY CAST(substr(case_id, 6) AS INTEGER)" + ).fetchall() return [str(row["case_id"]) for row in rows] def turn_ids(self, case_id: str) -> list[str]: @@ -352,41 +780,61 @@ def get_turns(self, case_id: str) -> list[TurnRecord]: self._get_case(connection, case_id) return self._get_turns(connection, case_id) - def find_case(self, post_id: str, root_comment_id: str) -> CaseRecord | None: + def find_cases(self, post_id: str, root_comment_id: str) -> list[CaseRecord]: with self._connect() as connection: - row = connection.execute( - "SELECT * FROM cases WHERE post_id = ? AND root_comment_id = ?", + rows = connection.execute( + """SELECT * FROM cases WHERE post_id = ? AND root_comment_id = ? + ORDER BY created_at, case_id""", (post_id, root_comment_id), - ).fetchone() - return _case_from_row(row) if row is not None else None + ).fetchall() + return [_case_from_row(row) for row in rows] - def list_open_summaries(self) -> list[dict[str, str]]: + def find_turn_duplicate(self, turn: TurnRecord) -> TurnRecord | None: + """Resolve a Turn by its strongest available immutable identity.""" with self._connect() as connection: - rows = connection.execute("SELECT * FROM cases ORDER BY case_id").fetchall() - output: list[dict[str, str]] = [] - for row in rows: - case = _case_from_row(row) - if case.status in CLOSED_STATUSES: - continue - parsed = parse_facebook_url(case.post_url) - if parsed.root_comment_id is None and parsed.reply_comment_id is None: - raise StorageError( - f"open case lacks a comment or reply permalink: {case.case_id}" + return _find_turn_duplicate(connection, turn) + + def list_open_summaries(self) -> list[dict[str, object]]: + with self._connect() as connection: + rows = connection.execute( + "SELECT * FROM cases " + "ORDER BY CAST(substr(case_id, 6) AS INTEGER)" + ).fetchall() + output: list[dict[str, object]] = [] + for row in rows: + case = _case_from_row(row) + if case.status in CLOSED_STATUSES: + continue + public_turns = [ + turn + for turn in self._get_turns(connection, case.case_id) + if turn.state not in {TurnState.DRAFT, TurnState.REPLACED} + ] + latest = max( + public_turns, + key=lambda turn: (turn.observed_at, int(turn.turn_id[1:])), + default=None, + ) + permalink = latest.exact_url if latest is not None else None + output.append( + { + "case_id": case.case_id, + "status": case.status.value, + "last_turn_id": latest.turn_id if latest is not None else None, + "last_comment_permalink": permalink, + "permalink_status": "supplied" if permalink else "missing", + } ) - output.append( - { - "case_id": case.case_id, - "status": case.status.value, - "exact_permalink": parsed.original_url, - } - ) return output def strategy_dataset(self) -> Mapping[str, object]: with self._connect() as connection: cases = [ _case_from_row(row) - for row in connection.execute("SELECT * FROM cases ORDER BY case_id") + for row in connection.execute( + "SELECT * FROM cases " + "ORDER BY CAST(substr(case_id, 6) AS INTEGER)" + ) if CaseStatus(str(row["status"])) in CLOSED_STATUSES ] turns = { @@ -446,6 +894,9 @@ def import_records( """Atomically import a validated snapshot into an empty database.""" _require_approval(approved) self._validate_bundle(cases, turns) + expected_case_ids = [f"Case-{index:03d}" for index in range(1, len(cases) + 1)] + if {case.case_id for case in cases} != set(expected_case_ids): + raise StorageError("database import requires a contiguous global Case sequence") try: with self._connect(writable=True) as connection: connection.execute("BEGIN IMMEDIATE") @@ -475,25 +926,274 @@ def import_records( return status def create_case( - self, case: CaseRecord, turns: list[TurnRecord], *, approved: bool + self, + case: CaseRecord, + turns: list[TurnRecord], + *, + approved: bool, + allocate_ids: bool = False, ) -> Mapping[str, object]: """Atomically create one Case and its initial public-turn graph.""" _require_approval(approved) - self._validate_bundle([case], turns) try: with self._connect(writable=True) as connection: connection.execute("BEGIN IMMEDIATE") - self._insert_case(connection, case) - for turn in turns: + stored_case = case + stored_turns = turns + existing_case_ids = [ + str(row["case_id"]) + for row in connection.execute("SELECT case_id FROM cases") + ] + if allocate_ids: + stored_case = replace( + case, case_id=next_case_id(existing_case_ids) + ) + stored_turns = [] + for turn in turns: + stored_turns.append( + replace( + turn, + case_id=stored_case.case_id, + turn_id=next_turn_id( + item.turn_id for item in stored_turns + ), + ) + ) + elif case.case_id != next_case_id(existing_case_ids): + raise StorageError( + "case creation requires the next global sequential Case ID" + ) + self._validate_bundle([stored_case], stored_turns) + self._insert_case(connection, stored_case) + for turn in stored_turns: self._insert_turn(connection, turn) connection.commit() except sqlite3.Error as error: raise StorageError(f"case creation failed: {error}") from error - actual_case = self.get_case(case.case_id) - actual_turns = self.get_turns(case.case_id) - if actual_case != case or actual_turns != sorted(turns, key=lambda item: item.turn_id): + actual_case = self.get_case(stored_case.case_id) + actual_turns = self.get_turns(stored_case.case_id) + if actual_case != stored_case or actual_turns != sorted( + stored_turns, key=lambda item: item.turn_id + ): raise StorageError("case creation read-back mismatch") - return {"created": True, "case_id": case.case_id, "turn_count": len(turns)} + return { + "created": True, + "case_id": stored_case.case_id, + "turn_count": len(stored_turns), + "readback": "verified", + } + + def split_case_branch( + self, + case_id: str, + branch_root_turn_id: str, + *, + new_case_title: str, + new_topic: str, + backup_destination: Path, + approved: bool, + ) -> Mapping[str, object]: + """Move one branch to a new Case while copying its shared ancestor path. + + The source is backed up and read back before an immediate transaction. + The new Case receives the next global Case ID and fresh case-local Turn + IDs; exact public text and URLs are preserved byte-for-byte. + """ + _require_approval(approved) + if not new_case_title.strip() or not new_topic.strip(): + raise StorageError("new Case title and topic are required") + target = backup_destination.resolve() + if target == self.path: + raise StorageError("backup destination must differ from the canonical database") + if target.exists(): + raise StorageError(f"backup destination already exists: {target}") + + with self._connect() as preflight: + source_case = self._get_case(preflight, case_id) + _branch_partition(self._get_turns(preflight, case_id), branch_root_turn_id) + if source_case.status in CLOSED_STATUSES: + raise StorageError("cannot split a closed Case") + target.parent.mkdir(parents=True, exist_ok=True) + + source_snapshot: dict[str, list[dict[str, object]]] | None = None + backup_verified = False + new_case: CaseRecord | None = None + updated_source: CaseRecord | None = None + new_turns: list[TurnRecord] = [] + remaining_turns: list[TurnRecord] = [] + moved_turns: list[TurnRecord] = [] + ancestors: list[TurnRecord] = [] + new_branch_root_turn_id = "" + try: + with self._connect(writable=True) as connection: + source_snapshot = _snapshot(connection) + source_case = self._get_case(connection, case_id) + source_turns = self._get_turns(connection, case_id) + if source_case.status in CLOSED_STATUSES: + raise StorageError("cannot split a closed Case") + ancestors, moved_turns, remaining_turns = _branch_partition( + source_turns, branch_root_turn_id + ) + + with sqlite3.connect(target) as backup: + connection.backup(backup) + with sqlite3.connect(target) as backup_read: + backup_read.row_factory = sqlite3.Row + _verify_database_shape( + backup_read, expected_signature=_SCHEMA_SIGNATURE + ) + if _snapshot(backup_read) != source_snapshot: + raise StorageError("branch split backup read-back mismatch") + backup_verified = True + + connection.execute("BEGIN IMMEDIATE") + if _snapshot(connection) != source_snapshot: + raise StorageError("database changed while branch split was starting") + existing_case_ids = [ + str(row["case_id"]) + for row in connection.execute("SELECT case_id FROM cases") + ] + new_case_id = next_case_id(existing_case_ids) + split_at = now_jerusalem().strftime("%Y-%m-%d %H:%M") + updated_source = replace(source_case, updated_at=split_at) + new_case = replace( + source_case, + case_id=new_case_id, + case_title=new_case_title.strip(), + topic=new_topic.strip(), + created_at=split_at, + updated_at=split_at, + ) + + selected_turns = [*ancestors, *moved_turns] + selected_ids = {turn.turn_id for turn in selected_turns} + selected_turns = [ + turn for turn in source_turns if turn.turn_id in selected_ids + ] + turn_id_map = { + turn.turn_id: f"T{index:03d}" + for index, turn in enumerate(selected_turns, start=1) + } + new_turns = [ + replace( + turn, + case_id=new_case_id, + turn_id=turn_id_map[turn.turn_id], + parent_turn_id=( + turn_id_map[turn.parent_turn_id] + if turn.parent_turn_id is not None + else None + ), + ) + for turn in selected_turns + ] + new_branch_root_turn_id = turn_id_map[branch_root_turn_id] + self._validate_bundle([updated_source], remaining_turns) + self._validate_bundle([new_case], new_turns) + + by_id = {turn.turn_id: turn for turn in source_turns} + + def depth(turn: TurnRecord) -> int: + value = 0 + parent_id = turn.parent_turn_id + while parent_id is not None: + value += 1 + parent_id = by_id[parent_id].parent_turn_id + return value + + for turn in sorted(moved_turns, key=depth, reverse=True): + connection.execute( + "DELETE FROM turns WHERE case_id = ? AND turn_id = ?", + (case_id, turn.turn_id), + ) + connection.execute( + "UPDATE cases SET updated_at = ? WHERE case_id = ?", + (split_at, case_id), + ) + self._insert_case(connection, new_case) + for turn in new_turns: + self._insert_turn(connection, turn) + + _verify_database_shape(connection, expected_signature=_SCHEMA_SIGNATURE) + if self._get_case(connection, case_id) != updated_source: + raise StorageError("branch split source Case read-back mismatch") + if self._get_turns(connection, case_id) != remaining_turns: + raise StorageError("branch split source Turn read-back mismatch") + if self._get_case(connection, new_case_id) != new_case: + raise StorageError("branch split new Case read-back mismatch") + if self._get_turns(connection, new_case_id) != new_turns: + raise StorageError("branch split new Turn read-back mismatch") + connection.commit() + except Exception as error: + recovery_errors: list[str] = [] + if target.exists() and backup_verified: + try: + with sqlite3.connect(target) as backup_read: + backup_read.row_factory = sqlite3.Row + _verify_database_shape( + backup_read, expected_signature=_SCHEMA_SIGNATURE + ) + if source_snapshot is not None and _snapshot( + backup_read + ) != source_snapshot: + raise StorageError("branch split backup verification failed") + except Exception as backup_error: + recovery_errors.append(str(backup_error)) + elif target.exists(): + try: + target.unlink() + except OSError as cleanup_error: + recovery_errors.append( + f"partial backup cleanup failed: {cleanup_error}" + ) + try: + with self._connect() as rolled_back: + _verify_database_shape( + rolled_back, expected_signature=_SCHEMA_SIGNATURE + ) + if source_snapshot is not None and _snapshot( + rolled_back + ) != source_snapshot: + raise StorageError("branch split rollback verification failed") + except Exception as rollback_error: + recovery_errors.append(str(rollback_error)) + if recovery_errors: + raise StorageError( + "branch split failed and recovery verification failed: " + + "; ".join(recovery_errors) + ) from error + backup_state = "retained" if backup_verified else "removed" + raise StorageError( + "branch split failed; rollback verified; " + f"backup {backup_state}: {error}" + ) from error + + if new_case is None or updated_source is None: + raise StorageError("branch split produced no committed records") + status = self.status() + if self.get_case(case_id) != updated_source: + raise StorageError("branch split committed source Case read-back mismatch") + if self.get_turns(case_id) != remaining_turns: + raise StorageError("branch split committed source Turn read-back mismatch") + if self.get_case(new_case.case_id) != new_case: + raise StorageError("branch split committed new Case read-back mismatch") + if self.get_turns(new_case.case_id) != new_turns: + raise StorageError("branch split committed new Turn read-back mismatch") + return { + "operation": "case-split-branch", + "source_case_id": case_id, + "new_case_id": new_case.case_id, + "source_branch_root_turn_id": branch_root_turn_id, + "new_branch_root_turn_id": new_branch_root_turn_id, + "copied_ancestor_count": len(ancestors), + "moved_branch_turn_count": len(moved_turns), + "source_turn_count": len(remaining_turns), + "new_turn_count": len(new_turns), + "database_backup": str(target), + "backup_verified": True, + "database_integrity": status["integrity"], + "committed_read_back": "verified", + } def add_turn( self, @@ -507,18 +1207,23 @@ def add_turn( ) -> Mapping[str, object]: """Atomically append one turn, optionally retire a Draft, and update Case state.""" _require_approval(approved) - validate_turn(turn) + promoted_draft = False try: with self._connect(writable=True) as connection: connection.execute("BEGIN IMMEDIATE") case = self._get_case(connection, turn.case_id) + existing_turns = self._get_turns(connection, turn.case_id) + stored_turn = replace( + turn, + turn_id=next_turn_id(item.turn_id for item in existing_turns), + ) + validate_turn(stored_turn) if (turn.post_id, turn.root_comment_id) != ( case.post_id, case.root_comment_id, ): raise StorageError(f"turn identity conflicts with Case: {turn.turn_id}") validate_transition(LifecycleTransition(case.status, target_status, reason)) - existing_turns = self._get_turns(connection, turn.case_id) if replace_draft_id is not None: draft = next( (item for item in existing_turns if item.turn_id == replace_draft_id), @@ -532,18 +1237,86 @@ def add_turn( raise StorageError( f"replacement target is not an Outgoing Draft: {replace_draft_id}" ) - connection.execute( - "UPDATE turns SET state = ? WHERE case_id = ? AND turn_id = ?", - (TurnState.REPLACED.value, turn.case_id, replace_draft_id), + same_turn_identity = ( + draft.parent_turn_id == stored_turn.parent_turn_id + and draft.direction is stored_turn.direction + and draft.exact_text == stored_turn.exact_text + and ( + draft.reply_comment_id is None + or stored_turn.reply_comment_id is None + or draft.reply_comment_id == stored_turn.reply_comment_id + ) ) - existing_turns = [ - item - if item.turn_id != replace_draft_id - else replace(item, state=TurnState.REPLACED) - for item in existing_turns - ] - validate_parent_graph([*existing_turns, turn]) - self._insert_turn(connection, turn) + if same_turn_identity: + stored_turn = replace( + stored_turn, + turn_id=draft.turn_id, + reply_comment_id=( + stored_turn.reply_comment_id or draft.reply_comment_id + ), + exact_url=stored_turn.exact_url or draft.exact_url, + url_supplied_at=( + stored_turn.url_supplied_at or draft.url_supplied_at + ), + ) + validate_turn(stored_turn) + duplicate = _find_turn_duplicate(connection, stored_turn) + if duplicate is not None and ( + duplicate.case_id, + duplicate.turn_id, + ) != (draft.case_id, draft.turn_id): + connection.rollback() + return _duplicate_turn_receipt(duplicate, stored_turn) + graph = [ + stored_turn if item.turn_id == draft.turn_id else item + for item in existing_turns + ] + validate_parent_graph(graph) + values = _turn_values(stored_turn) + connection.execute( + """UPDATE turns SET + parent_turn_id = :parent_turn_id, + parent_confidence = :parent_confidence, + participant_ref = :participant_ref, + direction = :direction, + kind = :kind, + state = :state, + exact_text = :exact_text, + post_id = :post_id, + root_comment_id = :root_comment_id, + reply_comment_id = :reply_comment_id, + exact_url = :exact_url, + url_supplied_at = :url_supplied_at, + observed_at = :observed_at, + notes = :notes + WHERE case_id = :case_id AND turn_id = :turn_id""", + values, + ) + promoted_draft = True + else: + duplicate = _find_turn_duplicate(connection, stored_turn) + if duplicate is not None: + connection.rollback() + return _duplicate_turn_receipt(duplicate, stored_turn) + connection.execute( + "UPDATE turns SET state = ? WHERE case_id = ? AND turn_id = ?", + (TurnState.REPLACED.value, turn.case_id, replace_draft_id), + ) + existing_turns = [ + item + if item.turn_id != replace_draft_id + else replace(item, state=TurnState.REPLACED) + for item in existing_turns + ] + validate_parent_graph([*existing_turns, stored_turn]) + self._insert_turn(connection, stored_turn) + else: + duplicate = _find_turn_duplicate(connection, stored_turn) + if duplicate is not None: + connection.rollback() + return _duplicate_turn_receipt(duplicate, stored_turn) + validate_parent_graph([*existing_turns, stored_turn]) + self._insert_turn(connection, stored_turn) connection.execute( "UPDATE cases SET status = ?, updated_at = ? WHERE case_id = ?", (target_status.value, updated_at, turn.case_id), @@ -551,24 +1324,28 @@ def add_turn( connection.commit() except sqlite3.Error as error: raise StorageError(f"turn write failed: {error}") from error - actual_turns = self.get_turns(turn.case_id) - actual = next((item for item in actual_turns if item.turn_id == turn.turn_id), None) + actual_turns = self.get_turns(stored_turn.case_id) + actual = next( + (item for item in actual_turns if item.turn_id == stored_turn.turn_id), None + ) actual_case = self.get_case(turn.case_id) if ( - actual != turn + actual != stored_turn or actual_case.status is not target_status or actual_case.updated_at != updated_at ): raise StorageError("turn write read-back mismatch") - if replace_draft_id is not None: + if replace_draft_id is not None and not promoted_draft: replaced = next( (item for item in actual_turns if item.turn_id == replace_draft_id), None ) if replaced is None or replaced.state is not TurnState.REPLACED: raise StorageError("Draft replacement read-back mismatch") return { + "created": not promoted_draft, "case_id": turn.case_id, - "turn_id": turn.turn_id, + "turn_id": stored_turn.turn_id, + "promoted": promoted_draft, "status": actual_case.status.value, "readback": "verified", } diff --git a/src/dialogue_lab/validation.py b/src/dialogue_lab/validation.py index 0046c7c..bce8542 100644 --- a/src/dialogue_lab/validation.py +++ b/src/dialogue_lab/validation.py @@ -5,7 +5,7 @@ from .enums import TurnDirection, TurnState from .errors import DialogueLabError from .facebook_url import parse_facebook_url -from .identifiers import CASE_ID_RE, TURN_ID_RE +from .identifiers import TURN_ID_RE, case_id_number from .lifecycle import CLOSED_STATUSES from .models import CaseRecord, TurnRecord @@ -13,8 +13,7 @@ def validate_case(record: CaseRecord) -> None: - if CASE_ID_RE.fullmatch(record.case_id) is None: - raise DialogueLabError(f"malformed Case ID: {record.case_id}") + case_id_number(record.case_id) required = { "case_title": record.case_title, "created_at": record.created_at, @@ -53,14 +52,15 @@ def validate_case(record: CaseRecord) -> None: def validate_turn(record: TurnRecord) -> None: - if CASE_ID_RE.fullmatch(record.case_id) is None: - raise DialogueLabError(f"malformed Case ID: {record.case_id}") + case_id_number(record.case_id) if TURN_ID_RE.fullmatch(record.turn_id) is None: raise DialogueLabError(f"malformed Turn ID: {record.turn_id}") if not PARTICIPANT_RE.fullmatch(record.participant_ref): raise DialogueLabError("Participant Ref must be USER or a case-local P-number") if not record.exact_text: raise DialogueLabError("Exact Text is required") + if record.reply_comment_id is not None and not record.reply_comment_id.strip(): + raise DialogueLabError("reply_comment_id must not be blank") if not record.observed_at: raise DialogueLabError("Observed At is required") if record.direction is TurnDirection.INCOMING and record.state is TurnState.DRAFT: @@ -73,3 +73,10 @@ def validate_turn(record: TurnRecord) -> None: raise DialogueLabError("Exact URL Post ID conflicts with Turn Post ID") if parsed.root_comment_id is not None and parsed.root_comment_id != record.root_comment_id: raise DialogueLabError("Exact URL Root Comment ID conflicts with Turn Root Comment ID") + if ( + parsed.reply_comment_id is not None + and parsed.reply_comment_id != record.reply_comment_id + ): + raise DialogueLabError( + "reply_comment_id must match the supplied Exact URL reply_comment_id" + ) diff --git a/tests/helpers.py b/tests/helpers.py index 998f913..8c0b556 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -12,7 +12,7 @@ def make_case(**overrides: object) -> CaseRecord: values: dict[str, object] = { - "case_id": "CASE-20260717-001", + "case_id": "Case-001", "case_title": "Synthetic policy discussion", "created_at": "2026-07-17 10:00", "updated_at": "2026-07-17 10:00", @@ -30,7 +30,7 @@ def make_case(**overrides: object) -> CaseRecord: def make_turn(**overrides: object) -> TurnRecord: values: dict[str, object] = { - "case_id": "CASE-20260717-001", + "case_id": "Case-001", "turn_id": "T001", "parent_turn_id": None, "parent_confidence": None, diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 4bb070f..e29ff7d 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -6,6 +6,7 @@ from dialogue_lab.cli import main from dialogue_lab.enums import CaseStatus, OutcomeClass, TurnDirection, TurnState from dialogue_lab.models import to_jsonable +from dialogue_lab.storage import SQLiteStore from tests.helpers import make_case, make_reply, make_turn @@ -31,7 +32,7 @@ def test_transactional_case_cli_lifecycle( db = ("--database", str(database)) initialized = _as_dict(_run_ok(capsys, *db, "db-init", "--approved")) assert initialized["integrity"] == "ok" - assert _as_dict(_run_ok(capsys, *db, "doctor"))["ok"] is True + assert _as_dict(_run_ok(capsys, *db, "check"))["ok"] is True case_payload = _as_dict(to_jsonable(make_case())) case_payload.pop("case_id") @@ -46,38 +47,44 @@ def test_transactional_case_cli_lifecycle( *db, "case-intake", str(intake_path), - "--date", - "2026-07-17", "--approved", ) ) assert intake == { "created": True, - "case_id": "CASE-20260717-001", + "case_id": "Case-001", "turn_count": 1, + "readback": "verified", } - duplicate = _as_dict( + second = _as_dict( _run_ok( capsys, *db, "case-intake", str(intake_path), - "--date", - "2026-07-17", "--approved", ) ) - assert duplicate["duplicate"] is True - assert duplicate["case_id"] == "CASE-20260717-001" + assert second["created"] is True + assert second["case_id"] == "Case-002" open_cases = _run_ok(capsys, *db, "case-list-open") assert open_cases == [ { - "case_id": "CASE-20260717-001", + "case_id": "Case-001", "status": "Posted", - "exact_permalink": case_payload["post_url"], - } + "last_turn_id": "T001", + "last_comment_permalink": None, + "permalink_status": "missing", + }, + { + "case_id": "Case-002", + "status": "Posted", + "last_turn_id": "T001", + "last_comment_permalink": None, + "permalink_status": "missing", + }, ] found = _as_dict( _run_ok( @@ -90,13 +97,27 @@ def test_transactional_case_cli_lifecycle( "456", ) ) - assert found["case_id"] == "CASE-20260717-001" + assert found["candidate_count"] == 2 + candidate_rows = found["candidates"] + assert isinstance(candidate_rows, list) + assert [item["case_id"] for item in candidate_rows] == [ + "Case-001", + "Case-002", + ] + definitive = _as_dict( + _run_ok(capsys, *db, "case-find", "--case-id", "Case-001") + ) + assert definitive["case_id"] == "Case-001" followup = make_reply( "ignored", "T001", observed_at="2026-07-17 11:00", exact_text="Synthetic follow-up.", + exact_url=( + "https://www.facebook.com/example/posts/123" + "?comment_id=456&reply_comment_id=789" + ), ) followup_path = _write(tmp_path / "followup.json", followup) followup_receipt = _as_dict( @@ -105,13 +126,17 @@ def test_transactional_case_cli_lifecycle( *db, "case-followup", "--case-id", - "CASE-20260717-001", + "Case-001", str(followup_path), "--approved", ) ) assert followup_receipt["turn_id"] == "T002" assert followup_receipt["status"] == "Active Exchange" + shown = _as_dict( + _run_ok(capsys, *db, "case-show", "--case-id", "Case-001") + ) + assert shown["turns"][1]["reply_comment_id"] == "789" # type: ignore[index] posted = make_reply( "ignored", @@ -129,7 +154,7 @@ def test_transactional_case_cli_lifecycle( *db, "case-record-posting", "--case-id", - "CASE-20260717-001", + "Case-001", str(posting_path), "--approved", ) @@ -158,53 +183,90 @@ def test_transactional_case_cli_lifecycle( *db, "case-close", "--case-id", - "CASE-20260717-001", + "Case-001", str(close_path), "--approved", ) ) assert close_receipt["status"] == "Closed - Substantive" - assert _run_ok(capsys, *db, "case-list-open") == [] + assert len(_run_ok(capsys, *db, "case-list-open")) == 1 # type: ignore[arg-type] dataset = _as_dict(_run_ok(capsys, *db, "strategy-dataset")) assert len(dataset["cases"]) == 1 # type: ignore[arg-type] - assert len(dataset["turns"]["CASE-20260717-001"]) == 3 # type: ignore[index] + assert len(dataset["turns"]["Case-001"]) == 3 # type: ignore[index] -def test_pure_validation_commands_and_migration_receipt( +def test_case_split_branch_cli_returns_verified_mapping( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - assert _as_dict( + database = tmp_path / "canonical.sqlite3" + store = SQLiteStore(database) + store.initialize(approved=True) + case = make_case(status=CaseStatus.ACTIVE_EXCHANGE) + turns = [ + make_turn(), + make_reply( + "T002", + "T001", + participant_ref="USER", + direction=TurnDirection.OUTGOING, + state=TurnState.POSTED, + exact_text="Shared reply.", + ), + make_reply("T003", "T002", exact_text="First branch."), + make_reply("T004", "T002", exact_text="Second branch."), + ] + store.create_case(case, turns, approved=True) + backup = tmp_path / "before-split.sqlite3" + + receipt = _as_dict( _run_ok( capsys, - "case-key", - "--post-id", - "123", - "--root-comment-id", - "456", + "--database", + str(database), + "case-split-branch", + "--case-id", + "Case-001", + "--branch-root-turn-id", + "T004", + "--new-case-title", + "Second branch Case", + "--new-topic", + "Second branch topic", + "--backup-destination", + str(backup), + "--approved", ) - )["case_key"] == "facebook:123:456" + ) + + assert receipt["source_case_id"] == "Case-001" + assert receipt["new_case_id"] == "Case-002" + assert receipt["new_branch_root_turn_id"] == "T003" + assert receipt["committed_read_back"] == "verified" + assert backup.is_file() + - ids_path = _write(tmp_path / "case-ids.json", ["CASE-20260717-001"]) +def test_pure_validation_commands_and_migration_receipt( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + ids_path = _write(tmp_path / "case-ids.json", ["Case-001"]) assert _as_dict( _run_ok( capsys, "next-case-id", - "--date", - "2026-07-17", "--existing", str(ids_path), ) - )["case_id"] == "CASE-20260717-002" + )["case_id"] == "Case-002" turns_path = _write( tmp_path / "turn-ids.json", - [{"case_id": "CASE-20260717-001", "turn_id": "T001"}], + [{"case_id": "Case-001", "turn_id": "T001"}], ) assert _as_dict( _run_ok( capsys, "next-turn-id", "--case-id", - "CASE-20260717-001", + "Case-001", "--existing", str(turns_path), ) @@ -258,7 +320,7 @@ def test_pure_validation_commands_and_migration_receipt( receipt_path = Path(__file__).parents[1] / "docs" / "migration-receipt.json.example" receipt = _as_dict(_run_ok(capsys, "migration-receipt", str(receipt_path))) - assert receipt["database_schema_version"] == 1 + assert receipt["database_schema_version_after"] == 1 def test_write_command_rejects_missing_approval( @@ -269,3 +331,22 @@ def test_write_command_rejects_missing_approval( error = json.loads(capsys.readouterr().err) assert error["category"] == "write_safety_error" assert not database.exists() + + initialized = SQLiteStore(database) + initialized.initialize(approved=True) + backup = tmp_path / "identity-backup.sqlite3" + assert ( + main( + [ + "--database", + str(database), + "db-migrate-identity", + "--backup-destination", + str(backup), + ] + ) + == 2 + ) + migration_error = json.loads(capsys.readouterr().err) + assert migration_error["category"] == "write_safety_error" + assert not backup.exists() diff --git a/tests/test_identity_identifiers.py b/tests/test_identity_identifiers.py index fe2cd6f..a0383d4 100644 --- a/tests/test_identity_identifiers.py +++ b/tests/test_identity_identifiers.py @@ -1,52 +1,33 @@ -from datetime import date - import pytest -from dialogue_lab.case_identity import find_duplicate_case, make_case_identity -from dialogue_lab.errors import DialogueLabError, IdentifierError +from dialogue_lab.case_identity import find_case, find_case_candidates +from dialogue_lab.errors import IdentifierError from dialogue_lab.identifiers import next_case_id, next_turn_id from tests.helpers import make_case -def test_same_post_and_root_produce_same_identity() -> None: - assert make_case_identity("post", "root") == make_case_identity("post", "root") - - -def test_different_roots_under_same_post_produce_different_identities() -> None: - assert make_case_identity("post", "root-1") != make_case_identity("post", "root-2") - - -def test_missing_identity_component_is_rejected() -> None: - with pytest.raises(DialogueLabError): - make_case_identity("post", " ") - +def test_case_id_is_definitive_and_root_lookup_returns_candidates() -> None: + first = make_case(case_id="Case-001") + second = make_case(case_id="Case-002") + assert find_case("Case-002", [first, second]) == second + assert find_case_candidates("123", "456", [first, second]) == [first, second] -def test_duplicate_case_lookup_reuses_existing_case() -> None: - case = make_case() - assert find_duplicate_case(case.identity, [case]) == case - -def test_duplicate_canonical_rows_are_rejected() -> None: - first = make_case(case_id="CASE-20260717-001") - second = make_case(case_id="CASE-20260717-002") - with pytest.raises(IdentifierError, match="duplicate canonical identity"): - find_duplicate_case(first.identity, [first, second]) - - -def test_case_ids_use_requested_jerusalem_date_and_sequence() -> None: - assert next_case_id([], on_date=date(2026, 7, 17)) == "CASE-20260717-001" - assert next_case_id( - ["CASE-20260717-001", "CASE-20260716-009"], on_date=date(2026, 7, 17) - ) == "CASE-20260717-002" +def test_case_ids_use_a_global_unbounded_sequence() -> None: + assert next_case_id([]) == "Case-001" + assert next_case_id(["Case-001", "Case-009"]) == "Case-010" + assert next_case_id(["Case-999"]) == "Case-1000" def test_malformed_and_duplicate_case_ids_are_rejected() -> None: with pytest.raises(IdentifierError, match="malformed"): - next_case_id(["CASE-17-1"], on_date=date(2026, 7, 17)) + next_case_id(["CASE-17-1"]) + with pytest.raises(IdentifierError, match="malformed"): + next_case_id(["Case-000"]) + with pytest.raises(IdentifierError, match="malformed"): + next_case_id(["Case-0001"]) with pytest.raises(IdentifierError, match="duplicate"): - next_case_id( - ["CASE-20260717-001", "CASE-20260717-001"], on_date=date(2026, 7, 17) - ) + next_case_id(["Case-001", "Case-001"]) def test_turn_ids_are_case_local_and_sequential() -> None: diff --git a/tests/test_migration_cli.py b/tests/test_migration_cli.py index adfc93a..92d27c7 100644 --- a/tests/test_migration_cli.py +++ b/tests/test_migration_cli.py @@ -10,13 +10,20 @@ def test_migration_receipt_requires_database_evidence_and_git_commit() -> None: payload: dict[str, object] = { + "operation": "db-migrate-identity", "cutover_timestamp": "2026-07-20T10:00:00+03:00", "timezone": "Asia/Jerusalem", - "database_schema_version": 1, + "database_schema_version_before": 1, + "database_schema_version_after": 1, "database_integrity": "ok", "database_backup": "external/snapshot.sqlite3", - "imported_case_count": 1, - "imported_turn_count": 2, + "migrated_case_count": 1, + "verified_turn_count": 2, + "first_case_id": "Case-001", + "last_case_id": "Case-001", + "case_id_map": {"CASE-20260720-001": "Case-001"}, + "backup_verified": True, + "committed_read_back": "verified", "repository_commit": "deadbeef", "test_results": "passed", "known_limitations": ["single local writer"], @@ -24,7 +31,7 @@ def test_migration_receipt_requires_database_evidence_and_git_commit() -> None: } receipt = migration_receipt_from_mapping(payload) serialized = to_jsonable(receipt) - assert serialized["database_schema_version"] == 1 + assert serialized["database_schema_version_after"] == 1 assert serialized["repository_commit"] == "deadbeef" @@ -48,5 +55,12 @@ def test_synthetic_eval_catalog_covers_local_storage_scenarios() -> None: path = Path(__file__).parents[1] / "evals" / "cases" / "synthetic-cases.json" cases = json.loads(path.read_text(encoding="utf-8")) ids = {case["id"] for case in cases} - assert len(cases) == 17 - assert {"new-case-intake", "sqlite-write-rollback", "schema-version-mismatch"} <= ids + assert len(cases) == 29 + assert { + "new-case-intake", + "same-root-separate-cases", + "reply-comment-turn-duplicate", + "identity-migration-readback", + "sqlite-write-rollback", + "schema-version-mismatch", + } <= ids diff --git a/tests/test_sqlite_storage.py b/tests/test_sqlite_storage.py index 5553754..6c49551 100644 --- a/tests/test_sqlite_storage.py +++ b/tests/test_sqlite_storage.py @@ -6,8 +6,8 @@ from dialogue_lab.enums import CaseStatus, OutcomeClass, TurnDirection, TurnState from dialogue_lab.errors import DialogueLabError, StorageError, WriteSafetyError -from dialogue_lab.storage import SQLiteStore -from dialogue_lab.validation import validate_case +from dialogue_lab.storage import _LEGACY_IDENTITY_SCHEMA_SIGNATURE, SQLiteStore +from dialogue_lab.validation import validate_case, validate_turn from tests.helpers import make_case, make_reply, make_turn @@ -24,7 +24,9 @@ def test_database_initialization_requires_approval_and_reports_integrity( assert status["case_count"] == 0 -def test_case_creation_is_atomic_unique_and_read_back(tmp_path: Path) -> None: +def test_case_creation_is_atomic_allows_root_candidates_and_reads_back( + tmp_path: Path, +) -> None: store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") store.initialize(approved=True) case = make_case() @@ -33,25 +35,138 @@ def test_case_creation_is_atomic_unique_and_read_back(tmp_path: Path) -> None: receipt = store.create_case(case, [turn], approved=True) assert receipt == { "created": True, - "case_id": "CASE-20260717-001", + "case_id": "Case-001", "turn_count": 1, + "readback": "verified", } assert store.get_case(case.case_id) == case assert store.get_turns(case.case_id) == [turn] - duplicate = make_case(case_id="CASE-20260717-002") - with pytest.raises(StorageError, match="UNIQUE"): - store.create_case(duplicate, [], approved=True) - assert store.status()["case_count"] == 1 + candidate = make_case(case_id="Case-002") + store.create_case(candidate, [], approved=True) + assert [item.case_id for item in store.find_cases("123", "456")] == [ + "Case-001", + "Case-002", + ] + + +def test_branch_split_allocates_case_copies_ancestors_and_verifies_backup( + tmp_path: Path, +) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case(status=CaseStatus.ACTIVE_EXCHANGE) + root = make_turn() + shared_reply = make_reply( + "T002", + "T001", + participant_ref="USER", + direction=TurnDirection.OUTGOING, + state=TurnState.POSTED, + exact_text="Shared reply.", + ) + first_branch = make_reply("T003", "T002", exact_text="First branch.") + second_branch = make_reply("T004", "T002", exact_text="Second branch.") + first_answer = make_reply( + "T005", + "T003", + participant_ref="USER", + direction=TurnDirection.OUTGOING, + state=TurnState.POSTED, + exact_text="First branch answer.", + ) + store.create_case( + case, + [root, shared_reply, first_branch, second_branch, first_answer], + approved=True, + ) + backup = tmp_path / "backups" / "before-split.sqlite3" + + with pytest.raises(WriteSafetyError, match="approval"): + store.split_case_branch( + "Case-001", + "T004", + new_case_title="Second branch Case", + new_topic="Second branch topic", + backup_destination=backup, + approved=False, + ) + + receipt = store.split_case_branch( + "Case-001", + "T004", + new_case_title="Second branch Case", + new_topic="Second branch topic", + backup_destination=backup, + approved=True, + ) + + assert receipt["new_case_id"] == "Case-002" + assert receipt["new_branch_root_turn_id"] == "T003" + assert receipt["backup_verified"] is True + assert SQLiteStore(backup).status()["case_count"] == 1 + assert [turn.turn_id for turn in store.get_turns("Case-001")] == [ + "T001", + "T002", + "T003", + "T005", + ] + new_turns = store.get_turns("Case-002") + assert [(turn.turn_id, turn.parent_turn_id, turn.exact_text) for turn in new_turns] == [ + ("T001", None, root.exact_text), + ("T002", "T001", shared_reply.exact_text), + ("T003", "T002", second_branch.exact_text), + ] + assert store.get_case("Case-002").case_title == "Second branch Case" + assert [item.case_id for item in store.find_cases("123", "456")] == [ + "Case-001", + "Case-002", + ] + + +def test_branch_split_failure_rolls_back_and_retains_verified_backup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case(status=CaseStatus.ACTIVE_EXCHANGE) + turns = [ + make_turn(), + make_reply("T002", "T001", exact_text="Shared reply."), + make_reply("T003", "T002", exact_text="First branch."), + make_reply("T004", "T002", exact_text="Second branch."), + ] + store.create_case(case, turns, approved=True) + backup = tmp_path / "backups" / "before-failed-split.sqlite3" + + def fail_insert(_connection: sqlite3.Connection, _record: object) -> None: + raise sqlite3.IntegrityError("synthetic insert failure") + + monkeypatch.setattr(store, "_insert_turn", fail_insert) + with pytest.raises(StorageError, match="rollback verified; backup retained"): + store.split_case_branch( + "Case-001", + "T004", + new_case_title="Second branch Case", + new_topic="Second branch topic", + backup_destination=backup, + approved=True, + ) + + assert backup.is_file() + assert SQLiteStore(backup).status()["case_count"] == 1 + assert store.case_ids() == ["Case-001"] + assert store.get_turns("Case-001") == turns + assert store.status()["integrity"] == "ok" def test_failed_import_rolls_back_every_row(tmp_path: Path) -> None: store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") store.initialize(approved=True) - first = make_case(case_id="CASE-20260717-001") - duplicate_identity = make_case(case_id="CASE-20260717-002") + first = make_case(case_id="Case-001") + duplicate_identity = make_case(case_id="Case-001") - with pytest.raises(StorageError, match="UNIQUE"): + with pytest.raises(StorageError, match="duplicate Case IDs"): store.import_records([first, duplicate_identity], [], approved=True) assert store.status()["case_count"] == 0 @@ -164,7 +279,9 @@ def test_close_case_writes_only_validated_outcome_and_reads_back(tmp_path: Path) assert store.get_case(case.case_id) == closed -def test_open_case_requires_and_returns_exact_comment_permalink(tmp_path: Path) -> None: +def test_open_case_returns_latest_turn_permalink_without_root_fallback( + tmp_path: Path, +) -> None: invalid = make_case(post_url="https://www.facebook.com/example/posts/123") with pytest.raises(DialogueLabError, match="comment or reply"): validate_case(invalid) @@ -177,13 +294,40 @@ def test_open_case_requires_and_returns_exact_comment_permalink(tmp_path: Path) "&__cft__[0]=tracking#comment" ) ) - store.create_case(case, [], approved=True) + root = make_turn(exact_url=case.post_url) + reply_url = ( + "https://www.facebook.com/example/posts/123" + "?comment_id=456&reply_comment_id=789" + ) + reply = make_reply( + "T002", + "T001", + exact_url=reply_url, + reply_comment_id="789", + observed_at="2026-07-17 11:00", + ) + store.create_case(case, [root, reply], approved=True) + missing_case = make_case(case_id="Case-002") + missing_turn = make_turn( + case_id="Case-002", + exact_text="Latest link was not supplied.", + ) + store.create_case(missing_case, [missing_turn], approved=True) assert store.list_open_summaries() == [ { "case_id": case.case_id, "status": "Posted", - "exact_permalink": case.post_url, - } + "last_turn_id": "T002", + "last_comment_permalink": reply_url, + "permalink_status": "supplied", + }, + { + "case_id": "Case-002", + "status": "Posted", + "last_turn_id": "T001", + "last_comment_permalink": None, + "permalink_status": "missing", + }, ] @@ -197,3 +341,277 @@ def test_backup_is_consistent_and_never_overwrites(tmp_path: Path) -> None: assert SQLiteStore(destination).status()["case_count"] == 1 with pytest.raises(StorageError, match="already exists"): store.backup(destination, approved=True) + + +def test_turn_duplicate_identity_uses_reply_id_then_null_parent_fallback( + tmp_path: Path, +) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case() + root = make_turn() + store.create_case(case, [root], approved=True) + + fallback_duplicate = replace(root, turn_id="ignored", observed_at="later") + receipt = store.add_turn( + fallback_duplicate, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at="later", + reason="duplicate supplied content", + replace_draft_id=None, + approved=True, + ) + assert receipt == { + "created": False, + "duplicate": True, + "duplicate_identity": "fallback", + "case_id": "Case-001", + "turn_id": "T001", + } + + strong = make_reply( + "ignored", + "T001", + reply_comment_id="789", + exact_url=( + "https://www.facebook.com/example/posts/123" + "?comment_id=456&reply_comment_id=789" + ), + exact_text="First observed reply text.", + ) + store.add_turn( + strong, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at=strong.observed_at, + reason="incoming public turn", + replace_draft_id=None, + approved=True, + ) + changed_text = replace(strong, turn_id="ignored", exact_text="Edited observation.") + duplicate = store.add_turn( + changed_text, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at=changed_text.observed_at, + reason="incoming public turn", + replace_draft_id=None, + approved=True, + ) + assert duplicate["duplicate"] is True + assert duplicate["duplicate_identity"] == "reply_comment_id" + assert duplicate["turn_id"] == "T002" + + other_case = make_case( + case_id="Case-002", + post_url="https://www.facebook.com/example/posts/123?comment_id=999", + root_comment_id="999", + ) + store.create_case(other_case, [], approved=True) + other_root = make_turn( + case_id="Case-002", + root_comment_id="999", + exact_text=root.exact_text, + ) + other_receipt = store.add_turn( + other_root, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at=other_root.observed_at, + reason="same fallback tuple in another Case", + replace_draft_id=None, + approved=True, + ) + assert other_receipt["created"] is True + cross_case = make_turn( + case_id="Case-002", + root_comment_id="999", + reply_comment_id="789", + exact_url=( + "https://www.facebook.com/example/posts/123" + "?comment_id=999&reply_comment_id=789" + ), + exact_text="Same Facebook reply in another explicit Case.", + ) + with pytest.raises(StorageError, match="different Case"): + store.add_turn( + cross_case, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at=cross_case.observed_at, + reason="incoming public turn", + replace_draft_id=None, + approved=True, + ) + + +def test_permalink_reply_id_cannot_be_omitted_or_conflict() -> None: + exact_url = ( + "https://www.facebook.com/example/posts/123" + "?comment_id=456&reply_comment_id=789" + ) + with pytest.raises(DialogueLabError, match="must match"): + validate_turn(make_turn(exact_url=exact_url)) + with pytest.raises(DialogueLabError, match="must match"): + validate_turn(make_turn(exact_url=exact_url, reply_comment_id="999")) + validate_turn(make_turn(exact_url=exact_url, reply_comment_id="789")) + + +def test_posting_unchanged_draft_promotes_and_enriches_same_turn(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "dialogue-lab.sqlite3") + store.initialize(approved=True) + case = make_case(status=CaseStatus.ACTIVE_EXCHANGE) + root = make_turn() + draft = make_reply( + "T002", + "T001", + participant_ref="USER", + direction=TurnDirection.OUTGOING, + state=TurnState.DRAFT, + exact_text="Unchanged exact wording.", + ) + store.create_case(case, [root, draft], approved=True) + posted = replace( + draft, + turn_id="ignored", + state=TurnState.POSTED, + reply_comment_id="900", + exact_url=( + "https://www.facebook.com/example/posts/123" + "?comment_id=456&reply_comment_id=900" + ), + url_supplied_at="2026-07-17 12:00", + observed_at="2026-07-17 12:00", + ) + + receipt = store.add_turn( + posted, + target_status=CaseStatus.ACTIVE_EXCHANGE, + updated_at=posted.observed_at, + reason="posting confirmed", + replace_draft_id="T002", + approved=True, + ) + + assert receipt["promoted"] is True + assert receipt["turn_id"] == "T002" + turns = store.get_turns(case.case_id) + assert len(turns) == 2 + assert turns[1].state is TurnState.POSTED + assert turns[1].reply_comment_id == "900" + + +def test_status_rejects_identity_index_tampering(tmp_path: Path) -> None: + path = tmp_path / "dialogue-lab.sqlite3" + store = SQLiteStore(path) + store.initialize(approved=True) + with sqlite3.connect(path) as connection: + connection.execute("DROP INDEX turns_fallback_identity_uq") + with pytest.raises(StorageError, match="indexes"): + store.status() + + +def test_identity_migration_stably_renumbers_and_preserves_turn_references( + tmp_path: Path, +) -> None: + path = tmp_path / "legacy.sqlite3" + store = SQLiteStore(path) + store.initialize(approved=True) + later_name = make_case(case_id="Case-002", created_at="2026-07-17 10:00") + earlier_name = make_case( + case_id="Case-001", + created_at="2026-07-17 10:00", + post_url="https://www.facebook.com/example/posts/123?comment_id=999", + root_comment_id="999", + ) + store.create_case( + earlier_name, + [make_turn(case_id="Case-001", root_comment_id="999")], + approved=True, + ) + store.create_case(later_name, [make_turn(case_id="Case-002")], approved=True) + with sqlite3.connect(path) as connection: + table_sql = str( + connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'cases'" + ).fetchone()[0] + ) + check_start = table_sql.index("case_id TEXT PRIMARY KEY CHECK") + check_end = table_sql.index(",\n case_title", check_start) + legacy_table_sql = ( + table_sql[:check_start] + + "case_id TEXT PRIMARY KEY" + + table_sql[check_end:] + ) + connection.execute("PRAGMA writable_schema = ON") + connection.execute( + "UPDATE sqlite_master SET sql = ? WHERE type = 'table' AND name = 'cases'", + (legacy_table_sql,), + ) + schema_version = int(connection.execute("PRAGMA schema_version").fetchone()[0]) + connection.execute(f"PRAGMA schema_version = {schema_version + 1}") + with sqlite3.connect(path) as connection: + connection.execute("PRAGMA foreign_keys = OFF") + connection.execute("PRAGMA ignore_check_constraints = ON") + connection.execute( + "UPDATE turns SET case_id = 'CASE-20260717-020' WHERE case_id = 'Case-002'" + ) + connection.execute( + "UPDATE cases SET case_id = 'CASE-20260717-020' WHERE case_id = 'Case-002'" + ) + connection.execute( + "UPDATE turns SET case_id = 'CASE-20260716-900' WHERE case_id = 'Case-001'" + ) + connection.execute( + "UPDATE cases SET case_id = 'CASE-20260716-900' WHERE case_id = 'Case-001'" + ) + connection.execute("DROP INDEX cases_root_candidates_idx") + connection.execute("DROP INDEX turns_reply_comment_id_uq") + connection.execute("DROP INDEX turns_fallback_identity_uq") + connection.execute( + "CREATE UNIQUE INDEX legacy_root_uq ON cases(post_id, root_comment_id)" + ) + connection.execute( + "UPDATE storage_metadata SET value = ? WHERE key = 'schema_signature'", + (_LEGACY_IDENTITY_SCHEMA_SIGNATURE,), + ) + + backup = tmp_path / "backups" / "before.sqlite3" + receipt = store.migrate_identity(backup, approved=True) + + assert receipt["case_id_map"] == { + "CASE-20260716-900": "Case-001", + "CASE-20260717-020": "Case-002", + } + assert receipt["database_schema_version_after"] == 1 + assert receipt["committed_read_back"] == "verified" + assert backup.is_file() + assert store.get_turns("Case-001")[0].case_id == "Case-001" + + already_backup = tmp_path / "backups" / "already.sqlite3" + with pytest.raises(StorageError, match="already applied"): + store.migrate_identity(already_backup, approved=True) + assert not already_backup.exists() + + conflict_path = tmp_path / "legacy-conflict.sqlite3" + with sqlite3.connect(backup) as source, sqlite3.connect(conflict_path) as target: + source.backup(target) + with sqlite3.connect(conflict_path) as connection: + connection.execute( + """INSERT INTO turns + SELECT case_id, 'T002', parent_turn_id, parent_confidence, + participant_ref, direction, kind, state, exact_text, post_id, + root_comment_id, reply_comment_id, exact_url, url_supplied_at, + observed_at, notes + FROM turns + WHERE case_id = 'CASE-20260716-900' AND turn_id = 'T001'""" + ) + failure_backup = tmp_path / "backups" / "failure.sqlite3" + conflict_store = SQLiteStore(conflict_path) + with pytest.raises(StorageError, match="rollback verified; backup retained"): + conflict_store.migrate_identity(failure_backup, approved=True) + assert failure_backup.is_file() + with sqlite3.connect(conflict_path) as connection: + assert connection.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + assert connection.execute( + "SELECT value FROM storage_metadata WHERE key = 'schema_signature'" + ).fetchone()[0] == _LEGACY_IDENTITY_SCHEMA_SIGNATURE + assert connection.execute( + "SELECT COUNT(*) FROM turns WHERE case_id = 'CASE-20260716-900'" + ).fetchone()[0] == 2 From 4126107870f550a726186b5a78fd2a690885b51c Mon Sep 17 00:00:00 2001 From: Codex <codex@local> Date: Wed, 22 Jul 2026 04:36:06 +0300 Subject: [PATCH 3/5] Update CI readiness command --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87fa97f..293fc21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,4 +25,4 @@ jobs: - run: uv run pytest - run: uv run ruff check . - run: uv run mypy - - run: uv run dialogue-lab doctor + - run: uv run dialogue-lab check From e06a8fccdc5ae4ae8707f6c25e68cc8be7ab0876 Mon Sep 17 00:00:00 2001 From: Codex <codex@local> Date: Wed, 22 Jul 2026 04:38:11 +0300 Subject: [PATCH 4/5] Initialize CI readiness database --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 293fc21..d1078dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,8 @@ permissions: jobs: checks: runs-on: ubuntu-latest + env: + DIALOGUE_LAB_DB: ${{ runner.temp }}/dialogue-lab-ci.sqlite3 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 @@ -25,4 +27,5 @@ jobs: - run: uv run pytest - run: uv run ruff check . - run: uv run mypy + - run: uv run dialogue-lab db-init --approved - run: uv run dialogue-lab check From 571634e08acaea3513e7f2447b4e51b30afe8e16 Mon Sep 17 00:00:00 2001 From: Codex <codex@local> Date: Wed, 22 Jul 2026 04:40:32 +0300 Subject: [PATCH 5/5] Scope CI database path to runner steps --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1078dd..38d87b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,6 @@ permissions: jobs: checks: runs-on: ubuntu-latest - env: - DIALOGUE_LAB_DB: ${{ runner.temp }}/dialogue-lab-ci.sqlite3 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 @@ -28,4 +26,8 @@ jobs: - run: uv run ruff check . - run: uv run mypy - run: uv run dialogue-lab db-init --approved + env: + DIALOGUE_LAB_DB: ${{ runner.temp }}/dialogue-lab-ci.sqlite3 - run: uv run dialogue-lab check + env: + DIALOGUE_LAB_DB: ${{ runner.temp }}/dialogue-lab-ci.sqlite3