From 7f65d11ae00514562567cdb0a50132ce6b29117b Mon Sep 17 00:00:00 2001 From: Martins Aloba Date: Sun, 17 May 2026 07:26:35 -0500 Subject: [PATCH 1/3] =?UTF-8?q?test(15-5):=20AI=20schema=20regression=20te?= =?UTF-8?q?st=20infrastructure=20=E2=80=94=20fixture=20loader=20+=203=20sy?= =?UTF-8?q?nthetic=20seed=20fixtures=20+=20runbook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dictation/synthetic-a2-mixed-001.json | 21 +++ .../synthetic-b1-listening-001.json | 34 ++++ .../synthetic-b1-formal-001.json | 27 +++ .../schemas/__tests__/fixture-replay.test.ts | 165 ++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100644 src/lib/schemas/__fixtures__/dictation/synthetic-a2-mixed-001.json create mode 100644 src/lib/schemas/__fixtures__/mock-test-section/synthetic-b1-listening-001.json create mode 100644 src/lib/schemas/__fixtures__/writing-evaluation/synthetic-b1-formal-001.json create mode 100644 src/lib/schemas/__tests__/fixture-replay.test.ts diff --git a/src/lib/schemas/__fixtures__/dictation/synthetic-a2-mixed-001.json b/src/lib/schemas/__fixtures__/dictation/synthetic-a2-mixed-001.json new file mode 100644 index 0000000..dca08a9 --- /dev/null +++ b/src/lib/schemas/__fixtures__/dictation/synthetic-a2-mixed-001.json @@ -0,0 +1,21 @@ +{ + "_synthetic": true, + "_note": "Story 15-5 seed fixture. Demonstrates the replay path works against dictationSetSchema. Replace with operator-captured real model output per runbooks/ai-fixture-capture.md.", + "sentences": [ + { + "sentence": "Bonjour, comment ça va aujourd'hui ?", + "translation": "Hello, how are you today?", + "difficulty": "easy" + }, + { + "sentence": "Je voudrais une baguette et deux croissants, s'il vous plaît.", + "translation": "I would like a baguette and two croissants, please.", + "difficulty": "medium" + }, + { + "sentence": "Malgré la pluie, nous avons décidé de continuer notre promenade.", + "translation": "Despite the rain, we decided to continue our walk.", + "difficulty": "hard" + } + ] +} diff --git a/src/lib/schemas/__fixtures__/mock-test-section/synthetic-b1-listening-001.json b/src/lib/schemas/__fixtures__/mock-test-section/synthetic-b1-listening-001.json new file mode 100644 index 0000000..141c185 --- /dev/null +++ b/src/lib/schemas/__fixtures__/mock-test-section/synthetic-b1-listening-001.json @@ -0,0 +1,34 @@ +{ + "_synthetic": true, + "_note": "Story 15-5 seed fixture. Demonstrates the replay path works against mockTestSectionSchema. Includes the load-bearing 'exactly 4 options + exactly 1 correct + unique ids' invariant per Story 9-7 + 9-7 R1 P9/P10. Replace with operator-captured real model output per runbooks/ai-fixture-capture.md.", + "passages": [ + { + "id": "passage-1", + "text": "Marie travaille dans une boulangerie depuis cinq ans. Elle commence à quatre heures du matin et finit à midi." + } + ], + "questions": [ + { + "question": "À quelle heure Marie commence-t-elle son travail ?", + "passageId": "passage-1", + "options": [ + { "id": "opt-a", "text": "À deux heures du matin", "isCorrect": false }, + { "id": "opt-b", "text": "À quatre heures du matin", "isCorrect": true }, + { "id": "opt-c", "text": "À six heures du matin", "isCorrect": false }, + { "id": "opt-d", "text": "À huit heures du matin", "isCorrect": false } + ], + "explanation": "Le texte indique qu'elle commence 'à quatre heures du matin'." + }, + { + "question": "Depuis combien de temps Marie travaille-t-elle à la boulangerie ?", + "passageId": "passage-1", + "options": [ + { "id": "opt-a", "text": "Deux ans", "isCorrect": false }, + { "id": "opt-b", "text": "Trois ans", "isCorrect": false }, + { "id": "opt-c", "text": "Cinq ans", "isCorrect": true }, + { "id": "opt-d", "text": "Dix ans", "isCorrect": false } + ], + "explanation": "Le texte dit 'depuis cinq ans'." + } + ] +} diff --git a/src/lib/schemas/__fixtures__/writing-evaluation/synthetic-b1-formal-001.json b/src/lib/schemas/__fixtures__/writing-evaluation/synthetic-b1-formal-001.json new file mode 100644 index 0000000..a6b2025 --- /dev/null +++ b/src/lib/schemas/__fixtures__/writing-evaluation/synthetic-b1-formal-001.json @@ -0,0 +1,27 @@ +{ + "_synthetic": true, + "_note": "Story 15-5 seed fixture. Demonstrates the replay path works against writingEvaluationSchema. Replace with operator-captured real model output per runbooks/ai-fixture-capture.md when available.", + "overallScore": 72, + "grammarScore": 75, + "cohesionScore": 70, + "lexicalRichnessScore": 68, + "registerScore": 78, + "errors": [ + { + "original": "Je suis allé au magasin pour acheter quelques pomme", + "correction": "Je suis allé au magasin pour acheter quelques pommes", + "explanation": "'pommes' takes plural agreement after 'quelques'", + "category": "grammar" + } + ], + "suggestions": [ + "Vary sentence openers — three sentences start with 'Je'", + "Consider using subordinate clauses to demonstrate B1+ syntactic range" + ], + "rewriteSuggestion": "Hier, je suis allé au magasin pour acheter des pommes. Quand je suis rentré, j'ai préparé une tarte.", + "tcfEstimatedScore": 12, + "vocabularyDiversityRatio": 0.42, + "connectorsUsed": ["et", "pour", "quand"], + "connectorsMissing": ["cependant", "néanmoins", "ainsi"], + "summary": "Solid B1 base; grammar largely accurate but vocabulary diversity and connector variety can grow." +} diff --git a/src/lib/schemas/__tests__/fixture-replay.test.ts b/src/lib/schemas/__tests__/fixture-replay.test.ts new file mode 100644 index 0000000..d45b27e --- /dev/null +++ b/src/lib/schemas/__tests__/fixture-replay.test.ts @@ -0,0 +1,165 @@ +/** + * Story 15-5 — AI schema regression test infrastructure. + * + * Walks `src/lib/schemas/__fixtures__//*.json` at suite setup time, + * loads each fixture, and replays it through the corresponding Zod parser. + * Each fixture file becomes a distinct Jest test case via `it.each` over + * the discovered files — adding a new fixture file is a pure data add (no + * code change needed). + * + * The `FIXTURE_SCHEMA_MAP` is the explicit dir-name → schema mapping. A + * future schema rename in `ai-responses.ts` requires updating both this + * map AND the fixtures directory name (caught by TypeScript on the + * `import` line). + * + * `_synthetic: true` marker on seed fixtures distinguishes them from + * operator-captured real model outputs. The replay logic strips the + * `_synthetic` + `_note` top-level fields BEFORE handing to the parser + * (those are metadata; not part of the schema contract). + */ + +import * as fs from "fs"; +import * as path from "path"; + +import { z } from "zod"; + +import { + writingEvaluationSchema, + dictationSetSchema, + mockTestSectionSchema, +} from "@/src/lib/schemas/ai-responses"; + +const FIXTURES_ROOT = path.resolve(__dirname, "../__fixtures__"); + +/** + * Explicit dir-name → Zod schema mapping. Add a new entry to extend + * regression coverage to a new schema; create the matching directory and + * drop fixture JSON files into it. + */ +const FIXTURE_SCHEMA_MAP: Record = { + "writing-evaluation": writingEvaluationSchema, + dictation: dictationSetSchema, + "mock-test-section": mockTestSectionSchema, +}; + +/** + * Metadata fields that exist on synthetic seed fixtures but are NOT part + * of the schema contract — stripped before replay. + */ +const METADATA_FIELDS = new Set(["_synthetic", "_note"]); + +interface DiscoveredFixture { + schemaName: string; + fileName: string; + fullPath: string; +} + +function discoverFixtures(): DiscoveredFixture[] { + const out: DiscoveredFixture[] = []; + if (!fs.existsSync(FIXTURES_ROOT)) return out; + const schemaDirs = fs.readdirSync(FIXTURES_ROOT, { withFileTypes: true }); + for (const dir of schemaDirs) { + if (!dir.isDirectory()) continue; + const schemaName = dir.name; + const schemaDirPath = path.join(FIXTURES_ROOT, schemaName); + const files = fs.readdirSync(schemaDirPath); + for (const file of files) { + if (!file.endsWith(".json")) continue; + out.push({ + schemaName, + fileName: file, + fullPath: path.join(schemaDirPath, file), + }); + } + } + return out; +} + +function stripMetadata(json: unknown): unknown { + if (json === null || typeof json !== "object" || Array.isArray(json)) return json; + const cleaned: Record = {}; + for (const [k, v] of Object.entries(json as Record)) { + if (METADATA_FIELDS.has(k)) continue; + cleaned[k] = v; + } + return cleaned; +} + +describe("Story 15-5 — AI schema regression fixture replay", () => { + const fixtures = discoverFixtures(); + + it("Case 1: FIXTURE_SCHEMA_MAP contains entries for the included seed schemas (3 minimum)", () => { + expect(Object.keys(FIXTURE_SCHEMA_MAP)).toEqual( + expect.arrayContaining(["writing-evaluation", "dictation", "mock-test-section"]) + ); + expect(Object.keys(FIXTURE_SCHEMA_MAP).length).toBeGreaterThanOrEqual(3); + }); + + it("Case 2: every fixture directory has a corresponding FIXTURE_SCHEMA_MAP entry (no orphan dirs)", () => { + const discoveredDirs = new Set(fixtures.map((f) => f.schemaName)); + for (const dir of discoveredDirs) { + expect(FIXTURE_SCHEMA_MAP[dir]).toBeDefined(); + } + }); + + it("Case 3: at least 1 seed fixture per included schema (3 schemas × ≥1 fixture)", () => { + for (const schemaName of Object.keys(FIXTURE_SCHEMA_MAP)) { + const count = fixtures.filter((f) => f.schemaName === schemaName).length; + expect(count).toBeGreaterThanOrEqual(1); + } + }); + + describe("Replay each discovered fixture through its Zod parser", () => { + if (fixtures.length === 0) { + // Defensive: if discovery returns empty (e.g., a bad merge), skip + // rather than fail. The infrastructure should still pass tests so + // 15-5 can ship before fixtures are populated. This branch is dead + // code given the seed fixtures above; included for resilience. + it.skip("no fixtures discovered — directory tree may be malformed", () => {}); + return; + } + it.each(fixtures)( + "fixture $schemaName/$fileName parses successfully against its Zod schema", + ({ schemaName, fullPath }) => { + const schema = FIXTURE_SCHEMA_MAP[schemaName]; + expect(schema).toBeDefined(); + const raw = fs.readFileSync(fullPath, "utf-8"); + const json = JSON.parse(raw); + const stripped = stripMetadata(json); + const result = schema.safeParse(stripped); + if (!result.success) { + // Detailed failure message so a regression PR shows which schema + // + which fixture + which Zod issues, not just "parse failed". + throw new Error( + `Fixture ${schemaName}/${path.basename(fullPath)} failed schema parse:\n` + + JSON.stringify(result.error.issues, null, 2) + ); + } + expect(result.success).toBe(true); + } + ); + }); + + it("Case 4: stripMetadata removes _synthetic + _note top-level fields without touching nested fields", () => { + const input = { + _synthetic: true, + _note: "test note", + data: { nested: { _synthetic: "should NOT be stripped at nested level" } }, + overallScore: 75, + }; + const out = stripMetadata(input) as Record; + expect(out._synthetic).toBeUndefined(); + expect(out._note).toBeUndefined(); + expect(out.data).toEqual({ + nested: { _synthetic: "should NOT be stripped at nested level" }, + }); + expect(out.overallScore).toBe(75); + }); + + it("Case 5: stripMetadata is a no-op on non-objects (defensive)", () => { + expect(stripMetadata(null)).toBeNull(); + expect(stripMetadata([1, 2, 3])).toEqual([1, 2, 3]); + expect(stripMetadata("string")).toBe("string"); + expect(stripMetadata(42)).toBe(42); + }); +}); From 87a6c9a856f67646880e794cb344039fbae2abab Mon Sep 17 00:00:00 2001 From: Martins Aloba Date: Sun, 17 May 2026 07:26:41 -0500 Subject: [PATCH 2/3] chore(15-5): spec + runbook + CLAUDE.md + sprint-status done --- CLAUDE.md | 2 + .../15-5-ai-schema-regression-tests.md | 81 ++++++++++++ .../sprint-status.yaml | 4 +- .../runbooks/ai-fixture-capture.md | 121 ++++++++++++++++++ 4 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 _bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md create mode 100644 _bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md diff --git a/CLAUDE.md b/CLAUDE.md index 2bc4428..d260889 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -201,6 +201,8 @@ A reusable pattern for capping in-memory data structures + UI work-budgets. Two **Lib unit tests — `srs.ts` SM-2 algorithm + `pronunciation.ts` `identifyWeakSounds` + `cache.ts` core API + `use-dictation.ts` `compareSentences`:** post-Epic-15.1, the FIRST Epic 15 (Test Coverage & QA Infrastructure) story closes the largest pure-function-test gap surfaced by the Story 12-10 audit refresh footnote. Per the coverage inventory at story-creation time, `activity.ts` was already fully tested (Story 9-2 + 12-3); `memory.ts` + `error-tracker.ts` had substantial partial coverage (sanitize via Story 9-4 prompt-injection.test.ts + dedup via Story 11-6 error-tracker-dedupe.test.ts + e2e clustering via 11-6); `pronunciation.ts` had history-cap coverage (Story 12-12) but NOT `identifyWeakSounds`. Story 15-1 lands **GAP-only scope** across **4 pure-function surfaces** (no source-module modifications — test-only): **(1) NEW [`src/lib/__tests__/srs.test.ts`](src/lib/__tests__/srs.test.ts)** (14 cases) pins the full SM-2 algorithm contract — 3 incorrect-response cases (quality 0/1/2 ease-factor deltas −0.80/−0.54/−0.32 + reset to `repetitions=0`, `intervalDays=1`); 3 correct-response cases (quality 3/4/5 ease-factor deltas −0.14/0/+0.10 + interval progression 1 → 6 → round(prev × ef)); 3 ease-factor 1.3-floor clamping cases (boundary + already-at-floor + just-above-floor); 3-consecutive-correct chain verification; 365-day interval cap; midnight-snap `nextReview` math via Date.now() before/after delta check; no-mutation invariant via input-snapshot equality + determinism via twice-call equivalence. **(2) NEW [`src/lib/__tests__/pronunciation.test.ts`](src/lib/__tests__/pronunciation.test.ts)** (10 cases) pins `identifyWeakSounds` contract — empty input + no-weak happy path + below-threshold-but-count<3 NOT flagged + single-phoneme-aggregated-across-results + same-phoneme-aggregated-across-words-in-one-result + threshold boundaries (`avgScore < 70` strict + `count >= 3` boundary) + multi-phoneme ranking ASCENDING by avgScore (worst first) + no-mutation deep-equal. **(3) NEW [`src/lib/__tests__/cache.test.ts`](src/lib/__tests__/cache.test.ts)** (20 cases) covers the AsyncStorage path (existing `cache-flush.test.ts` covers write-queue idempotency; existing `cache-secure-routing.test.ts` covers the Story 12-7 SecureStore fork) — `getCache` empty/fresh/TTL-fresh-boundary/TTL-expired-boundary/corrupted-JSON/throw paths each with `captureError` routing pinned; `setCache` envelope shape `{data, timestamp, ttlMs}` + DEFAULT_TTL_MS (1 hour) fallback + namespacing `@companion_cache::` (different userIds never collide); `setCache` → `getCache` round-trip with nested object structure; `invalidateCache` happy + error paths; `cacheWithFallback` 4-path matrix (fetcher-succeeds-writes-cache, fetcher-throws-stale-cache-hit-returns-fromCache:true, fetcher-throws-no-cache-rethrows-original, fetcher-throws-cache-also-throws-captures-fallback-read-error-rethrows-original); constant pins for `CACHE_KEYS` (5 keys) + `CACHE_TTL` (6 durations) + `SECURE_CACHE_KEYS` size===1 with PROFILE in / SKILLS+VOCABULARY out. **(4) NEW [`src/hooks/__tests__/use-dictation-compare.test.ts`](src/hooks/__tests__/use-dictation-compare.test.ts)** (15 cases) pins the pure word-by-word `compareSentences` helper from [`src/hooks/use-dictation.ts:88`](src/hooks/use-dictation.ts#L88) (despite living in a hook file, the function is pure — no React, no async, no side effects, so it fits 15-1 scope). Test mocks `@/src/hooks/use-audio-player` (transitive `expo-audio` native crash in Jest) + `@/src/lib/openai` (transitive supabase chain) at module load to keep the import boundary clean. Cases cover: perfect-match → 100% / one-wrong-word with `typed` populated → round(2/3 × 100)=67% / missing-word semantics (user shorter → past-user-length positions are "missing", overlap positions compare normally) / extra-word (user longer → extra tokens IGNORED; only original-length iterated) / empty-userInput → all "missing" / empty-original → empty wordResults + accuracy=0 + isFullyCorrect=true vacuously / both-empty same / case-insensitive ("Bonjour" vs "bonjour") with display preserving original capitalization / accent-insensitive ("café" vs "cafe") with display preserving accent / trailing punctuation stripped / internal comma stripped / multi-space collapses / leading+trailing whitespace trimmed / apostrophe stripped ("l'eau" matches "leau") / fresh wordResults array per call (no shared module-level cache). **3 operator decisions resolved per Recommended** (Q1 defer `assessPronunciation` Edge wrapper to 15-1-followup; Q2 defer `memory.ts` async/DB-touching paths to 15-2; Q3 defer `error-tracker.ts` non-dedup paths to 15-2) — keeps 15-1 pure-function-only per the Epic 14 retro lesson that test-writing stories are at scope-drift risk. **Explicitly out of scope** (filed for follow-up if motivated): `memory.ts` `extractFacts`/`persistMemories`/`retrieveMemories`; `error-tracker.ts` `getTopErrors`/`extractErrorsFromCorrections`/`persistErrorPatterns`/`getRecentResolvedError`; `pronunciation.ts` `assessPronunciation` Edge Function wrapper; `cache.ts` `enqueueWrite`/`flushWriteQueue` (already in `cache-flush.test.ts`)/`clearUserCache`/`clearAllCache` multi-key sweeps; `use-dictation.ts` `analyzeErrorPatterns`. **Cross-story invariants preserved by construction:** zero source-module modifications (Story 15-1 is test-only); Story 9-3 Sentry allowlist zero-diff (cache tests verify EXISTING `cache-get`/`cache-set`/`cache-invalidate`/`cache-fallback-read` feature tags fire; no new tags); Story 12-7 SecureStore fork unchanged (cache.test.ts uses non-secure key `"skills"` to exercise AsyncStorage path); all Epic 14 invariants orthogonal. **+59 net Jest cases** (2099 → 2158; spec target +50-65 — squarely in range). 4 new test files; 0 modified source files. All 5 design-system gates green (type-check 0 errors / lint 0 warnings / prettier clean / check:tokens clean / jest 113 suites / 2158 tests). Verified 2026-05-16, story 15-1. +**AI schema regression test infrastructure — fixture loader + replay harness:** post-Epic-15.5 ships the regression-test layer for `ai-responses.ts` Zod schemas. Pre-15-5 the existing `ai-responses.test.ts` covered schemas at the unit level (synthetic-input boundary tests); real-shape regressions (a prompt change shifting model output, a new field the model started emitting) had no CI gate. Story 15-5 lands **INFRASTRUCTURE-ONLY scope** — fixture loader + replay harness + 3 synthetic seed fixtures + operator runbook for real-fixture capture (deferred to operator action because (a) capturing 10 real outputs per prompt requires Sentry breadcrumb pulls / live OpenAI calls that burn cost-cap budget per Story 11-4 and (b) is inappropriate for an autopilot story). NEW [`src/lib/schemas/__fixtures__/`](src/lib/schemas/__fixtures__/) directory tree with subdirs `writing-evaluation/`, `dictation/`, `mock-test-section/` (matches the spec's "writing eval, mock test, dictation" example list) + 1 synthetic seed fixture per dir. NEW [`src/lib/schemas/__tests__/fixture-replay.test.ts`](src/lib/schemas/__tests__/fixture-replay.test.ts) (8 cases) walks the directory tree at suite setup, maps each schema-dir to its Zod parser via explicit `FIXTURE_SCHEMA_MAP` constant (catches schema rename via TypeScript import; auto-derive would silently fail), strips top-level `_synthetic` + `_note` metadata fields before parse (operator-readable bookkeeping markers not part of the schema contract), and uses `it.each` over discovered files so adding a new fixture is a **pure data add** (no code change needed). Infrastructure cases: `FIXTURE_SCHEMA_MAP` entries pin; no orphan dirs; ≥1 fixture per schema; `stripMetadata` strips top-level + preserves nested + is no-op on non-objects. Per-fixture cases: 3 synthetic seed fixtures parse successfully against their schemas (writing-evaluation B1 formal essay with 13-field shape incl. all 5 score bands + 1 grammar error + connectors used/missing + tcfEstimatedScore + summary; dictation A2 mixed 3-sentence set with easy/medium/hard difficulty enum; mock-test B1 listening 1-passage 2-question shape with 4-options/1-correct/unique-ids per `mcqQuestionSchema` Story 9-7 R1 P9/P10 invariant). NEW operator runbook [`_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md`](_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md) documents WHEN to capture (post-prompt-change / post-model-upgrade / post-schema-change), HOW (3 options — Sentry breadcrumb pull preferred; dev-mode temporary console.log; live OpenAI call last resort), naming convention (`-NNN.json`), WHERE (the new directory tree), HOW to verify (`npx jest fixture-replay` shows new test case), and the synthetic-vs-real bookkeeping convention. **3 operator decisions resolved per Recommended:** Q1 explicit `FIXTURE_SCHEMA_MAP` (vs auto-derive from dir name — explicit is TypeScript-checked + survives schema renames); Q2 `_synthetic: true` marker on seed fixtures (future tooling can filter to real-only when assessing regression confidence); Q3 start with 3 schemas matching the spec's example list (defer expansion to all 36 schemas to `15-5-followup-fixture-coverage-expansion`). **Cross-story invariants preserved:** `ai-responses.ts` schemas unchanged; existing `ai-responses.test.ts` unchanged; 0 source-module modifications beyond the test infrastructure + fixtures + runbook. **+8 net Jest cases** (3 fixture-replay cases via `it.each` + 5 infrastructure pins = 8 total per the new test file; pre-15-5 baseline 2159 → 2167). All 5 design-system gates green. Verified 2026-05-17, story 15-5. + ### Routing (`app/`) Expo Router file-based routing with three route groups: diff --git a/_bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md b/_bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md new file mode 100644 index 0000000..8033f00 --- /dev/null +++ b/_bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md @@ -0,0 +1,81 @@ +# Story 15.5: AI schema regression test infrastructure — fixture loader + replay harness for `ai-responses.ts` Zod schemas (capture deferred to operator) + +Status: review + +## Story + +As a developer, I want a **regression-test harness that loads recorded AI-response JSON fixtures from `src/lib/schemas/__fixtures__//*.json` and replays each through its corresponding Zod parser in `ai-responses.ts`** so that a future prompt change or model upgrade that breaks an existing real-shape response is caught in CI. + +## Background + +[`shippable-roadmap.md`](_bmad-output/planning-artifacts/shippable-roadmap.md) line 297 — Epic 15.5 deliverable: "AI schema regression tests — record 10 real model outputs per prompt (writing eval, mock test, dictation, etc); replay through Zod parsers in CI." + +**Coverage today:** `src/lib/schemas/__tests__/ai-responses.test.ts` covers the schemas at the unit level (synthetic-input boundary tests). Story 15-5 adds the regression-test layer: real-shaped fixtures from production model outputs replayed through the same parsers. + +**Why infrastructure-only scope:** capturing 10 real outputs per prompt requires either (a) operator action — pulling Sentry breadcrumbs or capturing live during dev/manual runs — or (b) running the live OpenAI/Azure APIs which would burn cost-cap budget per Story 11-4. Neither is appropriate for an autopilot story. The 15-5 deliverable splits cleanly: ship the infrastructure here; defer fixture capture to operator action documented in the runbook. + +## Acceptance Criteria + +### AC-A: Fixture loader + replay harness + +1. NEW directory `src/lib/schemas/__fixtures__/` with subdirectories one per Zod schema (e.g., `chat-completion/`, `writing-evaluation/`, `conversation-feedback/`). Each schema dir holds N JSON fixture files named `.json` (e.g., `b1-formal-prompt-001.json`). +2. NEW `src/lib/schemas/__tests__/fixture-replay.test.ts` that: + - Walks the `__fixtures__//` subdirectories at test setup time + - For each fixture file: loads the JSON, looks up the corresponding Zod schema by directory name, runs `schema.safeParse(json)`, asserts `success === true` + - **Each fixture becomes a distinct Jest test case** via `it.each` over discovered files. A future operator adding `__fixtures__/chat-completion/new-fixture.json` automatically gets a new test case without code changes. + - Empty fixture directory → schema is documented as "no regression fixtures yet" with a `describe.skip` block (NOT a hard test failure — we want to ship the infrastructure before fixtures exist). +3. The fixture-to-schema mapping is explicit (not auto-derived from directory name) — a constant `FIXTURE_SCHEMA_MAP: Record` at the top of the test file lists each schema dir mapped to its imported Zod parser. + +### AC-B: Seed fixtures (minimum 1 per included schema) + +4. Seed at least 1 fixture file per included schema directory (operator-captured shapes are out of scope; use a SYNTHETIC fixture per schema that demonstrates the replay path works). Mark each synthetic fixture with a top-level `"_synthetic": true` field to distinguish from operator-captured fixtures. +5. Start with 3 schema directories (matches the spec's "writing eval, mock test, dictation" example list): `writing-evaluation`, `mock-test-section`, `dictation`. + +### AC-C: Operator runbook + +6. NEW `_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md` documenting: + - WHEN to capture (after a prompt change OR after a model upgrade) + - HOW to capture (paste-from-Sentry vs run live OpenAI call vs use dev-mode logging breadcrumb) + - NAMING convention (`--NNN.json`) + - WHERE to store (`src/lib/schemas/__fixtures__//`) + - HOW to verify (run `npx jest fixture-replay` locally → new fixture should show as a new passing test case) + +### AC-D: Quality gates + +7. All 5 design-system gates green. +8. **Net test growth:** **+3 to +6 net Jest cases** (1 per included schema seed fixture + a few infrastructure pins). + +### AC-E: Cross-story invariants + +9. `ai-responses.ts` schemas unchanged (15-5 consumes them; doesn't modify). +10. Existing `ai-responses.test.ts` unchanged. +11. **No source-module modifications** beyond test infrastructure + runbook. + +## Operator Decisions + +| Q | Question | Recommended | +| --- | --- | --- | +| Q1 | Auto-derive schema from dir name (fragile to renames) vs explicit `FIXTURE_SCHEMA_MAP`? | **Explicit map** — clearer + survives schema renames + TypeScript-checked | +| Q2 | Include `_synthetic: true` marker on seed fixtures? | **Yes** — distinguishes operator-captured-real fixtures from infra-bootstrap seeds; future tooling can filter to real-only when assessing regression confidence | +| Q3 | Start with 3 schemas or all 36? | **3** — `writing-evaluation`, `mock-test-section`, `dictation` (matches the spec's example list). Adding more is a one-line dir create + map entry; defer to follow-ups. | + +## Out of Scope + +- Real fixture capture (operator action; runbook documents the procedure) +- All 36 schemas covered (defer to `15-5-followup-fixture-coverage-expansion`) +- Capture automation (e.g., adding a `dev-mode-fixture-recorder` to Sentry breadcrumbs) — defer + +## Tasks / Subtasks + +- [x] Task 1: Create `__fixtures__/` directory tree + 3 synthetic seed fixtures +- [x] Task 2: Write `fixture-replay.test.ts` with `FIXTURE_SCHEMA_MAP` + auto-discovery `it.each` +- [x] Task 3: Write runbook +- [x] Task 4: Quality gates + CLAUDE.md paragraph + sprint-status + +## Dev Agent Record + +### Agent Model Used + +### Completion Notes List + +### File List diff --git a/_bmad-output/implementation-artifacts/sprint-status.yaml b/_bmad-output/implementation-artifacts/sprint-status.yaml index 5405e25..bd5da5d 100644 --- a/_bmad-output/implementation-artifacts/sprint-status.yaml +++ b/_bmad-output/implementation-artifacts/sprint-status.yaml @@ -35,7 +35,7 @@ # - Dev moves story to 'review', then runs code-review (fresh context, different LLM recommended) generated: 2026-03-25 -last_updated: 2026-05-16 # Story 15-1 done. R1 review (5 patches: HIGH × 2 useFakeTimers time-flake fix + dictation empty-original semantic-trap TODO; MED × 3 Case 17 tightening + ref-inequality + typed-case-preservation Case 14a). 2159 tests (+1 net R1). All 5 gates green. PR #111. +last_updated: 2026-05-17 # Story 15-5 done. AI schema regression test infrastructure shipped; real fixture capture is operator action. Cumulative autopilot batch: 15-2 + 15-3 + 15-5 shipped. project: companion project_key: NOKEY tracking_system: file-system @@ -210,7 +210,7 @@ development_status: 15-2-hook-integration-tests: backlog 15-3-edge-function-deno-tests: backlog 15-4-golden-flow-e2e: backlog - 15-5-ai-schema-regression-tests: backlog + 15-5-ai-schema-regression-tests: done # 2026-05-17: PR pending. NEW fixture-replay infrastructure + 3 synthetic seed fixtures + operator runbook. +8 net Jest cases. Real fixture capture deferred to operator action per runbook. 15-6-ci-coverage-gating: backlog epic-15-retrospective: optional diff --git a/_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md b/_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md new file mode 100644 index 0000000..49e27d0 --- /dev/null +++ b/_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md @@ -0,0 +1,121 @@ +# AI Fixture Capture Runbook + +**Owner:** ai-integration agent / operator +**Frequency:** After any prompt change in `src/lib/prompts/*.ts` OR after a model upgrade (gpt-4o → gpt-4.x) OR after a schema change in `src/lib/schemas/ai-responses.ts`. + +--- + +## What is this? + +Story 15-5 shipped the **AI schema regression test infrastructure**: a fixture loader at `src/lib/schemas/__tests__/fixture-replay.test.ts` that walks `src/lib/schemas/__fixtures__//*.json` and replays each through its corresponding Zod parser. Real-shaped fixtures captured from production model outputs catch regressions that synthetic unit tests miss (e.g., an extra-field the model started emitting, or an enum value drift). + +Each fixture file becomes a distinct Jest test case automatically. **Adding a fixture is a pure data add** — no code change needed. + +--- + +## When to capture + +1. **Prompt change** — modifying any of `src/lib/prompts/*.ts` (e.g., `writing.ts`, `mock-test.ts`, `dictation.ts`). The new prompt may shift the model output shape; capture fresh fixtures to verify the schema still accepts the new responses. +2. **Model upgrade** — when switching the OpenAI/Azure model version. New models often subtly change JSON output shape. +3. **Schema change** — when modifying `src/lib/schemas/ai-responses.ts`. Run existing fixtures through the new schema to verify backward compatibility. + +--- + +## How to capture (3 options) + +### Option A: Sentry breadcrumb pull (preferred for production-shape fidelity) + +1. Open Sentry → Issues → filter `feature:chat-completion` (or `feature:writing-evaluation`, etc.). +2. Pick a recent breadcrumb with a successful AI response (look for `breadcrumb.category === "ai"` and `breadcrumb.level === "info"`). +3. Copy the raw JSON response body from the breadcrumb metadata. +4. Save to `src/lib/schemas/__fixtures__//-NNN.json`. + +### Option B: Dev-mode capture (when Sentry doesn't have the shape you want) + +1. Add a temporary `console.log` in the relevant `chatCompletionJSON` caller: `console.log("FIXTURE_CAPTURE:", JSON.stringify(parsed));` +2. Run the app in dev mode; trigger the AI call once. +3. Copy the logged JSON from the terminal. +4. **REMOVE the console.log before committing.** +5. Save to `src/lib/schemas/__fixtures__//-NNN.json`. + +### Option C: Live OpenAI call (most expensive — last resort) + +1. Write a one-off Node script that invokes `chatCompletionJSON` with a representative prompt. +2. Save the response. +3. **Counts against `daily_cost_ledger` per Story 11-4** — do this sparingly. + +--- + +## Naming convention + +Filename pattern: `-NNN.json` (e.g., `b2-formal-essay-003.json`, `a1-greeting-001.json`). + +- `` — short alphanumeric description matching the CEFR level + content theme + sequence index +- `NNN` — 3-digit zero-padded sequence number per descriptor + +The directory name is the schema name (lowercased, hyphenated) matching the keys in `FIXTURE_SCHEMA_MAP` at `src/lib/schemas/__tests__/fixture-replay.test.ts`. + +### Synthetic vs real-captured + +The Story 15-5 seed fixtures carry a top-level `"_synthetic": true` marker to distinguish them from real-captured fixtures. **When capturing real outputs, OMIT the `_synthetic` field.** The `stripMetadata` helper in the replay test strips `_synthetic` + `_note` fields before handing to the parser, so the presence/absence of the marker is purely operator-readable bookkeeping. + +--- + +## Where to store + +``` +src/lib/schemas/__fixtures__/ +├── writing-evaluation/ +│ └── synthetic-b1-formal-001.json ← seed +├── dictation/ +│ └── synthetic-a2-mixed-001.json ← seed +├── mock-test-section/ +│ └── synthetic-b1-listening-001.json ← seed +└── / ← add here as needed + └── -NNN.json +``` + +Adding a new schema to coverage requires: + +1. Create the directory: `mkdir -p src/lib/schemas/__fixtures__//` +2. Add an entry to `FIXTURE_SCHEMA_MAP` in `src/lib/schemas/__tests__/fixture-replay.test.ts`: + ```ts + const FIXTURE_SCHEMA_MAP: Record = { + "writing-evaluation": writingEvaluationSchema, + dictation: dictationSetSchema, + "mock-test-section": mockTestSectionSchema, + "": , // ← add this line + }; + ``` +3. Drop at least one fixture file into the new directory. + +--- + +## How to verify + +After adding a fixture: + +```bash +npx jest fixture-replay --no-coverage +``` + +You should see a new passing test case named `fixture / parses successfully against its Zod schema`. If the test FAILS: + +- Read the Zod issue list in the error output — it tells you which field is wrong and why. +- Common causes: model emitted an extra field the schema doesn't allow (Zod is `strict()` by default in many schemas); model omitted an optional-but-validated field; model returned a string where the schema expected a number. +- If the schema needs to be widened to accept the new shape, that's a schema change in `ai-responses.ts` — usually requires a follow-up story discussion. + +--- + +## Coverage target + +Story 15-5 spec target: 10 real fixtures per schema. Current state: 3 schemas × 1 synthetic seed = 3 fixtures total. **Operator action:** capture 9 more fixtures per schema during the next prompt-change or model-upgrade window. + +--- + +## Cross-story references + +- Story 15-5: shipped the infrastructure (this runbook + fixture-replay.test.ts + seed fixtures + 3 directories). +- Story 9-7: `chatCompletionJSON` parseRetries contract — failed schema parse retries the AI call once before throwing. +- Story 11-4: `daily_cost_ledger` — Option C (live capture) burns cost-cap budget. +- Story 15-6 (planned): jest `--coverage` CI gating; the fixture-replay tests count toward the coverage threshold. From ce6ec694d75af51502fcc2bc5d4f33d969e969e9 Mon Sep 17 00:00:00 2001 From: Martins Aloba Date: Sun, 17 May 2026 14:56:50 -0500 Subject: [PATCH 3/3] =?UTF-8?q?fix(15-5):=20R1=20patches=20HIGH=20=C3=97?= =?UTF-8?q?=205=20+=20MED=20=C3=97=205=20+=20LOW=20=C3=97=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3-agent adversarial review surfaced 12 actionable findings. HIGH: - BH-1: JSON.parse error path now wraps via loadFixture() so a malformed fixture's path appears in the error message, not just `SyntaxError at position N`. - BH-2: NEW Case 6 enforces `_synthetic: true` marker on every committed fixture — privacy defense until a future operator-action `.real-fixtures.txt` manifest exists for real captures. - BH-3 + EH-10: Runbook gains REQUIRED Privacy + GDPR section before Option A. User-derived French content (writing-evaluation errors[].original) may contain names/addresses/dates/contact info. Story 9-3 GDPR scrubber does NOT run on fixtures; Story 9-4 prompt wrapping does NOT protect captures; git history is irrevocable. Section mandates field-by-field sanitization + `_redacted: true` marker convention + prefer-synthetic-when-possible guidance. - EH-9 (load-bearing): NEW Case 7 parallel strict-mode probe. The Zod schemas use default `.strip()` which silently drops unknown fields — defeating the regression-detection goal of the story. Probe runs `schema.strict().safeParse()` per fixture and reports extra-field drift as SOFT console.warn (does not fail the test — synthetic seeds pass by construction; real captures may flag drift for operator triage). Also corrected the runbook's misleading "Zod is strict() by default" claim. MED: - BH-4: Case 2 walks fs.readdirSync directly so EMPTY orphan dirs fail loud (not just dirs with files). - BH-5: `FIXTURE_SCHEMA_MAP` uses `as const satisfies` for narrow value typing — value-typo at construction caught at compile time. - BH-6: `dirent.isFile()` filter so a directory named `foo.json` doesn't crash readFileSync with EISDIR. - BH-7 / EH-1: `stripMetadata` JSDoc documents the top-level-only contract — load-bearing for nested data passthrough. - EH-6: case-insensitive `.json` match so `.JSON` / `.Json` files don't silently skip. LOW: - BH-12: Belt-and-suspenders FIXTURES_ROOT existence pin (Case 0). - EH-2: Case 4 input gains nested `_note` for symmetric coverage alongside nested `_synthetic`. Deferred (filed as follow-ups): - 15-5-followup-real-fixture-manifest (operator-action allow-list) - 15-5-followup-passage-ref-integrity (BH-8 — pre-existing schema gap) - 15-5-followup-empty-fixture-boundary (EH-3 — empty-object edge) Story file housekeeping: Status: review → done; populated File List + Completion Notes per Acceptance Auditor APPROVE_WITH_NOTES. +3 net test cases (8 → 11). All 4 quality gates green: type-check 0 / lint 0 / prettier clean / jest 2170 tests pass. --- .../15-5-ai-schema-regression-tests.md | 22 +- .../runbooks/ai-fixture-capture.md | 32 ++- .../schemas/__tests__/fixture-replay.test.ts | 212 ++++++++++++++++-- 3 files changed, 246 insertions(+), 20 deletions(-) diff --git a/_bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md b/_bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md index 8033f00..b0855e4 100644 --- a/_bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md +++ b/_bmad-output/implementation-artifacts/15-5-ai-schema-regression-tests.md @@ -1,6 +1,6 @@ # Story 15.5: AI schema regression test infrastructure — fixture loader + replay harness for `ai-responses.ts` Zod schemas (capture deferred to operator) -Status: review +Status: done ## Story @@ -76,6 +76,26 @@ As a developer, I want a **regression-test harness that loads recorded AI-respon ### Agent Model Used +Claude Sonnet 4.6 (claude-sonnet-4-6) via /bmad-dev-story + /bmad-code-review workflows in autopilot mode. + ### Completion Notes List +- **Infrastructure shipped**: fixture-replay harness at `src/lib/schemas/__tests__/fixture-replay.test.ts` discovers `__fixtures__//*.json` via `it.each`, strips `_synthetic` + `_note` top-level metadata, parses through Zod schemas via `FIXTURE_SCHEMA_MAP`. 3 synthetic seed fixtures (writing-evaluation, dictation, mock-test-section). Operator runbook at `_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md`. +- **R1 patches applied** (HIGH × 5 + MED × 5 + LOW × 2): BH-1 JSON.parse error wrapping (surfaces fixture path on malformed JSON); BH-2 NEW Case 6 enforces `_synthetic: true` marker on every committed fixture (privacy defense); BH-3/EH-10 runbook gains REQUIRED Privacy + GDPR section before Option A — user-derived French content sanitization workflow with `_redacted: true` marker convention; BH-4 Case 2 walks `fs.readdirSync` directly so empty orphan dirs fail loud; BH-5 `FIXTURE_SCHEMA_MAP` uses `as const satisfies` for narrow value typing; BH-6 `dirent.isFile()` filter so a directory ending in `.json` doesn't crash with EISDIR; BH-7/EH-1 `stripMetadata` JSDoc documents top-level-only contract; BH-12 belt-and-suspenders FIXTURES_ROOT existence pin (Case 0); EH-2 Case 4 covers nested `_note` symmetrically with nested `_synthetic`; EH-6 case-insensitive `.json` match; EH-9 (load-bearing) NEW Case 7 parallel strict-mode probe — schemas use Zod default `.strip()` which silently drops unknown fields; strict-probe surfaces extra-field drift as SOFT console warning per fixture (does not fail test — synthetic seeds pass by construction; real captures may flag drift). Runbook's misleading "Zod is strict() by default" claim corrected. +- **Deferred** (filed as follow-ups): `15-5-followup-real-fixture-manifest` (operator-action `.real-fixtures.txt` allow-list for real captures); `15-5-followup-passage-ref-integrity` (BH-8 — mock-test passageId schema gap); `15-5-followup-empty-fixture-boundary` (EH-3 — empty-object edge case). +- **Quality gates green**: type-check 0 errors / lint 0 warnings / prettier clean / jest test passes (11/11). + ### File List + +**New:** + +- `src/lib/schemas/__tests__/fixture-replay.test.ts` — 11 Jest cases (8 original + 3 R1 patches: Case 0 FIXTURES_ROOT existence + Case 6 synthetic-marker + Case 7 strict-mode probe) +- `src/lib/schemas/__fixtures__/writing-evaluation/synthetic-b1-formal-001.json` +- `src/lib/schemas/__fixtures__/dictation/synthetic-a2-mixed-001.json` +- `src/lib/schemas/__fixtures__/mock-test-section/synthetic-b1-listening-001.json` +- `_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md` (round-1 includes new Privacy + GDPR section + Zod default-behavior correction) + +**Modified:** + +- `_bmad-output/implementation-artifacts/sprint-status.yaml` — 15-5 → done +- `CLAUDE.md` — Story 15-5 architecture paragraph appended diff --git a/_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md b/_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md index 49e27d0..95412fc 100644 --- a/_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md +++ b/_bmad-output/planning-artifacts/runbooks/ai-fixture-capture.md @@ -21,6 +21,36 @@ Each fixture file becomes a distinct Jest test case automatically. **Adding a fi --- +## Privacy + GDPR — REQUIRED before any real-capture commit (R1 BH-3 / EH-10) + +**⚠️ CRITICAL:** Real model outputs may contain user-derived French content. The `writing-evaluation` schema receives the user's actual essay text in the `errors[].original` field — which can carry names, addresses, dates, contact info, or other personally-identifiable details from the user's writing prompt. Committing real-captured fixtures to source without sanitization exfiltrates this data into the public git history. + +**Story 9-3 GDPR scrubber does NOT run on fixtures** — it operates on Sentry events at emit time, not on raw breadcrumb payloads pulled later. **Story 9-4 prompt-side `` wrapping does NOT protect captured outputs** — fixtures contain raw user text. **git history is irrevocable once pushed.** + +### Required sanitization steps before commit + +For each real-captured fixture: + +1. **Scan ALL user-derived string fields** for: + - Names (first, last, middle, nicknames) + - Place names (cities, streets, schools, employers) + - Dates (birthdays, anniversaries, specific calendar dates) + - Contact info (emails, phone numbers, social handles) + - Sensitive context (medical, financial, legal references) +2. **Replace** identifying tokens with neutral placeholders: `[NAME]`, `[CITY]`, `[DATE]`, `[EMAIL]`, etc. Preserve French grammatical structure (e.g., article agreement, verb conjugation). +3. **Add** a `_redacted: true` metadata marker to the fixture file alongside `_synthetic: false` (or `_synthetic: ` per the operator-action manifest in `15-5-followup-real-fixture-manifest`). +4. **Document** the redaction scope in `_note`: e.g., `_note: "Captured from production 2026-MM-DD; user name + city redacted to [NAME] / [CITY]"`. + +### Prefer synthetic-derived fixtures when possible + +For schemas covering user-derived content (writing-evaluation, conversation-feedback, post-conversation-analysis), prefer hand-authored synthetic fixtures using neutral prompts (e.g., "Le chat est sur la table"). Real captures should be reserved for schemas where the user-derived surface is minimal (dictation, mock-test-section — where user data only flows back as scores, not as text). + +### Enforcement + +Story 15-5 Case 6 currently REQUIRES `_synthetic: true` on every committed fixture. A real-captured fixture without an entry in a future `.real-fixtures.txt` operator-action manifest will fail CI loudly. The manifest is filed as `15-5-followup-real-fixture-manifest`. + +--- + ## How to capture (3 options) ### Option A: Sentry breadcrumb pull (preferred for production-shape fidelity) @@ -102,7 +132,7 @@ npx jest fixture-replay --no-coverage You should see a new passing test case named `fixture / parses successfully against its Zod schema`. If the test FAILS: - Read the Zod issue list in the error output — it tells you which field is wrong and why. -- Common causes: model emitted an extra field the schema doesn't allow (Zod is `strict()` by default in many schemas); model omitted an optional-but-validated field; model returned a string where the schema expected a number. +- Common causes: model omitted an optional-but-validated field; model returned a string where the schema expected a number; enum value drift (e.g., model emitted `"medium"` where schema accepts only `"low" | "high"`). **Note:** Zod's default behavior is `.strip()` — unknown extra fields are silently dropped, NOT rejected. The Story 15-5 `Case 7` parallel strict-mode probe surfaces extra-field drift as a SOFT console warning (does not fail the test). If the soft warning fires, decide whether to (a) update the schema to accept the new field, or (b) sanitize the fixture before commit. - If the schema needs to be widened to accept the new shape, that's a schema change in `ai-responses.ts` — usually requires a follow-up story discussion. --- diff --git a/src/lib/schemas/__tests__/fixture-replay.test.ts b/src/lib/schemas/__tests__/fixture-replay.test.ts index d45b27e..7f9e07d 100644 --- a/src/lib/schemas/__tests__/fixture-replay.test.ts +++ b/src/lib/schemas/__tests__/fixture-replay.test.ts @@ -10,12 +10,39 @@ * The `FIXTURE_SCHEMA_MAP` is the explicit dir-name → schema mapping. A * future schema rename in `ai-responses.ts` requires updating both this * map AND the fixtures directory name (caught by TypeScript on the - * `import` line). + * `import` line, with `satisfies` per R1 BH-5 narrowing). * * `_synthetic: true` marker on seed fixtures distinguishes them from * operator-captured real model outputs. The replay logic strips the * `_synthetic` + `_note` top-level fields BEFORE handing to the parser * (those are metadata; not part of the schema contract). + * + * R1 patches (HIGH × 5 + MED × 5 + LOW × 2): + * - BH-1: JSON.parse wrapped in try/catch so a malformed fixture's + * file path is surfaced in the error message, not buried. + * - BH-2: NEW Case 6 enforces every fixture carries `_synthetic: true` + * until an operator-action manifest is established for real captures. + * - BH-4: Case 2 walks `fs.readdirSync` directly so an empty orphan + * directory (no .json files yet) fails loudly, not silently. + * - BH-5: `FIXTURE_SCHEMA_MAP` typed with `as const satisfies` so + * value-typo at construction is caught at compile time. + * - BH-6: Filter via `dirent.isFile()` so a directory ending in `.json` + * doesn't crash `readFileSync` with EISDIR. + * - BH-7 / EH-1: `stripMetadata` JSDoc explicitly documents the + * top-level-only contract — load-bearing for nested data passthrough. + * - BH-12: Belt-and-suspenders existence check on FIXTURES_ROOT so a + * future relocation that mis-points the path fails loudly. + * - EH-2: Case 4 input gains a nested `_note` alongside nested + * `_synthetic` so both metadata keys are covered symmetrically. + * - EH-6: case-insensitive `.json` match so `.JSON` / `.Json` files + * don't silently skip. + * - EH-9: NEW Case 7 parallel strict-mode probe — the schemas use + * Zod default `.strip()` which silently drops unknown fields, + * defeating the regression-detection goal. The strict-mode probe + * runs `schema.strict().safeParse()` on each fixture and reports + * extra-field drift as a SOFT warning (does not fail the test) so + * real-fixture captures expose model output drift without breaking + * synthetic-seed parses that intentionally omit edge fields. */ import * as fs from "fs"; @@ -35,12 +62,16 @@ const FIXTURES_ROOT = path.resolve(__dirname, "../__fixtures__"); * Explicit dir-name → Zod schema mapping. Add a new entry to extend * regression coverage to a new schema; create the matching directory and * drop fixture JSON files into it. + * + * `as const satisfies` (R1 BH-5): preserves narrow value types so a typo + * like `writingEvalSchema` would fail compilation, while still enforcing + * the index signature. */ -const FIXTURE_SCHEMA_MAP: Record = { +const FIXTURE_SCHEMA_MAP = { "writing-evaluation": writingEvaluationSchema, dictation: dictationSetSchema, "mock-test-section": mockTestSectionSchema, -}; +} as const satisfies Record; /** * Metadata fields that exist on synthetic seed fixtures but are NOT part @@ -62,19 +93,37 @@ function discoverFixtures(): DiscoveredFixture[] { if (!dir.isDirectory()) continue; const schemaName = dir.name; const schemaDirPath = path.join(FIXTURES_ROOT, schemaName); - const files = fs.readdirSync(schemaDirPath); - for (const file of files) { - if (!file.endsWith(".json")) continue; + // R1 BH-6: pass `withFileTypes: true` so we can filter via + // `dirent.isFile()` — a directory named `foo.json` would otherwise be + // picked up by the string-ends-with check and crash `readFileSync` + // with EISDIR. + const entries = fs.readdirSync(schemaDirPath, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile()) continue; + // R1 EH-6: case-insensitive `.json` match so a copy-pasted fixture + // from a tool that produces `.JSON` doesn't silently skip. + if (!entry.name.toLowerCase().endsWith(".json")) continue; out.push({ schemaName, - fileName: file, - fullPath: path.join(schemaDirPath, file), + fileName: entry.name, + fullPath: path.join(schemaDirPath, entry.name), }); } } return out; } +/** + * Strip top-level metadata fields (`_synthetic`, `_note`) from a fixture + * JSON before handing to the Zod parser. + * + * **R1 BH-7 / EH-1 contract**: top-level only by design. Nested objects + * pass through unchanged because (a) the schema may legitimately want + * `_note` as a string field at some path, and (b) the metadata semantics + * apply to the fixture file as a whole, not to interior data shapes. A + * future "be helpful" refactor that deep-strips would silently swallow + * real model-emitted fields. Pinned by Case 4 nested-survival assertions. + */ function stripMetadata(json: unknown): unknown { if (json === null || typeof json !== "object" || Array.isArray(json)) return json; const cleaned: Record = {}; @@ -85,9 +134,30 @@ function stripMetadata(json: unknown): unknown { return cleaned; } +/** + * Read + parse a fixture file. R1 BH-1 — wrap `JSON.parse` so a + * syntactically-broken fixture surfaces its file path in the error, + * not just `SyntaxError at position N`. + */ +function loadFixture(fullPath: string, schemaName: string): unknown { + const raw = fs.readFileSync(fullPath, "utf-8"); + try { + return JSON.parse(raw); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error( + `Fixture ${schemaName}/${path.basename(fullPath)} contains invalid JSON: ${msg}` + ); + } +} + describe("Story 15-5 — AI schema regression fixture replay", () => { const fixtures = discoverFixtures(); + it("Case 0: FIXTURES_ROOT exists on disk (R1 BH-12 belt-and-suspenders defense against a future relocation that mis-points the path)", () => { + expect(fs.existsSync(FIXTURES_ROOT)).toBe(true); + }); + it("Case 1: FIXTURE_SCHEMA_MAP contains entries for the included seed schemas (3 minimum)", () => { expect(Object.keys(FIXTURE_SCHEMA_MAP)).toEqual( expect.arrayContaining(["writing-evaluation", "dictation", "mock-test-section"]) @@ -95,10 +165,19 @@ describe("Story 15-5 — AI schema regression fixture replay", () => { expect(Object.keys(FIXTURE_SCHEMA_MAP).length).toBeGreaterThanOrEqual(3); }); - it("Case 2: every fixture directory has a corresponding FIXTURE_SCHEMA_MAP entry (no orphan dirs)", () => { - const discoveredDirs = new Set(fixtures.map((f) => f.schemaName)); - for (const dir of discoveredDirs) { - expect(FIXTURE_SCHEMA_MAP[dir]).toBeDefined(); + it("Case 2: every fixture directory on disk has a corresponding FIXTURE_SCHEMA_MAP entry (R1 BH-4: walks fs directly so EMPTY orphan dirs also fail loudly, not just dirs with files)", () => { + // R1 BH-4: iterate the filesystem directly instead of deriving from + // `fixtures` (which only contains dirs with at least one .json file). + // An empty `__fixtures__/listening-comprehension/` dir created during + // scaffolding would silently pass the pre-patch test even without a + // map entry — now it fails loudly. + if (!fs.existsSync(FIXTURES_ROOT)) return; + const dirs = fs + .readdirSync(FIXTURES_ROOT, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + for (const dir of dirs) { + expect(FIXTURE_SCHEMA_MAP[dir as keyof typeof FIXTURE_SCHEMA_MAP]).toBeDefined(); } }); @@ -121,10 +200,9 @@ describe("Story 15-5 — AI schema regression fixture replay", () => { it.each(fixtures)( "fixture $schemaName/$fileName parses successfully against its Zod schema", ({ schemaName, fullPath }) => { - const schema = FIXTURE_SCHEMA_MAP[schemaName]; + const schema = FIXTURE_SCHEMA_MAP[schemaName as keyof typeof FIXTURE_SCHEMA_MAP]; expect(schema).toBeDefined(); - const raw = fs.readFileSync(fullPath, "utf-8"); - const json = JSON.parse(raw); + const json = loadFixture(fullPath, schemaName); const stripped = stripMetadata(json); const result = schema.safeParse(stripped); if (!result.success) { @@ -140,18 +218,28 @@ describe("Story 15-5 — AI schema regression fixture replay", () => { ); }); - it("Case 4: stripMetadata removes _synthetic + _note top-level fields without touching nested fields", () => { + it("Case 4: stripMetadata removes _synthetic + _note top-level fields without touching nested fields (R1 EH-2: symmetric coverage)", () => { const input = { _synthetic: true, _note: "test note", - data: { nested: { _synthetic: "should NOT be stripped at nested level" } }, + data: { + nested: { + _synthetic: "should NOT be stripped at nested level", + // R1 EH-2: parallel `_note` coverage so a future regression + // that deep-strips only one metadata key is also caught. + _note: "nested note also preserved", + }, + }, overallScore: 75, }; const out = stripMetadata(input) as Record; expect(out._synthetic).toBeUndefined(); expect(out._note).toBeUndefined(); expect(out.data).toEqual({ - nested: { _synthetic: "should NOT be stripped at nested level" }, + nested: { + _synthetic: "should NOT be stripped at nested level", + _note: "nested note also preserved", + }, }); expect(out.overallScore).toBe(75); }); @@ -162,4 +250,92 @@ describe("Story 15-5 — AI schema regression fixture replay", () => { expect(stripMetadata("string")).toBe("string"); expect(stripMetadata(42)).toBe(42); }); + + it("Case 6: every fixture in source carries `_synthetic: true` (R1 BH-2: synthetic-marker enforcement until operator-action manifest exists for real captures)", () => { + // Until a `.real-fixtures.txt` manifest is established (filed as + // `15-5-followup-real-fixture-manifest`), every fixture committed to + // source MUST be marked synthetic. This prevents an operator from + // accidentally committing a real-captured fixture containing + // user-derived French writing samples (privacy + GDPR concern; see + // runbook Privacy section R1 BH-3 / EH-10). + const missingMarker: string[] = []; + for (const fixture of fixtures) { + const json = loadFixture(fixture.fullPath, fixture.schemaName) as Record; + if (json._synthetic !== true) { + missingMarker.push(`${fixture.schemaName}/${fixture.fileName}`); + } + } + if (missingMarker.length > 0) { + throw new Error( + `Fixture(s) missing \`_synthetic: true\` marker (R1 BH-2 enforcement): ` + + missingMarker.join(", ") + + `. Either set it true (synthetic data) OR add the file to a future ` + + `\`.real-fixtures.txt\` manifest with explicit operator approval ` + + `(see \`15-5-followup-real-fixture-manifest\`).` + ); + } + expect(missingMarker.length).toBe(0); + }); + + it("Case 7: parallel strict-mode probe — fixtures stay PARSEABLE under schema.strict() (R1 EH-9: model output drift detection)", () => { + // R1 EH-9 (load-bearing): Zod default behavior is `.strip()` which + // silently drops unknown fields. The regression-detection goal of + // this story requires catching model-output drift — a new field + // emitted by the model that the schema doesn't know about. + // + // This probe wraps each fixture's parse with `schema.strict()` and + // accumulates a SOFT warning list. The current synthetic seeds are + // hand-authored to match the schemas exactly, so all three should + // pass strict parsing. Future real-captured fixtures that introduce + // unknown fields will surface as warnings — operator decides whether + // to update the schema OR sanitize the fixture. + // + // Implementation note: not all Zod types support .strict() at the + // root level (e.g., union/discriminated-union schemas); the probe + // wraps in try/catch so a non-strict-able schema doesn't fail the + // test infrastructure. If a fixture is rejected by .strict() AND + // the schema supports it, the failure is reported as a soft warning + // via console.warn but does NOT fail the test (per spec: soft signal + // for fixture-author triage, not a hard gate). + const warnings: string[] = []; + for (const fixture of fixtures) { + const schema = FIXTURE_SCHEMA_MAP[fixture.schemaName as keyof typeof FIXTURE_SCHEMA_MAP]; + const json = loadFixture(fixture.fullPath, fixture.schemaName); + const stripped = stripMetadata(json); + try { + // Only attempt strict-mode probe on schemas with a `.strict()` + // method (i.e., ZodObject). Unions/discriminated-unions skip. + const strictSchema = + typeof (schema as z.ZodTypeAny & { strict?: () => z.ZodTypeAny }).strict === "function" + ? (schema as z.ZodObject).strict() + : null; + if (!strictSchema) continue; + const result = strictSchema.safeParse(stripped); + if (!result.success) { + const extraFields = result.error.issues + .filter((i) => i.code === "unrecognized_keys") + .flatMap((i) => ("keys" in i ? (i as { keys: string[] }).keys : [])); + if (extraFields.length > 0) { + warnings.push( + `Fixture ${fixture.schemaName}/${fixture.fileName} has unknown field(s) under strict parse: ${extraFields.join(", ")}` + ); + } + } + } catch { + // Schema type doesn't support `.strict()` (e.g., ZodUnion at root). + // Silent skip — the soft-warning surface is best-effort. + } + } + if (warnings.length > 0) { + console.warn( + "[15-5 strict-mode probe] " + + warnings.length + + " fixture(s) have potential model-drift:\n " + + warnings.join("\n ") + ); + } + // Soft probe — never fails the test. Synthetic seeds pass strict + // by construction; real captures may surface drift as warnings. + expect(true).toBe(true); + }); });