From eefde1ff64828ab9980a5d590149f3d6d7dcec1c Mon Sep 17 00:00:00 2001 From: Ben Firestone Date: Fri, 10 Apr 2026 20:18:50 -0700 Subject: [PATCH 01/18] =?UTF-8?q?docs(spec):=20semantic=20detection=20labe?= =?UTF-8?q?ls=20=E2=80=94=20log=5Fcontent,=20stack=5Ftrace,=20diff=5Fpatch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design spec for adding 3 cross-cutting semantic detection labels and removing det_log_lines. Prompt-only change; validated via 5k stratified audit + targeted positive injection before committing to the $400-600 full 90k annotation run. Locks in v2 label definitions after adversarial + standard review of 4 checkpoint questions (density threshold, minimum frames, anti-markdown-list rule, co-fire guidance). --- ...-04-10-semantic-detection-labels-design.md | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 docs/superpowers/specs/2026-04-10-semantic-detection-labels-design.md diff --git a/docs/superpowers/specs/2026-04-10-semantic-detection-labels-design.md b/docs/superpowers/specs/2026-04-10-semantic-detection-labels-design.md new file mode 100644 index 0000000..ca616e0 --- /dev/null +++ b/docs/superpowers/specs/2026-04-10-semantic-detection-labels-design.md @@ -0,0 +1,190 @@ +# Semantic Detection Labels — log_content, stack_trace, diff_patch + +**Date:** 2026-04-10 +**Branch:** new branch off `main` post-PR#12-merge +**Workstream:** `training/` only (no `src/` changes — workspace boundary) +**Follows:** `docs/accuracy_runs/2026-04-10-iteration-15.md` + +## Summary + +Add 3 cross-cutting semantic detection labels — `log_content`, `stack_trace`, `diff_patch` — and remove `det_log_lines` from the detection label set. Validate with a 5k stratified audit plus targeted positive injection. This branch is a **prompt-only change**: no data pipeline runs, no model retrain, no Rust changes. It exists to lock in precise label definitions *before* the $400-600 full 90k annotation run in the next branch. + +The design principle driving every decision here: **optimize for label definition quality upfront so the 5k audit passes clean and the downstream expensive run isn't wasted on prompt bugs.** + +## Background + +- **iter15 deferred these labels.** That iteration added 9 programming-language detection labels (`csharp`, `swift`, `kotlin`, `r`, `lua`, etc.) and expanded the corpus for language coverage, but explicitly postponed the semantic labels as "the original goal of this workstream, deferred from this iteration." +- **The detection head has never been trained.** `model_config.json` lacks a `detection_map` field; the Rust inference code in `tier2.rs` skips the head gracefully when absent. This means schema changes to `DETECTION_LABELS` are free — no production artifact depends on the current set. +- **`log_lines` has no definition.** The existing prompt lists `log_lines` in the label set but has zero positive signals, anti-signals, or disambiguation text. iter15 called this out: "currently fires inconsistently." + +## Architectural Note: First Cross-Cutting Detection Labels + +All 40 existing detection labels are 1:1 with `ContentSubType`. Every detection label so far has been "is this sub_type present anywhere?" The 3 new labels are the **first detection-only labels** — they have no corresponding sub_type. + +This is correct by design: stack traces, unified diffs, and embedded log output are inherently cross-cutting phenomena. A stack trace is almost never a whole document; it's embedded in logs, bug reports, markdown docs, error emails. Same for diff patches (commit messages, PR descriptions, code review) and sample log output (tutorials, runbooks). Creating sub_types for them would be wrong — the primary sub_type of a bug report is still `markdown` or `plain`. + +The detection head was designed from the start for cross-cutting signal. This branch is the first use of that capability as intended. + +## Taxonomy Decision: Remove `det_log_lines`, Add `det_log_content` + +- **`log_lines` remains a `ContentSubType`** (unchanged). Meaning: "this row IS primarily log lines" (e.g., a 10k-line nginx access log). +- **`det_log_lines` is REMOVED from `DETECTION_LABELS`**. The `sub_type` column already carries this signal; duplicating it as a detection label creates name-collision ambiguity. +- **`det_log_content` is ADDED as a detection label**. Meaning: "this text contains log content anywhere, possibly embedded in prose/markdown/code." + +Removal is free because no trained model depends on `det_log_lines`. No annotated parquet with `det_*` columns exists yet (the 90k run is deferred to the next branch). + +## Label Definitions (v2) + +These are the exact strings to add to `SYSTEM_PROMPT` in `training/trainr/core/annotate_detections.py`, inserted into the `## Important distinctions` section. + +``` +- "log_content" = embedded log output (distinct from sub_type=log_lines, which + means the row IS primarily logs). Requires TWO OR MORE consecutive lines that + each independently match a log-line schema. A log-line schema is ONE of: + * ` ` where timestamp is + ISO 8601 (`2024-01-15T10:23:45Z`), apache-style + (`[15/Jan/2024:10:23:45 +0000]`), syslog-style (`Jan 15 10:23:45`), + or unix epoch (ms/seconds) + * A recognized named format: nginx/apache access log, syslog + (`timestamp host proc[pid]: msg`), dockerd/container log, + logfmt (`key=value key2=value2`) ONLY when co-occurring with a + timestamp OR severity field + * JSON log records: ≥2 consecutive JSON objects each containing BOTH a + timestamp field AND a `level`/`severity` field + Severity tokens must be UPPERCASE or BRACKETED (`INFO`, `[info]`, `WARN`, + `WARNING`, `ERROR`, `DEBUG`, `TRACE`, `FATAL`, `CRITICAL`) — lowercase + `error`/`info` in prose or code identifiers does NOT count. + + Do NOT fire on: single-line error messages (even inside code fences or + blockquotes), code that CALLS a logger (`log.info(...)`), sentences + describing logging behavior, changelogs with leading dates + (`2024-01-15 - fixed bug`), CSV/TSV with date columns, `ls -la` output, + git commit logs. Lines inside quotation marks or markdown blockquotes do + not contribute to the density count. + + If the embedded output is a PURE stack trace (no surrounding non-trace log + lines), fire "stack_trace": 1 but NOT "log_content": 1. If a stack trace + appears inside otherwise-normal log output, fire BOTH. + +- "stack_trace" = programmatic stack trace / traceback. Requires TWO OR MORE + frames where each frame carries a file:line locator OR a + package.Class.method locator (not just an `at` keyword). Strong signals + by language: + * Python: `Traceback (most recent call last):` header + `File "foo.py", + line N, in func` frames + final `ErrorType: message` line + * Java/JVM: `Exception in thread "..."` + `at com.pkg.Class.method(File.java:42)` + + optional `Caused by:` + `... N more` + * .NET: ` at Namespace.Class.Method() in File.cs:line 42` (leading + whitespace is distinctive) + * JS/Node: `Error: message` + `at fn (file:line:col)` / `at async fn` + * Go: `goroutine N [running]:` + `main.foo(0x0)\n\tpath/file.go:42 +0x1a` + * Rust: `thread 'main' panicked at` + numbered backtrace frames `0: ...`, + `1: ...` with file:line + * Ruby: `from file.rb:42:in 'method'` chain (one frame per line) + Truncation markers (`... N more frames`, `[truncated]`) do not disqualify + a trace. Do NOT fire on: single-line error messages, the literal phrase + "stack trace" in prose, tutorial pseudocode describing what a trace looks + like, profiler output tables, or prose like "at line 5 it crashed, at line + 6 it retried". If this trace appears inside log output, fire BOTH + "stack_trace": 1 AND "log_content": 1. + +- "diff_patch" = unified diff or git patch format. REQUIRED: at least one of + {`@@ -X,Y +A,B @@` hunk header, `diff --git a/... b/...` header, paired + `--- a/path` / `+++ b/path` file markers on adjacent lines}. Line prefixes + alone are NOT sufficient. Additional signals: `+`/`-`/space line prefixes + within a hunk, `index abc1234..def5678 100644` git metadata. The email-patch + header `From abc1234 Mon Sep 17 00:00:00 2001` is a signal ONLY when + co-occurring with `---`/`+++` or `@@` markers (otherwise a regular email + header). CRITICAL anti-signals: markdown bullet lists using `-` or `+`, + pro/con lists, code containing arithmetic `+`/`-`, isolated `+`/`-` lines + without hunk context, YAML frontmatter `---` without an adjacent `+++`. + Unified diffs only — context diffs (`*** file ***` separators) out of scope. +``` + +## Changes to `annotate_detections.py` + +1. **`DETECTION_LABELS` list:** remove `"log_lines"`, append `"log_content"`, `"stack_trace"`, `"diff_patch"` (place at end, matching iter15's convention of appending new labels). +2. **`SYSTEM_PROMPT` label list line:** remove `log_lines`, append the 3 new labels in the inline list. +3. **`SYSTEM_PROMPT` `## Important distinctions` section:** append the 3 definition blocks above. +4. **`SYSTEM_PROMPT` JSON example template at the bottom:** remove `"log_lines": 0`, add `"log_content": 0, "stack_trace": 0, "diff_patch": 0`. + +No other code changes. No routing table updates (routing is keyed by `sub_type`, unaffected). + +## Testing + +New file: `training/tests/test_annotate_detections.py` (pattern: iter15's `test_pull_real_data.py`). + +Unit tests: + +- `DETECTION_LABELS` contains `log_content`, `stack_trace`, `diff_patch` and does NOT contain `log_lines`. +- `parse_response` returns all 3 new labels with default 0. +- `parse_response` does NOT return a `log_lines` key. +- `parse_response` handles valid JSON with the new labels set to 1. + +Regression-guard tests on `SYSTEM_PROMPT` text (catch prompt drift): + +- Contains `"UPPERCASE or BRACKETED"` (log_content severity rule). +- Contains `"adjacent lines"` (diff_patch `---`/`+++` rule). +- Contains `".NET"` (stack_trace .NET format included). +- Contains `"PURE stack trace"` (log_content carveout for pure tracebacks). +- Does NOT contain `\"log_lines\":` (removed from JSON template). + +These text-presence tests are ugly but have the same role as iter15's R-regex precision tests: they catch silent prompt regressions during refactoring. + +## 5k Validation Audit + +**Sample composition:** +- **5k stratified sample** via existing `stratified_sample(n=5000)` — matches iter15's methodology, directly comparable metrics. +- **+~150 targeted positive candidates** (~50 per new label), tagged with source metadata, **excluded from the inter-annotator agreement metric** (injected positives would inflate it), reviewed manually for precision. + +**Injection queries (pre-filter the corpus, then sample):** +- `stack_trace`: regex `Traceback \(most recent call last\)`, `Exception in thread`, `goroutine \d+ \[`, `panicked at`, `\s+at [\w.]+\(.*\.java:` +- `diff_patch`: regex `^@@ -\d+,\d+ \+\d+,\d+ @@`, `^diff --git a/` +- `log_content`: text contains `\b(INFO|WARN|ERROR|DEBUG|TRACE|FATAL)\b` AND contains at least one timestamp-shaped substring. Generous superset — annotators judge. + +**Annotator models:** `gemini-3-flash`, `sonnet-4.6`, `gpt-5.4-mini` (iter15 precedent). + +**Pass criteria (ALL must hold — go/no-go on the $400-600 full run):** + +1. **Inter-annotator agreement ≥0.995** on the stratified 5k for ALL labels, existing AND new. This catches prompt-length regression on the 40 existing labels as well as the 3 new ones. +2. **Recall ≥0.90 on targeted positives** — for each new label, ≥90% of injected candidate rows must be correctly fired by at least 2 of 3 annotators. +3. **Zero obvious rule violations** in spot-check. Examples of violations: `log_content` firing on lowercase `error` in prose (uppercase rule ignored); `stack_trace` firing on a single `File "foo.py", line 42` line (frame count rule ignored); `diff_patch` firing on a markdown bullet list (anti-signal ignored); `log_content` firing on a pure traceback (pure-trace carveout ignored). + +**Cost estimate:** ~$20-25 total. Primary cost is the 3-model 5k audit; targeted injection adds negligible volume. + +## Risks and Mitigations + +- **Prompt length regression.** Adding 3 dense definition sections increases `SYSTEM_PROMPT` length by ~25%. Small models (nano/flash/mini) may behave differently with a longer prompt — existing label agreement could drift even without any definition changes to them. *Mitigation:* pass criterion #1 requires agreement on existing labels too, catching any regression. +- **.NET stack trace format over-interpretation.** The "leading whitespace is distinctive" hint may be read as "any indented `at Foo.Bar()` is a .NET frame." *Mitigation:* targeted injection includes known-good .NET traces; manual precision review catches overfire. +- **Pure-trace carveout for log_content is subtle.** "If the embedded output is a PURE stack trace... fire stack_trace but NOT log_content" is a conditional that small models may ignore. *Mitigation:* spot-check specifically for this pattern in the targeted stack_trace positives. +- **5k audit false-negative on precision.** At ~0.1% positive rate, even with targeted injection, precision measurement on negatives depends on spot-checks rather than ground truth. If the audit passes but precision is actually bad, the $400 full run will produce noisy data. *Mitigation:* explicit spot-check for rule violations in criterion #3; bias toward tightening over loosening on any ambiguity. + +## Implementation Order + +1. Edit `DETECTION_LABELS`, `SYSTEM_PROMPT`, and JSON template in `annotate_detections.py` +2. Write `test_annotate_detections.py` unit + regression-guard tests +3. Run unit tests (cheap, fast — should pass before any LLM calls) +4. Build injection candidate pool (grep the corpus with the injection queries; cap at 50 per label) +5. Stratified-sample 5k + concatenate injection candidates with a source-tag column +6. Run 3-model audit via `trainr data annotate-detections --routing --sample 5000 ...` plus a separate run on the injection pool +7. Compute agreement + recall metrics; run spot-check script for rule violations +8. **Gate:** if all 3 pass criteria hold → commit the `annotate_detections.py` changes, the new test file, the audit report (as a new doc under `docs/accuracy_runs/`), and this spec; open PR; merge. If not → iterate on definitions, re-audit. + +## Non-Goals / Explicit YAGNI + +Deliberately NOT in scope for this branch: + +- Full 90k annotation run — next branch +- Retrain with detection head — next branch +- Rust port / `src/` changes — third branch (workspace boundary) +- Additional log formats: Windows Event Log, systemd, CloudWatch +- Additional stack trace formats: PHP, Erlang/Elixir +- Additional diff dialects: SVN, hg, context diffs +- Class weighting in detection head loss — belongs to the retrain branch +- Additional semantic labels beyond the 3 — YAGNI until we see how these perform + +## Out-of-Scope Follow-Ups (for tracking) + +- `textclf-1dd1.013xtr` — dedup pipeline non-idempotence (iter15 finding) +- Per-label F1 tracking in training metrics — belongs to retrain branch +- `pos_weight` in `BCEWithLogitsLoss` for 0.1%-prevalence labels — belongs to retrain branch From b2d0594f4f31a8d695d986836261ade961c3d512 Mon Sep 17 00:00:00 2001 From: Ben Firestone Date: Fri, 10 Apr 2026 20:27:03 -0700 Subject: [PATCH 02/18] docs(plan): semantic detection labels implementation plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bite-sized TDD plan for iter16: prep (fix 3 stale iter15 tests), phased label changes (remove det_log_lines, add log_content, stack_trace, diff_patch with regression-guard tests), audit tooling (build sample, compute metrics), execution (build sample → 3-model annotate → report → gate → iterate), iteration report + PR. --- .../2026-04-10-semantic-detection-labels.md | 2093 +++++++++++++++++ 1 file changed, 2093 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-10-semantic-detection-labels.md diff --git a/docs/superpowers/plans/2026-04-10-semantic-detection-labels.md b/docs/superpowers/plans/2026-04-10-semantic-detection-labels.md new file mode 100644 index 0000000..2beecc3 --- /dev/null +++ b/docs/superpowers/plans/2026-04-10-semantic-detection-labels.md @@ -0,0 +1,2093 @@ +# Semantic Detection Labels Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add 3 cross-cutting semantic detection labels (`log_content`, `stack_trace`, `diff_patch`), remove `det_log_lines`, and validate via 5k audit before the downstream $400-600 90k annotation run. + +**Architecture:** Prompt-only change to `training/trainr/core/annotate_detections.py`. The 3 new labels are the first detection-only labels (not mirrored by `ContentSubType`). Validation is a 5k stratified audit + ~150 targeted positive injection with strict pass criteria: inter-annotator agreement ≥0.995, recall ≥0.90 on injected positives, zero rule violations in spot-check. + +**Tech Stack:** Python 3.10+, uv, polars, openai (OpenRouter), anthropic, pytest. Runs under `training/` workstream only — no Rust/`src/` changes. + +**Spec:** `docs/superpowers/specs/2026-04-10-semantic-detection-labels-design.md` + +**Working directory for all commands below:** `training/` (the Python workstream lives there). When running `uv` commands, `cd training/` first unless noted. + +--- + +## File Inventory + +**Modified:** +- `training/trainr/core/annotate_detections.py` — `DETECTION_LABELS` list, `SYSTEM_PROMPT` text, JSON template; new `SEMANTIC_LABELS` module constant +- `training/tests/test_annotate_detections.py` — fix 3 stale iter15 tests, add regression-guard tests for new labels + +**Created:** +- `training/trainr/core/build_audit_sample.py` — builds the 5k stratified + injection-pool input parquet +- `training/trainr/core/audit_semantic_labels.py` — computes agreement + recall metrics, outputs markdown report +- `training/tests/test_build_audit_sample.py` — smoke tests for the builder +- `docs/accuracy_runs/2026-04-10-iteration-16.md` — iteration report (written after the audit run) + +--- + +## Phase 0 — Repair Baseline Tests (prep) + +The existing `test_annotate_detections.py` has 3 failing tests carried over from iter15. They reference pre-iter15 label counts and ROUTING_TABLE assumptions that no longer hold. Fix them first so the baseline is green and subsequent TDD is not muddied by pre-existing noise. + +### Task 0.1: Verify baseline failure state + +**Files:** +- Read: `training/tests/test_annotate_detections.py` + +- [ ] **Step 1: Capture current failing tests** + +Run from `training/`: +```bash +uv run pytest tests/test_annotate_detections.py --tb=no -q +``` + +Expected: exactly 3 failures: +- `TestDetectionLabels::test_labels_defined` +- `TestRoutingTable::test_routing_table_covers_all_detection_labels` +- `TestRoutingAnnotation::test_routing_uses_correct_api_keys` + +If any other tests fail, STOP and investigate — the plan assumes these 3 as the starting state. + +### Task 0.2: Fix `test_labels_defined` (stale label count) + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` around the `TestDetectionLabels::test_labels_defined` method + +- [ ] **Step 1: Replace the stale count assertion** + +Replace: +```python + def test_labels_defined(self): + from trainr.core.annotate_detections import DETECTION_LABELS + + assert isinstance(DETECTION_LABELS, list) + assert len(DETECTION_LABELS) == 29 +``` + +With: +```python + def test_labels_defined(self): + from trainr.core.annotate_detections import DETECTION_LABELS + + assert isinstance(DETECTION_LABELS, list) + # Current baseline: 40 labels (pre-iter16). + # Phase 1 removes log_lines (→39), phases 3-5 add 3 semantic labels (→42). + assert len(DETECTION_LABELS) == 40 +``` + +- [ ] **Step 2: Run the single test to verify it passes** + +```bash +uv run pytest tests/test_annotate_detections.py::TestDetectionLabels::test_labels_defined -v +``` + +Expected: PASS. + +### Task 0.3: Fix `test_expected_labels_present` (stale label list) + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — `test_expected_labels_present` method + +- [ ] **Step 1: Update the expected list to match iter15's state** + +Replace the existing `expected = [...]` block with: +```python + expected = [ + "plain", "markdown", "rst", "latex", + "python", "javascript", "typescript", "rust", "go", "java", "c_cpp", "objc", + "csharp", "powershell", "ruby", "php", "swift", "kotlin", "r", "lua", "graphql", + "sql", "shell", "css", + "yaml", "toml", "ini", "dockerfile", "makefile", + "html", "xml", "sgml", + "csv", "tsv", "pipe_table", "fixed_width", + "json", "jsonl", "key_value", "log_lines", + ] +``` + +- [ ] **Step 2: Run the test** + +```bash +uv run pytest tests/test_annotate_detections.py::TestDetectionLabels::test_expected_labels_present -v +``` + +Expected: PASS. + +### Task 0.4: Fix `test_routing_table_covers_all_detection_labels` (architectural mismatch) + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — `TestRoutingTable::test_routing_table_covers_all_detection_labels` + +**Why:** `ROUTING_TABLE` is keyed by `sub_type`, not detection label. The iter15 labels (`csharp`, `swift`, ...) were added to `DETECTION_LABELS` but not to `ROUTING_TABLE` because they routed through fallback. The assertion was backwards — it should check that ROUTING_TABLE keys are a subset of sub_type-mirroring detection labels, NOT that every detection label has a routing entry. Phase 3-5 will add `log_content`/`stack_trace`/`diff_patch` which are explicitly detection-only and will never be in ROUTING_TABLE. + +- [ ] **Step 1: Replace the assertion direction** + +Replace: +```python + def test_routing_table_covers_all_detection_labels(self): + from trainr.core.annotate_detections import DETECTION_LABELS, ROUTING_TABLE + + for label in DETECTION_LABELS: + assert label in ROUTING_TABLE, f"ROUTING_TABLE missing entry for: {label}" +``` + +With: +```python + def test_routing_table_keys_are_valid_detection_labels(self): + """ROUTING_TABLE is keyed by sub_type. Every key must be a known + detection label that mirrors a sub_type (i.e., NOT a semantic + detection-only label like log_content). Labels not in ROUTING_TABLE + fall through to the default model in annotate_dataframe().""" + from trainr.core.annotate_detections import DETECTION_LABELS, ROUTING_TABLE + + for sub_type in ROUTING_TABLE.keys(): + assert sub_type in DETECTION_LABELS, ( + f"ROUTING_TABLE key {sub_type!r} is not in DETECTION_LABELS" + ) +``` + +- [ ] **Step 2: Run the test** + +```bash +uv run pytest tests/test_annotate_detections.py::TestRoutingTable::test_routing_table_keys_are_valid_detection_labels -v +``` + +Expected: PASS. + +### Task 0.5: Fix `test_routing_uses_correct_api_keys` (no anthropic entries) + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — `TestRoutingAnnotation::test_routing_uses_correct_api_keys` + +**Why:** iter15 migrated all ROUTING_TABLE entries to `openrouter` backend. The test searches for an anthropic entry and asserts one exists. That's a stale assumption. Make the test skip cleanly when no anthropic entry exists, so it becomes a no-op until a future iteration reintroduces one. + +- [ ] **Step 1: Replace the hard assertion with a conditional skip** + +Replace: +```python + anthropic_sub = None + for sub, (model, backend) in ROUTING_TABLE.items(): + if backend == "anthropic": + anthropic_sub = sub + break + assert anthropic_sub is not None, "No anthropic entries in ROUTING_TABLE" +``` + +With: +```python + anthropic_sub = None + for sub, (model, backend) in ROUTING_TABLE.items(): + if backend == "anthropic": + anthropic_sub = sub + break + if anthropic_sub is None: + pytest.skip( + "No anthropic entries in ROUTING_TABLE — test is a no-op " + "until a future iteration reintroduces one." + ) +``` + +- [ ] **Step 2: Run the test** + +```bash +uv run pytest tests/test_annotate_detections.py::TestRoutingAnnotation::test_routing_uses_correct_api_keys -v +``` + +Expected: SKIPPED. + +### Task 0.6: Verify full baseline is green + +- [ ] **Step 1: Run the full test file** + +```bash +uv run pytest tests/test_annotate_detections.py -v +``` + +Expected: all green (some SKIPPED is acceptable). + +- [ ] **Step 2: Commit** + +```bash +git add training/tests/test_annotate_detections.py +git commit -m "test(detections): repair stale iter15 test baseline + +- Update label count to 40 (was 29, pre-iter15). +- Refresh expected label list to include iter15's 9 language labels. +- Invert routing-table coverage assertion: ROUTING_TABLE keys must be + in DETECTION_LABELS, not the reverse. Sub_types without an entry + fall through to the default model by design, and iter16 introduces + detection-only semantic labels that will never have routing entries. +- Skip test_routing_uses_correct_api_keys when ROUTING_TABLE has no + anthropic entries (iter15 migrated everything to openrouter)." +``` + +--- + +## Phase 1 — Remove `log_lines` from Detection Labels + +### Task 1.1: Write the failing tests for log_lines removal + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — new test methods in `TestDetectionLabels` + +- [ ] **Step 1: Append these tests to `class TestDetectionLabels`** + +```python + def test_log_lines_removed_from_detection_labels(self): + from trainr.core.annotate_detections import DETECTION_LABELS + + assert "log_lines" not in DETECTION_LABELS, ( + "log_lines must be sub_type-only (not a detection label) after " + "iter16. See docs/superpowers/specs/" + "2026-04-10-semantic-detection-labels-design.md" + ) + + def test_log_lines_removed_from_system_prompt_label_list(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + # The inline label list line in SYSTEM_PROMPT begins with "Labels:" + label_list_line = next( + line for line in SYSTEM_PROMPT.splitlines() if line.startswith("Labels:") + ) + assert "log_lines" not in label_list_line + + def test_log_lines_removed_from_json_template(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + # The JSON example template at the bottom should not include log_lines + assert '"log_lines":' not in SYSTEM_PROMPT +``` + +- [ ] **Step 2: Also update `test_labels_defined` count expectation** + +Change the assertion in `test_labels_defined` from `== 40` to `== 39`: +```python + # After phase 1: log_lines removed → 39. + # After phases 3-5: 3 semantic labels added → 42. + assert len(DETECTION_LABELS) == 39 +``` + +- [ ] **Step 3: Also update `test_expected_labels_present`** + +Remove `"log_lines",` from the expected list inside `test_expected_labels_present`. + +- [ ] **Step 4: Run the tests and verify they FAIL** + +```bash +uv run pytest tests/test_annotate_detections.py::TestDetectionLabels -v +``` + +Expected: 4 failures (the three new tests plus `test_labels_defined`) because `log_lines` is still in the implementation. + +### Task 1.2: Remove `log_lines` from `DETECTION_LABELS` + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — `DETECTION_LABELS` list (around lines 31-42) + +- [ ] **Step 1: Delete `log_lines` from the list** + +Change: +```python + "json", "jsonl", "key_value", "log_lines", +] +``` + +To: +```python + "json", "jsonl", "key_value", +] +``` + +### Task 1.3: Remove `log_lines` from SYSTEM_PROMPT label list line + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — `SYSTEM_PROMPT` constant around lines 98-101 + +- [ ] **Step 1: Delete `log_lines` from the inline list** + +Find: +``` +Labels: plain, markdown, rst, latex, python, javascript, typescript, rust, go, \ +java, c_cpp, objc, csharp, powershell, ruby, php, swift, kotlin, r, lua, \ +graphql, sql, shell, css, yaml, toml, ini, dockerfile, makefile, html, xml, \ +sgml, csv, tsv, pipe_table, fixed_width, json, jsonl, key_value, log_lines +``` + +Change the last line to end with `key_value` instead of `key_value, log_lines`: +``` +Labels: plain, markdown, rst, latex, python, javascript, typescript, rust, go, \ +java, c_cpp, objc, csharp, powershell, ruby, php, swift, kotlin, r, lua, \ +graphql, sql, shell, css, yaml, toml, ini, dockerfile, makefile, html, xml, \ +sgml, csv, tsv, pipe_table, fixed_width, json, jsonl, key_value +``` + +### Task 1.4: Remove `log_lines` from SYSTEM_PROMPT JSON template + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — JSON template at the end of `SYSTEM_PROMPT` + +- [ ] **Step 1: Delete `"log_lines": 0` from the template** + +Find the JSON template block that ends in `..., "key_value": 0, "log_lines": 0}` and remove the `"log_lines": 0`, leaving the closing brace: + +```python +{"plain": 0, "markdown": 0, "rst": 0, "latex": 0, "python": 0, "javascript": 0, \ +"typescript": 0, "rust": 0, "go": 0, "java": 0, "c_cpp": 0, "objc": 0, \ +"csharp": 0, "powershell": 0, "ruby": 0, "php": 0, "swift": 0, "kotlin": 0, \ +"r": 0, "lua": 0, "graphql": 0, "sql": 0, "shell": 0, "css": 0, "yaml": 0, \ +"toml": 0, "ini": 0, "dockerfile": 0, "makefile": 0, "html": 0, "xml": 0, \ +"sgml": 0, "csv": 0, "tsv": 0, "pipe_table": 0, "fixed_width": 0, "json": 0, \ +"jsonl": 0, "key_value": 0}""" +``` + +### Task 1.5: Run all annotate_detections tests and commit + +- [ ] **Step 1: Run the file** + +```bash +uv run pytest tests/test_annotate_detections.py -v +``` + +Expected: all green. + +- [ ] **Step 2: Commit** + +```bash +git add training/trainr/core/annotate_detections.py training/tests/test_annotate_detections.py +git commit -m "feat(detections): remove det_log_lines (sub_type-only now) + +log_lines remains a ContentSubType. The detection label was redundant +with the sub_type column and will be replaced by a proper cross-cutting +det_log_content in subsequent commits. No trained model depends on +det_log_lines (detection head has never been trained)." +``` + +--- + +## Phase 2 — Introduce `SEMANTIC_LABELS` Constant + +Creates a module-level constant documenting which detection labels are cross-cutting (NOT mirrored by a `ContentSubType`). This constant drives the routing test and future label-set invariants. + +### Task 2.1: Write failing test for `SEMANTIC_LABELS` constant + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — new `TestSemanticLabels` class + +- [ ] **Step 1: Append this class after `TestDetectionLabels`** + +```python +class TestSemanticLabels: + """Detection labels that are NOT mirrored by a ContentSubType. + + These are cross-cutting phenomena (stack traces, diffs, embedded + log content) that can appear inside any primary sub_type. They + are the first detection-only labels in the classifier. + """ + + def test_constant_exists(self): + from trainr.core.annotate_detections import SEMANTIC_LABELS + + assert isinstance(SEMANTIC_LABELS, frozenset) + + def test_semantic_labels_are_subset_of_detection_labels(self): + from trainr.core.annotate_detections import ( + DETECTION_LABELS, SEMANTIC_LABELS, + ) + + for label in SEMANTIC_LABELS: + assert label in DETECTION_LABELS, ( + f"Semantic label {label!r} declared but not in DETECTION_LABELS" + ) + + def test_semantic_labels_are_not_in_routing_table(self): + """Semantic (detection-only) labels have no sub_type, so they + must not appear as keys in ROUTING_TABLE.""" + from trainr.core.annotate_detections import ( + ROUTING_TABLE, SEMANTIC_LABELS, + ) + + for label in SEMANTIC_LABELS: + assert label not in ROUTING_TABLE, ( + f"Semantic label {label!r} must not be in ROUTING_TABLE " + f"(ROUTING_TABLE is keyed by sub_type)" + ) +``` + +- [ ] **Step 2: Run and verify the tests FAIL with ImportError** + +```bash +uv run pytest tests/test_annotate_detections.py::TestSemanticLabels -v +``` + +Expected: failures due to missing `SEMANTIC_LABELS` import. + +### Task 2.2: Add the `SEMANTIC_LABELS` constant + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — after `DETECTION_LABELS` definition + +- [ ] **Step 1: Insert the constant after `DETECTION_LABELS`** + +Insert this block immediately after the `DETECTION_LABELS = [...]` definition (around line 42): + +```python +# --------------------------------------------------------------------------- +# Semantic (cross-cutting) detection labels — NOT mirrored by ContentSubType. +# These fire when the corresponding phenomenon is embedded anywhere in the +# text, regardless of the row's primary sub_type. Added in iter16 (2026-04-10). +# --------------------------------------------------------------------------- + +SEMANTIC_LABELS: frozenset[str] = frozenset() +"""Detection labels that have no corresponding ContentSubType. + +Empty until phases 3-5 append log_content, stack_trace, diff_patch. +""" +``` + +- [ ] **Step 2: Run and verify tests pass** + +```bash +uv run pytest tests/test_annotate_detections.py::TestSemanticLabels -v +``` + +Expected: PASS (empty frozenset vacuously satisfies all three tests). + +- [ ] **Step 3: Commit** + +```bash +git add training/trainr/core/annotate_detections.py training/tests/test_annotate_detections.py +git commit -m "feat(detections): introduce SEMANTIC_LABELS frozenset + +Documents which detection labels are NOT mirrored by a ContentSubType. +Empty in this commit — populated in subsequent phases with log_content, +stack_trace, and diff_patch." +``` + +--- + +## Phase 3 — Add `log_content` Label + +Each new semantic label gets its own phase (own commit) so that if the 5k audit reveals a prompt bug in one label, reverting/iterating is surgical. + +### Task 3.1: Write failing tests for log_content + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — new `TestLogContentLabel` class + +- [ ] **Step 1: Append this class** + +```python +class TestLogContentLabel: + """Regression-guard tests for the log_content semantic detection label.""" + + def test_in_detection_labels(self): + from trainr.core.annotate_detections import DETECTION_LABELS + + assert "log_content" in DETECTION_LABELS + + def test_in_semantic_labels(self): + from trainr.core.annotate_detections import SEMANTIC_LABELS + + assert "log_content" in SEMANTIC_LABELS + + def test_in_system_prompt_label_list(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + label_list_line = next( + line for line in SYSTEM_PROMPT.splitlines() if line.startswith("Labels:") + ) + assert "log_content" in label_list_line + + def test_in_json_template(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert '"log_content": 0' in SYSTEM_PROMPT + + def test_definition_includes_uppercase_severity_rule(self): + """Critical precision-tightening rule — if this is removed or + reworded, the false-positive rate on lowercase 'error'/'info' + in prose will explode.""" + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert "UPPERCASE or BRACKETED" in SYSTEM_PROMPT + + def test_definition_includes_density_rule(self): + """Density rule: two or more log-shaped lines in sequence.""" + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert "TWO OR MORE consecutive lines" in SYSTEM_PROMPT + + def test_definition_includes_pure_trace_carveout(self): + """Stack traces alone should fire stack_trace but NOT log_content.""" + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert "PURE stack trace" in SYSTEM_PROMPT +``` + +- [ ] **Step 2: Update `test_labels_defined` count** + +Bump from `== 39` to `== 40`: +```python + # After phase 1: 39. After phase 3: log_content added → 40. + # After phases 4-5: 2 more semantic labels → 42. + assert len(DETECTION_LABELS) == 40 +``` + +- [ ] **Step 3: Verify the tests FAIL** + +```bash +uv run pytest tests/test_annotate_detections.py::TestLogContentLabel -v +``` + +Expected: all 7 new tests fail. + +### Task 3.2: Add log_content to `DETECTION_LABELS` and `SEMANTIC_LABELS` + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` + +- [ ] **Step 1: Append to `DETECTION_LABELS`** + +Change the last line of `DETECTION_LABELS` from: +```python + "json", "jsonl", "key_value", +] +``` + +To: +```python + "json", "jsonl", "key_value", + # Added iter16 (2026-04-10): cross-cutting semantic detection labels + "log_content", +] +``` + +- [ ] **Step 2: Update `SEMANTIC_LABELS` frozenset** + +Change: +```python +SEMANTIC_LABELS: frozenset[str] = frozenset() +``` + +To: +```python +SEMANTIC_LABELS: frozenset[str] = frozenset({"log_content"}) +``` + +### Task 3.3: Add log_content to SYSTEM_PROMPT label list line + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — `SYSTEM_PROMPT` + +- [ ] **Step 1: Append to the inline list** + +Change the label list line ending from: +``` +sgml, csv, tsv, pipe_table, fixed_width, json, jsonl, key_value +``` + +To: +``` +sgml, csv, tsv, pipe_table, fixed_width, json, jsonl, key_value, \ +log_content +``` + +### Task 3.4: Add the log_content definition block to SYSTEM_PROMPT + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — end of `## Important distinctions` section in `SYSTEM_PROMPT` + +- [ ] **Step 1: Append the definition** + +Find the end of the `## Important distinctions` section (after the `graphql` definition, before `## Rules`). Append this definition block immediately before `## Rules`: + +``` +- "log_content" = embedded log output (distinct from sub_type=log_lines, \ +which means the row IS primarily logs). Requires TWO OR MORE consecutive \ +lines that each independently match a log-line schema. A log-line schema is \ +ONE of: + * ` ` where timestamp is ISO \ +8601 (`2024-01-15T10:23:45Z`), apache-style (`[15/Jan/2024:10:23:45 +0000]`), \ +syslog-style (`Jan 15 10:23:45`), or unix epoch (ms/seconds) + * A recognized named format: nginx/apache access log, syslog \ +(`timestamp host proc[pid]: msg`), dockerd/container log, logfmt \ +(`key=value key2=value2`) ONLY when co-occurring with a timestamp OR \ +severity field + * JSON log records: >=2 consecutive JSON objects each containing BOTH a \ +timestamp field AND a `level`/`severity` field +Severity tokens must be UPPERCASE or BRACKETED (`INFO`, `[info]`, `WARN`, \ +`WARNING`, `ERROR`, `DEBUG`, `TRACE`, `FATAL`, `CRITICAL`) — lowercase \ +`error`/`info` in prose or code identifiers does NOT count. +Do NOT fire on: single-line error messages (even inside code fences or \ +blockquotes), code that CALLS a logger (`log.info(...)`), sentences \ +describing logging behavior, changelogs with leading dates (`2024-01-15 - \ +fixed bug`), CSV/TSV with date columns, `ls -la` output, git commit logs. \ +Lines inside quotation marks or markdown blockquotes do not contribute to \ +the density count. +If the embedded output is a PURE stack trace (no surrounding non-trace log \ +lines), fire "stack_trace": 1 but NOT "log_content": 1. If a stack trace \ +appears inside otherwise-normal log output, fire BOTH. +``` + +### Task 3.5: Add log_content to JSON template + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — JSON template + +- [ ] **Step 1: Append to the JSON example** + +Change: +```python +"jsonl": 0, "key_value": 0}""" +``` + +To: +```python +"jsonl": 0, "key_value": 0, "log_content": 0}""" +``` + +### Task 3.6: Run tests and commit + +- [ ] **Step 1: Run the full file** + +```bash +uv run pytest tests/test_annotate_detections.py -v +``` + +Expected: all green. + +- [ ] **Step 2: Commit** + +```bash +git add training/trainr/core/annotate_detections.py training/tests/test_annotate_detections.py +git commit -m "feat(detections): add log_content semantic label + +First detection-only label — fires on embedded log output anywhere in a +row, distinct from sub_type=log_lines which means the row IS primarily +logs. Strict density+schema definition with uppercase-severity rule to +keep false positives bounded at the ~0.1% target class prevalence." +``` + +--- + +## Phase 4 — Add `stack_trace` Label + +### Task 4.1: Write failing tests for stack_trace + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — new `TestStackTraceLabel` class + +- [ ] **Step 1: Append this class** + +```python +class TestStackTraceLabel: + """Regression-guard tests for the stack_trace semantic detection label.""" + + def test_in_detection_labels(self): + from trainr.core.annotate_detections import DETECTION_LABELS + + assert "stack_trace" in DETECTION_LABELS + + def test_in_semantic_labels(self): + from trainr.core.annotate_detections import SEMANTIC_LABELS + + assert "stack_trace" in SEMANTIC_LABELS + + def test_in_system_prompt_label_list(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + label_list_line = next( + line for line in SYSTEM_PROMPT.splitlines() if line.startswith("Labels:") + ) + assert "stack_trace" in label_list_line + + def test_in_json_template(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert '"stack_trace": 0' in SYSTEM_PROMPT + + def test_definition_includes_two_frame_rule(self): + """Multi-frame requirement is the load-bearing anti-false-positive + rule for single-line errors and prose mentioning 'at line X'.""" + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert "TWO OR MORE" in SYSTEM_PROMPT + assert "frames" in SYSTEM_PROMPT + + def test_definition_includes_dotnet_format(self): + """The .NET format (` at Namespace.Class.Method() in File.cs:line 42`) + was flagged as highest-value addition by standard review.""" + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert ".NET" in SYSTEM_PROMPT + + def test_definition_mentions_cofire_with_log_content(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert 'fire BOTH' in SYSTEM_PROMPT +``` + +- [ ] **Step 2: Update `test_labels_defined` count** + +Bump from `== 40` to `== 41`. + +- [ ] **Step 3: Verify tests FAIL** + +```bash +uv run pytest tests/test_annotate_detections.py::TestStackTraceLabel -v +``` + +Expected: failures. + +### Task 4.2: Add stack_trace to the label set + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` + +- [ ] **Step 1: Append to `DETECTION_LABELS`** + +Change the semantic labels section: +```python + "json", "jsonl", "key_value", + # Added iter16 (2026-04-10): cross-cutting semantic detection labels + "log_content", +] +``` + +To: +```python + "json", "jsonl", "key_value", + # Added iter16 (2026-04-10): cross-cutting semantic detection labels + "log_content", "stack_trace", +] +``` + +- [ ] **Step 2: Update `SEMANTIC_LABELS`** + +Change: +```python +SEMANTIC_LABELS: frozenset[str] = frozenset({"log_content"}) +``` + +To: +```python +SEMANTIC_LABELS: frozenset[str] = frozenset({"log_content", "stack_trace"}) +``` + +### Task 4.3: Add stack_trace to SYSTEM_PROMPT label list line + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — `SYSTEM_PROMPT` + +- [ ] **Step 1: Append to the inline list** + +Change: +``` +sgml, csv, tsv, pipe_table, fixed_width, json, jsonl, key_value, \ +log_content +``` + +To: +``` +sgml, csv, tsv, pipe_table, fixed_width, json, jsonl, key_value, \ +log_content, stack_trace +``` + +### Task 4.4: Add the stack_trace definition block + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — `SYSTEM_PROMPT` `## Important distinctions` section + +- [ ] **Step 1: Append after the log_content definition** + +Insert immediately after the `log_content` definition block: + +``` +- "stack_trace" = programmatic stack trace / traceback. Requires TWO OR \ +MORE frames where each frame carries a file:line locator OR a \ +package.Class.method locator (not just an `at` keyword). Strong signals \ +by language: + * Python: `Traceback (most recent call last):` header + `File "foo.py", \ +line N, in func` frames + final `ErrorType: message` line + * Java/JVM: `Exception in thread "..."` + \ +`at com.pkg.Class.method(File.java:42)` + optional `Caused by:` + \ +`... N more` + * .NET: ` at Namespace.Class.Method() in File.cs:line 42` (leading \ +whitespace is distinctive) + * JS/Node: `Error: message` + `at fn (file:line:col)` / `at async fn` + * Go: `goroutine N [running]:` + `main.foo(0x0)\n\tpath/file.go:42 +0x1a` + * Rust: `thread 'main' panicked at` + numbered backtrace frames `0: ...`, \ +`1: ...` with file:line + * Ruby: `from file.rb:42:in 'method'` chain (one frame per line) +Truncation markers (`... N more frames`, `[truncated]`) do not disqualify \ +a trace. Do NOT fire on: single-line error messages, the literal phrase \ +"stack trace" in prose, tutorial pseudocode describing what a trace looks \ +like, profiler output tables, or prose like "at line 5 it crashed, at \ +line 6 it retried". If this trace appears inside log output, fire BOTH \ +"stack_trace": 1 AND "log_content": 1. +``` + +### Task 4.5: Add stack_trace to JSON template + +- [ ] **Step 1: Append to the JSON example** + +Change: +```python +"jsonl": 0, "key_value": 0, "log_content": 0}""" +``` + +To: +```python +"jsonl": 0, "key_value": 0, "log_content": 0, "stack_trace": 0}""" +``` + +### Task 4.6: Run tests and commit + +- [ ] **Step 1: Run** + +```bash +uv run pytest tests/test_annotate_detections.py -v +``` + +Expected: all green. + +- [ ] **Step 2: Commit** + +```bash +git add training/trainr/core/annotate_detections.py training/tests/test_annotate_detections.py +git commit -m "feat(detections): add stack_trace semantic label + +Multi-frame programmatic traceback detection across Python, Java/JVM, +.NET, JS/Node, Go, Rust, and Ruby. Requires two or more frames with +file:line or package.Class.method locators — single-line 'Error: foo' +messages and prose 'at line 5' do not qualify. Co-fires with log_content +when embedded in log output." +``` + +--- + +## Phase 5 — Add `diff_patch` Label + +### Task 5.1: Write failing tests for diff_patch + +**Files:** +- Modify: `training/tests/test_annotate_detections.py` — new `TestDiffPatchLabel` class + +- [ ] **Step 1: Append this class** + +```python +class TestDiffPatchLabel: + """Regression-guard tests for the diff_patch semantic detection label.""" + + def test_in_detection_labels(self): + from trainr.core.annotate_detections import DETECTION_LABELS + + assert "diff_patch" in DETECTION_LABELS + + def test_in_semantic_labels(self): + from trainr.core.annotate_detections import SEMANTIC_LABELS + + assert "diff_patch" in SEMANTIC_LABELS + + def test_in_system_prompt_label_list(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + label_list_line = next( + line for line in SYSTEM_PROMPT.splitlines() if line.startswith("Labels:") + ) + assert "diff_patch" in label_list_line + + def test_in_json_template(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert '"diff_patch": 0' in SYSTEM_PROMPT + + def test_definition_includes_required_markers(self): + """The REQUIRED marker set is the load-bearing anti-markdown-list + rule. If this is loosened, bullet lists will dominate false + positives at 0.1% class prevalence.""" + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert "REQUIRED" in SYSTEM_PROMPT + assert "@@ -X,Y +A,B @@" in SYSTEM_PROMPT + assert "diff --git" in SYSTEM_PROMPT + + def test_definition_includes_adjacent_file_markers_rule(self): + """--- and +++ must be on adjacent lines so YAML frontmatter + doesn't false-fire.""" + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert "adjacent lines" in SYSTEM_PROMPT + + def test_definition_excludes_markdown_bullet_lists(self): + from trainr.core.annotate_detections import SYSTEM_PROMPT + + assert "markdown bullet lists" in SYSTEM_PROMPT +``` + +- [ ] **Step 2: Update `test_labels_defined` count** + +Bump from `== 41` to `== 42`. + +- [ ] **Step 3: Verify tests FAIL** + +```bash +uv run pytest tests/test_annotate_detections.py::TestDiffPatchLabel -v +``` + +Expected: failures. + +### Task 5.2: Add diff_patch to the label set + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` + +- [ ] **Step 1: Append to `DETECTION_LABELS`** + +Change: +```python + "json", "jsonl", "key_value", + # Added iter16 (2026-04-10): cross-cutting semantic detection labels + "log_content", "stack_trace", +] +``` + +To: +```python + "json", "jsonl", "key_value", + # Added iter16 (2026-04-10): cross-cutting semantic detection labels + "log_content", "stack_trace", "diff_patch", +] +``` + +- [ ] **Step 2: Update `SEMANTIC_LABELS`** + +Change: +```python +SEMANTIC_LABELS: frozenset[str] = frozenset({"log_content", "stack_trace"}) +``` + +To: +```python +SEMANTIC_LABELS: frozenset[str] = frozenset( + {"log_content", "stack_trace", "diff_patch"} +) +``` + +### Task 5.3: Add diff_patch to SYSTEM_PROMPT label list line + +- [ ] **Step 1: Append** + +Change the label list ending from: +``` +log_content, stack_trace +``` + +To: +``` +log_content, stack_trace, diff_patch +``` + +### Task 5.4: Add the diff_patch definition block + +**Files:** +- Modify: `training/trainr/core/annotate_detections.py` — after stack_trace definition + +- [ ] **Step 1: Append after the stack_trace definition** + +Insert immediately after the `stack_trace` definition: + +``` +- "diff_patch" = unified diff or git patch format. REQUIRED: at least one \ +of {`@@ -X,Y +A,B @@` hunk header, `diff --git a/... b/...` header, paired \ +`--- a/path` / `+++ b/path` file markers on adjacent lines}. Line prefixes \ +alone are NOT sufficient. Additional signals: `+`/`-`/space line prefixes \ +within a hunk, `index abc1234..def5678 100644` git metadata. The \ +email-patch header `From abc1234 Mon Sep 17 00:00:00 2001` is a signal \ +ONLY when co-occurring with `---`/`+++` or `@@` markers (otherwise a \ +regular email header). CRITICAL anti-signals: markdown bullet lists using \ +`-` or `+`, pro/con lists, code containing arithmetic `+`/`-`, isolated \ +`+`/`-` lines without hunk context, YAML frontmatter `---` without an \ +adjacent `+++`. Unified diffs only — context diffs (`*** file ***` \ +separators) out of scope. +``` + +### Task 5.5: Add diff_patch to JSON template + +- [ ] **Step 1: Append** + +Change: +```python +"log_content": 0, "stack_trace": 0}""" +``` + +To: +```python +"log_content": 0, "stack_trace": 0, "diff_patch": 0}""" +``` + +### Task 5.6: Run tests and commit + +- [ ] **Step 1: Run full test file** + +```bash +uv run pytest tests/test_annotate_detections.py -v +``` + +Expected: all green. Label count is now 42. + +- [ ] **Step 2: Commit** + +```bash +git add training/trainr/core/annotate_detections.py training/tests/test_annotate_detections.py +git commit -m "feat(detections): add diff_patch semantic label + +Unified diff / git patch detection with strict required-marker rule: at +least one of (@@ hunk header, diff --git header, adjacent --- / +++ pair) +must be present. Line prefixes alone are insufficient — prevents markdown +bullet lists from dominating false positives at 0.1% class prevalence." +``` + +--- + +## Phase 6 — Audit Sample Builder + +### Task 6.1: Write smoke test for `build_audit_sample.py` + +**Files:** +- Create: `training/tests/test_build_audit_sample.py` + +- [ ] **Step 1: Create the test file** + +```python +"""Smoke tests for trainr.core.build_audit_sample.""" + +import tempfile +from pathlib import Path + +import polars as pl + + +def _make_fake_corpus(n_rows: int = 200) -> pl.DataFrame: + """Minimal corpus DataFrame with the columns build_audit_sample needs.""" + import random + + random.seed(42) + rows = [] + sub_types = ["markdown", "python", "plain", "json", "log_lines"] + + # Plain filler + for i in range(n_rows - 6): + rows.append({ + "text": f"filler text sample {i} with enough content to exist", + "sub_type": sub_types[i % len(sub_types)], + }) + + # Inject 2 known positives per label + rows.append({ + "text": ( + "Traceback (most recent call last):\n" + ' File "foo.py", line 10, in main\n' + " raise ValueError('oops')\n" + "ValueError: oops" + ), + "sub_type": "plain", + }) + rows.append({ + "text": ( + 'Exception in thread "main" java.lang.NullPointerException\n' + " at com.foo.Bar.baz(Bar.java:42)\n" + " at com.foo.App.main(App.java:12)" + ), + "sub_type": "plain", + }) + rows.append({ + "text": ( + "diff --git a/foo.py b/foo.py\n" + "--- a/foo.py\n" + "+++ b/foo.py\n" + "@@ -1,3 +1,3 @@\n" + "-old line\n" + "+new line\n" + " context" + ), + "sub_type": "plain", + }) + rows.append({ + "text": ( + "# Patch\n" + "@@ -10,4 +10,4 @@ def foo():\n" + "- return 1\n" + "+ return 2\n" + ), + "sub_type": "markdown", + }) + rows.append({ + "text": ( + "[2024-01-15T10:23:45Z] INFO request received\n" + "[2024-01-15T10:23:46Z] WARN slow query\n" + "[2024-01-15T10:23:47Z] ERROR timeout" + ), + "sub_type": "plain", + }) + rows.append({ + "text": ( + "Here is a log example:\n\n" + "2024-01-15 10:23:45 INFO app starting\n" + "2024-01-15 10:23:46 INFO listening on :8080\n" + ), + "sub_type": "markdown", + }) + + return pl.DataFrame(rows) + + +def test_build_audit_sample_stratified_count(): + from trainr.core.build_audit_sample import build_audit_sample + + corpus = _make_fake_corpus(n_rows=200) + with tempfile.TemporaryDirectory() as tmpdir: + input_path = Path(tmpdir) / "corpus.parquet" + output_path = Path(tmpdir) / "audit_input.parquet" + corpus.write_parquet(input_path) + + build_audit_sample( + input_path=str(input_path), + output_path=str(output_path), + stratified_n=100, + injection_per_label=3, + seed=42, + ) + + result = pl.read_parquet(output_path) + # stratified 100 + up to 3*3=9 injected (some may overlap) + assert 100 <= len(result) <= 112 + assert "audit_source" in result.columns + + +def test_build_audit_sample_injection_tags(): + """Injected rows must be tagged so the audit can exclude them from + the agreement metric and measure recall on them separately.""" + from trainr.core.build_audit_sample import build_audit_sample + + corpus = _make_fake_corpus(n_rows=200) + with tempfile.TemporaryDirectory() as tmpdir: + input_path = Path(tmpdir) / "corpus.parquet" + output_path = Path(tmpdir) / "audit_input.parquet" + corpus.write_parquet(input_path) + + build_audit_sample( + input_path=str(input_path), + output_path=str(output_path), + stratified_n=50, + injection_per_label=3, + seed=42, + ) + + result = pl.read_parquet(output_path) + sources = set(result["audit_source"].to_list()) + assert "stratified" in sources + assert "inject_stack_trace" in sources + assert "inject_diff_patch" in sources + assert "inject_log_content" in sources + + +def test_build_audit_sample_injection_regexes_match_positives(): + """Verify the known-positive fixture rows are selected by the + injection regexes (not passed over).""" + from trainr.core.build_audit_sample import build_audit_sample + + corpus = _make_fake_corpus(n_rows=200) + with tempfile.TemporaryDirectory() as tmpdir: + input_path = Path(tmpdir) / "corpus.parquet" + output_path = Path(tmpdir) / "audit_input.parquet" + corpus.write_parquet(input_path) + + build_audit_sample( + input_path=str(input_path), + output_path=str(output_path), + stratified_n=10, + injection_per_label=10, # Enough to catch all fixture positives + seed=42, + ) + + result = pl.read_parquet(output_path) + injected = result.filter(pl.col("audit_source") != "stratified") + # All 6 injected fixture rows should be found + inject_sources = injected["audit_source"].to_list() + assert inject_sources.count("inject_stack_trace") == 2 + assert inject_sources.count("inject_diff_patch") == 2 + assert inject_sources.count("inject_log_content") == 2 +``` + +- [ ] **Step 2: Verify the test FAILS (module does not exist yet)** + +```bash +uv run pytest tests/test_build_audit_sample.py -v +``` + +Expected: ImportError on `trainr.core.build_audit_sample`. + +### Task 6.2: Create `build_audit_sample.py` + +**Files:** +- Create: `training/trainr/core/build_audit_sample.py` + +- [ ] **Step 1: Write the module** + +```python +"""Build the 5k audit input for semantic detection label validation. + +Combines a stratified sample of the training corpus with targeted +injection of known-positive candidates for the 3 new semantic labels +(log_content, stack_trace, diff_patch). Injected rows are tagged with +an `audit_source` column so the downstream audit can: + +1. Compute inter-annotator agreement on the `stratified` rows only + (injected positives would inflate the stat). +2. Compute recall per new label on the `inject_*` rows. + +Usage: + uv run python -m trainr.core.build_audit_sample \ + --input data/curated/train/golden_train.parquet \ + --output data/audit/iter16_5k_input.parquet \ + --stratified-n 5000 \ + --injection-per-label 50 +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +import polars as pl + +from trainr.core.annotate_detections import stratified_sample + +# Regex patterns for targeted injection. Each pattern targets a label's +# high-precision surface features so that matches are very likely to be +# true positives for manual review. +INJECTION_PATTERNS: dict[str, list[str]] = { + "stack_trace": [ + r"Traceback \(most recent call last\)", + r"Exception in thread", + r"goroutine \d+ \[", + r"panicked at", + r"\s+at [\w.$]+\(.*\.java:\d+\)", + r"^\s+at \w+\.\w+\.\w+\(\) in .*\.cs:line \d+", # .NET + ], + "diff_patch": [ + r"^@@ -\d+(,\d+)? \+\d+(,\d+)? @@", + r"^diff --git a/", + ], + "log_content": [ + # Severity token + timestamp-shaped substring in same row. + # Left broad — this is a generous superset; annotators judge. + r"\b(INFO|WARN|ERROR|DEBUG|TRACE|FATAL)\b.*\d{4}-\d{2}-\d{2}", + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*\b(INFO|WARN|ERROR|DEBUG|TRACE|FATAL)\b", + r"\[\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}", # apache access log timestamp + ], +} + + +def find_injection_candidates( + corpus: pl.DataFrame, + label: str, + patterns: list[str], + n: int, + seed: int, +) -> pl.DataFrame: + """Return up to `n` rows from `corpus` whose `text` matches any pattern. + + Matches are case-insensitive only if the regex uses inline flags. + Uses polars' string.contains (regex) with OR composition across + patterns. Samples uniformly from the match set. + """ + # Build an OR regex + combined = "|".join(f"(?:{p})" for p in patterns) + matched = corpus.filter(pl.col("text").str.contains(combined)) + if len(matched) == 0: + return matched.with_columns(pl.lit(f"inject_{label}").alias("audit_source")) + if len(matched) > n: + matched = matched.sample(n=n, seed=seed) + return matched.with_columns(pl.lit(f"inject_{label}").alias("audit_source")) + + +def build_audit_sample( + input_path: str, + output_path: str, + stratified_n: int = 5000, + injection_per_label: int = 50, + seed: int = 42, +) -> pl.DataFrame: + """Build the audit input parquet. + + Args: + input_path: Path to the corpus parquet (golden_train.parquet). + output_path: Where to write the audit input parquet. + stratified_n: Size of the stratified sample. + injection_per_label: How many known-positive candidates to inject + per semantic label (log_content, stack_trace, diff_patch). + seed: RNG seed for sampling reproducibility. + + Returns: + The final DataFrame with an `audit_source` column distinguishing + 'stratified' rows from 'inject_