feat(darwin/repro): non-gating symptom-binding signal for conformant repros (ADR-175 §63, #47)

.github/workflows/ci.yml

@@ -147,6 +147,7 @@ jobs:
           node scripts/sota-attest.test.mjs
           node scripts/nightly-sota-gate.test.mjs
           node packages/darwin-mode/bench/swebench/solver-trajectory.test.mjs
+          node packages/darwin-mode/bench/swebench/test-critic.test.mjs
       - name: Native-router interop guard (ADR-043 — CJS default-unwrap regression)
         # tiny-dancer is CJS; vitest's interop masks the ESM `.default` unwrap, so
         # this runs under plain `node` (the real install condition) and fails if a

docs/adrs/ADR-175-dual-mode-test-driven-repair.md

@@ -60,8 +60,14 @@ Consumers evaluating these modes must account for their fundamentally different
 Direction matters: for the **benchmark**, a gamed self-repro can only *lower* the gold `FAIL_TO_PASS`
 score (it can't inflate a leaderboard number); for **product use with no ground truth there is no such
 backstop** — hence the `--pause-for-test-review` mode below. Partial mitigation today (ADR-174
-Test-Critic): the repro is validated to *fail on the unmodified code*; strengthening it to assert the
-issue's specific symptom is follow-up.
+Test-Critic): the repro is validated to *fail on the unmodified code*, and the failure must be a real
+assertion/exception (not an ImportError/collection error). Toward "assert the issue's *specific*
+symptom": `test-critic.symptomBindingScore` now computes a **non-gating** confidence that the repro's
+failure binds to the issue (does the trace raise an exception *type* the issue names? does the repro
+reference the issue's salient identifiers?), recorded per-instance (`row.symptomBinding`). It is
+deliberately **not** a gate — hard-rejecting low-binding repros risks lowering the load-bearing
+conformant resolve, so we ship the *measurement* first; a gate is a follow-up decision that a live
+A/B on the frozen holdout can now inform.
 
 ## Consequences
 - Product default = oracle-ON (TDR). Leaderboard/no-test = oracle-OFF. Both documented in the darwin

packages/darwin-mode/bench/swebench/repro-gate.mjs

@@ -47,14 +47,18 @@ export async function reproGateSolve({ writeRepro, solveRound, runRepro, maxRoun
 
   const rb = await writeRepro();
   cost += rb.cost || 0;
+  // ADR-175 §63 / #47: carry the non-gating symptom-binding confidence through so run reports can
+  // record how well the self-written repro binds to the issue's symptom (undefined if the writer
+  // didn't compute it, e.g. an invalid repro).
+  const symptomBinding = rb.symptomBinding;
 
   // Gate could not arm — no valid failing repro. Do ONE plain solve round so we still return a patch,
   // and report reproValid=false so this instance is not counted as repro-gated.
   if (!rb.valid) {
     const r = await solveRound({ round: 1, reproTrace: '', prevPatch: '' });
     cost += r.cost || 0;
     history.push({ round: 1, repro: 'invalid', resolvedInLoop: !!r.resolvedInLoop });
-    return { patch: r.patch || '', reproValid: false, reproPassed: false, rounds: 1, cost, history };
+    return { patch: r.patch || '', reproValid: false, reproPassed: false, rounds: 1, cost, history, symptomBinding };
   }
 
   const repro = rb.repro;
@@ -78,13 +82,13 @@ export async function reproGateSolve({ writeRepro, solveRound, runRepro, maxRoun
     const passed = !!(rr.ran && rr.passed);
     history.push({ round, ran: !!rr.ran, reproPassed: passed, resolvedInLoop: !!r.resolvedInLoop });
     if (passed) {
-      return { patch: r.patch, reproValid: true, reproPassed: true, rounds: round, cost, history };
+      return { patch: r.patch, reproValid: true, reproPassed: true, rounds: round, cost, history, symptomBinding };
     }
     // Feed the repro failure trace back so the next round iterates toward making the repro pass.
     reproTrace = (rr.logTail || '').slice(-2500);
   }
 
-  return { patch: lastPatch, reproValid: true, reproPassed: false, rounds: history.length, cost, history };
+  return { patch: lastPatch, reproValid: true, reproPassed: false, rounds: history.length, cost, history, symptomBinding };
 }
 
 /**

packages/darwin-mode/bench/swebench/solve-agentic.mjs

@@ -519,6 +519,8 @@ async function runInstance(inst) {
       // ADR-195 #2: reproduction-first path replaces the base solve (it owns the iterate loop).
       const rg = await runReproGate(inst, llm, localizeHint);
       patch = rg.patch; row.tier = 'repro'; row.reproValid = rg.reproValid; row.reproPassed = rg.reproPassed; row.reproRounds = rg.rounds; row.resolved = !!rg.reproPassed;
+      // ADR-175 §63 / #47: record the non-gating symptom-binding confidence of the self-written repro.
+      if (rg.symptomBinding) row.symptomBinding = rg.symptomBinding;
     } else {
       // ADR-205 perf (--early-escalate): abort the cheap attempt at half budget if it has not even
       // ATTEMPTED an edit — the dominant unrecoverable signature (all-exploration, zero edits).

packages/darwin-mode/bench/swebench/test-critic.mjs

@@ -32,6 +32,70 @@ function classify(r) {
   return 'failed'; // raised a real exception/assertion → reproduces the bug ✓
 }
 
+// ── ADR-175 §63 / GH #47 — SYMPTOM-BINDING signal (NON-GATING, measurement only). ───────────────
+// The `failed` verdict above proves the repro raises a REAL exception on the buggy code — but not that
+// the failure is THE issue's symptom rather than an unrelated/friendlier assertion the agent authored
+// (the Goodhart trap #47 flags: a self-written test with narrower scope or weaker assertions). ADR-175
+// names "strengthening it to assert the issue's specific symptom" as follow-up. Hard-GATING on this
+// would risk rejecting valid repros → lower the load-bearing conformant resolve, so this is deliberately
+// a NON-GATING confidence signal: the measurement you'd want BEFORE deciding whether a gate is safe.
+// Pure (issue text + repro source + failure trace → a bounded score) so it unit-tests offline.
+
+const EXC_RE = /\b([A-Z][A-Za-z0-9_]*(?:Error|Exception|Warning))\b/g;
+
+/** Distinct exception/error TYPE names named in a blob (e.g. "raises TypeError"). */
+function exceptionTypes(text) {
+  const out = new Set();
+  for (const m of String(text || '').matchAll(EXC_RE)) out.add(m[1]);
+  return [...out];
+}
+
+/** Salient identifiers the issue calls out: `backtick-quoted` tokens + `foo()` call names (len ≥ 3).
+ *  Conservative on purpose — noise here only weakens a NON-gating score, never rejects a repro. */
+function issueIdentifiers(problemStatement) {
+  const s = String(problemStatement || '');
+  const out = new Set();
+  for (const m of s.matchAll(/`([A-Za-z_][A-Za-z0-9_.]{2,})`/g)) out.add(m[1].split('.').pop());
+  for (const m of s.matchAll(/\b([A-Za-z_][A-Za-z0-9_]{2,})\s*\(/g)) out.add(m[1]);
+  // Drop generic English/code stopwords that would match trivially.
+  const STOP = new Set(['the', 'and', 'for', 'this', 'that', 'with', 'not', 'test', 'def', 'from', 'import', 'return', 'value', 'error', 'function', 'method', 'class', 'object', 'result']);
+  return [...out].filter(w => !STOP.has(w.toLowerCase()));
+}
+
+/**
+ * Heuristic confidence that a conformant repro exercises the ISSUE'S symptom (not just *a* failure).
+ * NON-GATING (ADR-175 §63 / #47) — recorded for analysis, never used to reject a repro.
+ *
+ *   { assessable, score, issueExceptions, boundExceptionType, matchedExceptions,
+ *     issueIdentifierCount, matchedIdentifiers }
+ *
+ * score ∈ [0,1]: +0.5 when the failure trace raises an exception TYPE the issue names
+ * (`boundExceptionType`), +0.5 × the fraction of (up to 3) salient issue identifiers the repro
+ * references. `assessable=false` when the issue names neither an exception nor a salient identifier
+ * (score null) — we don't fabricate confidence from nothing.
+ */
+export function symptomBindingScore(problemStatement, repro, logTail) {
+  const issueExc = exceptionTypes(problemStatement);
+  const traceExc = exceptionTypes(logTail);
+  const idents = issueIdentifiers(problemStatement);
+  const reproText = String(repro || '');
+
+  const matchedExceptions = issueExc.filter(e => traceExc.includes(e));
+  const boundExceptionType = matchedExceptions.length > 0;
+
+  const salient = idents.slice(0, 3);
+  const matchedIdentifiers = salient.filter(id => reproText.includes(id));
+
+  const assessable = issueExc.length > 0 || idents.length > 0;
+  if (!assessable) {
+    return { assessable: false, score: null, issueExceptions: issueExc, boundExceptionType: false, matchedExceptions: [], issueIdentifierCount: idents.length, matchedIdentifiers: [] };
+  }
+  const excPart = boundExceptionType ? 0.5 : 0;
+  const idPart = salient.length > 0 ? 0.5 * (matchedIdentifiers.length / salient.length) : 0;
+  const score = Math.round((excPart + idPart) * 1000) / 1000;
+  return { assessable: true, score, issueExceptions: issueExc, boundExceptionType, matchedExceptions, issueIdentifierCount: idents.length, matchedIdentifiers };
+}
+
 // Framework-aware repro guidance — the recurring repro-gap (django/sympy ~30% of Lite need scaffolding
 // before a self-contained pytest can import the project). Keyed off the SWE-bench instance prefix.
 function frameworkHint(instanceId) {
@@ -65,7 +129,12 @@ export async function buildReproTest(instanceId, problemStatement, llm, opts = {
     });
     lastTail = r.logTail;
     const verdict = r.ran ? classify(r) : 'error';
-    if (verdict === 'failed') return { valid: true, repro, attempts: att, cost, logTail: r.logTail };
+    if (verdict === 'failed') {
+      // ADR-175 §63 / #47: attach the NON-GATING symptom-binding confidence (valid stays true — this
+      // never rejects a repro; it's recorded so a future decision to gate can be measured, not guessed).
+      const symptomBinding = symptomBindingScore(problemStatement, repro, r.logTail);
+      return { valid: true, repro, attempts: att, cost, logTail: r.logTail, symptomBinding };
+    }
     feedback = verdict === 'passed'
       ? `\n--- attempt ${att}: your test PASSED on the unmodified buggy code, so it does NOT reproduce the bug. Rewrite it to assert the CORRECT behavior described in the issue, so it FAILS on the current code. ---`
       : `\n--- attempt ${att}: your test could not run (${verdict}). Output:\n${r.logTail.slice(-800)}\nFix imports/collection so a single test function runs. ---`;

packages/darwin-mode/bench/swebench/test-critic.test.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: MIT
+// Pure-function tests for the ADR-175 §63 / GH #47 symptom-binding signal in test-critic.mjs, plus the
+// non-gating passthrough through repro-gate.mjs. NO network / NO Docker / NO LLM. Run: node test-critic.test.mjs
+import assert from 'node:assert';
+import { symptomBindingScore } from './test-critic.mjs';
+import { reproGateSolve } from './repro-gate.mjs';
+
+let pass = 0;
+const t = (name, fn) => { try { fn(); pass++; console.log(`  ok  ${name}`); } catch (e) { console.error(`  FAIL ${name}: ${e.message}`); process.exitCode = 1; } };
+const ta = async (name, fn) => { try { await fn(); pass++; console.log(`  ok  ${name}`); } catch (e) { console.error(`  FAIL ${name}: ${e.message}`); process.exitCode = 1; } };
+
+console.log('test-critic symptom-binding (ADR-175 §63 / #47) unit tests:');
+
+t('binds when the failure raises an exception TYPE the issue names, and the repro references the issue identifier', () => {
+  const issue = 'Calling `frobnicate()` on an empty list raises TypeError instead of returning None.';
+  const repro = 'from mod import frobnicate\ndef test():\n    assert frobnicate([]) is None\n';
+  const trace = 'Traceback (most recent call last):\n  ...\nTypeError: object of type NoneType has no len()';
+  const s = symptomBindingScore(issue, repro, trace);
+  assert.equal(s.assessable, true);
+  assert.equal(s.boundExceptionType, true, 'TypeError named in issue AND raised in trace');
+  assert.deepEqual(s.matchedExceptions, ['TypeError']);
+  assert(s.matchedIdentifiers.includes('frobnicate'), 'repro references the issue identifier');
+  assert.equal(s.score, 1, 'exception-bound (0.5) + all salient identifiers matched (0.5)');
+});
+
+t('does NOT bind the exception type when the trace raises a DIFFERENT exception than the issue names', () => {
+  const issue = 'Calling `frobnicate()` raises TypeError on empty input.';
+  const repro = 'from mod import frobnicate\ndef test():\n    assert frobnicate([]) == 0\n';
+  const trace = 'AssertionError: expected 0';
+  const s = symptomBindingScore(issue, repro, trace);
+  assert.equal(s.boundExceptionType, false, 'issue names TypeError but trace only has AssertionError');
+  assert.equal(s.score, 0.5, 'no exception bind (0) + identifier match (0.5)');
+});
+
+t('is NOT assessable when the issue names neither an exception nor a salient identifier', () => {
+  const s = symptomBindingScore('The rendered output looks wrong to me.', 'def test():\n    assert render() == "x"\n', 'AssertionError');
+  assert.equal(s.assessable, false);
+  assert.equal(s.score, null, 'no fabricated confidence when there is nothing to bind to');
+});
+
+t('filters generic stopwords so they do not count as issue identifiers', () => {
+  // `value` and `result` are backtick-quoted but are stopwords → not salient identifiers; no exception named.
+  const s = symptomBindingScore('The `value` in the `result` is off.', 'def test(): assert value == result', 'AssertionError');
+  assert.equal(s.issueIdentifierCount, 0, 'stopwords are dropped');
+  assert.equal(s.assessable, false);
+});
+
+t('caps salient identifiers at 3 and scores the matched fraction', () => {
+  const issue = 'The functions `alpha()`, `beta()`, `gamma()`, `delta()` all misbehave.';
+  const repro = 'assert alpha() and beta()';  // 2 of the first 3 salient (alpha, beta, gamma) referenced
+  const s = symptomBindingScore(issue, repro, 'AssertionError');
+  assert.equal(s.boundExceptionType, false);
+  assert.equal(s.score, 0.333, `matched 2 of 3 salient identifiers → 0.5×2/3 rounded to 3dp; got ${s.score}`);
+});
+
+await ta('reproGateSolve passes the writer symptomBinding through to its result (non-gating)', async () => {
+  const sb = { assessable: true, score: 0.5, boundExceptionType: false };
+  const r = await reproGateSolve({
+    writeRepro: async () => ({ valid: true, repro: 'def test(): assert x', cost: 0.01, symptomBinding: sb }),
+    solveRound: async ({ round }) => ({ patch: `p${round}`, cost: 0.01 }),
+    runRepro: ({ patch }) => ({ ran: true, passed: patch === 'p1', logTail: '' }),
+    maxRounds: 2,
+  });
+  assert.equal(r.reproPassed, true);
+  assert.deepEqual(r.symptomBinding, sb, 'the gate result carries the writer\'s symptom-binding');
+});
+
+await ta('reproGateSolve tolerates a writer that computed no symptomBinding (undefined) — never throws', async () => {
+  const r = await reproGateSolve({
+    writeRepro: async () => ({ valid: false, repro: '', cost: 0 }),   // invalid repro → no symptomBinding
+    solveRound: async () => ({ patch: 'p', cost: 0 }),
+    runRepro: () => ({ ran: true, passed: false, logTail: '' }),
+    maxRounds: 1,
+  });
+  assert.equal(r.reproValid, false);
+  assert.equal(r.symptomBinding, undefined);
+});
+
+console.log(`\n${pass} passed`);