test(eval): add fix agent human /fs-fix functional eval

eval/fix/cases/001-human-fs-fix-add/annotations.yaml

@@ -0,0 +1,17 @@
+state: open
+
+expected_files:
+  - calc.py
+
+labels:
+  forbidden: []
+
+max_turns: 80
+max_cost_usd: 8.00
+
+# Human reference only; not consumed by judges (unlike triage/review quality).
+fix_expectations: |
+  Human-triggered fix (TRIGGER_SOURCE=eval-human): case input.yaml supplies
+  human_instruction describing the add() bug. Success is a new commit on the
+  existing PR branch that touches calc.py (post-fix push). PRE_AGENT_HEAD is
+  the fixture commit SHA.

eval/fix/cases/001-human-fs-fix-add/input.yaml

@@ -0,0 +1,41 @@
+forge: github
+# Single-line bug report supplied to the fix agent as HUMAN_INSTRUCTION
+# (forwarded by setup-fixture.sh → run-fullsend.sh). Kept here with the
+# fixture rather than hardcoded in the shared runner.
+#
+# Written as a factual bug report (what's wrong / what's failing), not as
+# imperative directives to an agent — avoids the agent-instruction-pattern
+# compliance check while still giving the fix agent enough signal to act on.
+# Scope (touch only calc.py) is enforced by annotations.yaml's expected_files,
+# not by wording here.
+human_instruction: "calc.py's add() function returns a - b instead of a + b, causing the existing tests in tests/test_calc.py to fail."
+fixture:
+  type: pull_request
+  title: "feat: add basic calculator"
+  body: |
+    Adds a tiny calculator module with tests.
+
+    Note: add() currently appears wrong in review — please fix via /fs-fix if needed.
+  base: main
+  files:
+    - path: calc.py
+      content: |
+        # Tiny calculator — intentional off-by-operator bug for the fix eval.
+
+
+        def add(a: int, b: int) -> int:
+            """Return the sum of a and b."""
+            return a - b  # BUG: should be a + b
+    - path: tests/test_calc.py
+      content: |
+        """Tests for calc module."""
+
+        from calc import add
+
+
+        def test_add() -> None:
+            assert add(2, 3) == 5
+
+
+        def test_add_negative() -> None:
+            assert add(-1, -2) == -3

eval/fix/cases/001-human-fs-fix-add/repo/README.md

@@ -0,0 +1,3 @@
+# tiny-calc (fix eval)
+
+Base branch is intentionally empty of product code. The PR under test adds a buggy calculator.

eval/fix/eval.yaml

@@ -0,0 +1,185 @@
+name: fix-eval
+description: >
+  Functional test of the fullsend fix agent pipeline (pre → sandbox → post)
+  for a human /fs-fix trigger on an existing PR. Validates the post-script
+  pushes a new commit that touches the expected files.
+
+skill: fix
+
+execution:
+  mode: case
+  # Harness hard-kill. Keep slightly above runner.env.EVAL_TIMEOUT so the
+  # script-level timeout can fire first and trap cleanup EXIT still runs.
+  timeout: 1800  # fix harness timeout_minutes is 25; leave headroom
+  parallelism: 1
+  env:
+    EVAL_ORG: $EVAL_ORG
+    GH_TOKEN: $GH_TOKEN
+    FULLSEND_DIR: $FULLSEND_DIR
+    GOOGLE_APPLICATION_CREDENTIALS: $GOOGLE_APPLICATION_CREDENTIALS
+    ANTHROPIC_VERTEX_PROJECT_ID: $ANTHROPIC_VERTEX_PROJECT_ID
+    GOOGLE_CLOUD_PROJECT: $GOOGLE_CLOUD_PROJECT
+    CLOUD_ML_REGION: $CLOUD_ML_REGION
+
+hooks:
+  before_each:
+    - command: "setup-fixture.sh"
+      timeout: 120
+      description: "Create ephemeral repo and PR fixture"
+
+  after_each:
+    - command: "capture-fixture.sh"
+      timeout: 60
+      description: "Capture PR head/files for judges"
+    - command: "teardown-fixture.sh"
+      timeout: 30
+      on_failure: continue
+      description: "Delete ephemeral repo"
+
+runner:
+  type: cli
+  command:
+    - "run-fullsend.sh"
+    - "{agent}"
+    - "{workspace}"
+    - "{output_dir}"
+  env:
+    FULLSEND_DIR: $FULLSEND_DIR
+    GH_TOKEN: $GH_TOKEN
+    # Script-level timeout (slightly under execution.timeout) so trap cleanup runs.
+    EVAL_TIMEOUT: "1700"
+    GOOGLE_APPLICATION_CREDENTIALS: $GOOGLE_APPLICATION_CREDENTIALS
+    ANTHROPIC_VERTEX_PROJECT_ID: $ANTHROPIC_VERTEX_PROJECT_ID
+    GOOGLE_CLOUD_PROJECT: $GOOGLE_CLOUD_PROJECT
+    CLOUD_ML_REGION: $CLOUD_ML_REGION
+
+models:
+  skill: claude-opus-4-6
+  judge: claude-opus-4-6
+
+dataset:
+  path: cases
+  schema: |
+    Each case directory contains:
+    - input.yaml: PR fixture (base repo + PR files with the bug).
+    - annotations.yaml: Expected files and budgets.
+    - repo/: Base branch contents pushed to main before the PR.
+
+outputs:
+  - path: output
+    schema: |
+      fixture-state.json: Captured PR state including head_sha, files,
+      and pre_agent_head for new_commit / expected_files judges.
+
+# forbidden_labels / max_turns / max_cost below are shared verbatim with
+# eval/code/eval.yaml and eval/review/eval.yaml — update all three if changing.
+judges:
+  - name: new_commit
+    description: Post-script must push at least one new commit on the PR head
+    check: |
+      import json
+      state = json.loads(outputs["files"]["output/fixture-state.json"])
+      head = state.get("head_sha")
+      if not head:
+          return False, "head_sha missing from fixture-state.json"
+      baseline = state.get("pre_agent_head")
+      if not baseline:
+          # Fallback: annotations may declare fixture_head_sha after a dry run
+          baseline = outputs.get("annotations", {}).get("fixture_head_sha")
+      if not baseline:
+          return False, "pre_agent_head missing — cannot prove a new commit"
+      if str(head) == str(baseline):
+          if state.get("head_sha_poll_exhausted"):
+              return False, (
+                  f"PR head unchanged ({head}) after polling exhausted — "
+                  "could not confirm within the polling window; this may be "
+                  "a push-propagation delay rather than a confirmed missing push"
+              )
+          return False, f"PR head unchanged ({head}) — fix post-script did not push"
+      return True, f"New commit on PR: {baseline} -> {head}"
+
+  - name: expected_files
+    description: >
+      The fix run's own commit(s) — not the fixture PR's pre-existing files —
+      must touch the paths listed in annotations.expected_files. Checked via
+      files_changed_since_pre_agent_head (a compare of pre_agent_head...head_sha),
+      not the PR-aggregate files list, which would pass even if the fix agent
+      pushed nothing relevant (see PR #381 review discussion).
+    check: |
+      import json
+      state = json.loads(outputs["files"]["output/fixture-state.json"])
+      expected = outputs.get("annotations", {}).get("expected_files") or []
+      if not expected:
+          return True, "No expected_files declared"
+      if state.get("files_changed_since_pre_agent_head_failed"):
+          return False, "Could not fetch files changed since pre_agent_head"
+      changed = set(state.get("files_changed_since_pre_agent_head") or [])
+      missing = [p for p in expected if p not in changed]
+      if missing:
+          reason = f"Expected files missing from fix commit(s): {missing} (changed since pre_agent_head: {sorted(changed)})"
+          if state.get("head_sha_poll_exhausted"):
+              reason += " — head_sha polling exhausted; this may be a push-propagation delay rather than a confirmed missing push"
+          return False, reason
+      return True, f"All expected files present in fix commit(s): {expected}"
+
+  - name: forbidden_labels
+    description: Labels listed in annotations.yaml forbidden list must NOT be present
+    check: |
+      import json
+      state = json.loads(outputs["files"]["output/fixture-state.json"])
+      actual = [l.lower() for l in state.get("labels", [])]
+      forbidden = outputs.get("annotations", {}).get("labels", {}).get("forbidden", [])
+      if not forbidden:
+          return True, "No forbidden labels specified"
+      present = [l for l in forbidden if l.lower() in actual]
+      if present:
+          return False, f"Forbidden labels present: {present} (actual: {actual})"
+      return True, f"No forbidden labels found (checked: {forbidden})"
+
+  - name: max_turns
+    description: Agent must complete within the declared turn budget
+    check: |
+      import json
+      raw = outputs["files"].get("output/metrics.json")
+      if not raw:
+          return False, "metrics.json not found"
+      metrics = json.loads(raw)
+      actual = metrics.get("num_turns")
+      if actual is None:
+          return False, "num_turns not present in metrics.json"
+      limit = outputs.get("annotations", {}).get("max_turns")
+      if limit is None:
+          return False, "max_turns not declared in annotations.yaml"
+      if int(actual) > int(limit):
+          return False, f"Exceeded max_turns: {actual} > {limit}"
+      return True, f"Turns OK: {actual} <= {limit}"
+
+  - name: max_cost
+    description: Agent must complete within the declared cost budget
+    check: |
+      import json
+      raw = outputs["files"].get("output/metrics.json")
+      if not raw:
+          return False, "metrics.json not found"
+      metrics = json.loads(raw)
+      actual = metrics.get("total_cost_usd")
+      if actual is None:
+          return False, "total_cost_usd not present in metrics.json"
+      limit = outputs.get("annotations", {}).get("max_cost_usd")
+      if limit is None:
+          return False, "max_cost_usd not declared in annotations.yaml"
+      if float(actual) > float(limit):
+          return False, f"Exceeded max_cost_usd: {actual} > {limit}"
+      return True, f"Cost OK: {actual} <= {limit}"
+
+thresholds:
+  new_commit:
+    min_pass_rate: 1.0
+  expected_files:
+    min_pass_rate: 1.0
+  forbidden_labels:
+    min_pass_rate: 1.0
+  max_turns:
+    min_pass_rate: 1.0
+  max_cost:
+    min_pass_rate: 1.0