test: add code agent eval framework with 4 functional test cases

eval/code/cases/001-sort-direction/annotations.yaml

@@ -0,0 +1,41 @@
+# Expected behavior after the code agent pipeline runs.
+state: open
+
+labels:
+  forbidden:
+    - needs-info
+    - wontfix
+
+max_turns: 50
+max_cost_usd: 5.00
+
+code_expectations: |
+  This is a trivial sort direction bug. The `get_tasks_by_priority()`
+  function in `tasks/manager.py` sorts tasks by priority in ascending
+  order but the docstring and intended behavior is descending (highest
+  first).
+
+  A correct implementation must:
+  1. Add `reverse=True` to the `sorted()` call, or negate the sort key,
+     or use an equivalent approach that produces descending order
+  2. Not change the function signature or return type
+  3. Not modify files beyond what is needed for the fix
+
+  The existing test `test_highest_priority_first` must pass after the fix.
+  The test asserts that the first element has the highest priority.
+
+  A score of 5 requires:
+  - The fix is minimal (ideally just adding reverse=True)
+  - The existing tests pass after the fix
+  - No unnecessary files changed
+  - Clean commit message referencing the issue number
+
+  A score of 3 is appropriate if:
+  - The fix is correct but the approach is unnecessarily indirect
+  - Extra files were modified that the issue did not require
+  - Tests were not explicitly verified
+
+  A score of 1 if:
+  - The sort order is still wrong after the change
+  - The function signature or behavior changed in unintended ways
+  - The fix broke other tests or introduced regressions

eval/code/cases/001-sort-direction/input.yaml

@@ -0,0 +1,40 @@
+forge: github
+fixture:
+  type: issue
+  title: "Tasks sorted in wrong order — low priority appears first"
+  body: |
+    ## Bug Report
+
+    **What happened:**
+    When I call `get_tasks_by_priority()`, the tasks come back sorted with
+    the lowest priority first. The function's docstring says "highest first"
+    but the actual behavior is the opposite.
+
+    **Expected behavior:**
+    Tasks should be sorted with the highest priority first (priority 5 before
+    priority 1).
+
+    **Steps to reproduce:**
+    ```python
+    from tasks.manager import get_tasks_by_priority
+
+    tasks = [
+        {"name": "Fix login", "priority": 3},
+        {"name": "Update docs", "priority": 1},
+        {"name": "Security patch", "priority": 5},
+    ]
+    result = get_tasks_by_priority(tasks)
+    print([t["name"] for t in result])
+    # Prints: ['Update docs', 'Fix login', 'Security patch']
+    # Expected: ['Security patch', 'Fix login', 'Update docs']
+    ```
+
+    **Environment:**
+    - Python 3.12
+    - No external dependencies
+
+    The existing test `test_highest_priority_first` in `tests/test_manager.py`
+    also fails, confirming the bug.
+  labels:
+    - ready-to-code
+    - bug

eval/code/cases/001-sort-direction/repo/Makefile

@@ -0,0 +1,7 @@
+.PHONY: test lint
+
+test:
+	pytest tests/ -v
+
+lint:
+	python -m py_compile tasks/manager.py

eval/code/cases/001-sort-direction/repo/README.md

@@ -0,0 +1,22 @@
+# Task Manager
+
+A simple task management library.
+
+## Usage
+
+```python
+from tasks.manager import get_tasks_by_priority, add_task, get_tasks_by_status
+
+tasks = [
+    {"name": "Fix login", "priority": 3, "status": "open"},
+    {"name": "Update docs", "priority": 1, "status": "open"},
+]
+
+sorted_tasks = get_tasks_by_priority(tasks)
+```
+
+## Testing
+
+```bash
+pytest tests/
+```

eval/code/cases/001-sort-direction/repo/tasks/manager.py

@@ -0,0 +1,18 @@
+"""Task management utilities."""
+
+
+def get_tasks_by_priority(tasks):
+    """Return tasks sorted by priority (highest first)."""
+    return sorted(tasks, key=lambda t: t["priority"])
+
+
+def add_task(tasks, name, priority=1, status="open"):
+    """Add a new task to the list."""
+    task = {"name": name, "priority": priority, "status": status}
+    tasks.append(task)
+    return task
+
+
+def get_tasks_by_status(tasks, status):
+    """Return tasks matching the given status."""
+    return [t for t in tasks if t["status"] == status]

eval/code/cases/001-sort-direction/repo/tests/test_manager.py

@@ -0,0 +1,49 @@
+"""Tests for task management utilities."""
+
+from tasks.manager import add_task, get_tasks_by_priority, get_tasks_by_status
+
+
+def test_highest_priority_first():
+    tasks = [
+        {"name": "Low", "priority": 1, "status": "open"},
+        {"name": "High", "priority": 5, "status": "open"},
+        {"name": "Medium", "priority": 3, "status": "open"},
+    ]
+    result = get_tasks_by_priority(tasks)
+    assert result[0]["name"] == "High"
+    assert result[-1]["name"] == "Low"
+
+
+def test_preserves_all_tasks():
+    tasks = [
+        {"name": "A", "priority": 2, "status": "open"},
+        {"name": "B", "priority": 1, "status": "open"},
+    ]
+    result = get_tasks_by_priority(tasks)
+    assert len(result) == 2
+
+
+def test_single_task():
+    tasks = [{"name": "Only", "priority": 1, "status": "open"}]
+    result = get_tasks_by_priority(tasks)
+    assert result[0]["name"] == "Only"
+
+
+def test_add_task():
+    tasks = []
+    task = add_task(tasks, "New task", priority=3)
+    assert task["name"] == "New task"
+    assert task["priority"] == 3
+    assert task["status"] == "open"
+    assert len(tasks) == 1
+
+
+def test_get_tasks_by_status():
+    tasks = [
+        {"name": "Open", "priority": 1, "status": "open"},
+        {"name": "Closed", "priority": 2, "status": "closed"},
+        {"name": "Also open", "priority": 3, "status": "open"},
+    ]
+    result = get_tasks_by_status(tasks, "open")
+    assert len(result) == 2
+    assert all(t["status"] == "open" for t in result)

eval/code/cases/002-docs-guidance/annotations.yaml

@@ -0,0 +1,40 @@
+state: open
+
+labels:
+  forbidden:
+    - needs-info
+    - wontfix
+
+max_turns: 30
+max_cost_usd: 3.00
+
+code_expectations: |
+  This is a docs-only change. The agent must add an "## Error Handling"
+  section to AGENTS.md after the existing "## Code Style" section.
+
+  A correct implementation must:
+  1. Add a new heading "## Error Handling" in the right location
+     (after "## Code Style", before "## Testing")
+  2. Cover all four requested topics: specific exceptions, logging
+     before re-raise, traceback preservation, structured API errors
+  3. NOT modify any .go files — this is purely a docs change
+  4. Match the existing Markdown formatting style of AGENTS.md
+
+  A score of 5 requires:
+  - Section placed in exactly the right location
+  - All four topics covered concisely
+  - Zero modifications to .go files or any non-AGENTS.md file
+  - Clean commit message referencing the issue
+  - Professional tone matching the existing document
+
+  A score of 3 is appropriate if:
+  - Section is added but in the wrong location (e.g., at the end)
+  - Only 2-3 of the 4 topics are covered
+  - Minor formatting inconsistencies with existing sections
+  - Some non-docs files were touched but not meaningfully changed
+
+  A score of 1 if:
+  - No "## Error Handling" section was added
+  - Go source code was modified
+  - The content is mostly unrelated to error handling
+  - The AGENTS.md structure was damaged or rearranged

eval/code/cases/002-docs-guidance/input.yaml

@@ -0,0 +1,27 @@
+forge: github
+fixture:
+  type: issue
+  title: "Add error handling guidance to AGENTS.md"
+  body: |
+    ## Enhancement Request
+
+    Our AGENTS.md is missing guidance about error handling conventions.
+    New contributors keep writing overly broad try/except blocks that
+    swallow errors silently.
+
+    **What to add:**
+    Add a section titled "## Error Handling" after the existing
+    "## Code Style" section in AGENTS.md. The section should cover:
+
+    1. Always catch specific exception types, never bare `except:`
+    2. Log errors before re-raising — use `logger.error()` not `print()`
+    3. Include the original traceback when wrapping exceptions
+    4. Return structured error responses from API handlers, not raw strings
+
+    **Constraints:**
+    - Do NOT modify any Go source files — this is a docs-only change
+    - Keep the guidance concise (aim for 10-20 lines of markdown)
+    - Match the existing tone and formatting of other AGENTS.md sections
+  labels:
+    - ready-to-code
+    - documentation

eval/code/cases/002-docs-guidance/repo/AGENTS.md

@@ -0,0 +1,29 @@
+# Agent Guidelines
+
+This document describes conventions for contributing to the
+notification service.
+
+## Project Structure
+
+- `cmd/server/` — HTTP server entry point
+- `internal/notify/` — Core notification logic
+- `internal/notify/notify_test.go` — Tests
+
+## Code Style
+
+- Use `gofmt` for formatting — do not override with custom rules
+- Prefer returning errors over panicking
+- Keep functions under 40 lines; extract helpers when they grow
+- Name variables descriptively: `userEmail` not `ue`
+
+## Testing
+
+- Every exported function must have at least one test
+- Use table-driven tests for functions with multiple input variations
+- Run `make test` before committing
+
+## Commit Messages
+
+- Use conventional commit format: `feat:`, `fix:`, `docs:`, `test:`
+- Reference the issue number in the body, not the subject line
+- Keep subject under 72 characters

eval/code/cases/002-docs-guidance/repo/Makefile

@@ -0,0 +1,14 @@
+.PHONY: test lint check-docs
+
+test:
+	go test ./...
+
+lint:
+	gofmt -l .
+
+check-docs:
+	@echo "Checking AGENTS.md structure..."
+	@grep -q "## Error Handling" AGENTS.md && echo "PASS: Error Handling section found" || (echo "FAIL: Missing '## Error Handling' section" && exit 1)
+	@grep -q "specific exception" AGENTS.md && echo "PASS: Specific exceptions topic found" || (echo "FAIL: Missing specific exceptions guidance" && exit 1)
+	@grep -q "traceback\|stacktrace\|stack trace" AGENTS.md && echo "PASS: Traceback topic found" || (echo "FAIL: Missing traceback guidance" && exit 1)
+	@echo "All docs checks passed"

eval/code/cases/002-docs-guidance/repo/README.md

@@ -0,0 +1,18 @@
+# Notification Service
+
+A lightweight notification service that sends email and Slack alerts.
+
+## Quick Start
+
+```bash
+go run cmd/server/main.go
+```
+
+## Development
+
+```bash
+make test    # run tests
+make lint    # check formatting
+```
+
+See AGENTS.md for coding conventions.

eval/code/cases/002-docs-guidance/repo/cmd/server/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+
+	"github.com/example/notify/internal/notify"
+)
+
+func main() {
+	svc := notify.NewService()
+
+	http.HandleFunc("/send", func(w http.ResponseWriter, r *http.Request) {
+		channel := r.URL.Query().Get("channel")
+		msg := r.URL.Query().Get("msg")
+		if err := svc.Send(channel, msg); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		fmt.Fprintln(w, "sent")
+	})
+
+	log.Println("listening on :8080")
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}

eval/code/cases/002-docs-guidance/repo/go.mod

@@ -0,0 +1,3 @@
+module github.com/example/notify
+
+go 1.21

eval/code/cases/002-docs-guidance/repo/internal/notify/notify.go

@@ -0,0 +1,30 @@
+package notify
+
+import "fmt"
+
+// Service handles sending notifications to various channels.
+type Service struct {
+	sent []string
+}
+
+// NewService creates a notification service.
+func NewService() *Service {
+	return &Service{}
+}
+
+// Send delivers a message to the given channel.
+func (s *Service) Send(channel, message string) error {
+	if channel == "" {
+		return fmt.Errorf("channel is required")
+	}
+	if message == "" {
+		return fmt.Errorf("message is required")
+	}
+	s.sent = append(s.sent, fmt.Sprintf("%s: %s", channel, message))
+	return nil
+}
+
+// SentCount returns the number of messages sent.
+func (s *Service) SentCount() int {
+	return len(s.sent)
+}