diff --git a/.bully.yml b/.bully.yml index 24821fa..5d11040 100644 --- a/.bully.yml +++ b/.bully.yml @@ -97,10 +97,12 @@ rules: ruff-clean: description: > - Python files must pass `ruff check`. --force-exclude makes ruff - respect pyproject's extend-exclude even when a single file is - passed, so test fixtures under bench/fixtures/ are skipped. + Python files must pass `ruff check`. Requires ruff on PATH (see + the `dev` extras in pyproject.toml). When ruff isn't installed the + rule passes with a stderr hint rather than blocking edits. + --force-exclude makes ruff respect pyproject's extend-exclude so + test fixtures under bench/fixtures/ are skipped. engine: script scope: "*.py" severity: error - script: "ruff check --force-exclude {file}" + script: "command -v ruff >/dev/null 2>&1 || { echo 'bully: ruff not on PATH; pip install -e .[dev]' >&2; exit 0; }; ruff check --force-exclude {file}" diff --git a/pipeline/bench.py b/pipeline/bench.py index 2b03e2b..852e0a8 100644 --- a/pipeline/bench.py +++ b/pipeline/bench.py @@ -86,6 +86,11 @@ def discover_fixtures(root: Path) -> list[Fixture]: BENCH_MODEL = "claude-sonnet-4-6" +# USD per million tokens (Sonnet 4.6). Used to render the --full summary's +# cost column. Bump alongside any BENCH_MODEL change. +BENCH_INPUT_PRICE_PER_MTOK = 3.0 +BENCH_OUTPUT_PRICE_PER_MTOK = 15.0 + def _repo_root() -> Path: """Return the project root (directory holding the `agents/` dir). @@ -146,6 +151,44 @@ def count_tokens(payload: dict, *, system: str, use_api: bool = True) -> tuple[i return len(json.dumps(payload)) + len(system), "proxy" +def full_dispatch(payload: dict, *, system: str, max_tokens: int = 1024) -> tuple[int, int, str]: + """Make a real `messages.create` call to capture input + output tokens. + + Returns (input_tokens, output_tokens, method) where method is 'full' + on a real round-trip. Falls back to count_tokens for input and + `output_tokens=0` if the API call can't be made (no key, no SDK, or + transient failure). Method in the fallback case is whatever + count_tokens returned ('count_tokens' or 'proxy'). + + Real model cost is paid each call -- use sparingly (e.g. one bench + run for calibration, not per-edit). + """ + api_key = os.environ.get("ANTHROPIC_API_KEY") + anthropic = _import_anthropic() + if api_key and anthropic is not None: + try: + client = anthropic.Anthropic(api_key=api_key) + resp = client.messages.create( + model=BENCH_MODEL, + max_tokens=max_tokens, + system=system, + messages=[{"role": "user", "content": json.dumps(payload)}], + ) + return int(resp.usage.input_tokens), int(resp.usage.output_tokens), "full" + except Exception: + pass + tokens, method = count_tokens(payload, system=system, use_api=True) + return tokens, 0, method + + +def _estimate_cost_usd(input_tokens: int, output_tokens: int) -> float: + """Rough USD cost using BENCH_*_PRICE_PER_MTOK constants.""" + return ( + input_tokens * BENCH_INPUT_PRICE_PER_MTOK / 1_000_000 + + output_tokens * BENCH_OUTPUT_PRICE_PER_MTOK / 1_000_000 + ) + + class PhaseTimer: """Callable that records elapsed time for each named phase. @@ -200,10 +243,15 @@ def run_fixture( iterations: int = 5, use_api: bool = True, skip_cold_start: bool = False, + full: bool = False, ) -> dict: """Run one fixture: warm + N timed + cold-start + token count. Returns a per-fixture result dict suitable for the history JSONL. + + When `full=True` and the Anthropic SDK + API key are available, makes + one real `messages.create` round-trip per fixture to capture real + output tokens. Costs actual model spend -- use for calibration runs. """ # Import here to avoid circular import at module load. # When bench runs as pipeline.bench (package context), `import pipeline` @@ -278,11 +326,23 @@ def run_fixture( if semantic: system = load_evaluator_system_prompt() payload = pl.build_semantic_payload(fx.file_path, fx.diff, passed, semantic) - tokens, method = count_tokens( - payload["_evaluator_input"], system=system, use_api=use_api - ) + if full: + input_tokens, output_tokens, method = full_dispatch( + payload["_evaluator_input"], system=system + ) + tokens_record = { + "input": input_tokens, + "output": output_tokens, + "method": method, + "cost_usd": _estimate_cost_usd(input_tokens, output_tokens), + } + else: + input_tokens, method = count_tokens( + payload["_evaluator_input"], system=system, use_api=use_api + ) + tokens_record = {"input": input_tokens, "method": method} else: - tokens, method = 0, "n/a-no-semantic-rules" + tokens_record = {"input": 0, "method": "n/a-no-semantic-rules"} return { "name": fx.name, @@ -291,7 +351,7 @@ def run_fixture( "wall_ms_p95": _percentile(wall_ms, 95), "phases_ms": phases_ms, "cold_start_ms": cold_start_ms, - "tokens": {"input": tokens, "method": method}, + "tokens": tokens_record, } finally: if prior_trust is None: @@ -343,8 +403,14 @@ def run_mode_a( iterations: int = 5, skip_cold_start: bool = False, emit_json: bool = False, + full: bool = False, ) -> int: - """Run every fixture in `fixtures_dir`, append a record to `history_path`.""" + """Run every fixture in `fixtures_dir`, append a record to `history_path`. + + When `full=True`, each fixture's semantic payload triggers a real + Anthropic `messages.create` call so the record carries real output + tokens (and a per-fixture `cost_usd` estimate). Costs real money. + """ fixtures = discover_fixtures(fixtures_dir) if not fixtures: sys.stderr.write(f"bench: no fixtures found in {fixtures_dir}\n") @@ -358,14 +424,26 @@ def run_mode_a( iterations=iterations, use_api=use_api, skip_cold_start=skip_cold_start, + full=full, ) ) total_wall = sum(r["wall_ms_p50"] for r in results) cold_vals = [r["cold_start_ms"] for r in results if r.get("cold_start_ms") is not None] total_cold = sum(cold_vals) if cold_vals else None - total_tokens = sum(r["tokens"]["input"] for r in results) + total_input_tokens = sum(r["tokens"]["input"] for r in results) + total_output_tokens = sum(r["tokens"].get("output", 0) for r in results) + total_cost_usd = sum(r["tokens"].get("cost_usd", 0.0) for r in results) methods = {r["tokens"]["method"] for r in results if r["tokens"]["input"]} + aggregates = { + "total_wall_ms_p50": total_wall, + "total_cold_start_ms": total_cold, + "total_input_tokens": total_input_tokens, + "tokens_method": next(iter(methods)) if len(methods) == 1 else "mixed", + } + if full: + aggregates["total_output_tokens"] = total_output_tokens + aggregates["total_cost_usd"] = total_cost_usd record = { "ts": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), "git_sha": _git_sha(), @@ -374,12 +452,7 @@ def run_mode_a( "anthropic_sdk_version": _anthropic_sdk_version(), "machine": f"{platform.system().lower()}-{platform.release()}", "fixtures": results, - "aggregates": { - "total_wall_ms_p50": total_wall, - "total_cold_start_ms": total_cold, - "total_input_tokens": total_tokens, - "tokens_method": next(iter(methods)) if len(methods) == 1 else "mixed", - }, + "aggregates": aggregates, } history_path.parent.mkdir(parents=True, exist_ok=True) with history_path.open("a", encoding="utf-8") as f: @@ -397,21 +470,41 @@ def _print_mode_a_summary(record: dict) -> None: print(f"bench run @ {record['ts']} (sha={record['git_sha'] or '?'})") print(f" python={record['python_version']} machine={record['machine']}") print() - print(f" {'fixture':<32} {'wall_p50_ms':>12} {'cold_ms':>10} {'tokens':>9} method") + is_full = "total_cost_usd" in record["aggregates"] + if is_full: + header = ( + f" {'fixture':<32} {'wall_p50_ms':>12} {'cold_ms':>10} " + f"{'in_tok':>8} {'out_tok':>8} {'cost_usd':>10} method" + ) + else: + header = f" {'fixture':<32} {'wall_p50_ms':>12} {'cold_ms':>10} {'tokens':>9} method" + print(header) for r in record["fixtures"]: cold = r.get("cold_start_ms") cold_str = f"{cold:.1f}" if isinstance(cold, (int, float)) else "-" - print( - f" {r['name']:<32} {r['wall_ms_p50']:>12.2f} {cold_str:>10} " - f"{r['tokens']['input']:>9} {r['tokens']['method']}" - ) + tok = r["tokens"] + if is_full: + out_tok = tok.get("output", 0) + cost = tok.get("cost_usd", 0.0) + print( + f" {r['name']:<32} {r['wall_ms_p50']:>12.2f} {cold_str:>10} " + f"{tok['input']:>8} {out_tok:>8} {cost:>10.4f} {tok['method']}" + ) + else: + print( + f" {r['name']:<32} {r['wall_ms_p50']:>12.2f} {cold_str:>10} " + f"{tok['input']:>9} {tok['method']}" + ) agg = record["aggregates"] print() - print( + line = ( f" totals: wall_p50={agg['total_wall_ms_p50']:.1f}ms " f"cold={agg['total_cold_start_ms'] or 0:.1f}ms " f"tokens={agg['total_input_tokens']} ({agg['tokens_method']})" ) + if is_full: + line += f" output={agg['total_output_tokens']} cost=${agg['total_cost_usd']:.4f}" + print(line) def run_compare(*, history_path: Path) -> int: @@ -593,6 +686,14 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Skip Anthropic API call; use char-count proxy for token counts.", ) + parser.add_argument( + "--full", + action="store_true", + help=( + "Mode A only: make one real Anthropic messages.create per fixture " + "to capture real output tokens + dollar cost. Costs real money." + ), + ) parser.add_argument( "--fixtures-dir", default="bench/fixtures", @@ -620,6 +721,7 @@ def main(argv: list[str] | None = None) -> int: history_path=Path(args.history), use_api=not args.no_tokens, emit_json=args.json, + full=args.full, ) diff --git a/pipeline/tests/test_bench.py b/pipeline/tests/test_bench.py index 6fee771..6bbef90 100644 --- a/pipeline/tests/test_bench.py +++ b/pipeline/tests/test_bench.py @@ -585,3 +585,162 @@ def test_real_fixtures_complete_successfully(tmp_path, monkeypatch): # Auto-generated skip fixture should short-circuit (wall time near zero). skip_fx = next(f for f in record["fixtures"] if "auto-generated-skip" in f["name"]) assert skip_fx["wall_ms_p50"] < 10.0 + + +def test_full_dispatch_real_api_path(monkeypatch): + """full_dispatch calls messages.create and returns (input, output, 'full').""" + import bench + + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + + class FakeUsage: + input_tokens = 1200 + output_tokens = 85 + + class FakeResp: + usage = FakeUsage() + + class FakeMessages: + def create(self, **kwargs): + assert kwargs["model"] == bench.BENCH_MODEL + assert kwargs["max_tokens"] == 1024 + assert kwargs["system"] == "s" + return FakeResp() + + class FakeClient: + def __init__(self, api_key=None): + self.messages = FakeMessages() + + class FakeAnthropic: + Anthropic = FakeClient + + monkeypatch.setattr(bench, "_import_anthropic", lambda: FakeAnthropic) + + input_tok, output_tok, method = bench.full_dispatch({"a": 1}, system="s") + assert input_tok == 1200 + assert output_tok == 85 + assert method == "full" + + +def test_full_dispatch_falls_back_to_count_tokens_on_failure(monkeypatch): + """Any exception from messages.create falls back to (count_tokens, 0, method).""" + import bench + + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.setattr(bench, "_import_anthropic", lambda: None) + + input_tok, output_tok, method = bench.full_dispatch({"a": 1}, system="s") + assert output_tok == 0 + assert method == "proxy" + # Proxy = len(json.dumps(payload)) + len(system) + import json as _json + + assert input_tok == len(_json.dumps({"a": 1})) + len("s") + + +def test_estimate_cost_usd_matches_pricing_constants(): + """_estimate_cost_usd uses the published per-million rates.""" + import bench + + # 1M input + 1M output should sum the two constants. + cost = bench._estimate_cost_usd(1_000_000, 1_000_000) + assert cost == bench.BENCH_INPUT_PRICE_PER_MTOK + bench.BENCH_OUTPUT_PRICE_PER_MTOK + + +def test_run_fixture_full_mode_captures_output_and_cost(tmp_path, monkeypatch): + """run_fixture(full=True) records input + output + cost_usd.""" + import bench + from bench import Fixture, run_fixture + + fx_dir = tmp_path / "fx" + fx_dir.mkdir() + cfg = fx_dir / "config.yml" + cfg.write_text( + "rules:\n" + " sem:\n" + " description: a semantic rule\n" + " engine: semantic\n" + " scope: '**/*.py'\n" + " severity: warning\n" + ) + (fx_dir / "fixture.json").write_text( + '{"name": "fx", "description": "x", "file_path": "a.py", "edit_type": "Edit", "diff": ""}' + ) + target = tmp_path / "a.py" + target.write_text("x = 1\n") + monkeypatch.chdir(tmp_path) + + # Stub full_dispatch so the test doesn't hit the real API. + monkeypatch.setattr( + bench, + "full_dispatch", + lambda payload, *, system, max_tokens=1024: (1500, 100, "full"), + ) + + fx = Fixture( + name="fx", + description="x", + file_path=str(target), + edit_type="Edit", + diff="", + config_path=cfg, + ) + result = run_fixture(fx, iterations=2, skip_cold_start=True, full=True) + assert result["tokens"]["method"] == "full" + assert result["tokens"]["input"] == 1500 + assert result["tokens"]["output"] == 100 + # cost = 1500 * 3/M + 100 * 15/M + expected = 1500 * 3.0 / 1_000_000 + 100 * 15.0 / 1_000_000 + assert abs(result["tokens"]["cost_usd"] - expected) < 1e-12 + + +def test_run_mode_a_full_mode_aggregates_output_and_cost(tmp_path, monkeypatch): + """run_mode_a(full=True) aggregates output tokens + cost_usd.""" + import json as _json + + import bench + from bench import run_mode_a + + fixtures_root = tmp_path / "fixtures" + for name in ("a", "b"): + d = fixtures_root / name + d.mkdir(parents=True) + (d / "config.yml").write_text( + "rules:\n" + " sem:\n" + " description: d\n" + " engine: semantic\n" + " scope: '*.py'\n" + " severity: warning\n" + ) + (d / "fixture.json").write_text( + '{"name": "' + name + '", "description": "x",' + ' "file_path": "x.py", "edit_type": "Edit", "diff": ""}' + ) + (tmp_path / "x.py").write_text("x = 1\n") + monkeypatch.chdir(tmp_path) + + monkeypatch.setattr( + bench, + "full_dispatch", + lambda payload, *, system, max_tokens=1024: (500, 50, "full"), + ) + + history_path = tmp_path / "history.jsonl" + rc = run_mode_a( + fixtures_dir=fixtures_root, + history_path=history_path, + iterations=2, + skip_cold_start=True, + full=True, + emit_json=True, + ) + assert rc == 0 + record = _json.loads(history_path.read_text().strip().splitlines()[-1]) + agg = record["aggregates"] + assert agg["total_input_tokens"] == 1000 + assert agg["total_output_tokens"] == 100 + assert agg["tokens_method"] == "full" + # 1000 * 3/M + 100 * 15/M + expected = 1000 * 3.0 / 1_000_000 + 100 * 15.0 / 1_000_000 + assert abs(agg["total_cost_usd"] - expected) < 1e-12