From 274f525cbac7edd481749ae7d8a82c8350508be4 Mon Sep 17 00:00:00 2001 From: stefanc-ai2 Date: Fri, 24 Apr 2026 13:47:47 -0700 Subject: [PATCH 1/2] custom pricing --- pyproject.toml | 2 +- src/agenteval/local_cost.py | 21 +++++++++++++++++++++ src/agenteval/log.py | 9 ++++++++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5df092b..3b0e46e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agent-eval" -version = "0.1.48" +version = "0.1.49" description = "Agent evaluation toolkit" readme = "README.md" requires-python = ">=3.10" diff --git a/src/agenteval/local_cost.py b/src/agenteval/local_cost.py index 5b7cafd..a1c9440 100644 --- a/src/agenteval/local_cost.py +++ b/src/agenteval/local_cost.py @@ -26,6 +26,13 @@ ), # cost is for xai/grok-3 https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json "grok-3": CostPerToken(input_cost_per_token=3e-06, output_cost_per_token=1.5e-05), + # Flat historical Anthropic API pricing for Sonnet 4 calls without the 1M context beta. + "claude-sonnet-4-20250514": CostPerToken( + input_cost_per_token=3e-06, output_cost_per_token=1.5e-05 + ), + "anthropic/claude-sonnet-4-20250514": CostPerToken( + input_cost_per_token=3e-06, output_cost_per_token=1.5e-05 + ), # using https://artificialanalysis.ai/models/qwen3-8b-instruct "Qwen3-8B-SciQA-SFT": CostPerToken( input_cost_per_token=1.8e-07, output_cost_per_token=7e-07 @@ -40,6 +47,7 @@ class CostPerTokenWithCache(BaseModel): input_cost_per_token: float output_cost_per_token: float cache_read_input_token_cost: float + cache_write_input_token_cost: float | None = None # Like CUSTOM_PRICING, but for models that also have a cache read discount. @@ -47,6 +55,19 @@ class CostPerTokenWithCache(BaseModel): # so costs are computed manually in compute_model_cost. # key represents model name as found in inspect model_usage CUSTOM_PRICING_WITH_CACHE = { + # Costs from https://platform.claude.com/docs/en/about-claude/pricing + "claude-3-5-haiku-20241022": CostPerTokenWithCache( + input_cost_per_token=8e-07, + output_cost_per_token=4e-06, + cache_read_input_token_cost=8e-08, + cache_write_input_token_cost=1e-06, + ), + "anthropic/claude-3-5-haiku-20241022": CostPerTokenWithCache( + input_cost_per_token=8e-07, + output_cost_per_token=4e-06, + cache_read_input_token_cost=8e-08, + cache_write_input_token_cost=1e-06, + ), # costs from https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart "moonshotai/kimi-k2.5-0127": CostPerTokenWithCache( input_cost_per_token=6e-07, diff --git a/src/agenteval/log.py b/src/agenteval/log.py index 8120909..5b5c44f 100644 --- a/src/agenteval/log.py +++ b/src/agenteval/log.py @@ -117,10 +117,17 @@ def compute_model_cost(model_usages: list[ModelUsageWithName]) -> float | None: pricing = CUSTOM_PRICING_WITH_CACHE[model_usage.model] cache_read_tokens = model_usage.usage.input_tokens_cache_read or 0 - text_tokens = input_tokens - cache_read_tokens + cache_write_tokens = model_usage.usage.input_tokens_cache_write or 0 + cache_write_cost = pricing.cache_write_input_token_cost + if cache_write_cost is None: + text_tokens = input_tokens - cache_read_tokens + cache_write_cost = 0 + else: + text_tokens = input_tokens prompt_cost = ( text_tokens * pricing.input_cost_per_token + cache_read_tokens * pricing.cache_read_input_token_cost + + cache_write_tokens * cache_write_cost ) completion_cost = output_tokens * pricing.output_cost_per_token From 6b957f0bc42eece66d9143c01adfa2a89449f36e Mon Sep 17 00:00:00 2001 From: stefanc-ai2 Date: Fri, 24 Apr 2026 16:51:58 -0700 Subject: [PATCH 2/2] remove sonnet 4, up litellm --- pyproject.toml | 2 +- src/agenteval/local_cost.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3b0e46e..c3b69c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "inspect-ai==0.3.203", # pin litellm so that we know what model costs we're using # see the Development.md doc before changing - "litellm>=1.67.4.post1,<=1.83.10", + "litellm>=1.67.4.post1,<=1.83.13", "pydantic>=2.0.0", # For leaderboard "huggingface_hub", diff --git a/src/agenteval/local_cost.py b/src/agenteval/local_cost.py index a1c9440..5fd20c3 100644 --- a/src/agenteval/local_cost.py +++ b/src/agenteval/local_cost.py @@ -26,13 +26,6 @@ ), # cost is for xai/grok-3 https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json "grok-3": CostPerToken(input_cost_per_token=3e-06, output_cost_per_token=1.5e-05), - # Flat historical Anthropic API pricing for Sonnet 4 calls without the 1M context beta. - "claude-sonnet-4-20250514": CostPerToken( - input_cost_per_token=3e-06, output_cost_per_token=1.5e-05 - ), - "anthropic/claude-sonnet-4-20250514": CostPerToken( - input_cost_per_token=3e-06, output_cost_per_token=1.5e-05 - ), # using https://artificialanalysis.ai/models/qwen3-8b-instruct "Qwen3-8B-SciQA-SFT": CostPerToken( input_cost_per_token=1.8e-07, output_cost_per_token=7e-07