diff --git a/pyproject.toml b/pyproject.toml index 5df092b..c3b69c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agent-eval" -version = "0.1.48" +version = "0.1.49" description = "Agent evaluation toolkit" readme = "README.md" requires-python = ">=3.10" @@ -14,7 +14,7 @@ dependencies = [ "inspect-ai==0.3.203", # pin litellm so that we know what model costs we're using # see the Development.md doc before changing - "litellm>=1.67.4.post1,<=1.83.10", + "litellm>=1.67.4.post1,<=1.83.13", "pydantic>=2.0.0", # For leaderboard "huggingface_hub", diff --git a/src/agenteval/local_cost.py b/src/agenteval/local_cost.py index 5b7cafd..5fd20c3 100644 --- a/src/agenteval/local_cost.py +++ b/src/agenteval/local_cost.py @@ -40,6 +40,7 @@ class CostPerTokenWithCache(BaseModel): input_cost_per_token: float output_cost_per_token: float cache_read_input_token_cost: float + cache_write_input_token_cost: float | None = None # Like CUSTOM_PRICING, but for models that also have a cache read discount. @@ -47,6 +48,19 @@ class CostPerTokenWithCache(BaseModel): # so costs are computed manually in compute_model_cost. # key represents model name as found in inspect model_usage CUSTOM_PRICING_WITH_CACHE = { + # Costs from https://platform.claude.com/docs/en/about-claude/pricing + "claude-3-5-haiku-20241022": CostPerTokenWithCache( + input_cost_per_token=8e-07, + output_cost_per_token=4e-06, + cache_read_input_token_cost=8e-08, + cache_write_input_token_cost=1e-06, + ), + "anthropic/claude-3-5-haiku-20241022": CostPerTokenWithCache( + input_cost_per_token=8e-07, + output_cost_per_token=4e-06, + cache_read_input_token_cost=8e-08, + cache_write_input_token_cost=1e-06, + ), # costs from https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart "moonshotai/kimi-k2.5-0127": CostPerTokenWithCache( input_cost_per_token=6e-07, diff --git a/src/agenteval/log.py b/src/agenteval/log.py index 8120909..5b5c44f 100644 --- a/src/agenteval/log.py +++ b/src/agenteval/log.py @@ -117,10 +117,17 @@ def compute_model_cost(model_usages: list[ModelUsageWithName]) -> float | None: pricing = CUSTOM_PRICING_WITH_CACHE[model_usage.model] cache_read_tokens = model_usage.usage.input_tokens_cache_read or 0 - text_tokens = input_tokens - cache_read_tokens + cache_write_tokens = model_usage.usage.input_tokens_cache_write or 0 + cache_write_cost = pricing.cache_write_input_token_cost + if cache_write_cost is None: + text_tokens = input_tokens - cache_read_tokens + cache_write_cost = 0 + else: + text_tokens = input_tokens prompt_cost = ( text_tokens * pricing.input_cost_per_token + cache_read_tokens * pricing.cache_read_input_token_cost + + cache_write_tokens * cache_write_cost ) completion_cost = output_tokens * pricing.output_cost_per_token