Anthropic custom pricing

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "agent-eval"
-version = "0.1.48"
+version = "0.1.49"
 description = "Agent evaluation toolkit"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -14,7 +14,7 @@ dependencies = [
   "inspect-ai==0.3.203",
   # pin litellm so that we know what model costs we're using
   # see the Development.md doc before changing
-  "litellm>=1.67.4.post1,<=1.83.10",
+  "litellm>=1.67.4.post1,<=1.83.13",
   "pydantic>=2.0.0",
   # For leaderboard
   "huggingface_hub",

src/agenteval/local_cost.py

@@ -40,13 +40,27 @@ class CostPerTokenWithCache(BaseModel):
     input_cost_per_token: float
     output_cost_per_token: float
     cache_read_input_token_cost: float
+    cache_write_input_token_cost: float | None = None
 
 
 # Like CUSTOM_PRICING, but for models that also have a cache read discount.
 # cost_per_token with usage_object doesn't work for these models in litellm 1.75.8,
 # so costs are computed manually in compute_model_cost.
 # key represents model name as found in inspect model_usage
 CUSTOM_PRICING_WITH_CACHE = {
+    # Costs from https://platform.claude.com/docs/en/about-claude/pricing
+    "claude-3-5-haiku-20241022": CostPerTokenWithCache(
+        input_cost_per_token=8e-07,
+        output_cost_per_token=4e-06,
+        cache_read_input_token_cost=8e-08,
+        cache_write_input_token_cost=1e-06,
+    ),
+    "anthropic/claude-3-5-haiku-20241022": CostPerTokenWithCache(
+        input_cost_per_token=8e-07,
+        output_cost_per_token=4e-06,
+        cache_read_input_token_cost=8e-08,
+        cache_write_input_token_cost=1e-06,
+    ),
     # costs from https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart
     "moonshotai/kimi-k2.5-0127": CostPerTokenWithCache(
         input_cost_per_token=6e-07,

src/agenteval/log.py

@@ -117,10 +117,17 @@ def compute_model_cost(model_usages: list[ModelUsageWithName]) -> float | None:
 
                 pricing = CUSTOM_PRICING_WITH_CACHE[model_usage.model]
                 cache_read_tokens = model_usage.usage.input_tokens_cache_read or 0
-                text_tokens = input_tokens - cache_read_tokens
+                cache_write_tokens = model_usage.usage.input_tokens_cache_write or 0
+                cache_write_cost = pricing.cache_write_input_token_cost
+                if cache_write_cost is None:
+                    text_tokens = input_tokens - cache_read_tokens
+                    cache_write_cost = 0
+                else:
+                    text_tokens = input_tokens
                 prompt_cost = (
                     text_tokens * pricing.input_cost_per_token
                     + cache_read_tokens * pricing.cache_read_input_token_cost
+                    + cache_write_tokens * cache_write_cost
                 )
                 completion_cost = output_tokens * pricing.output_cost_per_token