diff --git a/pyproject.toml b/pyproject.toml index edc9989..8a91fd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agent-eval" -version = "0.1.50" +version = "0.1.51" description = "Agent evaluation toolkit" readme = "README.md" requires-python = ">=3.10" @@ -14,7 +14,7 @@ dependencies = [ "inspect-ai==0.3.203", # pin litellm so that we know what model costs we're using # see the Development.md doc before changing - "litellm>=1.67.4.post1,<=1.83.13", + "litellm>=1.67.4.post1,<=1.88.1", "pydantic>=2.0.0", # For leaderboard "huggingface_hub", diff --git a/src/agenteval/cli.py b/src/agenteval/cli.py index f4adafa..d7a6e9c 100644 --- a/src/agenteval/cli.py +++ b/src/agenteval/cli.py @@ -160,7 +160,9 @@ def prep_litellm_cost_map(): # This snippet is mostly lifted from # https://github.com/BerriAI/litellm/blob/b9621c760d3355e06dd17ec89b9eb6776755392e/litellm/litellm_core_utils/get_model_cost_map.py#L16 # See the Development.md before changing. - desired_model_costs_url = "https://raw.githubusercontent.com/BerriAI/litellm/319193604cfccb1092a17f58ec4ca049115da446/litellm/model_prices_and_context_window_backup.json" + # SHA below is the v1.88.1 release tag of BerriAI/litellm (upper bound of the + # litellm pin in pyproject.toml); keep the two in sync when bumping. + desired_model_costs_url = "https://raw.githubusercontent.com/BerriAI/litellm/9c117a58be8fb8066e833aa92dd98fdf15771e50/litellm/model_prices_and_context_window_backup.json" response = httpx.get(desired_model_costs_url, timeout=5) response.raise_for_status() desired_model_costs = response.json()