🔧(evals) add run_eval management command

.codespellrc

@@ -11,6 +11,7 @@ skip =
     *.tsbuildinfo,
     **/uv.lock,
     ./docker/files/etc/mime.types,
+    ./src/backend/chat/evals,
 check-filenames = true
 ignore-words-list =
     afterAll,

.gitignore

@@ -71,6 +71,12 @@ src/backend/core/templates/mail/
 db.sqlite3
 .mypy_cache
 
+# Eval runs (local experiments — only baseline snapshots under baselines/ are committed)
+src/backend/chat/evals/runs/**
+!src/backend/chat/evals/runs/.gitkeep
+src/backend/chat/evals/dashboard/dashboard.html
+
+
 # Site media
 /data/
 

CHANGELOG.md

@@ -394,6 +394,7 @@ and this project adheres to
 - ✨(langfuse) allow user to score messages from LLM #6
 - ✨(onboarding) add activation code logic for launch #62
 - 💄(chat) add code highlighting for LLM responses #67
+- 🔧(evals) add run_evals management command
 
 [unreleased]: https://github.com/suitenumerique/conversations/compare/v0.0.20...main
 [0.0.20]: https://github.com/suitenumerique/conversations/compare/v0.0.20

Makefile

@@ -50,6 +50,13 @@ COMPOSE_RUN_CROWDIN = $(COMPOSE_RUN) crowdin crowdin
 MANAGE              = $(COMPOSE_RUN_APP) python manage.py
 MAIL_YARN           = $(COMPOSE_RUN) -w /app/src/mail node yarn
 
+# -- Evals (git metadata read on the host; Docker has no .git mount)
+EVAL_GIT_COMMIT     = $(shell git rev-parse HEAD 2>/dev/null)
+EVAL_GIT_BRANCH     = $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
+EVAL_GIT_DIRTY      = $(shell git status --porcelain 2>/dev/null | grep -q . && echo 1 || echo 0)
+EVAL_GIT_ENV        = -e EVAL_GIT_COMMIT=$(EVAL_GIT_COMMIT) -e EVAL_GIT_BRANCH=$(EVAL_GIT_BRANCH) -e EVAL_GIT_DIRTY=$(EVAL_GIT_DIRTY)
+COMPOSE_RUN_EVAL    = $(COMPOSE_RUN) $(EVAL_GIT_ENV) app-dev
+
 # -- Frontend
 PATH_FRONT          = ./src/frontend
 PATH_FRONT_CONVERSATIONS  = $(PATH_FRONT)/apps/conversations
@@ -254,6 +261,30 @@ fetch_model_health: ## check the health of the models (usage: make fetch_model_h
 	@$(MANAGE) fetch_model_health $(FETCH_MODEL_HEALTH_ARGS)
 .PHONY: fetch_model_health
 
+eval: ## run behavioral evals (usage: make eval EVAL_ARGS="--dataset url_hallucination --verbose")
+	@$(COMPOSE_RUN_EVAL) python manage.py run_evals $(EVAL_ARGS)
+.PHONY: eval
+
+eval-debug: ## run behavioral evals with debugpy on port 5678 (attach VS Code before the command runs)
+	@$(COMPOSE) run --rm -p 5678:5678 $(EVAL_GIT_ENV) app-dev python -m debugpy --listen 0.0.0.0:5678 --wait-for-client manage.py run_evals $(EVAL_ARGS)
+.PHONY: eval-debug
+
+eval-baseline: ## mark a saved eval run as baseline (usage: make eval-baseline EVAL_ARGS="--run latest")
+	@$(MANAGE) create_eval_baseline $(EVAL_ARGS)
+.PHONY: eval-baseline
+
+eval-compare: ## compare saved eval runs (usage: make eval-compare EVAL_ARGS="--fail-on-regression")
+	@$(MANAGE) compare_evals $(EVAL_ARGS)
+.PHONY: eval-compare
+
+eval-dashboard: ## generate the HTML dashboard for saved eval runs
+	@$(MANAGE) generate_eval_dashboard
+.PHONY: eval-dashboard
+
+eval-reset: ## wipe saved eval runs, baselines, and dashboard (usage: make eval-reset EVAL_ARGS="--keep-baselines")
+	@$(MANAGE) reset_evals $(EVAL_ARGS)
+.PHONY: eval-reset
+
 # -- Database
 
 dbshell: ## connect to database shell

src/backend/.pylintrc

@@ -19,7 +19,7 @@ ignore-patterns=
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
-jobs=0
+jobs=1
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.

src/backend/chat/agents/base.py

@@ -127,6 +127,12 @@ def prepare_custom_model(configuration: "chat.llm_configuration.LLModel"):
     """
     # pylint: disable=import-outside-toplevel
 
+    # Model settings declared in the LLM configuration file (temperature, top_p,
+    # max_tokens, ...). Unset fields are omitted so provider defaults still apply.
+    _configured_settings = (
+        configuration.settings.model_dump(exclude_none=True) if configuration.settings else {}
+    )
+
     match configuration.provider.kind:
         case "mistral":
             import pydantic_ai.models.mistral as mistral_models  # noqa: PLC0415
@@ -155,13 +161,19 @@ def prepare_custom_model(configuration: "chat.llm_configuration.LLModel"):
                 # null and require a valid value or omission, so we pass neutral
                 # defaults to keep the request valid.
                 settings=mistral_models.MistralModelSettings(
-                    presence_penalty=0.0,
-                    frequency_penalty=0.0,
-                    stop_sequences=[],
+                    **{
+                        "presence_penalty": 0.0,
+                        "frequency_penalty": 0.0,
+                        "stop_sequences": [],
+                        **_configured_settings,
+                    }
                 ),
             )
         case "openai":
-            from pydantic_ai.models.openai import OpenAIChatModel  # noqa: PLC0415
+            from pydantic_ai.models.openai import (  # noqa: PLC0415
+                OpenAIChatModel,
+                OpenAIChatModelSettings,
+            )
             from pydantic_ai.profiles.openai import OpenAIModelProfile  # noqa: PLC0415
             from pydantic_ai.providers.openai import OpenAIProvider  # noqa: PLC0415
 
@@ -186,6 +198,10 @@ def prepare_custom_model(configuration: "chat.llm_configuration.LLModel"):
             else:
                 profile = None
 
+            _openai_settings = (
+                OpenAIChatModelSettings(**_configured_settings) if _configured_settings else None
+            )
+
             if configuration.provider.co2_handling == "albert":
                 return AlbertOpenAIChatModel(
                     model_name=configuration.model_name,
@@ -194,6 +210,7 @@ def prepare_custom_model(configuration: "chat.llm_configuration.LLModel"):
                         base_url=configuration.provider.base_url,
                         api_key=configuration.provider.api_key,
                     ),
+                    settings=_openai_settings,
                 )
 
             return OpenAIChatModel(
@@ -203,6 +220,7 @@ def prepare_custom_model(configuration: "chat.llm_configuration.LLModel"):
                     base_url=configuration.provider.base_url,
                     api_key=configuration.provider.api_key,
                 ),
+                settings=_openai_settings,
             )
         case _:
             raise ImproperlyConfigured(