From b50d520d5184da084d5ffc3c3e2fd5c89fb151d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-F=C3=A9lix=20Nothias?= Date: Mon, 29 Jun 2026 15:31:30 +0200 Subject: [PATCH 1/2] feat(verifier): add optional cheaper model tier for mechanical judge calls --- config.py | 10 ++++ docs/reference/configuration.md | 1 + sources/evaluators/base.py | 63 +++++++++++++++++++----- sources/evaluators/verifier_claims.py | 10 ++-- sources/evaluators/verifier_per_claim.py | 10 ++-- tests/judge_extraction_test.py | 38 ++++++++++++++ tests/verifier_importance_test.py | 2 +- 7 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 tests/judge_extraction_test.py diff --git a/config.py b/config.py index f37c4dd..2289ed6 100644 --- a/config.py +++ b/config.py @@ -50,6 +50,11 @@ def __init__(self): self.workflow_llm_model: str = "openrouter/z-ai/glm-5.1" self.smolagent_model_id: str = "deepseek/deepseek-v4-flash" self.judge_model = "openrouter/mistralai/mixtral-8x22b-instruct" + # Optional cheaper model for the verifier's mechanical calls (claim + # extraction, dedup, importance rating, file selection, verifier-script + # generation). None reuses judge_model, so behaviour is unchanged + # unless set. Final claim verdicts stay on judge_model. + self.judge_extraction_model: str | None = None self.capsule_namer_model = "openrouter/deepseek/deepseek-v4-flash" self.engine_name: str = "litellm" # for smolagent @@ -221,6 +226,7 @@ def jsonify( "workflow_llm_model": self.workflow_llm_model, "smolagent_model_id": self.smolagent_model_id, "judge_model": self.judge_model, + "judge_extraction_model": self.judge_extraction_model, "engine_name": self.engine_name, "openrouter_provider": self.openrouter_provider, "prompt_planner": self.prompt_planner, @@ -262,6 +268,9 @@ def from_json(self, data: dict[str, Any]) -> None: ) self.smolagent_model_id = data.get("smolagent_model_id", self.smolagent_model_id) self.judge_model = data.get("judge_model", self.judge_model) + self.judge_extraction_model = data.get( + "judge_extraction_model", self.judge_extraction_model + ) self.engine_name = data.get("engine_name", self.engine_name) self.openrouter_provider = data.get("openrouter_provider", self.openrouter_provider) self.prompt_planner = data.get("prompt_planner", self.prompt_planner) @@ -342,6 +351,7 @@ def __str__(self) -> str: lines.append(f" workflow_llm_model={self.workflow_llm_model}") lines.append(f" smolagent_model_id={self.smolagent_model_id}") lines.append(f" judge_model={self.judge_model}") + lines.append(f" judge_extraction_model={self.judge_extraction_model or '(= judge_model)'}") lines.append(f" capsule_namer_model={self.capsule_namer_model}") lines.append(f" engine_name={self.engine_name}") lines.append(f" prompt_planner={self.prompt_planner}") diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 391642f..1eb795d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -24,6 +24,7 @@ Ports must be in `[0, 65535]` and `port_min ≤ port_max`. | `workflow_llm_model` | `str` | `openai/gpt-5.5` | Workflow synthesis. | | `smolagent_model_id` | `str` | `openrouter/deepseek/deepseek-v3.2` | Execution agents inside the sandbox. | | `judge_model` | `str` | `openai/gpt-5.5` | Verifier soft-claim verdicts. | +| `judge_extraction_model` | `str \| None` | `None` | Cheaper model for the verifier's mechanical calls (claim extraction, dedup, importance, file selection, verifier-script generation). Falls back to `judge_model` when unset. | | `capsule_namer_model` | `str` | `deepseek/deepseek-chat` | Generates human-readable capsule names. | | `engine_name` | `str` | `litellm` | SmolAgents engine — keep as `litellm`. | | `reasoning_effort` | `str` | `medium` | `minimal | low | medium | high` for models that support it. | diff --git a/sources/evaluators/base.py b/sources/evaluators/base.py index ec16387..9f65d39 100644 --- a/sources/evaluators/base.py +++ b/sources/evaluators/base.py @@ -123,17 +123,19 @@ def __init__(self, config: "Config") -> None: self.workflow_dir.mkdir(parents=True, exist_ok=True) self.judge_model = config.judge_model + # Optional cheaper tier for the verifier's mechanical extraction + # calls. Falls back to judge_model when unset, so the default + # behaviour (one model for everything) is unchanged. + self.judge_extraction_model = ( + getattr(config, "judge_extraction_model", None) or self.judge_model + ) try: - provider, model = self.judge_model.split("/", 1) if "/" in self.judge_model else ("openai", self.judge_model) - self.llm_config = LLMConfig().from_dict({ - "model": model, - "provider": provider, - "temperature": 0.2, - "reasoning_effort": config.reasoning_effort, - "max_tokens": getattr(config, 'max_tokens', 8192), - "openrouter_provider": config.openrouter_provider_for(self.judge_model), - "openrouter_quantizations": config.openrouter_quantizations_for(self.judge_model), - }) + self.llm_config = self._build_judge_llm_config(self.judge_model, config) + self.extraction_llm_config = ( + self.llm_config + if self.judge_extraction_model == self.judge_model + else self._build_judge_llm_config(self.judge_extraction_model, config) + ) except Exception as e: raise EvaluatorError(f"Failed to initialize LLM configuration: {str(e)}") from e @@ -308,13 +310,45 @@ def _get_judge_system_prompt(self) -> str: "the JSON object, no prose, no markdown fences, no commentary." ) - def _call_judge(self, uuid: str, agent_name: str, prompt: str) -> str: + def _build_judge_llm_config(self, model_id: str, config: "Config") -> LLMConfig: + """Build an LLMConfig for a judge/extraction model id ("provider/model"). + + Centralises judge LLM construction so the strong judge and the optional + cheaper extraction tier are configured identically (temperature, + reasoning effort, OpenRouter provider/quantization routing). + + Args: + model_id: Model identifier, optionally "provider/model" (defaults to + the ``openai`` provider when no "/" is present). + config: The run config (supplies reasoning_effort, max_tokens and the + OpenRouter routing lookups). + + Returns: + A configured :class:`LLMConfig` for that model. + """ + provider, model = model_id.split("/", 1) if "/" in model_id else ("openai", model_id) + return LLMConfig().from_dict({ + "model": model, + "provider": provider, + "temperature": 0.2, + "reasoning_effort": config.reasoning_effort, + "max_tokens": getattr(config, 'max_tokens', 8192), + "openrouter_provider": config.openrouter_provider_for(model_id), + "openrouter_quantizations": config.openrouter_quantizations_for(model_id), + }) + + def _call_judge(self, uuid: str, agent_name: str, prompt: str, + use_extraction_model: bool = False) -> str: """One judge round-trip; raises whatever the LLM provider raises. Args: uuid: Workflow identifier (used to scope per-uuid memory). agent_name: Logical name for this judge call (used for memory file). prompt: User-side prompt to send to the judge. + use_extraction_model: Route this call to the cheaper + ``judge_extraction_model`` tier (for mechanical extraction calls + such as claim extraction, importance rating, file selection and + verifier-script generation). Defaults to False (strong judge). Returns: Raw text response from the LLM provider. @@ -325,7 +359,7 @@ def _call_judge(self, uuid: str, agent_name: str, prompt: str) -> str: agent_name=agent_name, memory_path=memory_path, system_msg=self._get_judge_system_prompt(), - config=self.llm_config, + config=self.extraction_llm_config if use_extraction_model else self.llm_config, ) return provider(prompt) @@ -334,6 +368,7 @@ def _call_judge_for_json( uuid: str, agent_name: str, prompt: str, + use_extraction_model: bool = False, ) -> tuple[Any, str | None]: """Call the judge expecting JSON; one retry on parse failure. @@ -353,7 +388,9 @@ def _call_judge_for_json( for attempt in (1, 2): agent = agent_name if attempt == 1 else f"{agent_name}_retry" try: - raw = self._call_judge(uuid, agent, cur_prompt) + raw = self._call_judge( + uuid, agent, cur_prompt, use_extraction_model=use_extraction_model + ) except Exception as e: last_err = f"judge call failed: {type(e).__name__}: {e}" self.logger.warning(f"[{agent}] {last_err}") diff --git a/sources/evaluators/verifier_claims.py b/sources/evaluators/verifier_claims.py index b5b1191..4055bfd 100644 --- a/sources/evaluators/verifier_claims.py +++ b/sources/evaluators/verifier_claims.py @@ -129,7 +129,8 @@ def _extract_claims( prompt = source.build(source_ctx) t_src = time.time() data, err = self._call_judge_for_json( - uuid, f"verifier_extract_claims_{label}", prompt + uuid, f"verifier_extract_claims_{label}", prompt, + use_extraction_model=True, ) if err is not None: self.logger.warning( @@ -302,7 +303,8 @@ def _run_dedup_pass( """Single cheap LLM call returning only ids to drop as near-duplicates.""" prompt = self._build_dedup_prompt(goal, claims, grounding) data, err = self._call_judge_for_json( - uuid, "verifier_dedup_claims", prompt + uuid, "verifier_dedup_claims", prompt, + use_extraction_model=True, ) if err is not None or not isinstance(data, dict): self.logger.warning( @@ -351,7 +353,9 @@ def _rate_one_importance_batch( """Rate one batch; default-importance fallback on parse/call error.""" prompt = self._build_importance_batch_prompt(goal, batch, grounding) agent_name = f"verifier_rate_importance_b{batch_idx}" - data, err = self._call_judge_for_json(uuid, agent_name, prompt) + data, err = self._call_judge_for_json( + uuid, agent_name, prompt, use_extraction_model=True + ) if err is not None or not isinstance(data, dict): self.logger.warning( f"importance batch {batch_idx} failed for {uuid} " diff --git a/sources/evaluators/verifier_per_claim.py b/sources/evaluators/verifier_per_claim.py index 43cd949..8f5038d 100644 --- a/sources/evaluators/verifier_per_claim.py +++ b/sources/evaluators/verifier_per_claim.py @@ -371,7 +371,8 @@ def _llm_select_files( ) cid = str(claim.get("id", "unknown")) parsed, err = self._call_judge_for_json( - uuid, f"verifier_select_files_{cid}", prompt + uuid, f"verifier_select_files_{cid}", prompt, + use_extraction_model=True, ) if err or not isinstance(parsed, dict): self.logger.debug(f"file selection failed for {cid}: {err or 'non-dict JSON'}") @@ -485,7 +486,9 @@ def _call_and_parse_verifier( """Call the judge for a verifier spec; soft-fail to ``executable: False``.""" suffix = "" if attempt == 1 else "_retry" agent_name = f"verifier_gen_{claim['id']}{suffix}" - spec, err = self._call_judge_for_json(uuid, agent_name, prompt) + spec, err = self._call_judge_for_json( + uuid, agent_name, prompt, use_extraction_model=True + ) if err is not None: return {"executable": False, "reason": err} if not isinstance(spec, dict): @@ -805,7 +808,8 @@ def _llm_packages_needed_for_claim( """Ask the judge which missing pip packages are genuinely required.""" prompt = self._build_package_check_prompt(claim, stderr) parsed, err = self._call_judge_for_json( - uuid, f"verifier_pkg_check_{claim['id']}", prompt + uuid, f"verifier_pkg_check_{claim['id']}", prompt, + use_extraction_model=True, ) if err is not None or not isinstance(parsed, dict): self.logger.debug( diff --git a/tests/judge_extraction_test.py b/tests/judge_extraction_test.py new file mode 100644 index 0000000..10e1593 --- /dev/null +++ b/tests/judge_extraction_test.py @@ -0,0 +1,38 @@ +"""Tests for the optional cheap judge-extraction model tier. + +Covers the config surface (default and round-trip). The routing of the +verifier's mechanical judge calls through this tier is exercised by the +verifier tests; this file has no import dependency on the evaluator stack.""" + +import sys +import tempfile +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent)) + +from config import Config + + +def test_judge_extraction_defaults_to_none(): + """Unset means reuse judge_model, so behaviour is unchanged by default.""" + assert Config().judge_extraction_model is None + + +def test_judge_extraction_round_trips(tmp_path): + """A configured extraction model survives dump and load.""" + cheap = "openrouter/deepseek/deepseek-v4-flash" + config = Config() + config.judge_extraction_model = cheap + path = tmp_path / "cfg.json" + config.dump(str(path)) + + loaded = Config() + loaded.load(str(path)) + assert loaded.judge_extraction_model == cheap + + +if __name__ == "__main__": + test_judge_extraction_defaults_to_none() + with tempfile.TemporaryDirectory() as d: + test_judge_extraction_round_trips(Path(d)) + print("judge_extraction tests passed") diff --git a/tests/verifier_importance_test.py b/tests/verifier_importance_test.py index f5dd4ad..583c30d 100644 --- a/tests/verifier_importance_test.py +++ b/tests/verifier_importance_test.py @@ -185,7 +185,7 @@ def test_declare_claim_importance_drops_duplicates_and_clamps_scale() -> None: """Rater output is respected: drop ids removed, importance clamped to [1,10].""" v = _StubVerifier() - def _fake_judge(_uuid, _agent, _prompt): + def _fake_judge(_uuid, _agent, _prompt, **_kwargs): return ({ "drop_ids": ["dup"], "importance": [ From 38554aa09fbff6497e760c8bc22d07591f2c11fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-F=C3=A9lix=20Nothias?= Date: Mon, 29 Jun 2026 15:54:34 +0200 Subject: [PATCH 2/2] fix(verifier): keep quality-sensitive judge calls on the strong model --- config.py | 9 +++++---- docs/reference/configuration.md | 2 +- sources/evaluators/base.py | 14 ++++++++------ sources/evaluators/verifier_claims.py | 7 ++----- sources/evaluators/verifier_per_claim.py | 7 ++----- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/config.py b/config.py index 2289ed6..86201a6 100644 --- a/config.py +++ b/config.py @@ -50,10 +50,11 @@ def __init__(self): self.workflow_llm_model: str = "openrouter/z-ai/glm-5.1" self.smolagent_model_id: str = "deepseek/deepseek-v4-flash" self.judge_model = "openrouter/mistralai/mixtral-8x22b-instruct" - # Optional cheaper model for the verifier's mechanical calls (claim - # extraction, dedup, importance rating, file selection, verifier-script - # generation). None reuses judge_model, so behaviour is unchanged - # unless set. Final claim verdicts stay on judge_model. + # Optional cheaper model for the verifier's genuinely mechanical calls + # (claim dedup, missing-package checks). None reuses judge_model, so + # behaviour is unchanged unless set. Calls that need judgement + # (extraction, importance, file selection, verifier-script generation, + # verdicts) always stay on judge_model. self.judge_extraction_model: str | None = None self.capsule_namer_model = "openrouter/deepseek/deepseek-v4-flash" self.engine_name: str = "litellm" # for smolagent diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1eb795d..ffec7d0 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -24,7 +24,7 @@ Ports must be in `[0, 65535]` and `port_min ≤ port_max`. | `workflow_llm_model` | `str` | `openai/gpt-5.5` | Workflow synthesis. | | `smolagent_model_id` | `str` | `openrouter/deepseek/deepseek-v3.2` | Execution agents inside the sandbox. | | `judge_model` | `str` | `openai/gpt-5.5` | Verifier soft-claim verdicts. | -| `judge_extraction_model` | `str \| None` | `None` | Cheaper model for the verifier's mechanical calls (claim extraction, dedup, importance, file selection, verifier-script generation). Falls back to `judge_model` when unset. | +| `judge_extraction_model` | `str \| None` | `None` | Cheaper model for the verifier's genuinely mechanical calls (claim dedup, missing-package checks). Calls needing judgement stay on `judge_model`. Falls back to `judge_model` when unset. | | `capsule_namer_model` | `str` | `deepseek/deepseek-chat` | Generates human-readable capsule names. | | `engine_name` | `str` | `litellm` | SmolAgents engine — keep as `litellm`. | | `reasoning_effort` | `str` | `medium` | `minimal | low | medium | high` for models that support it. | diff --git a/sources/evaluators/base.py b/sources/evaluators/base.py index 9f65d39..cd813d5 100644 --- a/sources/evaluators/base.py +++ b/sources/evaluators/base.py @@ -123,9 +123,9 @@ def __init__(self, config: "Config") -> None: self.workflow_dir.mkdir(parents=True, exist_ok=True) self.judge_model = config.judge_model - # Optional cheaper tier for the verifier's mechanical extraction - # calls. Falls back to judge_model when unset, so the default - # behaviour (one model for everything) is unchanged. + # Optional cheaper tier for the verifier's genuinely mechanical + # calls (claim dedup, missing-package checks). Falls back to + # judge_model when unset, so default behaviour is unchanged. self.judge_extraction_model = ( getattr(config, "judge_extraction_model", None) or self.judge_model ) @@ -346,9 +346,11 @@ def _call_judge(self, uuid: str, agent_name: str, prompt: str, agent_name: Logical name for this judge call (used for memory file). prompt: User-side prompt to send to the judge. use_extraction_model: Route this call to the cheaper - ``judge_extraction_model`` tier (for mechanical extraction calls - such as claim extraction, importance rating, file selection and - verifier-script generation). Defaults to False (strong judge). + ``judge_extraction_model`` tier. Reserved for the genuinely + mechanical calls (claim dedup, missing-package checks); calls + that need judgement — claim extraction, importance rating, file + selection, verifier-script generation, soft verdicts — stay on + judge_model. Defaults to False (strong judge). Returns: Raw text response from the LLM provider. diff --git a/sources/evaluators/verifier_claims.py b/sources/evaluators/verifier_claims.py index 4055bfd..3365edb 100644 --- a/sources/evaluators/verifier_claims.py +++ b/sources/evaluators/verifier_claims.py @@ -129,8 +129,7 @@ def _extract_claims( prompt = source.build(source_ctx) t_src = time.time() data, err = self._call_judge_for_json( - uuid, f"verifier_extract_claims_{label}", prompt, - use_extraction_model=True, + uuid, f"verifier_extract_claims_{label}", prompt ) if err is not None: self.logger.warning( @@ -353,9 +352,7 @@ def _rate_one_importance_batch( """Rate one batch; default-importance fallback on parse/call error.""" prompt = self._build_importance_batch_prompt(goal, batch, grounding) agent_name = f"verifier_rate_importance_b{batch_idx}" - data, err = self._call_judge_for_json( - uuid, agent_name, prompt, use_extraction_model=True - ) + data, err = self._call_judge_for_json(uuid, agent_name, prompt) if err is not None or not isinstance(data, dict): self.logger.warning( f"importance batch {batch_idx} failed for {uuid} " diff --git a/sources/evaluators/verifier_per_claim.py b/sources/evaluators/verifier_per_claim.py index 8f5038d..cbacb97 100644 --- a/sources/evaluators/verifier_per_claim.py +++ b/sources/evaluators/verifier_per_claim.py @@ -371,8 +371,7 @@ def _llm_select_files( ) cid = str(claim.get("id", "unknown")) parsed, err = self._call_judge_for_json( - uuid, f"verifier_select_files_{cid}", prompt, - use_extraction_model=True, + uuid, f"verifier_select_files_{cid}", prompt ) if err or not isinstance(parsed, dict): self.logger.debug(f"file selection failed for {cid}: {err or 'non-dict JSON'}") @@ -486,9 +485,7 @@ def _call_and_parse_verifier( """Call the judge for a verifier spec; soft-fail to ``executable: False``.""" suffix = "" if attempt == 1 else "_retry" agent_name = f"verifier_gen_{claim['id']}{suffix}" - spec, err = self._call_judge_for_json( - uuid, agent_name, prompt, use_extraction_model=True - ) + spec, err = self._call_judge_for_json(uuid, agent_name, prompt) if err is not None: return {"executable": False, "reason": err} if not isinstance(spec, dict):