diff --git a/config.py b/config.py index f37c4dd..86201a6 100644 --- a/config.py +++ b/config.py @@ -50,6 +50,12 @@ def __init__(self): self.workflow_llm_model: str = "openrouter/z-ai/glm-5.1" self.smolagent_model_id: str = "deepseek/deepseek-v4-flash" self.judge_model = "openrouter/mistralai/mixtral-8x22b-instruct" + # Optional cheaper model for the verifier's genuinely mechanical calls + # (claim dedup, missing-package checks). None reuses judge_model, so + # behaviour is unchanged unless set. Calls that need judgement + # (extraction, importance, file selection, verifier-script generation, + # verdicts) always stay on judge_model. + self.judge_extraction_model: str | None = None self.capsule_namer_model = "openrouter/deepseek/deepseek-v4-flash" self.engine_name: str = "litellm" # for smolagent @@ -221,6 +227,7 @@ def jsonify( "workflow_llm_model": self.workflow_llm_model, "smolagent_model_id": self.smolagent_model_id, "judge_model": self.judge_model, + "judge_extraction_model": self.judge_extraction_model, "engine_name": self.engine_name, "openrouter_provider": self.openrouter_provider, "prompt_planner": self.prompt_planner, @@ -262,6 +269,9 @@ def from_json(self, data: dict[str, Any]) -> None: ) self.smolagent_model_id = data.get("smolagent_model_id", self.smolagent_model_id) self.judge_model = data.get("judge_model", self.judge_model) + self.judge_extraction_model = data.get( + "judge_extraction_model", self.judge_extraction_model + ) self.engine_name = data.get("engine_name", self.engine_name) self.openrouter_provider = data.get("openrouter_provider", self.openrouter_provider) self.prompt_planner = data.get("prompt_planner", self.prompt_planner) @@ -342,6 +352,7 @@ def __str__(self) -> str: lines.append(f" workflow_llm_model={self.workflow_llm_model}") lines.append(f" smolagent_model_id={self.smolagent_model_id}") lines.append(f" judge_model={self.judge_model}") + lines.append(f" judge_extraction_model={self.judge_extraction_model or '(= judge_model)'}") lines.append(f" capsule_namer_model={self.capsule_namer_model}") lines.append(f" engine_name={self.engine_name}") lines.append(f" prompt_planner={self.prompt_planner}") diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 391642f..ffec7d0 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -24,6 +24,7 @@ Ports must be in `[0, 65535]` and `port_min ≤ port_max`. | `workflow_llm_model` | `str` | `openai/gpt-5.5` | Workflow synthesis. | | `smolagent_model_id` | `str` | `openrouter/deepseek/deepseek-v3.2` | Execution agents inside the sandbox. | | `judge_model` | `str` | `openai/gpt-5.5` | Verifier soft-claim verdicts. | +| `judge_extraction_model` | `str \| None` | `None` | Cheaper model for the verifier's genuinely mechanical calls (claim dedup, missing-package checks). Calls needing judgement stay on `judge_model`. Falls back to `judge_model` when unset. | | `capsule_namer_model` | `str` | `deepseek/deepseek-chat` | Generates human-readable capsule names. | | `engine_name` | `str` | `litellm` | SmolAgents engine — keep as `litellm`. | | `reasoning_effort` | `str` | `medium` | `minimal | low | medium | high` for models that support it. | diff --git a/sources/evaluators/base.py b/sources/evaluators/base.py index ec16387..cd813d5 100644 --- a/sources/evaluators/base.py +++ b/sources/evaluators/base.py @@ -123,17 +123,19 @@ def __init__(self, config: "Config") -> None: self.workflow_dir.mkdir(parents=True, exist_ok=True) self.judge_model = config.judge_model + # Optional cheaper tier for the verifier's genuinely mechanical + # calls (claim dedup, missing-package checks). Falls back to + # judge_model when unset, so default behaviour is unchanged. + self.judge_extraction_model = ( + getattr(config, "judge_extraction_model", None) or self.judge_model + ) try: - provider, model = self.judge_model.split("/", 1) if "/" in self.judge_model else ("openai", self.judge_model) - self.llm_config = LLMConfig().from_dict({ - "model": model, - "provider": provider, - "temperature": 0.2, - "reasoning_effort": config.reasoning_effort, - "max_tokens": getattr(config, 'max_tokens', 8192), - "openrouter_provider": config.openrouter_provider_for(self.judge_model), - "openrouter_quantizations": config.openrouter_quantizations_for(self.judge_model), - }) + self.llm_config = self._build_judge_llm_config(self.judge_model, config) + self.extraction_llm_config = ( + self.llm_config + if self.judge_extraction_model == self.judge_model + else self._build_judge_llm_config(self.judge_extraction_model, config) + ) except Exception as e: raise EvaluatorError(f"Failed to initialize LLM configuration: {str(e)}") from e @@ -308,13 +310,47 @@ def _get_judge_system_prompt(self) -> str: "the JSON object, no prose, no markdown fences, no commentary." ) - def _call_judge(self, uuid: str, agent_name: str, prompt: str) -> str: + def _build_judge_llm_config(self, model_id: str, config: "Config") -> LLMConfig: + """Build an LLMConfig for a judge/extraction model id ("provider/model"). + + Centralises judge LLM construction so the strong judge and the optional + cheaper extraction tier are configured identically (temperature, + reasoning effort, OpenRouter provider/quantization routing). + + Args: + model_id: Model identifier, optionally "provider/model" (defaults to + the ``openai`` provider when no "/" is present). + config: The run config (supplies reasoning_effort, max_tokens and the + OpenRouter routing lookups). + + Returns: + A configured :class:`LLMConfig` for that model. + """ + provider, model = model_id.split("/", 1) if "/" in model_id else ("openai", model_id) + return LLMConfig().from_dict({ + "model": model, + "provider": provider, + "temperature": 0.2, + "reasoning_effort": config.reasoning_effort, + "max_tokens": getattr(config, 'max_tokens', 8192), + "openrouter_provider": config.openrouter_provider_for(model_id), + "openrouter_quantizations": config.openrouter_quantizations_for(model_id), + }) + + def _call_judge(self, uuid: str, agent_name: str, prompt: str, + use_extraction_model: bool = False) -> str: """One judge round-trip; raises whatever the LLM provider raises. Args: uuid: Workflow identifier (used to scope per-uuid memory). agent_name: Logical name for this judge call (used for memory file). prompt: User-side prompt to send to the judge. + use_extraction_model: Route this call to the cheaper + ``judge_extraction_model`` tier. Reserved for the genuinely + mechanical calls (claim dedup, missing-package checks); calls + that need judgement — claim extraction, importance rating, file + selection, verifier-script generation, soft verdicts — stay on + judge_model. Defaults to False (strong judge). Returns: Raw text response from the LLM provider. @@ -325,7 +361,7 @@ def _call_judge(self, uuid: str, agent_name: str, prompt: str) -> str: agent_name=agent_name, memory_path=memory_path, system_msg=self._get_judge_system_prompt(), - config=self.llm_config, + config=self.extraction_llm_config if use_extraction_model else self.llm_config, ) return provider(prompt) @@ -334,6 +370,7 @@ def _call_judge_for_json( uuid: str, agent_name: str, prompt: str, + use_extraction_model: bool = False, ) -> tuple[Any, str | None]: """Call the judge expecting JSON; one retry on parse failure. @@ -353,7 +390,9 @@ def _call_judge_for_json( for attempt in (1, 2): agent = agent_name if attempt == 1 else f"{agent_name}_retry" try: - raw = self._call_judge(uuid, agent, cur_prompt) + raw = self._call_judge( + uuid, agent, cur_prompt, use_extraction_model=use_extraction_model + ) except Exception as e: last_err = f"judge call failed: {type(e).__name__}: {e}" self.logger.warning(f"[{agent}] {last_err}") diff --git a/sources/evaluators/verifier_claims.py b/sources/evaluators/verifier_claims.py index b5b1191..3365edb 100644 --- a/sources/evaluators/verifier_claims.py +++ b/sources/evaluators/verifier_claims.py @@ -302,7 +302,8 @@ def _run_dedup_pass( """Single cheap LLM call returning only ids to drop as near-duplicates.""" prompt = self._build_dedup_prompt(goal, claims, grounding) data, err = self._call_judge_for_json( - uuid, "verifier_dedup_claims", prompt + uuid, "verifier_dedup_claims", prompt, + use_extraction_model=True, ) if err is not None or not isinstance(data, dict): self.logger.warning( diff --git a/sources/evaluators/verifier_per_claim.py b/sources/evaluators/verifier_per_claim.py index 43cd949..cbacb97 100644 --- a/sources/evaluators/verifier_per_claim.py +++ b/sources/evaluators/verifier_per_claim.py @@ -805,7 +805,8 @@ def _llm_packages_needed_for_claim( """Ask the judge which missing pip packages are genuinely required.""" prompt = self._build_package_check_prompt(claim, stderr) parsed, err = self._call_judge_for_json( - uuid, f"verifier_pkg_check_{claim['id']}", prompt + uuid, f"verifier_pkg_check_{claim['id']}", prompt, + use_extraction_model=True, ) if err is not None or not isinstance(parsed, dict): self.logger.debug( diff --git a/tests/judge_extraction_test.py b/tests/judge_extraction_test.py new file mode 100644 index 0000000..10e1593 --- /dev/null +++ b/tests/judge_extraction_test.py @@ -0,0 +1,38 @@ +"""Tests for the optional cheap judge-extraction model tier. + +Covers the config surface (default and round-trip). The routing of the +verifier's mechanical judge calls through this tier is exercised by the +verifier tests; this file has no import dependency on the evaluator stack.""" + +import sys +import tempfile +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent)) + +from config import Config + + +def test_judge_extraction_defaults_to_none(): + """Unset means reuse judge_model, so behaviour is unchanged by default.""" + assert Config().judge_extraction_model is None + + +def test_judge_extraction_round_trips(tmp_path): + """A configured extraction model survives dump and load.""" + cheap = "openrouter/deepseek/deepseek-v4-flash" + config = Config() + config.judge_extraction_model = cheap + path = tmp_path / "cfg.json" + config.dump(str(path)) + + loaded = Config() + loaded.load(str(path)) + assert loaded.judge_extraction_model == cheap + + +if __name__ == "__main__": + test_judge_extraction_defaults_to_none() + with tempfile.TemporaryDirectory() as d: + test_judge_extraction_round_trips(Path(d)) + print("judge_extraction tests passed") diff --git a/tests/verifier_importance_test.py b/tests/verifier_importance_test.py index f5dd4ad..583c30d 100644 --- a/tests/verifier_importance_test.py +++ b/tests/verifier_importance_test.py @@ -185,7 +185,7 @@ def test_declare_claim_importance_drops_duplicates_and_clamps_scale() -> None: """Rater output is respected: drop ids removed, importance clamped to [1,10].""" v = _StubVerifier() - def _fake_judge(_uuid, _agent, _prompt): + def _fake_judge(_uuid, _agent, _prompt, **_kwargs): return ({ "drop_ids": ["dup"], "importance": [