From 14872784099b9570a6c7bd013af62f4337636685 Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:34:34 -0700 Subject: [PATCH 1/4] feat: add vera.py CLI orchestrator with centralized config schema New top-level vera.py implements the CLI surface from docs/architecture.md and docs/vera-cli-use-cases.md: generate, judge, score, pool, pipeline, and resume subcommands, sharing a single FLAG_SPECS registry so -c/-u/-j/--config etc. are defined exactly once and reused across subcommands. utils/config_schema.py centralizes the config.json shape (ModelSpec, GenerationConfig, JudgingConfig, RunConfig, RubricBundleManifest) that both CLI shorthand and --config/VERA_RUN_CONFIG resolve into, per the Phase 1 migration plan. Business-logic wiring into the existing generate/judge engines is left for a follow-up change; each subcommand validates its inputs, resolves and prints the canonical RunConfig, and stops. --- utils/config_schema.py | 219 ++++++++++++++++++++ vera.py | 448 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 667 insertions(+) create mode 100644 utils/config_schema.py create mode 100644 vera.py diff --git a/utils/config_schema.py b/utils/config_schema.py new file mode 100644 index 000000000..b5f6b0048 --- /dev/null +++ b/utils/config_schema.py @@ -0,0 +1,219 @@ +"""Centralized `config.json` schema for `vera.py`. + +Single source of truth for the run-config shape described in +docs/vera-cli-use-cases.md ("config.json shape") and docs/architecture.md +("Rubric bundle manifest"). CLI flags and `--config` both resolve into a +`RunConfig` here so there is exactly one canonical representation of "what +this run does," regardless of which input form produced it. + +Per docs/architecture.md's "Stable interfaces" section, this file's schema +is a stable interface once Phase 3 formalizes it β€” see the ESCALATE section +there before changing the shape of `RunConfig`/`GenerationConfig`/ +`JudgingConfig`. Phase 1 (this file, as first written) is explicitly the +*informal* shape. +""" + +from __future__ import annotations + +import dataclasses +from typing import Any, Optional + + +@dataclasses.dataclass +class ModelSpec: + """One model entry in a `generation.user`/`generation.chatbot`/`judging.models`. + + `name` is always a specific model identifier (e.g. "claude-sonnet-2026xxxx"), + never a bare provider name. `extra_params` holds bespoke sampling knobs + (temperature, top_p, max_tokens, ...) -- config-only, never expressible via + `-u`/`-c`/`-j` shorthand. + """ + + name: str + repeats: int = 1 + extra_params: dict[str, Any] = dataclasses.field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "repeats": self.repeats, **self.extra_params} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "ModelSpec": + data = dict(data) + name = data.pop("name") + repeats = data.pop("repeats", 1) + return cls(name=name, repeats=repeats, extra_params=data) + + @classmethod + def from_shorthand(cls, token: str) -> "ModelSpec": + """Parse `-u`/`-c`/`-j` shorthand: "[:]".""" + name, sep, repeats_str = token.partition(":") + if not name: + raise ValueError(f"invalid model shorthand: {token!r}") + repeats = int(repeats_str) if sep else 1 + return cls(name=name, repeats=repeats) + + +@dataclasses.dataclass +class RubricSpec: + """One entry in `judging.rubrics[]`. + + `name` resolves to a rubric bundle manifest (see + docs/architecture.md#rubric-bundle-manifest) via `--target`/manifest + lookup, not a bare `.tsv` path. `models` optionally overrides + `judging.models` for this rubric only. + """ + + name: str + models: list[ModelSpec] = dataclasses.field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {"name": self.name} + if self.models: + d["models"] = [m.to_dict() for m in self.models] + return d + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "RubricSpec": + return cls( + name=data["name"], + models=[ModelSpec.from_dict(m) for m in data.get("models", [])], + ) + + +@dataclasses.dataclass +class GenerationConfig: + """`generation` block. Orthogonal to `JudgingConfig` -- see RunConfig.""" + + chatbot: Optional[ModelSpec] = None + user: list[ModelSpec] = dataclasses.field(default_factory=list) + personas: list[str] = dataclasses.field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {} + if self.chatbot is not None: + d["chatbot"] = self.chatbot.to_dict() + if self.user: + d["user"] = [m.to_dict() for m in self.user] + if self.personas: + d["personas"] = self.personas + return d + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "GenerationConfig": + chatbot = data.get("chatbot") + return cls( + chatbot=ModelSpec.from_dict(chatbot) if chatbot else None, + user=[ModelSpec.from_dict(m) for m in data.get("user", [])], + personas=list(data.get("personas", [])), + ) + + +@dataclasses.dataclass +class JudgingConfig: + """`judging` block. Orthogonal to `GenerationConfig` -- see RunConfig. + + `rubrics` is a list from day one (per docs/architecture.md's migration + Phase 0-4 notes) even though only a length-1 list is supported/validated + until Phase 4. + """ + + models: list[ModelSpec] = dataclasses.field(default_factory=list) + rubrics: list[RubricSpec] = dataclasses.field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {} + if self.models: + d["models"] = [m.to_dict() for m in self.models] + if self.rubrics: + d["rubrics"] = [r.to_dict() for r in self.rubrics] + return d + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "JudgingConfig": + return cls( + models=[ModelSpec.from_dict(m) for m in data.get("models", [])], + rubrics=[RubricSpec.from_dict(r) for r in data.get("rubrics", [])], + ) + + +@dataclasses.dataclass +class RunConfig: + """The canonical resolved form of a `vera.py` invocation. + + Whether a run was invoked via CLI shorthand or `--config`, it always + resolves to exactly one `RunConfig` -- printed at run start for + terminal/CI-log visibility (docs/vera-cli-use-cases.md#config-mechanism). + + `target` mirrors the `--target ` shorthand: set only when the + invocation used `--target` (or the input config's own top-level `target` + field) instead of independently specifying `generation.personas` and + `judging.rubrics`. Setting `target` alongside explicit + `generation.personas`/`judging.rubrics` is an error -- see + `docs/architecture.md#rubric-bundle-manifest`. + """ + + generation: Optional[GenerationConfig] = None + judging: Optional[JudgingConfig] = None + target: Optional[str] = None + sample: Optional[int] = None + + def __post_init__(self) -> None: + if self.target is not None and self.generation and self.generation.personas: + raise ValueError( + "target is mutually exclusive with generation.personas " + "-- target expands to both personas and rubrics itself" + ) + if self.target is not None and self.judging and self.judging.rubrics: + raise ValueError( + "target is mutually exclusive with judging.rubrics " + "-- target expands to both personas and rubrics itself" + ) + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {} + if self.generation is not None: + d["generation"] = self.generation.to_dict() + if self.judging is not None: + d["judging"] = self.judging.to_dict() + if self.target is not None: + d["target"] = self.target + if self.sample is not None: + d["sample"] = self.sample + return d + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "RunConfig": + generation = data.get("generation") + judging = data.get("judging") + return cls( + generation=GenerationConfig.from_dict(generation) if generation else None, + judging=JudgingConfig.from_dict(judging) if judging else None, + target=data.get("target"), + sample=data.get("sample"), + ) + + +@dataclasses.dataclass +class RubricBundleManifest: + """A rubric bundle manifest (docs/architecture.md#rubric-bundle-manifest). + + Paths (`rubric_file`, `rubric_prompt_beginning_file`, `question_prompt_file`, + entries in `personas`) resolve relative to the manifest's own folder -- + never relative to `$ROOT` or the CLI's working directory. `personas` is + informational-only except when resolved via `--target`, per the + docs/architecture.md `--target` note. + """ + + rubric_file: str + rubric_prompt_beginning_file: str + question_prompt_file: str + personas: list[str] = dataclasses.field(default_factory=list) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "RubricBundleManifest": + return cls( + rubric_file=data["rubric_file"], + rubric_prompt_beginning_file=data["rubric_prompt_beginning_file"], + question_prompt_file=data["question_prompt_file"], + personas=list(data.get("personas", [])), + ) diff --git a/vera.py b/vera.py new file mode 100644 index 000000000..dc44cf1ef --- /dev/null +++ b/vera.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""VERA-MH unified CLI orchestrator. + +Per docs/architecture.md's "CLI surface" section, `vera.py` is the single +root-level orchestrator: subcommands parse arguments and delegate to domain +runners; they contain no business logic themselves. + +Subcommands: generate, judge, score, pool, pipeline, resume -- see +docs/vera-cli-use-cases.md for the full CLI/config design this implements. + +This is Phase 1 of the migration (docs/architecture.md#migration-from-current-layout): +argument parsing and config resolution land now; wiring into the existing +`generate`/`judge` engines is tracked separately and stubbed here for now. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Callable, Optional + +from utils.config_schema import ( + GenerationConfig, + JudgingConfig, + ModelSpec, + RubricSpec, + RunConfig, +) + +PROG = "vera" + +VERA_RUN_CONFIG_ENV = "VERA_RUN_CONFIG" + + +# --------------------------------------------------------------------------- +# Centralized flag registry. +# +# Every CLI flag is defined exactly once here and referenced by name from +# whichever subcommand(s) need it, so `-c`/`-u`/`-j`/`--config`/etc. can never +# drift into slightly-different definitions across subcommands. +# --------------------------------------------------------------------------- + +FLAG_SPECS: dict[str, dict[str, Any]] = { + "chatbot": { + "flags": ("-c", "--chatbot"), + "kwargs": { + "metavar": "", + "help": "Chatbot (provider/agent) LLM under test. No default.", + }, + }, + "user": { + "flags": ("-u", "--user"), + "kwargs": { + "nargs": "+", + "metavar": "[:]", + "help": "User-side LLM(s), e.g. `-u gpt:1 sonnet:2`. `repeats` default 1.", + }, + }, + "judge": { + "flags": ("-j", "--judge"), + "kwargs": { + "nargs": "+", + "metavar": "[:]", + "help": "Judge LLM(s), e.g. `-j claude:1 gpt:2`. `repeats` defaults to 1.", + }, + }, + "personas": { + "flags": ("--personas",), + "kwargs": { + "nargs": "+", + "metavar": "", + "help": "Persona file(s) for generation. Mutually exclusive with --target.", + }, + }, + "target": { + "flags": ("--target",), + "kwargs": { + "metavar": "", + "help": ( + "Resolve to a rubric bundle manifest and set both " + "generation personas and the judging rubric from it in one shot. " + "Use `--target all` to run every known evaluator." + ), + }, + }, + "rubric": { + "flags": ("--rubric",), + "kwargs": { + "nargs": "+", + "metavar": "", + "help": "Rubric bundle manifest path(s) (see architecture.md).", + }, + }, + "conversations": { + "flags": ("--conversations",), + "kwargs": { + "nargs": "+", + "metavar": "", + "help": "Existing transcript folder(s) to judge.", + }, + }, + "evaluations": { + "flags": ("--evaluations",), + "kwargs": { + "nargs": "+", + "metavar": "", + "help": "Evaluation folder(s) to pool together.", + }, + }, + "results": { + "flags": ("-r", "--results"), + "kwargs": { + "metavar": "", + "help": "Path to an existing results.csv to score.", + }, + }, + "config": { + "flags": ("--config",), + "kwargs": { + "metavar": "", + "help": ( + "JSON config file (or `-` for stdin). Mutually exclusive with " + f"CLI model/persona/rubric flags and the {VERA_RUN_CONFIG_ENV} env var." + ), + }, + }, + "sample": { + "flags": ("--sample",), + "kwargs": { + "type": int, + "metavar": "N", + "help": "Smoke-test override: cap personas/rubrics/judges to N.", + }, + }, + "print": { + "flags": ("--print",), + "kwargs": { + "action": "store_true", + "help": "Print the resolved flag-string and exit, without running.", + }, + }, +} + + +def add_flags( + parser: argparse.ArgumentParser | argparse._MutuallyExclusiveGroup, *names: str +) -> None: + """Attach flags from FLAG_SPECS to a parser or argument group by name.""" + for name in names: + spec = FLAG_SPECS[name] + parser.add_argument(*spec["flags"], **spec["kwargs"]) + + +# --------------------------------------------------------------------------- +# Shared shorthand parsing. +# --------------------------------------------------------------------------- + + +def parse_model_list(tokens: Optional[list[str]]) -> list[ModelSpec]: + if not tokens: + return [] + return [ModelSpec.from_shorthand(t) for t in tokens] + + +def parse_single_model(token: Optional[str]) -> Optional[ModelSpec]: + if token is None: + return None + return ModelSpec.from_shorthand(token) + + +# --------------------------------------------------------------------------- +# Config resolution: CLI flags and --config both funnel into one RunConfig. +# --------------------------------------------------------------------------- + + +class ConfigError(ValueError): + """Raised when CLI flags and --config/env conflict, or a field is missing.""" + + +def _load_config_json(args: argparse.Namespace) -> Optional[dict[str, Any]]: + """Load raw config JSON from --config or VERA_RUN_CONFIG, if given.""" + config_arg = getattr(args, "config", None) + env_config = os.environ.get(VERA_RUN_CONFIG_ENV) + + if config_arg and env_config: + raise ConfigError(f"--config and {VERA_RUN_CONFIG_ENV} are mutually exclusive") + + if config_arg: + if config_arg == "-": + return json.loads(sys.stdin.read()) + with open(config_arg) as f: + return json.load(f) + + if env_config: + return json.loads(env_config) + + return None + + +def _cli_flags_given(args: argparse.Namespace, names: tuple[str, ...]) -> list[str]: + return [n for n in names if getattr(args, n, None)] + + +def resolve_run_config( + args: argparse.Namespace, cli_flag_names: tuple[str, ...] +) -> RunConfig: + """Resolve a subcommand's parsed args into one canonical RunConfig. + + CLI flags and --config/VERA_RUN_CONFIG are strictly either/or -- never + combined for the same run (docs/vera-cli-use-cases.md#config-mechanism). + """ + config_json = _load_config_json(args) + given_cli_flags = _cli_flags_given(args, cli_flag_names) + + if config_json is not None: + if given_cli_flags: + raise ConfigError( + "--config/" + f"{VERA_RUN_CONFIG_ENV} cannot be combined with CLI flags: " + f"{', '.join(given_cli_flags)}" + ) + return RunConfig.from_dict(config_json) + + return _run_config_from_cli(args) + + +def _run_config_from_cli(args: argparse.Namespace) -> RunConfig: + generation: Optional[GenerationConfig] = None + chatbot = parse_single_model(getattr(args, "chatbot", None)) + user = parse_model_list(getattr(args, "user", None)) + personas = getattr(args, "personas", None) or [] + if chatbot is not None or user or personas: + generation = GenerationConfig(chatbot=chatbot, user=user, personas=personas) + + judging: Optional[JudgingConfig] = None + judge_models = parse_model_list(getattr(args, "judge", None)) + rubric_paths = getattr(args, "rubric", None) or [] + rubrics = [RubricSpec(name=r) for r in rubric_paths] + if judge_models or rubrics: + judging = JudgingConfig(models=judge_models, rubrics=rubrics) + + return RunConfig( + generation=generation, + judging=judging, + target=getattr(args, "target", None), + sample=getattr(args, "sample", None), + ) + + +def print_resolved_config(run_config: RunConfig) -> None: + print(json.dumps(run_config.to_dict(), indent=2)) + + +# --------------------------------------------------------------------------- +# Subcommand handlers. +# +# Phase 1 only replaces the front end (docs/architecture.md's migration +# table) -- these delegate to the existing generate/judge engines in a +# follow-up change. For now they resolve + print the config and stop. +# --------------------------------------------------------------------------- + + +def _not_yet_wired(command: str) -> None: + print( + f"vera {command}: argument parsing and config resolution only for now -- " + "wiring into the existing generate/judge engine lands in a follow-up " + "change (docs/architecture.md, migration Phase 1).", + file=sys.stderr, + ) + + +def cmd_generate(args: argparse.Namespace) -> int: + run_config = resolve_run_config( + args, cli_flag_names=("chatbot", "user", "personas", "target") + ) + if run_config.generation is None or run_config.generation.chatbot is None: + raise ConfigError( + "generate requires a chatbot (-c/--chatbot or generation.chatbot)" + ) + if not run_config.generation.user: + raise ConfigError( + "generate requires at least one user model (-u/--user or generation.user)" + ) + if not run_config.generation.personas and not run_config.target: + raise ConfigError("generate requires --personas or --target") + print_resolved_config(run_config) + if args.print: + return 0 + _not_yet_wired("generate") + return 0 + + +def cmd_judge(args: argparse.Namespace) -> int: + run_config = resolve_run_config(args, cli_flag_names=("judge", "rubric")) + if run_config.judging is None or not run_config.judging.models: + raise ConfigError( + "judge requires at least one judge model (-j/--judge or judging.models)" + ) + if not run_config.judging.rubrics: + raise ConfigError("judge requires --rubric or judging.rubrics") + if not args.conversations: + raise ConfigError("judge requires --conversations") + print_resolved_config(run_config) + if args.print: + return 0 + _not_yet_wired("judge") + return 0 + + +def cmd_score(args: argparse.Namespace) -> int: + if not args.results: + raise ConfigError("score requires -r/--results") + if args.print: + print(f"vera score -r {args.results}") + return 0 + _not_yet_wired("score") + return 0 + + +def cmd_pool(args: argparse.Namespace) -> int: + if not args.evaluations: + raise ConfigError("pool requires --evaluations") + if args.print: + print(f"vera pool --evaluations {' '.join(args.evaluations)}") + return 0 + _not_yet_wired("pool") + return 0 + + +def cmd_pipeline(args: argparse.Namespace) -> int: + run_config = resolve_run_config( + args, + cli_flag_names=("chatbot", "user", "judge", "personas", "target", "rubric"), + ) + if run_config.target is None: + if run_config.generation is None or run_config.generation.chatbot is None: + raise ConfigError( + "pipeline requires a chatbot (-c/--chatbot or generation.chatbot)" + ) + if not run_config.generation.user: + raise ConfigError( + "pipeline requires at least one user model (-u/--user or " + "generation.user)" + ) + if run_config.judging is None or not run_config.judging.models: + raise ConfigError( + "pipeline requires at least one judge model (-j/--judge or " + "judging.models)" + ) + print_resolved_config(run_config) + if args.print: + return 0 + _not_yet_wired("pipeline") + return 0 + + +def cmd_resume(args: argparse.Namespace) -> int: + if not args.config: + raise ConfigError("resume requires --config ") + if args.print: + print(f"vera resume --config {args.config}") + return 0 + _not_yet_wired("resume") + return 0 + + +# --------------------------------------------------------------------------- +# Parser construction. +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog=PROG, description="VERA-MH unified CLI orchestrator." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + def add_subcommand( + name: str, help_text: str, handler: Callable[[argparse.Namespace], int] + ) -> argparse.ArgumentParser: + sub = subparsers.add_parser(name, help=help_text) + sub.set_defaults(handler=handler) + return sub + + generate = add_subcommand("generate", "Simulate conversations.", cmd_generate) + add_flags( + generate, "chatbot", "user", "personas", "target", "config", "sample", "print" + ) + + judge = add_subcommand( + "judge", "Evaluate existing transcripts against a rubric.", cmd_judge + ) + add_flags(judge, "judge", "rubric", "conversations", "config", "sample", "print") + + score = add_subcommand( + "score", "Aggregate results.csv into scores and visualizations.", cmd_score + ) + add_flags(score, "results", "print") + + pool = add_subcommand( + "pool", + "Concatenate multiple evaluation folders into one pooled result.", + cmd_pool, + ) + add_flags(pool, "evaluations", "print") + + pipeline = add_subcommand( + "pipeline", + "Full generate -> judge -> score workflow for one chatbot.", + cmd_pipeline, + ) + add_flags( + pipeline, + "chatbot", + "user", + "judge", + "personas", + "target", + "rubric", + "config", + "sample", + "print", + ) + + resume = add_subcommand( + "resume", + "Resume an incomplete run from its config.json + state.json.", + cmd_resume, + ) + add_flags(resume, "config", "print") + + return parser + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + return args.handler(args) + except ConfigError as e: + parser.error(str(e)) + return 2 # pragma: no cover - argparse.error() exits before this + + +if __name__ == "__main__": + sys.exit(main()) From 8b93c59d2b1c66eba0b6e9f2a265677a2c17ffa8 Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:26:33 -0700 Subject: [PATCH 2/4] feat: wire unified VERA CLI to domain services --- README.md | 51 +- docs/design/vera-cli-runtime-wiring.md | 42 ++ generate.py | 159 +---- generate_conversations/__init__.py | 3 +- generate_conversations/service.py | 152 +++++ judge/__init__.py | 2 + judge/runner.py | 64 ++- judge/score.py | 108 ++-- .../test_generate_cli.py | 61 +- tests/unit/judge/test_runner_extra_params.py | 47 ++ tests/unit/test_vera_cli.py | 352 ++++++++++++ utils/config_schema.py | 25 +- utils/rubric_manifest.py | 19 +- vera.py | 543 +++++++++++++----- 14 files changed, 1258 insertions(+), 370 deletions(-) create mode 100644 docs/design/vera-cli-runtime-wiring.md create mode 100644 generate_conversations/service.py create mode 100644 tests/unit/test_vera_cli.py diff --git a/README.md b/README.md index ff9b733ce..ebe19a741 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ We value every interaction that follows the [Code of Conduct](https://www.contri - [Getting Started](#getting-started) - [Environment setup](#environment-setup) +- [Unified CLI](#unified-cli) - [Connecting your own LLM, Agent, or API](#connecting-your-own-llm-or-api) - [Recommended settings](#recommended-settings) - [Reliable VERA-MH score (automated)](#reliable-vera-mh-score-automated) @@ -32,7 +33,7 @@ We value every interaction that follows the [Code of Conduct](https://www.contri # Getting started -This page covers [Environment setup](#environment-setup), optional [custom provider wiring](#connecting-your-own-llm-or-api), [Recommended settings](#recommended-settings) for comparable scores, the [automated pooled pipeline](#reliable-vera-mh-score-automated), and [Running VERA-MH step by step](#running-vera-mh-step-by-step) (`run_pipeline.py`, `generate.py`, `judge.py`, scoring, comparison, and improvement reports). +This page covers [Environment setup](#environment-setup), the [unified CLI](#unified-cli), optional [custom provider wiring](#connecting-your-own-llm-or-api), [Recommended settings](#recommended-settings) for comparable scores, the [automated pooled pipeline](#reliable-vera-mh-score-automated), and [Running VERA-MH step by step](#running-vera-mh-step-by-step) (`run_pipeline.py`, `generate.py`, `judge.py`, scoring, comparison, and improvement reports). ## Environment setup @@ -58,6 +59,54 @@ This page covers [Environment setup](#environment-setup), optional [custom provi pre-commit install ``` +## Unified CLI + +`vera.py` provides one command surface for generation, judging, scoring, pooling, +and the end-to-end pipeline. It calls parser-independent domain functions; the +legacy `generate.py`, `judge.py`, and scoring CLIs are not runtime dependencies: + +```bash +uv run python vera.py generate \ + -c gpt-4o \ + -u claude-sonnet-4-5-20250929 \ + --personas data/SI/personas.tsv + +uv run python vera.py judge \ + -j gpt-5.4 \ + --rubric data/SI/rubric_manifest.json \ + --conversations output/ + +uv run python vera.py pipeline \ + -c gpt-4o \ + -u claude-sonnet-4-5-20250929 \ + -j gpt-5.4 \ + --target SI +``` + +Run-defining CLI flags and JSON config are strictly either/or. `--sample` is the +sole debug-only flag that may accompany `--config`. For standalone judging, put +conversation paths in `judging.conversations` when using config: + +```json +{ + "judging": { + "models": [{"name": "gpt-5.4", "repeats": 1}], + "rubrics": [{"name": "SI"}], + "conversations": ["output/example-run"] + } +} +``` + +Then run `uv run python vera.py judge --config run.json`. Relative paths inside +config resolve from the repository root. `--sample N` is a debug-only cap and is +never serialized into the resolved run config. `vera resume` is reserved but fails +explicitly until the checksum/state recovery contract is implemented. + +Generation always consumes persona files. `--target` is shorthand that resolves a +rubric manifest into both its persona files and judging rubric. A target manifest +without `personas` remains valid for judge-only use, but cannot be used to generate; +VERA fails explicitly rather than silently selecting default personas. + ## Connecting your own LLM, Agent, or API Use this when the **provider** you want to evaluate (the mental-health chatbot under test) is **not** already available as a built-in model name in `generate.py`β€”for example a private HTTP API, an internal gateway, or a new cloud provider. diff --git a/docs/design/vera-cli-runtime-wiring.md b/docs/design/vera-cli-runtime-wiring.md new file mode 100644 index 000000000..cd943be9e --- /dev/null +++ b/docs/design/vera-cli-runtime-wiring.md @@ -0,0 +1,42 @@ +# VERA CLI runtime wiring + +## Context + +`vera.py` already defined the Phase 1 command and config shape from the architecture +contract on `feat/VERA_2.0`, but its handlers stopped after parsing. Wiring those +handlers exposed one ambiguity: standalone judging needs conversation paths, while +AD-17 requires an invocation to use either JSON config or run-defining command-line +flags, never both. + +## Decision + +- `JudgingConfig` owns `conversations`, mirroring `--conversations`. +- Config-sourced paths resolve from the repository root, as required by AD-28. +- `--sample` remains an invocation-only debug cap and is removed from serialized + `RunConfig`. Per AD-17, it is the sole flag allowed alongside `--config`; + `--debug` and `--print` remain CLI-only. +- `vera.py` delegates to parser-independent application functions: + `generate_conversations.run_generation`, `judge.run_judging`, and + `judge.score.score_results_file`. It never imports or dynamically loads the + legacy CLI entry points. +- The generation application function accepts resolved persona files, never a + rubric manifest. `--target` is an orchestration convenience: `vera.py` reads + its manifest, validates that it defines personas, and passes those paths to + generation while passing the manifest itself only to judging. +- A manifest's `personas` field remains optional for judge-only use. It is + required contextually when that manifest is selected through `--target` for + generation; a missing list fails explicitly and never falls back to SI data. +- The legacy scripts are compatibility adapters while callers migrate. Their + parsers are not architectural dependencies and can be deleted separately. + The legacy `generate.py --rubric-manifest` adapter performs its own manifest + translation before calling the generation application function. +- Pooling already exposes `pool_evaluation_directories`; `vera.py` calls that + function rather than its script entry point. +- `vera resume` fails explicitly until the architecture's checksum, state ownership, + and partial-write recovery contract is implemented. + +## Compatibility + +Existing config files without `judging.conversations` remain valid for `pipeline`, +where conversations come from the generation stage. A standalone `judge` config must +now include that field instead of combining `--config` with `--conversations`. diff --git a/generate.py b/generate.py index f38d7f7ba..92109e422 100644 --- a/generate.py +++ b/generate.py @@ -2,19 +2,12 @@ import argparse import asyncio -import os import sys -from datetime import datetime from typing import Any, Dict, List, Optional -from generate_conversations import ConversationRunner +from generate_conversations import run_generation from llm_clients.llm_interface import DEFAULT_START_PROMPT from utils.debug import set_debug -from utils.naming import ( - build_generation_run_folder_name, - model_token_for_run_folder, - parse_generation_run_folder_name, -) from utils.rubric_manifest import ( load_manifest_persona_context_template, load_manifest_personas, @@ -40,141 +33,35 @@ async def main( session_types: Optional[List[str]] = None, resume: bool = False, rubric_manifest: Optional[str] = None, + persona_files: Optional[List[str]] = None, ) -> tuple[List[Dict[str, Any]], str]: - """ - Generate conversations and return results. - - Args: - # TODO: should the extra config be separated? - persona_model_config: Configuration dictionary for the persona model - agent_model_config: Configuration dictionary for the agent model - persona_extra_run_params: Extra parameters for the persona model - agent_extra_run_params: Extra parameters for the agent model - max_turns: Maximum turns per conversation - runs_per_prompt: Number of runs per prompt - persona_names: List of persona names to use. If None, uses all personas. - verbose: Whether to print status messages - output_folder: Parent directory for new runs (default ``output/``), or the - existing ``p_*`` run folder when ``resume`` is True. - max_total_words: Optional maximum total words across all responses - max_concurrent: Maximum number of concurrent conversations. If None, runs all - conversations concurrently. - max_personas: Optional maximum number of personas to load from CSV. If None, - loads all personas. - persona_speaks_first: If True (default), persona speaks first; else provider - speaks first. max_turns is adjusted so the provider always speaks last. - rubric_manifest: Optional path to a rubric bundle manifest (see - docs/architecture.md#rubric-bundle-manifest). When set, personas load - from the manifest's ``personas`` list instead of the default - ``data/SI/personas.tsv`` -- Phase 0's generation-side counterpart to - ``judge.py --rubrics``, so a manifest attaches personas and rubric - together. Only the first entry is used if the manifest lists more - than one. - - Returns: - List of conversation results - - Raises: - ValueError: Configuration error - Exception: Other errors - """ - if verbose: - print("πŸ”„ Generating conversations with the following parameters:") - print(f" - Persona model: {persona_model_config}") - print(f" - Agent model: {agent_model_config}") - print(f" - Persona extra run params: {persona_extra_run_params}") - print(f" - Agent extra run params: {agent_extra_run_params}") - print(f" - Max turns: {max_turns}") - print(f" - Runs per prompt: {runs_per_prompt}") - print(f" - Persona names: {persona_names}") - print(f" - Output folder: {output_folder}") - print(f" - Run ID: {run_id}") - print(f" - Max concurrent: {max_concurrent}") - print(f" - Max total words: {max_total_words}") - print(f" - Max personas: {max_personas}") - print(f" - Persona speaks first: {persona_speaks_first}") - print(f" - Resume: {resume}") - - # Generate default folder name if not provided - if output_folder is None: - output_folder = "output" - - persona_prompt_path = "data/SI/personas.tsv" + """Compatibility wrapper around the parser-independent generation service.""" + if rubric_manifest and persona_files: + raise ValueError("rubric_manifest and persona_files are mutually exclusive") + + resolved_persona_files = persona_files or ["data/SI/personas.tsv"] persona_context_template_path = "data/SI/persona_context_template.txt" if rubric_manifest: - manifest_personas = await load_manifest_personas(rubric_manifest) - if not manifest_personas: + resolved_persona_files = await load_manifest_personas(rubric_manifest) + if not resolved_persona_files: raise ValueError( f"Rubric bundle manifest {rubric_manifest} has no personas listed" ) - if len(manifest_personas) > 1: - print( - f"Warning: manifest lists multiple persona files " - f"({manifest_personas}); multi-persona-file support is not yet " - f"implemented, using only the first: {manifest_personas[0]}", - file=sys.stderr, - ) - persona_prompt_path = manifest_personas[0] persona_context_template_path = await load_manifest_persona_context_template( rubric_manifest ) - if resume: - if not os.path.isdir(output_folder): - raise ValueError( - "Resume mode requires --output to point to an existing run folder." - ) - run_folder_name = os.path.basename(os.path.normpath(output_folder)) - run_meta = parse_generation_run_folder_name(run_folder_name) - expected_persona = model_token_for_run_folder(persona_model_config["model"]) - expected_agent = model_token_for_run_folder(agent_model_config["model"]) - - if run_meta["persona"] != expected_persona: - raise ValueError( - "Resume folder persona model does not match current --user-agent. " - f"Expected p_{expected_persona}, got p_{run_meta['persona']}." - ) - if run_meta["agent"] != expected_agent: - raise ValueError( - "Resume folder provider model does not match current --provider-agent. " - f"Expected a_{expected_agent}, got a_{run_meta['agent']}." - ) - if run_meta["turns"] != max_turns: - raise ValueError( - "Resume folder max turns does not match current --turns. " - f"Expected t{max_turns}, got t{run_meta['turns']}." - ) - if run_meta["runs"] != runs_per_prompt: - raise ValueError( - "Resume folder runs-per-prompt does not match current --runs. " - f"Expected r{runs_per_prompt}, got r{run_meta['runs']}." - ) - if run_id is None: - run_id = run_folder_name - elif run_id != run_folder_name: - raise ValueError( - "Resume mode requires --run-id to match the run folder name when set." - ) - elif run_id is None: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - run_id = build_generation_run_folder_name( - persona_model_config["model"], - agent_model_config["model"], - max_turns, - runs_per_prompt, - timestamp, - ) - output_folder = f"{output_folder}/{run_id}" - # TODO: do we want to give a message if the folder already exists? - os.makedirs(output_folder, exist_ok=True) - - # Configuration - runner = ConversationRunner( + return await run_generation( persona_model_config=persona_model_config, agent_model_config=agent_model_config, + persona_files=resolved_persona_files, + persona_extra_run_params=persona_extra_run_params, + agent_extra_run_params=agent_extra_run_params, max_turns=max_turns, runs_per_prompt=runs_per_prompt, - folder_name=output_folder, + persona_names=persona_names, + verbose=verbose, + output_folder=output_folder, run_id=run_id, max_concurrent=max_concurrent, max_total_words=max_total_words, @@ -182,23 +69,9 @@ async def main( persona_speaks_first=persona_speaks_first, session_types=session_types, resume=resume, - persona_prompt_path=persona_prompt_path, persona_context_template_path=persona_context_template_path, ) - # Run conversations - results = await runner.run_conversations(persona_names=persona_names) - - if verbose: - skipped_n = sum(1 for r in results if r.get("skipped")) - ok_n = len(results) - skipped_n - msg = f"βœ… Generated {ok_n} conversations β†’ {output_folder}/" - if skipped_n: - msg += f" ({skipped_n} skipped)" - print(msg) - - return results, output_folder - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate LLM conversations") diff --git a/generate_conversations/__init__.py b/generate_conversations/__init__.py index 2538fc825..a931dfd77 100644 --- a/generate_conversations/__init__.py +++ b/generate_conversations/__init__.py @@ -1,5 +1,6 @@ """Generate Conversations Package - LLM Conversation Simulation""" from .runner import ConversationRunner +from .service import run_generation -__all__ = ["ConversationRunner"] +__all__ = ["ConversationRunner", "run_generation"] diff --git a/generate_conversations/service.py b/generate_conversations/service.py new file mode 100644 index 000000000..f30ba0570 --- /dev/null +++ b/generate_conversations/service.py @@ -0,0 +1,152 @@ +"""Application service for conversation generation. + +This module owns generation orchestration independently of any command-line +parser. CLI entry points are adapters around :func:`run_generation`. +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime +from typing import Any, Dict, List, Optional + +from utils.naming import ( + build_generation_run_folder_name, + model_token_for_run_folder, + parse_generation_run_folder_name, +) + +from .runner import ConversationRunner + + +async def run_generation( + persona_model_config: Dict[str, Any], + agent_model_config: Dict[str, Any], + persona_files: List[str], + persona_extra_run_params: Optional[Dict[str, Any]] = None, + agent_extra_run_params: Optional[Dict[str, Any]] = None, + max_turns: int = 3, + runs_per_prompt: int = 2, + persona_names: Optional[List[str]] = None, + verbose: bool = True, + output_folder: Optional[str] = None, + run_id: Optional[str] = None, + max_concurrent: Optional[int] = None, + max_total_words: Optional[int] = None, + max_personas: Optional[int] = None, + persona_speaks_first: bool = True, + session_types: Optional[List[str]] = None, + resume: bool = False, + persona_context_template_path: str = "data/SI/persona_context_template.txt", +) -> tuple[List[Dict[str, Any]], str]: + """Generate conversations from already-resolved persona file paths.""" + persona_extra_run_params = persona_extra_run_params or {} + agent_extra_run_params = agent_extra_run_params or {} + + if verbose: + print("πŸ”„ Generating conversations with the following parameters:") + print(f" - Persona model: {persona_model_config}") + print(f" - Agent model: {agent_model_config}") + print(f" - Persona extra run params: {persona_extra_run_params}") + print(f" - Agent extra run params: {agent_extra_run_params}") + print(f" - Max turns: {max_turns}") + print(f" - Runs per prompt: {runs_per_prompt}") + print(f" - Persona names: {persona_names}") + print(f" - Output folder: {output_folder}") + print(f" - Run ID: {run_id}") + print(f" - Max concurrent: {max_concurrent}") + print(f" - Max total words: {max_total_words}") + print(f" - Max personas: {max_personas}") + print(f" - Persona speaks first: {persona_speaks_first}") + print(f" - Resume: {resume}") + + if output_folder is None: + output_folder = "output" + + if not persona_files: + raise ValueError("generation requires at least one persona file") + if len(persona_files) > 1: + print( + f"Warning: multiple persona files passed ({persona_files}); " + "multi-persona-file support is not yet implemented, using only " + f"the first: {persona_files[0]}", + file=sys.stderr, + ) + persona_prompt_path = persona_files[0] + + if resume: + if not os.path.isdir(output_folder): + raise ValueError( + "Resume mode requires --output to point to an existing run folder." + ) + run_folder_name = os.path.basename(os.path.normpath(output_folder)) + run_meta = parse_generation_run_folder_name(run_folder_name) + expected_persona = model_token_for_run_folder(persona_model_config["model"]) + expected_agent = model_token_for_run_folder(agent_model_config["model"]) + + if run_meta["persona"] != expected_persona: + raise ValueError( + "Resume folder persona model does not match current --user-agent. " + f"Expected p_{expected_persona}, got p_{run_meta['persona']}." + ) + if run_meta["agent"] != expected_agent: + raise ValueError( + "Resume folder provider model does not match current --provider-agent. " + f"Expected a_{expected_agent}, got a_{run_meta['agent']}." + ) + if run_meta["turns"] != max_turns: + raise ValueError( + "Resume folder max turns does not match current --turns. " + f"Expected t{max_turns}, got t{run_meta['turns']}." + ) + if run_meta["runs"] != runs_per_prompt: + raise ValueError( + "Resume folder runs-per-prompt does not match current --runs. " + f"Expected r{runs_per_prompt}, got r{run_meta['runs']}." + ) + if run_id is None: + run_id = run_folder_name + elif run_id != run_folder_name: + raise ValueError( + "Resume mode requires --run-id to match the run folder name when set." + ) + elif run_id is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + run_id = build_generation_run_folder_name( + persona_model_config["model"], + agent_model_config["model"], + max_turns, + runs_per_prompt, + timestamp, + ) + output_folder = f"{output_folder}/{run_id}" + os.makedirs(output_folder, exist_ok=True) + + runner = ConversationRunner( + persona_model_config=persona_model_config, + agent_model_config=agent_model_config, + max_turns=max_turns, + runs_per_prompt=runs_per_prompt, + folder_name=output_folder, + run_id=run_id, + max_concurrent=max_concurrent, + max_total_words=max_total_words, + max_personas=max_personas, + persona_speaks_first=persona_speaks_first, + session_types=session_types, + resume=resume, + persona_prompt_path=persona_prompt_path, + persona_context_template_path=persona_context_template_path, + ) + results = await runner.run_conversations(persona_names=persona_names) + + if verbose: + skipped_n = sum(1 for result in results if result.get("skipped")) + ok_n = len(results) - skipped_n + message = f"βœ… Generated {ok_n} conversations β†’ {output_folder}/" + if skipped_n: + message += f" ({skipped_n} skipped)" + print(message) + + return results, output_folder diff --git a/judge/__init__.py b/judge/__init__.py index 8805a5c86..17f6f004f 100644 --- a/judge/__init__.py +++ b/judge/__init__.py @@ -4,10 +4,12 @@ from .runner import ( judge_conversations, judge_single_conversation, + run_judging, ) __all__ = [ "LLMJudge", "judge_conversations", "judge_single_conversation", + "run_judging", ] diff --git a/judge/runner.py b/judge/runner.py index 3dafe0ee0..806511c67 100644 --- a/judge/runner.py +++ b/judge/runner.py @@ -5,6 +5,7 @@ import asyncio import os +import sys from asyncio import Queue from datetime import datetime from pathlib import Path @@ -12,8 +13,10 @@ import pandas as pd +from utils.conversation_layout import resolve_conversation_input + from .llm_judge import LLMJudge -from .rubric_config import ConversationData, RubricConfig +from .rubric_config import ConversationData, RubricConfig, load_conversations from .score_utils import build_dataframe_from_tsv_files from .utils import ( build_evaluation_run_folder_path, @@ -25,6 +28,65 @@ EVALUATION_SEPARATOR = ":" +async def run_judging( + conversation_folder: str, + rubric_manifest: str, + judge_models: Dict[str, int], + *, + judge_model_extra_params: Optional[Dict[str, Any]] = None, + limit: Optional[int] = None, + output_root: Optional[str] = None, + max_concurrent: Optional[int] = None, + per_judge: bool = False, + verbose_workers: bool = False, + debug: bool = False, +) -> str: + """Judge a conversation folder without depending on a CLI parser.""" + if debug: + from utils.debug import set_debug + + set_debug(True) + + models_str = ", ".join(f"{model}x{count}" for model, count in judge_models.items()) + print(f"🎯 LLM Judge | Models: {models_str}") + print("πŸ“š Loading rubric configuration...") + rubric_config = await RubricConfig.load_bundle(rubric_manifest) + + transcripts_dir, gen_run, conv_basename = resolve_conversation_input( + conversation_folder + ) + print(f"πŸ“‚ Loading conversations from {transcripts_dir}...") + conversations = await load_conversations(transcripts_dir, limit=limit) + print(f"βœ… Loaded {len(conversations)} conversations") + + if output_root is None: + if gen_run is not None: + output_root = os.path.join(gen_run, "evaluations") + else: + output_root = "evaluations" + print( + "Note: flat conversation folder; writing evaluations under " + "evaluations/. New runs use output/p_*__/conversations/.", + file=sys.stderr, + ) + + _, output_folder = await judge_conversations( + judge_models=judge_models, + conversations=conversations, + rubric_config=rubric_config, + output_root=output_root, + max_concurrent=max_concurrent, + conversation_folder_name=conv_basename, + verbose=True, + judge_model_extra_params=judge_model_extra_params, + per_judge=per_judge, + verbose_workers=verbose_workers, + resume=False, + ) + print(f"Evaluation output: {output_folder}/") + return output_folder + + def _parse_evaluation_to_dict(evaluation: Dict[str, Any]) -> Dict[str, Any]: """ Parse evaluation results into a flat dictionary. diff --git a/judge/score.py b/judge/score.py index 058a56aeb..d7504be95 100644 --- a/judge/score.py +++ b/judge/score.py @@ -476,8 +476,62 @@ def _rebuild_dataframe_if_needed(results_csv_path: Path) -> bool: return False +def score_results_file( + results_csv: str, + *, + output_json: Optional[str] = None, + personas_tsv: str = "data/SI/personas.tsv", + skip_risk_analysis: bool = False, +) -> int: + """Score one results file and write all standard derived artifacts.""" + results_csv_path = Path(results_csv) + if not results_csv_path.exists(): + print(f"Error: Results CSV file not found: {results_csv}") + return 1 + + if not _rebuild_dataframe_if_needed(results_csv_path): + if not has_dimension_data(read_judge_results_csv(results_csv_path)): + return 1 + + results = score_results(str(results_csv_path), output_path=output_json) + print_scores(results) + + scores_dir = _scores_output_dir(str(results_csv_path)) + json_path = Path(output_json) if output_json else scores_dir / "scores.json" + print(f"\nβœ… Scores saved to: {json_path}") + + viz_path = scores_dir / "scores_visualization.png" + try: + create_visualizations(results, viz_path) + except Exception as error: + print(f"⚠️ Warning: Could not create standard visualizations: {error}") + + if not skip_risk_analysis: + personas_tsv_path = Path(personas_tsv) + if not personas_tsv_path.exists(): + print(f"⚠️ Warning: Personas TSV file not found: {personas_tsv}") + print( + " Skipping risk-level analysis. Use --skip-risk-analysis " + "to suppress this warning." + ) + else: + try: + risk_results = score_results_by_risk( + str(results_csv_path), str(personas_tsv_path) + ) + risk_viz_path = scores_dir / "scores_by_risk_visualization.png" + create_risk_level_visualizations(risk_results, risk_viz_path) + except Exception as error: + print(f"⚠️ Warning: Could not create risk-level analysis: {error}") + import traceback + + traceback.print_exc() + + return 0 + + def main(): - """Main entry point for scoring script.""" + """Parse legacy CLI arguments and call the scoring service.""" parser = argparse.ArgumentParser( description=( "Score evaluation results from judge/runner.py output " @@ -515,54 +569,12 @@ def main(): args = parser.parse_args() - results_csv_path = Path(args.results_csv) - if not results_csv_path.exists(): - print(f"Error: Results CSV file not found: {args.results_csv}") - return 1 - - if not _rebuild_dataframe_if_needed(results_csv_path): - # If rebuild failed, exit - if not has_dimension_data(read_judge_results_csv(results_csv_path)): - return 1 - - results = score_results(str(results_csv_path), output_path=args.output_json) - print_scores(results) - - scores_dir = _scores_output_dir(str(results_csv_path)) - json_path = ( - Path(args.output_json) if args.output_json else scores_dir / "scores.json" + return score_results_file( + args.results_csv, + output_json=args.output_json, + personas_tsv=args.personas_tsv, + skip_risk_analysis=args.skip_risk_analysis, ) - print(f"\nβœ… Scores saved to: {json_path}") - - viz_path = scores_dir / "scores_visualization.png" - try: - create_visualizations(results, viz_path) - except Exception as e: - print(f"⚠️ Warning: Could not create standard visualizations: {e}") - - # Create risk-level analysis and visualization if not skipped - if not args.skip_risk_analysis: - personas_tsv_path = Path(args.personas_tsv) - if not personas_tsv_path.exists(): - print(f"⚠️ Warning: Personas TSV file not found: {args.personas_tsv}") - print( - " Skipping risk-level analysis. Use --skip-risk-analysis " - "to suppress this warning." - ) - else: - try: - risk_results = score_results_by_risk( - str(results_csv_path), str(personas_tsv_path) - ) - risk_viz_path = scores_dir / "scores_by_risk_visualization.png" - create_risk_level_visualizations(risk_results, risk_viz_path) - except Exception as e: - print(f"⚠️ Warning: Could not create risk-level analysis: {e}") - import traceback - - traceback.print_exc() - - return 0 if __name__ == "__main__": diff --git a/tests/unit/generate_conversations/test_generate_cli.py b/tests/unit/generate_conversations/test_generate_cli.py index c21decd59..d57dcf30b 100644 --- a/tests/unit/generate_conversations/test_generate_cli.py +++ b/tests/unit/generate_conversations/test_generate_cli.py @@ -7,6 +7,7 @@ import pytest import generate +from generate_conversations.service import run_generation @pytest.mark.asyncio @@ -18,7 +19,7 @@ async def test_main_resume_uses_existing_run_folder(tmp_path: Path) -> None: persona_model_config = {"model": "mock-persona"} agent_model_config = {"model": "mock-agent", "name": "mock-agent"} - with patch("generate.ConversationRunner") as mock_runner_cls: + with patch("generate_conversations.service.ConversationRunner") as mock_runner_cls: mock_runner = mock_runner_cls.return_value mock_runner.run_conversations = AsyncMock(return_value=[]) @@ -82,7 +83,7 @@ async def test_main_rubric_manifest_loads_personas_from_manifest( persona_model_config = {"model": "mock-persona"} agent_model_config = {"model": "mock-agent", "name": "mock-agent"} - with patch("generate.ConversationRunner") as mock_runner_cls: + with patch("generate_conversations.service.ConversationRunner") as mock_runner_cls: mock_runner = mock_runner_cls.return_value mock_runner.run_conversations = AsyncMock(return_value=[]) @@ -110,7 +111,7 @@ async def test_main_no_rubric_manifest_uses_default_personas(tmp_path: Path) -> persona_model_config = {"model": "mock-persona"} agent_model_config = {"model": "mock-agent", "name": "mock-agent"} - with patch("generate.ConversationRunner") as mock_runner_cls: + with patch("generate_conversations.service.ConversationRunner") as mock_runner_cls: mock_runner = mock_runner_cls.return_value mock_runner.run_conversations = AsyncMock(return_value=[]) @@ -132,6 +133,60 @@ async def test_main_no_rubric_manifest_uses_default_personas(tmp_path: Path) -> ) +@pytest.mark.asyncio +async def test_main_persona_files_uses_direct_cli_selection(tmp_path: Path) -> None: + """The unified CLI can select a persona file without a rubric manifest.""" + persona_file = tmp_path / "personas.tsv" + + with patch("generate_conversations.service.ConversationRunner") as mock_runner_cls: + mock_runner = mock_runner_cls.return_value + mock_runner.run_conversations = AsyncMock(return_value=[]) + + await generate.main( + persona_model_config={"model": "mock-persona"}, + agent_model_config={"model": "mock-agent", "name": "mock-agent"}, + output_folder=str(tmp_path / "out"), + run_id="run1", + persona_files=[str(persona_file)], + verbose=False, + ) + + kwargs = mock_runner_cls.call_args.kwargs + assert kwargs["persona_prompt_path"] == str(persona_file) + assert ( + kwargs["persona_context_template_path"] + == "data/SI/persona_context_template.txt" + ) + + +@pytest.mark.asyncio +async def test_main_rejects_persona_files_with_rubric_manifest( + tmp_path: Path, +) -> None: + with pytest.raises(ValueError, match="mutually exclusive"): + await generate.main( + persona_model_config={"model": "mock-persona"}, + agent_model_config={"model": "mock-agent", "name": "mock-agent"}, + output_folder=str(tmp_path / "out"), + run_id="run1", + persona_files=[str(tmp_path / "personas.tsv")], + rubric_manifest=str(tmp_path / "manifest.json"), + verbose=False, + ) + + +@pytest.mark.asyncio +async def test_domain_generation_requires_resolved_persona_file() -> None: + with pytest.raises(ValueError, match="requires at least one persona file"): + await run_generation( + persona_model_config={"model": "mock-persona"}, + agent_model_config={"model": "mock-agent", "name": "mock-agent"}, + persona_files=[], + run_id="run1", + verbose=False, + ) + + @pytest.mark.asyncio async def test_main_rubric_manifest_without_personas_raises_value_error( tmp_path: Path, diff --git a/tests/unit/judge/test_runner_extra_params.py b/tests/unit/judge/test_runner_extra_params.py index db2554d6d..2da24b4a9 100644 --- a/tests/unit/judge/test_runner_extra_params.py +++ b/tests/unit/judge/test_runner_extra_params.py @@ -11,6 +11,7 @@ _create_evaluation_jobs, batch_evaluate_with_individual_judges, judge_conversations, + run_judging, ) from judge.utils import build_judge_task_log_path, judge_evaluation_tsv_filename @@ -24,6 +25,52 @@ } +@pytest.mark.unit +@pytest.mark.asyncio +async def test_run_judging_is_parser_independent(tmp_path: Path) -> None: + """The application service accepts values directly and returns its run path.""" + generation_run = tmp_path / "p_user__a_agent__t3__r1__20260731_120000" + conversations_dir = generation_run / "conversations" + conversations_dir.mkdir(parents=True) + rubric_config = MagicMock() + conversations = [MagicMock()] + expected_output = str(generation_run / "evaluations" / "j_run") + + with ( + patch( + "judge.runner.RubricConfig.load_bundle", + new_callable=AsyncMock, + return_value=rubric_config, + ) as load_bundle, + patch( + "judge.runner.load_conversations", + new_callable=AsyncMock, + return_value=conversations, + ) as load_conversations, + patch( + "judge.runner.judge_conversations", + new_callable=AsyncMock, + return_value=([], expected_output), + ) as judge_batch, + ): + output = await run_judging( + str(generation_run), + "rubric_manifest.json", + {"judge": 2}, + judge_model_extra_params={"temperature": 0}, + limit=3, + ) + + assert output == expected_output + load_bundle.assert_awaited_once_with("rubric_manifest.json") + load_conversations.assert_awaited_once_with(str(conversations_dir), limit=3) + assert judge_batch.await_args is not None + assert judge_batch.await_args.kwargs["output_root"] == str( + generation_run / "evaluations" + ) + assert judge_batch.await_args.kwargs["judge_models"] == {"judge": 2} + + def _conversation(tmp_path: Path, index: int = 0) -> ConversationData: """Single ConversationData for tests.""" if index == 0: diff --git a/tests/unit/test_vera_cli.py b/tests/unit/test_vera_cli.py new file mode 100644 index 000000000..c785c9a9e --- /dev/null +++ b/tests/unit/test_vera_cli.py @@ -0,0 +1,352 @@ +"""Tests for the unified ``vera.py`` CLI adapters and source contract.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +import vera +from utils.config_schema import ( + GenerationConfig, + JudgingConfig, + ModelSpec, + RubricSpec, + RunConfig, +) + + +@pytest.fixture(autouse=True) +def clear_env_config(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(vera.VERA_RUN_CONFIG_ENV, raising=False) + + +def _write_config(tmp_path: Path, data: dict) -> Path: + path = tmp_path / "run.json" + path.write_text(json.dumps(data), encoding="utf-8") + return path + + +def test_model_shorthand_preserves_colon_model_names() -> None: + assert ModelSpec.from_shorthand("llama3:8b") == ModelSpec(name="llama3:8b") + assert ModelSpec.from_shorthand("gpt-5:3") == ModelSpec(name="gpt-5", repeats=3) + + +def test_judge_rejects_config_mixed_with_conversations(tmp_path: Path) -> None: + config = _write_config( + tmp_path, + { + "judging": { + "models": [{"name": "gpt-5"}], + "rubrics": [{"name": "SI"}], + "conversations": ["output/conversations"], + } + }, + ) + + with pytest.raises(SystemExit) as error: + vera.main( + [ + "judge", + "--config", + str(config), + "--conversations", + "other/conversations", + ] + ) + + assert error.value.code == 2 + + +def test_config_allows_cli_sample_only( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + config = _write_config( + tmp_path, + { + "generation": { + "chatbot": {"name": "gpt-5"}, + "user": [{"name": "claude-sonnet-5"}], + "personas": ["data/SI/personas.tsv"], + } + }, + ) + + with patch.object( + vera, "_run_generation", new_callable=AsyncMock + ) as run_generation: + result = vera.main(["generate", "--config", str(config), "--sample", "1"]) + + assert result == 0 + rendered = json.loads(capsys.readouterr().out) + assert "sample" not in rendered + assert "debug" not in rendered + assert rendered["generation"]["personas"] == [ + str((vera.ROOT / "data/SI/personas.tsv").resolve()) + ] + assert run_generation.await_args is not None + assert run_generation.await_args.kwargs["sample"] == 1 + + +@pytest.mark.parametrize("control", ["--debug", "--print"]) +def test_config_rejects_non_sample_cli_controls(tmp_path: Path, control: str) -> None: + config = _write_config( + tmp_path, + { + "generation": { + "chatbot": {"name": "gpt-5"}, + "user": [{"name": "claude-sonnet-5"}], + "personas": ["data/SI/personas.tsv"], + } + }, + ) + + with pytest.raises(SystemExit) as error: + vera.main(["generate", "--config", str(config), control]) + + assert error.value.code == 2 + + +@pytest.mark.parametrize("field", ["sample", "debug", "print"]) +def test_debug_controls_are_not_config_fields(tmp_path: Path, field: str) -> None: + config = _write_config( + tmp_path, + { + "generation": { + "chatbot": {"name": "gpt-5"}, + "user": [{"name": "claude-sonnet-5"}], + "personas": ["data/SI/personas.tsv"], + }, + field: 1, + }, + ) + + with pytest.raises(SystemExit) as error: + vera.main(["generate", "--config", str(config), "--print"]) + + assert error.value.code == 2 + + +def test_judge_config_owns_conversation_paths(tmp_path: Path) -> None: + config = _write_config( + tmp_path, + { + "judging": { + "models": [{"name": "gpt-5"}], + "rubrics": [{"name": "SI"}], + "conversations": ["output/conversations"], + } + }, + ) + + args = vera.build_parser().parse_args(["judge", "--config", str(config)]) + run_config = vera.resolve_run_config( + args, cli_flag_names=("judge", "rubric", "conversations") + ) + assert run_config.judging is not None + assert run_config.judging.conversations == [ + str((vera.ROOT / "output/conversations").resolve()) + ] + + +@pytest.mark.asyncio +async def test_generation_delegates_to_domain_function() -> None: + run_config = RunConfig( + generation=GenerationConfig( + chatbot=ModelSpec(name="chatbot", extra_params={"temperature": 0.2}), + user=[ModelSpec(name="user", repeats=3, extra_params={"top_p": 0.8})], + personas=["personas.tsv"], + ) + ) + + with patch( + "generate_conversations.run_generation", new_callable=AsyncMock + ) as run_generation: + run_generation.return_value = ([], "output/generated") + outputs = await vera._run_generation(run_config, sample=2, debug=False) + + assert outputs == ["output/generated"] + assert run_generation.await_args is not None + kwargs = run_generation.await_args.kwargs + assert kwargs["persona_model_config"] == {"model": "user", "top_p": 0.8} + assert kwargs["agent_model_config"] == { + "model": "chatbot", + "name": "chatbot", + "temperature": 0.2, + } + assert kwargs["runs_per_prompt"] == 3 + assert kwargs["max_personas"] == 2 + assert kwargs["persona_files"] == ["personas.tsv"] + + +@pytest.mark.asyncio +async def test_generation_target_resolves_manifest_personas(tmp_path: Path) -> None: + manifest_dir = tmp_path / "target" + manifest_dir.mkdir() + manifest = manifest_dir / "rubric_manifest.json" + manifest.write_text( + json.dumps( + { + "rubric_file": "rubric.tsv", + "rubric_prompt_beginning_file": "rubric_prompt.txt", + "question_prompt_file": "question_prompt.txt", + "personas": ["personas.tsv"], + } + ), + encoding="utf-8", + ) + run_config = RunConfig( + generation=GenerationConfig( + chatbot=ModelSpec(name="chatbot"), + user=[ModelSpec(name="user")], + ), + target="target", + ) + + with ( + patch.object(vera, "_target_manifests", return_value=[manifest]), + patch( + "generate_conversations.run_generation", new_callable=AsyncMock + ) as run_generation, + ): + run_generation.return_value = ([], "output/generated") + outputs = await vera._run_generation(run_config, sample=None, debug=False) + + assert outputs == ["output/generated"] + assert run_generation.await_args is not None + assert run_generation.await_args.kwargs["persona_files"] == [ + str(manifest_dir / "personas.tsv") + ] + assert "rubric_manifest" not in run_generation.await_args.kwargs + + +@pytest.mark.asyncio +async def test_generation_target_without_personas_has_actionable_error( + tmp_path: Path, +) -> None: + manifest = tmp_path / "rubric_manifest.json" + manifest.write_text( + json.dumps( + { + "rubric_file": "rubric.tsv", + "rubric_prompt_beginning_file": "rubric_prompt.txt", + "question_prompt_file": "question_prompt.txt", + } + ), + encoding="utf-8", + ) + run_config = RunConfig( + generation=GenerationConfig( + chatbot=ModelSpec(name="chatbot"), + user=[ModelSpec(name="user")], + ), + target="target", + ) + + with ( + patch.object(vera, "_target_manifests", return_value=[manifest]), + patch( + "generate_conversations.run_generation", new_callable=AsyncMock + ) as run_generation, + pytest.raises(vera.ConfigError, match="manifest .* defines no personas"), + ): + await vera._run_generation(run_config, sample=None, debug=False) + + run_generation.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_judging_delegates_to_domain_function() -> None: + run_config = RunConfig( + judging=JudgingConfig( + models=[ModelSpec(name="judge", repeats=2)], + rubrics=[RubricSpec(name="data/SI/rubric_manifest.json")], + conversations=["output/conversations"], + ) + ) + with patch("judge.run_judging", new_callable=AsyncMock) as run_judging: + run_judging.return_value = "output/evaluations/j_run" + outputs = await vera._run_judging(run_config, sample=1, debug=False) + + assert outputs == ["output/evaluations/j_run"] + assert run_judging.await_args is not None + kwargs = run_judging.await_args.kwargs + assert kwargs["conversation_folder"] == "output/conversations" + assert kwargs["judge_models"] == {"judge": 1} + assert kwargs["limit"] == 1 + assert kwargs["rubric_manifest"] == str( + (vera.ROOT / "data/SI/rubric_manifest.json").resolve() + ) + + +def test_score_delegates_to_score_adapter() -> None: + with patch.object(vera, "_run_scoring") as run_scoring: + assert vera.main(["score", "--results", "results.csv"]) == 0 + + run_scoring.assert_called_once_with("results.csv") + + +def test_score_adapter_delegates_to_domain_function() -> None: + with patch("judge.score.score_results_file", return_value=0) as score_file: + vera._run_scoring("results.csv") + + score_file.assert_called_once_with( + "results.csv", + personas_tsv=str(vera.ROOT / "data" / "SI" / "personas.tsv"), + ) + + +def test_pool_delegates_to_existing_pool_function() -> None: + args = argparse.Namespace(evaluations=["one", "two"], print=False) + with patch( + "scripts.pool_vera_scores.pool_evaluation_directories" + ) as pool_evaluation_directories: + assert vera.cmd_pool(args) == 0 + + pool_evaluation_directories.assert_called_once_with( + ["one", "two"], + vera.ROOT / "output", + personas_tsv=vera.ROOT / "data" / "SI" / "personas.tsv", + ) + + +def test_pipeline_chains_generation_judging_and_scoring() -> None: + with ( + patch.object( + vera, + "_run_generation", + new_callable=AsyncMock, + return_value=["output/generated"], + ) as run_generation, + patch.object( + vera, + "_run_judging", + new_callable=AsyncMock, + return_value=["output/evaluations/j_run"], + ) as run_judging, + patch.object(vera, "_run_scoring") as run_scoring, + ): + result = vera.main( + [ + "pipeline", + "-c", + "chatbot", + "-u", + "user", + "-j", + "judge", + "--personas", + "data/SI/personas.tsv", + "--rubric", + "data/SI/rubric_manifest.json", + ] + ) + + assert result == 0 + run_generation.assert_awaited_once() + assert run_judging.await_args is not None + assert run_judging.await_args.kwargs["conversations"] == ["output/generated"] + run_scoring.assert_called_once_with("output/evaluations/j_run/results.csv") diff --git a/utils/config_schema.py b/utils/config_schema.py index b5f6b0048..efd01d2f5 100644 --- a/utils/config_schema.py +++ b/utils/config_schema.py @@ -33,6 +33,14 @@ class ModelSpec: repeats: int = 1 extra_params: dict[str, Any] = dataclasses.field(default_factory=dict) + def __post_init__(self) -> None: + if not self.name: + raise ValueError("model name cannot be empty") + if isinstance(self.repeats, bool) or not isinstance(self.repeats, int): + raise ValueError("model repeats must be an integer") + if self.repeats < 1: + raise ValueError("model repeats must be at least 1") + def to_dict(self) -> dict[str, Any]: return {"name": self.name, "repeats": self.repeats, **self.extra_params} @@ -46,10 +54,15 @@ def from_dict(cls, data: dict[str, Any]) -> "ModelSpec": @classmethod def from_shorthand(cls, token: str) -> "ModelSpec": """Parse `-u`/`-c`/`-j` shorthand: "[:]".""" - name, sep, repeats_str = token.partition(":") + name, sep, repeats_str = token.rpartition(":") + if not sep or not repeats_str.isdigit(): + name = token + repeats_str = "1" if not name: raise ValueError(f"invalid model shorthand: {token!r}") - repeats = int(repeats_str) if sep else 1 + repeats = int(repeats_str) + if repeats < 1: + raise ValueError(f"model repeats must be at least 1: {token!r}") return cls(name=name, repeats=repeats) @@ -119,6 +132,7 @@ class JudgingConfig: models: list[ModelSpec] = dataclasses.field(default_factory=list) rubrics: list[RubricSpec] = dataclasses.field(default_factory=list) + conversations: list[str] = dataclasses.field(default_factory=list) def to_dict(self) -> dict[str, Any]: d: dict[str, Any] = {} @@ -126,6 +140,8 @@ def to_dict(self) -> dict[str, Any]: d["models"] = [m.to_dict() for m in self.models] if self.rubrics: d["rubrics"] = [r.to_dict() for r in self.rubrics] + if self.conversations: + d["conversations"] = self.conversations return d @classmethod @@ -133,6 +149,7 @@ def from_dict(cls, data: dict[str, Any]) -> "JudgingConfig": return cls( models=[ModelSpec.from_dict(m) for m in data.get("models", [])], rubrics=[RubricSpec.from_dict(r) for r in data.get("rubrics", [])], + conversations=list(data.get("conversations", [])), ) @@ -155,7 +172,6 @@ class RunConfig: generation: Optional[GenerationConfig] = None judging: Optional[JudgingConfig] = None target: Optional[str] = None - sample: Optional[int] = None def __post_init__(self) -> None: if self.target is not None and self.generation and self.generation.personas: @@ -177,8 +193,6 @@ def to_dict(self) -> dict[str, Any]: d["judging"] = self.judging.to_dict() if self.target is not None: d["target"] = self.target - if self.sample is not None: - d["sample"] = self.sample return d @classmethod @@ -189,7 +203,6 @@ def from_dict(cls, data: dict[str, Any]) -> "RunConfig": generation=GenerationConfig.from_dict(generation) if generation else None, judging=JudgingConfig.from_dict(judging) if judging else None, target=data.get("target"), - sample=data.get("sample"), ) diff --git a/utils/rubric_manifest.py b/utils/rubric_manifest.py index c205191a9..b0c7b7159 100644 --- a/utils/rubric_manifest.py +++ b/utils/rubric_manifest.py @@ -1,12 +1,9 @@ """Shared rubric bundle manifest reading. -A rubric bundle manifest (docs/architecture.md#rubric-bundle-manifest) -attaches a rubric and the personas it's validated for as one unit. Both -`generate.py` (personas half) and `judge/rubric_config.py` (rubric half) -read the same manifest file -- this lives in `utils/` (the leaf layer) -rather than in `judge/` so `generate.py` never has to import a `judge/` -module to read it (`generate/`/`judge/` must never import each other, per -docs/architecture.md's Layer model). +Manifest personas are optional and informational for ordinary judging. Explicit +consumers such as ``vera --target`` and legacy ``generate.py --rubric-manifest`` +may resolve them into generation inputs. This helper lives in the leaf ``utils`` +layer so generation never imports judging code. """ from __future__ import annotations @@ -57,11 +54,9 @@ async def load_manifest(manifest_path: str) -> dict[str, Any]: async def load_manifest_personas(manifest_path: str) -> list[str]: """Read a rubric bundle manifest's `personas` list. - Used by `generate.py --rubric-manifest` (Phase 0's generation-side - counterpart to `judge.py --rubrics`, see docs/architecture.md's Phase 0 - migration entry) to select personas from the same manifest that - `judge.py` loads the rubric from. `personas` is optional in the - manifest and defaults to an empty list. + Used by ``vera --target`` and the legacy ``generate.py --rubric-manifest`` + adapter. ``personas`` is optional in the manifest and defaults to an empty + list; each caller decides whether its invocation requires personas. Entries resolve relative to the manifest's own folder (never `$ROOT` or the caller's working directory), per docs/architecture.md#rubric-bundle-manifest diff --git a/vera.py b/vera.py index dc44cf1ef..6de430155 100644 --- a/vera.py +++ b/vera.py @@ -1,25 +1,21 @@ #!/usr/bin/env python3 """VERA-MH unified CLI orchestrator. -Per docs/architecture.md's "CLI surface" section, `vera.py` is the single -root-level orchestrator: subcommands parse arguments and delegate to domain -runners; they contain no business logic themselves. - -Subcommands: generate, judge, score, pool, pipeline, resume -- see -docs/vera-cli-use-cases.md for the full CLI/config design this implements. - -This is Phase 1 of the migration (docs/architecture.md#migration-from-current-layout): -argument parsing and config resolution land now; wiring into the existing -`generate`/`judge` engines is tracked separately and stubbed here for now. +The CLI resolves either command-line flags or JSON config into one ``RunConfig`` +and delegates to parser-independent domain functions. It deliberately contains +orchestration only; domain behavior stays in the existing modules. """ from __future__ import annotations import argparse +import asyncio import json import os import sys -from typing import Any, Callable, Optional +from collections.abc import Callable +from pathlib import Path +from typing import Any, Optional from utils.config_schema import ( GenerationConfig, @@ -30,18 +26,10 @@ ) PROG = "vera" - +ROOT = Path(__file__).resolve().parent VERA_RUN_CONFIG_ENV = "VERA_RUN_CONFIG" -# --------------------------------------------------------------------------- -# Centralized flag registry. -# -# Every CLI flag is defined exactly once here and referenced by name from -# whichever subcommand(s) need it, so `-c`/`-u`/`-j`/`--config`/etc. can never -# drift into slightly-different definitions across subcommands. -# --------------------------------------------------------------------------- - FLAG_SPECS: dict[str, dict[str, Any]] = { "chatbot": { "flags": ("-c", "--chatbot"), @@ -55,7 +43,7 @@ "kwargs": { "nargs": "+", "metavar": "[:]", - "help": "User-side LLM(s), e.g. `-u gpt:1 sonnet:2`. `repeats` default 1.", + "help": "User-side LLM(s), e.g. `-u gpt:1 sonnet:2`.", }, }, "judge": { @@ -63,7 +51,7 @@ "kwargs": { "nargs": "+", "metavar": "[:]", - "help": "Judge LLM(s), e.g. `-j claude:1 gpt:2`. `repeats` defaults to 1.", + "help": "Judge LLM(s), e.g. `-j claude:1 gpt:2`.", }, }, "personas": { @@ -71,7 +59,7 @@ "kwargs": { "nargs": "+", "metavar": "", - "help": "Persona file(s) for generation. Mutually exclusive with --target.", + "help": "Persona file(s). Mutually exclusive with --target.", }, }, "target": { @@ -79,9 +67,8 @@ "kwargs": { "metavar": "", "help": ( - "Resolve to a rubric bundle manifest and set both " - "generation personas and the judging rubric from it in one shot. " - "Use `--target all` to run every known evaluator." + "Select personas and rubric(s) from a named rubric bundle. " + "Use `--target all` for every discovered bundle." ), }, }, @@ -90,7 +77,7 @@ "kwargs": { "nargs": "+", "metavar": "", - "help": "Rubric bundle manifest path(s) (see architecture.md).", + "help": "Rubric bundle manifest path(s).", }, }, "conversations": { @@ -121,8 +108,8 @@ "kwargs": { "metavar": "", "help": ( - "JSON config file (or `-` for stdin). Mutually exclusive with " - f"CLI model/persona/rubric flags and the {VERA_RUN_CONFIG_ENV} env var." + "JSON config file (or `-` for stdin). Cannot be combined with " + f"run-defining CLI flags or {VERA_RUN_CONFIG_ENV}." ), }, }, @@ -131,121 +118,158 @@ "kwargs": { "type": int, "metavar": "N", - "help": "Smoke-test override: cap personas/rubrics/judges to N.", + "help": "Debug-only cap for personas, rubrics, and judges.", + }, + }, + "debug": { + "flags": ("-d", "--debug"), + "kwargs": { + "action": "store_true", + "help": "Enable debug logging for a CLI-defined run.", }, }, "print": { "flags": ("--print",), "kwargs": { "action": "store_true", - "help": "Print the resolved flag-string and exit, without running.", + "help": "Print the resolved config and exit without running.", }, }, } +class ConfigError(ValueError): + """Raised for invalid or ambiguous CLI/config input.""" + + def add_flags( parser: argparse.ArgumentParser | argparse._MutuallyExclusiveGroup, *names: str ) -> None: - """Attach flags from FLAG_SPECS to a parser or argument group by name.""" + """Attach centrally-defined flags to a parser.""" for name in names: spec = FLAG_SPECS[name] parser.add_argument(*spec["flags"], **spec["kwargs"]) -# --------------------------------------------------------------------------- -# Shared shorthand parsing. -# --------------------------------------------------------------------------- - - def parse_model_list(tokens: Optional[list[str]]) -> list[ModelSpec]: - if not tokens: - return [] - return [ModelSpec.from_shorthand(t) for t in tokens] + return [ModelSpec.from_shorthand(token) for token in tokens or []] def parse_single_model(token: Optional[str]) -> Optional[ModelSpec]: - if token is None: - return None - return ModelSpec.from_shorthand(token) - - -# --------------------------------------------------------------------------- -# Config resolution: CLI flags and --config both funnel into one RunConfig. -# --------------------------------------------------------------------------- - - -class ConfigError(ValueError): - """Raised when CLI flags and --config/env conflict, or a field is missing.""" + return ModelSpec.from_shorthand(token) if token is not None else None def _load_config_json(args: argparse.Namespace) -> Optional[dict[str, Any]]: - """Load raw config JSON from --config or VERA_RUN_CONFIG, if given.""" config_arg = getattr(args, "config", None) env_config = os.environ.get(VERA_RUN_CONFIG_ENV) if config_arg and env_config: raise ConfigError(f"--config and {VERA_RUN_CONFIG_ENV} are mutually exclusive") - - if config_arg: - if config_arg == "-": - return json.loads(sys.stdin.read()) - with open(config_arg) as f: - return json.load(f) - - if env_config: - return json.loads(env_config) - + try: + if config_arg: + if config_arg == "-": + return json.loads(sys.stdin.read()) + with open(config_arg, encoding="utf-8") as config_file: + return json.load(config_file) + if env_config: + return json.loads(env_config) + except (json.JSONDecodeError, OSError) as error: + raise ConfigError(f"could not load config: {error}") from error return None def _cli_flags_given(args: argparse.Namespace, names: tuple[str, ...]) -> list[str]: - return [n for n in names if getattr(args, n, None)] + return [ + name for name in names if getattr(args, name, None) not in (None, False, []) + ] + + +def _root_path(path: str) -> str: + candidate = Path(path) + if not candidate.is_absolute(): + candidate = ROOT / candidate + return str(candidate.resolve()) + + +def _resolve_config_paths(run_config: RunConfig) -> RunConfig: + """Apply the architecture's $ROOT rule to config-sourced path fields.""" + if run_config.generation: + run_config.generation.personas = [ + _root_path(path) for path in run_config.generation.personas + ] + if run_config.judging: + run_config.judging.conversations = [ + _root_path(path) for path in run_config.judging.conversations + ] + for rubric in run_config.judging.rubrics: + if rubric.name.lower() != "all" and ( + "/" in rubric.name or rubric.name.endswith(".json") + ): + rubric.name = _root_path(rubric.name) + return run_config def resolve_run_config( args: argparse.Namespace, cli_flag_names: tuple[str, ...] ) -> RunConfig: - """Resolve a subcommand's parsed args into one canonical RunConfig. + """Resolve exactly one run-definition source into a canonical config. - CLI flags and --config/VERA_RUN_CONFIG are strictly either/or -- never - combined for the same run (docs/vera-cli-use-cases.md#config-mechanism). + ``--sample`` is AD-17's sole named exception and may accompany config. + ``--debug`` and ``--print`` remain CLI-only controls. """ config_json = _load_config_json(args) - given_cli_flags = _cli_flags_given(args, cli_flag_names) + given_cli_flags = _cli_flags_given(args, (*cli_flag_names, "debug", "print")) + config_sourced = config_json is not None - if config_json is not None: - if given_cli_flags: + if config_sourced and given_cli_flags: + rendered = ", ".join(f"--{name.replace('_', '-')}" for name in given_cli_flags) + raise ConfigError( + f"config input cannot be combined with run-defining CLI flags: {rendered}" + ) + if config_sourced: + debug_fields = sorted({"sample", "debug", "print"}.intersection(config_json)) + if debug_fields: raise ConfigError( - "--config/" - f"{VERA_RUN_CONFIG_ENV} cannot be combined with CLI flags: " - f"{', '.join(given_cli_flags)}" + "debug/execution controls belong on the command line, not in config: " + f"{', '.join(debug_fields)}" ) - return RunConfig.from_dict(config_json) - return _run_config_from_cli(args) + try: + run_config = ( + RunConfig.from_dict(config_json) + if config_sourced + else _run_config_from_cli(args) + ) + except (KeyError, TypeError, ValueError) as error: + raise ConfigError(f"invalid run config: {error}") from error + + setattr(args, "_config_sourced", config_sourced) + return _resolve_config_paths(run_config) if config_sourced else run_config def _run_config_from_cli(args: argparse.Namespace) -> RunConfig: - generation: Optional[GenerationConfig] = None chatbot = parse_single_model(getattr(args, "chatbot", None)) - user = parse_model_list(getattr(args, "user", None)) - personas = getattr(args, "personas", None) or [] - if chatbot is not None or user or personas: - generation = GenerationConfig(chatbot=chatbot, user=user, personas=personas) + users = parse_model_list(getattr(args, "user", None)) + personas = list(getattr(args, "personas", None) or []) + generation = None + if chatbot or users or personas: + generation = GenerationConfig(chatbot=chatbot, user=users, personas=personas) - judging: Optional[JudgingConfig] = None judge_models = parse_model_list(getattr(args, "judge", None)) - rubric_paths = getattr(args, "rubric", None) or [] - rubrics = [RubricSpec(name=r) for r in rubric_paths] - if judge_models or rubrics: - judging = JudgingConfig(models=judge_models, rubrics=rubrics) + rubrics = [RubricSpec(name=path) for path in getattr(args, "rubric", None) or []] + conversations = list(getattr(args, "conversations", None) or []) + judging = None + if judge_models or rubrics or conversations: + judging = JudgingConfig( + models=judge_models, + rubrics=rubrics, + conversations=conversations, + ) return RunConfig( generation=generation, judging=judging, target=getattr(args, "target", None), - sample=getattr(args, "sample", None), ) @@ -253,59 +277,245 @@ def print_resolved_config(run_config: RunConfig) -> None: print(json.dumps(run_config.to_dict(), indent=2)) -# --------------------------------------------------------------------------- -# Subcommand handlers. -# -# Phase 1 only replaces the front end (docs/architecture.md's migration -# table) -- these delegate to the existing generate/judge engines in a -# follow-up change. For now they resolve + print the config and stop. -# --------------------------------------------------------------------------- +def _validate_sample(sample: Optional[int]) -> None: + if sample is not None and sample < 1: + raise ConfigError("--sample must be at least 1") -def _not_yet_wired(command: str) -> None: - print( - f"vera {command}: argument parsing and config resolution only for now -- " - "wiring into the existing generate/judge engine lands in a follow-up " - "change (docs/architecture.md, migration Phase 1).", - file=sys.stderr, - ) +def _manifest_catalog() -> list[Path]: + return sorted((ROOT / "data").glob("**/rubric_manifest.json")) -def cmd_generate(args: argparse.Namespace) -> int: - run_config = resolve_run_config( - args, cli_flag_names=("chatbot", "user", "personas", "target") +def _resolve_named_manifest(name: str) -> Path: + candidate = Path(name) + if candidate.is_file(): + return candidate.resolve() + + matches = [ + path + for path in _manifest_catalog() + if path.parent.name.casefold() == name.casefold() + or path.stem.casefold() == name.casefold() + ] + if len(matches) == 1: + return matches[0].resolve() + if not matches: + raise ConfigError(f"unknown rubric target or manifest: {name!r}") + raise ConfigError(f"ambiguous rubric target {name!r}: {matches}") + + +def _target_manifests(target: Optional[str]) -> list[Path]: + if target is None: + return [] + if target.casefold() == "all": + manifests = [path.resolve() for path in _manifest_catalog()] + if not manifests: + raise ConfigError("--target all found no rubric bundle manifests") + return manifests + return [_resolve_named_manifest(target)] + + +def _rubric_manifest(rubric: RubricSpec) -> str: + return str(_resolve_named_manifest(rubric.name)) + + +def _enable_debug(enabled: bool) -> None: + if enabled: + from utils.debug import set_debug + + set_debug(True) + + +def _model_config(spec: ModelSpec, *, chatbot: bool = False) -> dict[str, Any]: + config = {"model": spec.name, **spec.extra_params} + if chatbot: + config["name"] = spec.name + return config + + +async def _run_generation( + run_config: RunConfig, *, sample: Optional[int], debug: bool +) -> list[str]: + from generate_conversations import run_generation + from utils.rubric_manifest import load_manifest_personas + + generation = run_config.generation + if generation is None or generation.chatbot is None: + raise ConfigError("generation configuration is missing") + if generation.chatbot.repeats != 1: + raise ConfigError("generation.chatbot repeats must be 1") + + _enable_debug(debug) + persona_sources: list[list[str]] = [] + manifests = _target_manifests(run_config.target) + if manifests: + for manifest in manifests: + manifest_personas = await load_manifest_personas(str(manifest)) + if not manifest_personas: + raise ConfigError( + f"Target {run_config.target!r} cannot generate conversations " + f"because its manifest {manifest} defines no personas. Add " + "personas to the manifest, or select personas and rubrics " + "independently." + ) + persona_sources.append(manifest_personas) + else: + persona_sources.extend([persona] for persona in generation.personas) + if sample is not None: + persona_sources = persona_sources[:sample] + + output_folders: list[str] = [] + for user in generation.user: + for persona_files in persona_sources: + _, output_folder = await run_generation( + persona_model_config=_model_config(user), + agent_model_config=_model_config(generation.chatbot, chatbot=True), + persona_files=persona_files, + persona_extra_run_params=dict(user.extra_params), + agent_extra_run_params=dict(generation.chatbot.extra_params), + runs_per_prompt=user.repeats, + max_personas=sample, + output_folder="output", + ) + output_folders.append(output_folder) + return output_folders + + +def _group_models_by_params( + models: list[ModelSpec], +) -> list[tuple[list[ModelSpec], dict[str, Any]]]: + grouped: dict[str, tuple[list[ModelSpec], dict[str, Any]]] = {} + for model in models: + key = json.dumps(model.extra_params, sort_keys=True, default=str) + if key not in grouped: + grouped[key] = ([], dict(model.extra_params)) + grouped[key][0].append(model) + return list(grouped.values()) + + +async def _run_judging( + run_config: RunConfig, + *, + sample: Optional[int], + debug: bool, + conversations: Optional[list[str]] = None, +) -> list[str]: + judging = run_config.judging + if judging is None: + raise ConfigError("judging configuration is missing") + + _enable_debug(debug) + from judge import run_judging + + conversation_folders = list(conversations or judging.conversations) + rubrics = list(judging.rubrics) + if run_config.target: + rubrics = [ + RubricSpec(name=str(path)) for path in _target_manifests(run_config.target) + ] + if sample is not None: + rubrics = rubrics[:sample] + + outputs: list[str] = [] + for conversation_folder in conversation_folders: + for rubric in rubrics: + models = list(rubric.models or judging.models) + if sample is not None: + models = models[:sample] + for grouped_models, extra_params in _group_models_by_params(models): + judge_models: dict[str, int] = {} + for model in grouped_models: + repeats = min(model.repeats, sample) if sample else model.repeats + judge_models[model.name] = repeats + output = await run_judging( + conversation_folder=conversation_folder, + rubric_manifest=_rubric_manifest(rubric), + judge_models=judge_models, + judge_model_extra_params=extra_params, + limit=sample, + max_concurrent=None, + per_judge=False, + verbose_workers=False, + debug=debug, + ) + if output: + outputs.append(output) + return outputs + + +def _run_scoring(results_csv: str) -> None: + from judge.score import score_results_file + + status = score_results_file( + results_csv, + personas_tsv=str(ROOT / "data" / "SI" / "personas.tsv"), ) - if run_config.generation is None or run_config.generation.chatbot is None: + if status: + raise ConfigError(f"could not score results file: {results_csv}") + + +def _validate_generation(run_config: RunConfig) -> None: + generation = run_config.generation + if generation is None or generation.chatbot is None: raise ConfigError( "generate requires a chatbot (-c/--chatbot or generation.chatbot)" ) - if not run_config.generation.user: + if not generation.user: raise ConfigError( "generate requires at least one user model (-u/--user or generation.user)" ) - if not run_config.generation.personas and not run_config.target: + if not generation.personas and not run_config.target: raise ConfigError("generate requires --personas or --target") + + +def _models_for_rubric(judging: JudgingConfig, rubric: RubricSpec) -> list[ModelSpec]: + return rubric.models or judging.models + + +def _validate_judging( + run_config: RunConfig, *, require_conversations: bool = True +) -> None: + judging = run_config.judging + if judging is None: + raise ConfigError("judge requires a judging configuration") + rubrics = judging.rubrics or [ + RubricSpec(name=str(path)) for path in _target_manifests(run_config.target) + ] + if not rubrics: + raise ConfigError("judge requires --rubric, judging.rubrics, or --target") + if any(not _models_for_rubric(judging, rubric) for rubric in rubrics): + raise ConfigError( + "judge requires judge models (-j/--judge, judging.models, or rubric models)" + ) + if require_conversations and not judging.conversations: + raise ConfigError( + "judge requires conversations from --conversations or judging.conversations" + ) + + +def cmd_generate(args: argparse.Namespace) -> int: + run_config = resolve_run_config( + args, cli_flag_names=("chatbot", "user", "personas", "target") + ) + _validate_sample(args.sample) + _validate_generation(run_config) print_resolved_config(run_config) if args.print: return 0 - _not_yet_wired("generate") + asyncio.run(_run_generation(run_config, sample=args.sample, debug=args.debug)) return 0 def cmd_judge(args: argparse.Namespace) -> int: - run_config = resolve_run_config(args, cli_flag_names=("judge", "rubric")) - if run_config.judging is None or not run_config.judging.models: - raise ConfigError( - "judge requires at least one judge model (-j/--judge or judging.models)" - ) - if not run_config.judging.rubrics: - raise ConfigError("judge requires --rubric or judging.rubrics") - if not args.conversations: - raise ConfigError("judge requires --conversations") + run_config = resolve_run_config( + args, cli_flag_names=("judge", "rubric", "conversations") + ) + _validate_sample(args.sample) + _validate_judging(run_config) print_resolved_config(run_config) if args.print: return 0 - _not_yet_wired("judge") + asyncio.run(_run_judging(run_config, sample=args.sample, debug=args.debug)) return 0 @@ -315,7 +525,7 @@ def cmd_score(args: argparse.Namespace) -> int: if args.print: print(f"vera score -r {args.results}") return 0 - _not_yet_wired("score") + _run_scoring(args.results) return 0 @@ -325,7 +535,14 @@ def cmd_pool(args: argparse.Namespace) -> int: if args.print: print(f"vera pool --evaluations {' '.join(args.evaluations)}") return 0 - _not_yet_wired("pool") + + from scripts.pool_vera_scores import pool_evaluation_directories + + pool_evaluation_directories( + args.evaluations, + ROOT / "output", + personas_tsv=ROOT / "data" / "SI" / "personas.tsv", + ) return 0 @@ -334,25 +551,27 @@ def cmd_pipeline(args: argparse.Namespace) -> int: args, cli_flag_names=("chatbot", "user", "judge", "personas", "target", "rubric"), ) - if run_config.target is None: - if run_config.generation is None or run_config.generation.chatbot is None: - raise ConfigError( - "pipeline requires a chatbot (-c/--chatbot or generation.chatbot)" - ) - if not run_config.generation.user: - raise ConfigError( - "pipeline requires at least one user model (-u/--user or " - "generation.user)" - ) - if run_config.judging is None or not run_config.judging.models: - raise ConfigError( - "pipeline requires at least one judge model (-j/--judge or " - "judging.models)" - ) + _validate_sample(args.sample) + _validate_generation(run_config) + _validate_judging(run_config, require_conversations=False) print_resolved_config(run_config) if args.print: return 0 - _not_yet_wired("pipeline") + + async def run_pipeline() -> list[str]: + generated = await _run_generation( + run_config, sample=args.sample, debug=args.debug + ) + return await _run_judging( + run_config, + sample=args.sample, + debug=args.debug, + conversations=generated, + ) + + evaluation_folders = asyncio.run(run_pipeline()) + for folder in evaluation_folders: + _run_scoring(str(Path(folder) / "results.csv")) return 0 @@ -362,13 +581,10 @@ def cmd_resume(args: argparse.Namespace) -> int: if args.print: print(f"vera resume --config {args.config}") return 0 - _not_yet_wired("resume") - return 0 - - -# --------------------------------------------------------------------------- -# Parser construction. -# --------------------------------------------------------------------------- + raise ConfigError( + "resume execution is deferred until the config checksum/state contract " + "described in docs/architecture.md is implemented" + ) def build_parser() -> argparse.ArgumentParser: @@ -380,19 +596,36 @@ def build_parser() -> argparse.ArgumentParser: def add_subcommand( name: str, help_text: str, handler: Callable[[argparse.Namespace], int] ) -> argparse.ArgumentParser: - sub = subparsers.add_parser(name, help=help_text) - sub.set_defaults(handler=handler) - return sub + subcommand = subparsers.add_parser(name, help=help_text) + subcommand.set_defaults(handler=handler) + return subcommand generate = add_subcommand("generate", "Simulate conversations.", cmd_generate) add_flags( - generate, "chatbot", "user", "personas", "target", "config", "sample", "print" + generate, + "chatbot", + "user", + "personas", + "target", + "config", + "sample", + "debug", + "print", ) judge = add_subcommand( "judge", "Evaluate existing transcripts against a rubric.", cmd_judge ) - add_flags(judge, "judge", "rubric", "conversations", "config", "sample", "print") + add_flags( + judge, + "judge", + "rubric", + "conversations", + "config", + "sample", + "debug", + "print", + ) score = add_subcommand( "score", "Aggregate results.csv into scores and visualizations.", cmd_score @@ -401,7 +634,7 @@ def add_subcommand( pool = add_subcommand( "pool", - "Concatenate multiple evaluation folders into one pooled result.", + "Concatenate evaluation folders into one pooled result.", cmd_pool, ) add_flags(pool, "evaluations", "print") @@ -421,16 +654,16 @@ def add_subcommand( "rubric", "config", "sample", + "debug", "print", ) resume = add_subcommand( "resume", - "Resume an incomplete run from its config.json + state.json.", + "Resume an incomplete run from config.json + state.json.", cmd_resume, ) - add_flags(resume, "config", "print") - + add_flags(resume, "config", "debug", "print") return parser @@ -439,10 +672,10 @@ def main(argv: Optional[list[str]] = None) -> int: args = parser.parse_args(argv) try: return args.handler(args) - except ConfigError as e: - parser.error(str(e)) - return 2 # pragma: no cover - argparse.error() exits before this + except ConfigError as error: + parser.error(str(error)) + return 2 # pragma: no cover if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) From 3d35ceff12bc3461ef17503e6eb4e3501d3fd8a4 Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:35:53 -0700 Subject: [PATCH 3/4] no-mistakes(review): Resolve manifest persona context template for vera --target generation --- tests/unit/test_vera_cli.py | 43 +++++++++++++++++++++++++++++++++++++ vera.py | 22 +++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_vera_cli.py b/tests/unit/test_vera_cli.py index c785c9a9e..e44790c0d 100644 --- a/tests/unit/test_vera_cli.py +++ b/tests/unit/test_vera_cli.py @@ -221,6 +221,49 @@ async def test_generation_target_resolves_manifest_personas(tmp_path: Path) -> N str(manifest_dir / "personas.tsv") ] assert "rubric_manifest" not in run_generation.await_args.kwargs + assert "persona_context_template_path" not in run_generation.await_args.kwargs + + +@pytest.mark.asyncio +async def test_generation_target_resolves_manifest_persona_context_template( + tmp_path: Path, +) -> None: + manifest_dir = tmp_path / "target" + manifest_dir.mkdir() + manifest = manifest_dir / "rubric_manifest.json" + manifest.write_text( + json.dumps( + { + "rubric_file": "rubric.tsv", + "rubric_prompt_beginning_file": "rubric_prompt.txt", + "question_prompt_file": "question_prompt.txt", + "personas": ["personas.tsv"], + "persona_context_template_file": "persona_context_template.txt", + } + ), + encoding="utf-8", + ) + run_config = RunConfig( + generation=GenerationConfig( + chatbot=ModelSpec(name="chatbot"), + user=[ModelSpec(name="user")], + ), + target="target", + ) + + with ( + patch.object(vera, "_target_manifests", return_value=[manifest]), + patch( + "generate_conversations.run_generation", new_callable=AsyncMock + ) as run_generation, + ): + run_generation.return_value = ([], "output/generated") + await vera._run_generation(run_config, sample=None, debug=False) + + assert run_generation.await_args is not None + assert run_generation.await_args.kwargs["persona_context_template_path"] == str( + manifest_dir / "persona_context_template.txt" + ) @pytest.mark.asyncio diff --git a/vera.py b/vera.py index 6de430155..13e6c9562 100644 --- a/vera.py +++ b/vera.py @@ -337,7 +337,10 @@ async def _run_generation( run_config: RunConfig, *, sample: Optional[int], debug: bool ) -> list[str]: from generate_conversations import run_generation - from utils.rubric_manifest import load_manifest_personas + from utils.rubric_manifest import ( + load_manifest_persona_context_template, + load_manifest_personas, + ) generation = run_config.generation if generation is None or generation.chatbot is None: @@ -347,6 +350,7 @@ async def _run_generation( _enable_debug(debug) persona_sources: list[list[str]] = [] + persona_context_templates: list[Optional[str]] = [] manifests = _target_manifests(run_config.target) if manifests: for manifest in manifests: @@ -359,14 +363,27 @@ async def _run_generation( "independently." ) persona_sources.append(manifest_personas) + try: + persona_context_templates.append( + await load_manifest_persona_context_template(str(manifest)) + ) + except ValueError: + persona_context_templates.append(None) else: persona_sources.extend([persona] for persona in generation.personas) + persona_context_templates.extend(None for _ in generation.personas) if sample is not None: persona_sources = persona_sources[:sample] + persona_context_templates = persona_context_templates[:sample] output_folders: list[str] = [] for user in generation.user: - for persona_files in persona_sources: + for persona_files, context_template in zip( + persona_sources, persona_context_templates + ): + extra_kwargs: dict[str, Any] = {} + if context_template is not None: + extra_kwargs["persona_context_template_path"] = context_template _, output_folder = await run_generation( persona_model_config=_model_config(user), agent_model_config=_model_config(generation.chatbot, chatbot=True), @@ -376,6 +393,7 @@ async def _run_generation( runs_per_prompt=user.repeats, max_personas=sample, output_folder="output", + **extra_kwargs, ) output_folders.append(output_folder) return output_folders From 6ecd4aeded996bf8a11383800bc927621b1e099a Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:40:48 -0700 Subject: [PATCH 4/4] no-mistakes(document): docs: point AGENTS.md architecture map at unified vera.py CLI --- AGENTS.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 06e9cc437..faa0983e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,7 @@ cp .env.example .env # Add API keys (ANTHROPIC_API_KEY, OPENAI_API_KEY, et | Area | Key paths | When to edit | |------|-----------|--------------| +| **Unified CLI** | `vera.py`, `utils/config_schema.py` | Command/config wiring shared across generate, judge, score, pool, pipeline | | **Generation** | `generate.py`, `generate_conversations/` | Conversation simulation, turns, personas | | **Judging** | `judge.py`, `judge/` | Rubric scoring, TSV output, question navigation | | **LLM providers** | `llm_clients/`, `llm_clients/llm_factory.py` | New models, custom HTTP/API providers | @@ -38,7 +39,7 @@ cp .env.example .env # Add API keys (ANTHROPIC_API_KEY, OPENAI_API_KEY, et | **Config** | `utils/model_config_loader.py`, `llm_clients/config.py` | Model name resolution, API keys | | **Shared utils** | `utils/` | Naming, logging, conversation layout | -**Entry points:** `generate.py` (simulate), `judge.py` (evaluate), `run_pipeline.py` (full workflow), `judge/score.py` (scoring/visualization). +**Entry points:** `vera.py` (unified CLI: generate/judge/score/pool/pipeline, see [README's Unified CLI section](./README.md#unified-cli)); `generate.py`, `judge.py`, `run_pipeline.py`, `judge/score.py` remain as legacy per-step scripts and compatibility adapters (see [docs/design/vera-cli-runtime-wiring.md](./docs/design/vera-cli-runtime-wiring.md)). **Temporary experiments:** `tmp_tests/` (not committed). **Permanent tests:** `tests/`. @@ -76,6 +77,8 @@ uv run pytest tests/integration/ ## Key Commands +`vera.py` is the unified CLI (generate/judge/score/pool/pipeline); see [README's Unified CLI section](./README.md#unified-cli). The commands below use the legacy per-step scripts, still valid as compatibility adapters. + ```bash # End-to-end pipeline (preferred for full workflows) uv run python run_pipeline.py \ @@ -167,6 +170,7 @@ One canonical home per concern β€” cross-link, don't copy paragraphs. - **Judge behavior:** [docs/judge.md](./docs/judge.md) - **Structured output:** [docs/structured-output.md](./docs/structured-output.md) - **Pre-commit hooks:** [docs/pre-commit-hooks.md](./docs/pre-commit-hooks.md) +- **Unified CLI wiring:** [docs/design/vera-cli-runtime-wiring.md](./docs/design/vera-cli-runtime-wiring.md) - **Claude Code commands:** [CLAUDE.md](./CLAUDE.md), [.claude/commands/](./.claude/commands/) ## Docker