diff --git a/ergon_cli/ergon_cli/domains/examples/catalogue.py b/ergon_cli/ergon_cli/domains/examples/catalogue.py index 06b78fb06..546ccaa66 100644 --- a/ergon_cli/ergon_cli/domains/examples/catalogue.py +++ b/ergon_cli/ergon_cli/domains/examples/catalogue.py @@ -9,7 +9,7 @@ display_name="MiniF2F local llama.cpp", short_description=("Run three MiniF2F Lean proof tasks with a managed local llama.cpp server."), purpose=("Run three MiniF2F Lean proof tasks with local llama.cpp and an E2B Lean sandbox."), - script_path="examples/getting_started/01_minif2f_local_llamacpp/run.py", + script_path="examples/getting_started/01_minif2f_local_llamacpp/submit.py", prerequisites=( "E2B_API_KEY is configured in Ergon's .env file or process environment.", "A local GGUF model path or Hugging Face GGUF ref is available for llama.cpp.", @@ -77,8 +77,63 @@ ), ) +EXPERIMENT_API_MINIF2F = ExampleDefinition( + slug="experiment-api-minif2f", + display_name="Experiment API MiniF2F", + short_description="Submit a single MiniF2F environment through the experiment API.", + purpose="Demonstrate the smallest complete Environment + Experiment submit flow.", + script_path="examples/experiment_api/01_minif2f_single_environment/submit.py", + prerequisites=("Database settings are configured.", "Provider credentials are available."), + options=(), +) + +EXPERIMENT_API_MIXED_ENVIRONMENTS = ExampleDefinition( + slug="experiment-api-mixed-environments", + display_name="Experiment API mixed environments", + short_description="Submit one experiment spanning several builtin environments.", + purpose="Demonstrate heterogeneous environment composition for training-style launches.", + script_path="examples/experiment_api/02_mixed_environment_training/submit.py", + prerequisites=("Database settings are configured.", "Provider credentials are available."), + options=(), +) + +EXPERIMENT_API_STREAMING = ExampleDefinition( + slug="experiment-api-streaming", + display_name="Experiment API streaming", + short_description="Submit from a streaming SWE-bench candidate buffer.", + purpose="Demonstrate streamed source buffering with a larger candidate pool.", + script_path="examples/experiment_api/03_streaming_hf_dataset/submit.py", + prerequisites=("Database settings are configured.", "SWE-bench data access is available."), + options=(), +) + +EXPERIMENT_API_CURRICULUM_SAMPLER = ExampleDefinition( + slug="experiment-api-curriculum-sampler", + display_name="Experiment API curriculum sampler", + short_description="Submit with a custom sampler that orders a retained candidate pool.", + purpose="Demonstrate sampler customization without changing environment code.", + script_path="examples/experiment_api/04_curriculum_sampler/submit.py", + prerequisites=("Database settings are configured.", "Provider credentials are available."), + options=(), +) + +EXPERIMENT_API_SWEBENCH = ExampleDefinition( + slug="experiment-api-swebench", + display_name="Experiment API SWE-bench", + short_description="Submit SWE-bench samples with row-dependent runtime configs.", + purpose="Demonstrate row-dependent worker, evaluator, and sandbox selection.", + script_path="examples/experiment_api/05_row_dependent_runtime_configs/submit.py", + prerequisites=("Database settings are configured.", "SWE-bench data access is available."), + options=(), +) + EXAMPLES: dict[str, ExampleDefinition] = { MINIF2F_LOCAL_LLAMACPP.slug: MINIF2F_LOCAL_LLAMACPP, + EXPERIMENT_API_MINIF2F.slug: EXPERIMENT_API_MINIF2F, + EXPERIMENT_API_MIXED_ENVIRONMENTS.slug: EXPERIMENT_API_MIXED_ENVIRONMENTS, + EXPERIMENT_API_STREAMING.slug: EXPERIMENT_API_STREAMING, + EXPERIMENT_API_CURRICULUM_SAMPLER.slug: EXPERIMENT_API_CURRICULUM_SAMPLER, + EXPERIMENT_API_SWEBENCH.slug: EXPERIMENT_API_SWEBENCH, } diff --git a/ergon_cli/ergon_cli/domains/examples/runner.py b/ergon_cli/ergon_cli/domains/examples/runner.py index 7fa04848d..8f5e3fdea 100644 --- a/ergon_cli/ergon_cli/domains/examples/runner.py +++ b/ergon_cli/ergon_cli/domains/examples/runner.py @@ -65,9 +65,15 @@ def _script_args(command: ExampleCommand) -> list[str]: def _repo_root(script_path: str) -> Path: configured = os.environ.get("ERGON_REPO_ROOT") - candidates: list[Path] = [] if configured: - candidates.append(Path(configured).resolve()) + candidate = Path(configured).resolve() + if (candidate / script_path).is_file(): + return candidate + raise ExampleRunError( + f"ERGON_REPO_ROOT is set, but the configured checkout does not contain {script_path}." + ) + + candidates: list[Path] = [] cwd = Path.cwd().resolve() candidates.append(cwd) candidates.extend(cwd.parents) diff --git a/ergon_cli/tests/unit/cli/test_examples_cli.py b/ergon_cli/tests/unit/cli/test_examples_cli.py index d89c0434b..f614a7a80 100644 --- a/ergon_cli/tests/unit/cli/test_examples_cli.py +++ b/ergon_cli/tests/unit/cli/test_examples_cli.py @@ -11,6 +11,7 @@ import ergon_cli.domains.examples.preflight as examples_preflight import ergon_cli.domains.examples.runner as examples_runner from ergon_cli.domains.examples.commands import handle_examples +from ergon_cli.domains.examples.catalogue import EXAMPLES from ergon_cli.main import build_parser from ergon_cli.domains.examples.preflight import ExampleSetupError, PreflightResult @@ -68,6 +69,36 @@ def test_examples_subcommands_are_registered_in_main_parser() -> None: assert run_args.max_iterations == 4 +def test_experiment_api_examples_are_catalogued() -> None: + expected = { + "experiment-api-minif2f", + "experiment-api-swebench", + "experiment-api-mixed-environments", + "experiment-api-curriculum-sampler", + "experiment-api-streaming", + } + + assert expected.issubset(EXAMPLES) + + +def test_example_submission_summary_includes_sampler_invocation( + capsys: pytest.CaptureFixture[str], +) -> None: + from examples.getting_started._shared.launch import print_submission_summary + + print_submission_summary( + experiment_id="exp-1", + sampler_invocation_id="sampler-1", + batch_id=None, + sample_ids=["sample-1"], + ) + + out = capsys.readouterr().out + assert "experiment_id" in out + assert "sampler_invocation_id" in out + assert "sample_ids" in out + + def test_examples_run_accepts_base_model_for_single_command_launch() -> None: args = build_parser().parse_args( [ @@ -160,7 +191,7 @@ def test_info_prints_prerequisites_options_and_script_path( assert "E2B_API_KEY" in out assert "--limit" in out assert "--base-url" in out - assert "examples/getting_started/01_minif2f_local_llamacpp/run.py" in out + assert "examples/getting_started/01_minif2f_local_llamacpp/submit.py" in out def test_check_runs_preflight_without_launching( @@ -301,7 +332,7 @@ def fake_preflight(*, base_url: str) -> PreflightResult: assert ( Path(command[5]) .as_posix() - .endswith("examples/getting_started/01_minif2f_local_llamacpp/run.py") + .endswith("examples/getting_started/01_minif2f_local_llamacpp/submit.py") ) assert command[6:] == [ "--limit", @@ -351,7 +382,7 @@ def test_run_can_use_configured_repo_root( tmp_path: Path, ) -> None: fake_repo = tmp_path / "repo" - script = fake_repo / "examples/getting_started/01_minif2f_local_llamacpp/run.py" + script = fake_repo / "examples/getting_started/01_minif2f_local_llamacpp/submit.py" script.parent.mkdir(parents=True) (fake_repo / "examples/pyproject.toml").write_text("[project]\nname='fake'\nversion='0'\n") script.write_text("print('ok')\n") diff --git a/ergon_core/tests/unit/architecture/test_definition_identity_naming.py b/ergon_core/tests/unit/architecture/test_definition_identity_naming.py index 14467245a..ea917c163 100644 --- a/ergon_core/tests/unit/architecture/test_definition_identity_naming.py +++ b/ergon_core/tests/unit/architecture/test_definition_identity_naming.py @@ -19,6 +19,39 @@ ) EXCLUDED_FILES = {Path(__file__).resolve()} EXPERIMENT_ID_PATTERN = re.compile(r"\b(?:experiment" r"_id|experiment" r"Id)\b") +ALLOWED_EXPERIMENT_ID_PATH_PREFIXES = ( + ROOT / "ergon_core" / "ergon_core" / "api" / "experiment", + ROOT / "ergon_core" / "ergon_core" / "core" / "application" / "experiments", + ROOT / "ergon_core" / "ergon_core" / "core" / "persistence" / "experiments", + ROOT / "ergon_core" / "ergon_core" / "core" / "views" / "experiments", + ROOT / "ergon_core" / "ergon_core" / "core" / "views" / "samples", + ROOT / "ergon_core" / "tests" / "integration" / "experiments", + ROOT / "ergon_core" / "tests" / "unit" / "api", + ROOT / "ergon_core" / "tests" / "unit" / "core" / "application" / "experiments", + ROOT / "ergon_core" / "tests" / "unit" / "read_models", + ROOT / "ergon_core" / "tests" / "unit" / "rest_api", + ROOT / "tests" / "examples", +) +ALLOWED_EXPERIMENT_ID_FILES = { + ROOT + / "ergon_core" + / "ergon_core" + / "core" + / "infrastructure" + / "http" + / "routes" + / "experiments.py", + ROOT + / "ergon_core" + / "ergon_core" + / "core" + / "infrastructure" + / "http" + / "routes" + / "samples.py", + ROOT / "ergon_core" / "ergon_core" / "core" / "persistence" / "telemetry" / "models.py", + ROOT / "ergon_core" / "tests" / "unit" / "state" / "test_type_invariants.py", +} ALLOWED_EXPERIMENT_ID_PATTERNS_BY_FILE = { ROOT / "ergon_core" / "ergon_core" / "api" / "experiment" / "experiment.py": ( re.compile(r"experiment_id: UUID"), @@ -90,6 +123,13 @@ } +def _allows_experiment_identity_path(path: Path) -> bool: + resolved = path.resolve() + if resolved in ALLOWED_EXPERIMENT_ID_FILES: + return True + return any(resolved.is_relative_to(prefix) for prefix in ALLOWED_EXPERIMENT_ID_PATH_PREFIXES) + + def test_definition_identity_uses_definition_name() -> None: hits: list[str] = [] for root in DEFINITION_ROOTS: @@ -107,6 +147,8 @@ def test_definition_identity_uses_definition_name() -> None: text = path.read_text() except UnicodeDecodeError: continue + if _allows_experiment_identity_path(path): + continue for line_number, line in enumerate(text.splitlines(), start=1): if EXPERIMENT_ID_PATTERN.search(line): allowed = ALLOWED_EXPERIMENT_ID_PATTERNS_BY_FILE.get(path.resolve(), ()) diff --git a/examples/experiment_api/01_minif2f_single_environment/submit.py b/examples/experiment_api/01_minif2f_single_environment/submit.py new file mode 100644 index 000000000..9f18c7e71 --- /dev/null +++ b/examples/experiment_api/01_minif2f_single_environment/submit.py @@ -0,0 +1,37 @@ +"""Smallest complete MiniF2F experiment submission example.""" + +from __future__ import annotations + +import asyncio + +from ergon_builtins.benchmarks.minif2f.sandbox import LeanSandbox +from ergon_builtins.benchmarks.minif2f.worker_factory import ( + make_minif2f_rubric, + make_minif2f_worker, +) +from ergon_builtins.environments import MiniF2FEnvironment +from ergon_core.api import Experiment, RandomSampler +from experiment_api._shared import experiment_submission_service + + +async def main() -> None: + worker = make_minif2f_worker(model="openai:gpt-4o") + env = MiniF2FEnvironment( + name="mini-validation", + split="validation", + limit=10, + worker=worker, + evaluators=[make_minif2f_rubric()], + sandbox=LeanSandbox(), + ) + experiment = Experiment(name="mini-validation-smoke", environments=[env]) + result = await experiment.submit( + service=experiment_submission_service(), + k=10, + sampler=RandomSampler(seed=0), + ) + print(result.model_dump_json(indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/experiment_api/02_mixed_environment_training/submit.py b/examples/experiment_api/02_mixed_environment_training/submit.py new file mode 100644 index 000000000..041914fb5 --- /dev/null +++ b/examples/experiment_api/02_mixed_environment_training/submit.py @@ -0,0 +1,78 @@ +"""Heterogeneous training-style experiment with four builtin environments.""" + +from __future__ import annotations + +import asyncio + +from ergon_builtins.benchmarks.gdpeval.benchmark import _default_gdpeval_sandbox +from ergon_builtins.benchmarks.gdpeval.worker_factory import ( + make_gdpeval_rubric, + make_gdpeval_worker, +) +from ergon_builtins.benchmarks.minif2f.sandbox import LeanSandbox +from ergon_builtins.benchmarks.minif2f.worker_factory import make_minif2f_rubric +from ergon_builtins.benchmarks.researchrubrics.benchmark import _default_research_sandbox +from ergon_builtins.benchmarks.researchrubrics.worker_factory import make_research_rubric +from ergon_builtins.benchmarks.swebench_verified.benchmark import _default_swebench_sandbox +from ergon_builtins.benchmarks.swebench_verified.worker_factory import make_swebench_rubric +from ergon_builtins.environments import ( + GDPEvalEnvironment, + MiniF2FEnvironment, + ResearchRubricsEnvironment, + SweBenchVerifiedEnvironment, +) +from ergon_core.api import Experiment, RandomSampler +from experiment_api._shared import experiment_submission_service + + +async def main() -> None: + worker = make_gdpeval_worker(model="openai:gpt-4o") + experiment = Experiment( + name="generalist-mixed-training", + environments=[ + MiniF2FEnvironment( + name="mini-validation", + split="validation", + limit=10, + worker=worker, + evaluators=[make_minif2f_rubric()], + sandbox=LeanSandbox(), + ), + ResearchRubricsEnvironment( + name="research-validation", + split="validation", + limit=10, + worker=worker, + evaluators=[make_research_rubric()], + sandbox=_default_research_sandbox(), + ), + GDPEvalEnvironment( + name="gdp-validation", + split="validation", + limit=10, + worker=worker, + evaluators=[make_gdpeval_rubric()], + sandbox=_default_gdpeval_sandbox(), + ), + SweBenchVerifiedEnvironment( + name="swebench-train", + split="train", + streaming=True, + limit=100, + worker=worker, + evaluators=[make_swebench_rubric()], + sandbox=_default_swebench_sandbox(), + ), + ], + ) + result = await experiment.submit( + service=experiment_submission_service(), + k=32, + candidate_pool_size=128, + sampler=RandomSampler(seed=0), + ) + print(result.model_dump_json(indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/experiment_api/03_streaming_hf_dataset/submit.py b/examples/experiment_api/03_streaming_hf_dataset/submit.py new file mode 100644 index 000000000..8b7807952 --- /dev/null +++ b/examples/experiment_api/03_streaming_hf_dataset/submit.py @@ -0,0 +1,37 @@ +"""Streaming SWE-bench candidate-buffering example.""" + +from __future__ import annotations + +import asyncio + +from ergon_builtins.benchmarks.swebench_verified.benchmark import _default_swebench_sandbox +from ergon_builtins.benchmarks.swebench_verified.worker_factory import ( + make_swebench_rubric, + make_swebench_worker, +) +from ergon_builtins.environments import SweBenchVerifiedEnvironment +from ergon_core.api import Experiment, RandomSampler +from experiment_api._shared import experiment_submission_service + + +async def main() -> None: + env = SweBenchVerifiedEnvironment( + name="swebench-train", + split="train", + streaming=True, + worker=make_swebench_worker(model="openai:gpt-4o"), + evaluators=[make_swebench_rubric()], + sandbox=_default_swebench_sandbox(), + ) + experiment = Experiment(name="swebench-streaming-buffer", environments=[env]) + result = await experiment.submit( + service=experiment_submission_service(), + k=8, + candidate_pool_size=32, + sampler=RandomSampler(seed=0), + ) + print(result.model_dump_json(indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/experiment_api/04_curriculum_sampler/submit.py b/examples/experiment_api/04_curriculum_sampler/submit.py new file mode 100644 index 000000000..a0577a121 --- /dev/null +++ b/examples/experiment_api/04_curriculum_sampler/submit.py @@ -0,0 +1,53 @@ +"""Custom sampler example that prefers easier samples from a larger pool.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence + +from ergon_builtins.environments import MiniF2FEnvironment +from ergon_core.api import Experiment, RandomSampler, Sample, SamplingContext +from experiment_api._shared import experiment_submission_service + + +class EasyFirstSampler(RandomSampler): + name: str = "easy-first" + + async def select( + self, + *, + samples: Sequence[Sample], + k: int, + context: SamplingContext, + ) -> Sequence[Sample]: + del k, context + return sorted(samples, key=_difficulty_key) + + +def _difficulty_key(sample: Sample) -> tuple[int, str]: + difficulty = sample.metadata.get("difficulty", 0) + if isinstance(difficulty, int): + score = difficulty + elif isinstance(difficulty, float): + score = int(difficulty) + elif isinstance(difficulty, str): + score = int(difficulty) + else: + score = 0 + return (score, sample.sample_key) + + +async def main() -> None: + env = MiniF2FEnvironment(name="mini-validation", split="validation", limit=64) + experiment = Experiment(name="mini-curriculum", environments=[env]) + result = await experiment.submit( + service=experiment_submission_service(), + k=16, + candidate_pool_size=64, + sampler=EasyFirstSampler(seed=0), + ) + print(result.model_dump_json(indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/experiment_api/05_row_dependent_runtime_configs/submit.py b/examples/experiment_api/05_row_dependent_runtime_configs/submit.py new file mode 100644 index 000000000..fbbf37f0a --- /dev/null +++ b/examples/experiment_api/05_row_dependent_runtime_configs/submit.py @@ -0,0 +1,57 @@ +"""Row-dependent worker, evaluator, and sandbox selection example.""" + +from __future__ import annotations + +import asyncio + +from ergon_builtins.benchmarks.swebench_verified.benchmark import _default_swebench_sandbox +from ergon_builtins.benchmarks.swebench_verified.sandbox import SWEBenchSandbox +from ergon_builtins.benchmarks.swebench_verified.task_schemas import SWEBenchInstance +from ergon_builtins.benchmarks.swebench_verified.worker_factory import ( + make_swebench_rubric, + make_swebench_worker, +) +from ergon_builtins.environments import SweBenchVerifiedEnvironment +from ergon_core.api import Evaluator, Experiment, RandomSampler, Sandbox, Worker +from experiment_api._shared import experiment_submission_service + + +def worker_for_row(row: SWEBenchInstance) -> Worker: + if row.repo.startswith("django/"): + return make_swebench_worker(model="openai:gpt-4o") + return make_swebench_worker(model="openai:gpt-4o-mini") + + +def evaluators_for_row(row: SWEBenchInstance) -> list[Evaluator]: + del row + return [make_swebench_rubric()] + + +def sandbox_for_row(row: SWEBenchInstance) -> Sandbox: + if row.repo.startswith("django/"): + return SWEBenchSandbox() + return _default_swebench_sandbox() + + +async def main() -> None: + env = SweBenchVerifiedEnvironment( + name="swebench-row-dependent", + split="train", + streaming=True, + limit=100, + worker=worker_for_row, + evaluators=evaluators_for_row, + sandbox=sandbox_for_row, + ) + experiment = Experiment(name="row-dependent-runtime-configs", environments=[env]) + result = await experiment.submit( + service=experiment_submission_service(), + k=8, + candidate_pool_size=32, + sampler=RandomSampler(seed=0), + ) + print(result.model_dump_json(indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/experiment_api/__init__.py b/examples/experiment_api/__init__.py new file mode 100644 index 000000000..fed2eb9f5 --- /dev/null +++ b/examples/experiment_api/__init__.py @@ -0,0 +1 @@ +"""Runnable examples for the public experiment authoring API.""" diff --git a/examples/experiment_api/_shared.py b/examples/experiment_api/_shared.py new file mode 100644 index 000000000..f403e08ce --- /dev/null +++ b/examples/experiment_api/_shared.py @@ -0,0 +1,12 @@ +"""Shared helpers for experiment API examples.""" + +from __future__ import annotations + +from ergon_core.core.application.experiments.submission import ExperimentSubmissionService +from ergon_core.core.persistence.shared.db import ensure_db, get_session + + +def experiment_submission_service() -> ExperimentSubmissionService: + """Create the concrete submission service used by example scripts.""" + ensure_db() + return ExperimentSubmissionService.for_session(get_session()) diff --git a/examples/getting_started/01_minif2f_local_llamacpp/README.md b/examples/getting_started/01_minif2f_local_llamacpp/README.md index 2a4bd4085..b65bea090 100644 --- a/examples/getting_started/01_minif2f_local_llamacpp/README.md +++ b/examples/getting_started/01_minif2f_local_llamacpp/README.md @@ -81,7 +81,7 @@ The plain Python script remains the source of benchmark behavior and is useful when you want to read or edit the example directly: ```bash -uv run --project examples python examples/getting_started/01_minif2f_local_llamacpp/run.py +uv run --project examples python examples/getting_started/01_minif2f_local_llamacpp/submit.py ``` The script defaults to: @@ -94,7 +94,7 @@ The script defaults to: You can override the same values on the command line: ```bash -uv run --project examples python examples/getting_started/01_minif2f_local_llamacpp/run.py \ +uv run --project examples python examples/getting_started/01_minif2f_local_llamacpp/submit.py \ --limit 3 \ --base-url http://localhost:8080 \ --model local-proof-model \ @@ -112,11 +112,11 @@ The script: 1. Resolves `--base-model` from either a local path or Hugging Face GGUF ref. 2. Starts `llama-server` for managed runs, then discovers the served model from `/v1/models`. 3. Checks Ergon settings for `E2B_API_KEY` and verifies the llama.cpp endpoint. -4. Builds `MiniF2FBenchmark(limit=3, worker_factory=make_worker)`. -5. Binds `make_minif2f_worker(model="llamacpp:", max_iterations=12)`. -6. Persists the benchmark definition with `persist_benchmark`. -7. Launches a sample with `launch_sample`. -8. Prints the definition id, sample id, model target, and observation commands. +4. Binds `make_minif2f_worker(model="llamacpp:", max_iterations=12)`. +5. Builds `MiniF2FEnvironment(name="mini-validation", limit=3, worker=...)`. +6. Wraps that environment in `Experiment(name="minif2f-local-llamacpp", ...)`. +7. Submits the experiment with `experiment.submit(...)`. +8. Prints the experiment id, sample ids, model target, and observation commands. MiniF2F is a real theorem-proving benchmark. Local model quality, quantization, and context length strongly affect proof success. A terminal sample with failed diff --git a/examples/getting_started/01_minif2f_local_llamacpp/run.py b/examples/getting_started/01_minif2f_local_llamacpp/run.py index 4f9a86f20..de1fdbeb7 100644 --- a/examples/getting_started/01_minif2f_local_llamacpp/run.py +++ b/examples/getting_started/01_minif2f_local_llamacpp/run.py @@ -1,238 +1,10 @@ -"""Launch the MiniF2F getting-started example against local llama.cpp.""" +"""Compatibility entrypoint for the MiniF2F getting-started example.""" from __future__ import annotations -import argparse -import asyncio -import sys -from collections.abc import Sequence -from uuid import UUID +import importlib -from ergon_builtins.benchmarks.minif2f.benchmark import MiniF2FBenchmark -from ergon_builtins.benchmarks.minif2f.worker_factory import make_minif2f_worker -from ergon_core.api.worker import Worker -from ergon_core.core.application.experiments.service import launch_run as launch_sample, persist_benchmark -from getting_started._shared.env import ( - DEFAULT_LLAMA_CPP_BASE_URL, - DEFAULT_MINIF2F_LIMIT, - DEFAULT_MINIF2F_MAX_ITERATIONS, - ExampleSetupError, - base_url_from_model_target, - build_llamacpp_model_target, - env_optional_str, - env_positive_int, - env_str, - preflight_llamacpp_and_e2b, -) -from getting_started._shared.llamacpp import ManagedLlamaServer, start_llama_server -from getting_started._shared.launch import first_sample_id -from getting_started._shared.model_cache import resolve_base_model -from getting_started._shared.observe import cli_status_command, dashboard_sample_url - - -def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: - """Parse command-line arguments for the example.""" - parser = argparse.ArgumentParser( - description="Run three MiniF2F Lean proof tasks with local llama.cpp and E2B." - ) - parser.add_argument( - "--limit", - type=_positive_int, - default=env_positive_int("ERGON_MINIF2F_LIMIT", DEFAULT_MINIF2F_LIMIT), - help="Number of MiniF2F tasks to launch.", - ) - parser.add_argument( - "--base-url", - default=env_str("ERGON_LLAMA_CPP_BASE_URL", DEFAULT_LLAMA_CPP_BASE_URL), - help="Base URL for the llama.cpp OpenAI-compatible server.", - ) - parser.add_argument( - "--model", - default=env_optional_str("ERGON_LLAMA_CPP_MODEL"), - help="Optional llama.cpp model name; encoded as # on the target.", - ) - parser.add_argument( - "--model-target", - default=None, - help="Optional full Ergon model target. Overrides --base-url and --model.", - ) - parser.add_argument( - "--max-iterations", - type=_positive_int, - default=env_positive_int( - "ERGON_MINIF2F_MAX_ITERATIONS", - DEFAULT_MINIF2F_MAX_ITERATIONS, - ), - help="Maximum ReAct tool iterations per MiniF2F task.", - ) - parser.add_argument( - "--base-model", - default=None, - help=( - "Local GGUF path or Hugging Face ':' ref. " - "Starts a managed llama.cpp server for this sample." - ), - ) - parser.add_argument( - "--model-cache-dir", - default=None, - help="Directory for downloaded Hugging Face GGUF files.", - ) - parser.add_argument( - "--llama-server-bin", - default="llama-server", - help="llama.cpp server command to run with --base-model.", - ) - parser.add_argument( - "--host", - default="127.0.0.1", - help="Host for the managed llama.cpp server.", - ) - parser.add_argument( - "--port", - type=_positive_int, - default=8080, - help="Port for the managed llama.cpp server.", - ) - parser.add_argument( - "--startup-timeout", - type=_positive_int, - default=60, - help="Seconds to wait for a managed llama.cpp server to become ready.", - ) - parser.add_argument( - "--keep-llama-server", - action="store_true", - help="Leave the managed llama.cpp server running after the example exits.", - ) - args = parser.parse_args(argv) - _validate_model_routing(args, parser) - return args - - -def model_target_from_args(args: argparse.Namespace) -> str: - """Build the worker model target from parsed CLI args.""" - return build_llamacpp_model_target( - base_url=args.base_url, - model_name=args.model, - model_target=args.model_target, - ) - - -def preflight_base_url_from_args(args: argparse.Namespace) -> str: - """Choose the llama.cpp-compatible endpoint to check before launch.""" - if args.model_target is None: - return args.base_url - base_url = base_url_from_model_target(args.model_target) - if base_url is None: - raise ExampleSetupError( - "--model-target must include an http(s) endpoint for this local llama.cpp example." - ) - return base_url - - -async def async_main(argv: Sequence[str] | None = None) -> int: - """Run the example and return a process exit code.""" - server: ManagedLlamaServer | None = None - keep_llama_server = False - try: - args = parse_args(argv) - keep_llama_server = args.keep_llama_server - if args.base_model is not None: - print(f"Resolving base model: {args.base_model}") - base_model_path = resolve_base_model( - args.base_model, - cache_dir=args.model_cache_dir, - ) - print(f"Using base model: {base_model_path}") - print(f"Starting llama.cpp on {args.host}:{args.port}") - server = start_llama_server( - base_model=str(base_model_path), - llama_server_bin=args.llama_server_bin, - host=args.host, - port=args.port, - startup_timeout=args.startup_timeout, - ) - args.base_url = server.base_url - args.model = server.discovered_model - - model_target = model_target_from_args(args) - preflight_llamacpp_and_e2b(base_url=preflight_base_url_from_args(args)) - except ExampleSetupError as exc: - if server is not None: - server.close(keep_running=keep_llama_server) - print(f"Setup error: {exc}", file=sys.stderr) - return 2 - - def make_worker() -> Worker: - return make_minif2f_worker( - model=model_target, - max_iterations=args.max_iterations, - ) - - try: - benchmark = MiniF2FBenchmark(limit=args.limit, worker_factory=make_worker) - handle = persist_benchmark(benchmark) - sample_result = await launch_sample(handle.definition_id) - sample_id = first_sample_id(sample_result) - _print_launch_summary( - definition_id=handle.definition_id, - sample_id=sample_id, - model_target=model_target, - limit=args.limit, - max_iterations=args.max_iterations, - ) - return 0 - finally: - if server is not None: - server.close(keep_running=keep_llama_server) - - -def main(argv: Sequence[str] | None = None) -> int: - """Synchronous entrypoint for direct script execution.""" - return asyncio.run(async_main(argv)) - - -def _positive_int(value: str) -> int: - try: - parsed = int(value) - except ValueError as exc: - raise argparse.ArgumentTypeError(f"{value!r} is not a positive integer") from exc - if parsed < 1: - raise argparse.ArgumentTypeError(f"{value!r} is not a positive integer") - return parsed - - -def _validate_model_routing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: - if args.base_model is None: - return - conflicts = { - "--model": args.model is not None, - "--model-target": args.model_target is not None, - } - for flag, is_set in conflicts.items(): - if is_set: - parser.error(f"--base-model cannot be combined with {flag}") - - -def _print_launch_summary( - *, - definition_id: object, - sample_id: UUID, - model_target: str, - limit: int, - max_iterations: int, -) -> None: - print("MiniF2F llama.cpp sample launched") - print(f"Definition id: {definition_id}") - print(f"Sample id: {sample_id}") - print(f"Model target: {model_target}") - print(f"Limit: {limit}") - print(f"Max iterations: {max_iterations}") - print(f"CLI status: {cli_status_command(sample_id)}") - url = dashboard_sample_url(sample_id) - if url: - print(f"Dashboard: {url}") +main = importlib.import_module("getting_started.01_minif2f_local_llamacpp.submit").main if __name__ == "__main__": diff --git a/examples/getting_started/01_minif2f_local_llamacpp/submit.py b/examples/getting_started/01_minif2f_local_llamacpp/submit.py new file mode 100644 index 000000000..463459563 --- /dev/null +++ b/examples/getting_started/01_minif2f_local_llamacpp/submit.py @@ -0,0 +1,289 @@ +"""Submit the MiniF2F getting-started example against local llama.cpp.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from collections.abc import Sequence +from uuid import UUID + +from ergon_builtins.benchmarks.minif2f.sandbox import LeanSandbox +from ergon_builtins.benchmarks.minif2f.worker_factory import ( + make_minif2f_rubric, + make_minif2f_worker, +) +from ergon_builtins.environments import MiniF2FEnvironment +from ergon_core.api import Experiment, RandomSampler +from ergon_core.api.worker import Worker +from ergon_core.core.application.experiments.submission import ExperimentSubmissionService +from ergon_core.core.persistence.shared.db import ensure_db, get_session +from getting_started._shared.env import ( + DEFAULT_LLAMA_CPP_BASE_URL, + DEFAULT_MINIF2F_LIMIT, + DEFAULT_MINIF2F_MAX_ITERATIONS, + ExampleSetupError, + base_url_from_model_target, + build_llamacpp_model_target, + env_optional_str, + env_positive_int, + env_str, + preflight_llamacpp_and_e2b, +) +from getting_started._shared.llamacpp import ManagedLlamaServer, start_llama_server +from getting_started._shared.launch import print_submission_summary +from getting_started._shared.model_cache import resolve_base_model +from getting_started._shared.observe import cli_status_command, dashboard_sample_url + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse command-line arguments for the example.""" + parser = argparse.ArgumentParser( + description="Submit three MiniF2F Lean proof samples with local llama.cpp and E2B." + ) + parser.add_argument( + "--limit", + type=_positive_int, + default=env_positive_int("ERGON_MINIF2F_LIMIT", DEFAULT_MINIF2F_LIMIT), + help="Number of MiniF2F samples to submit.", + ) + parser.add_argument( + "--base-url", + default=env_str("ERGON_LLAMA_CPP_BASE_URL", DEFAULT_LLAMA_CPP_BASE_URL), + help="Base URL for the llama.cpp OpenAI-compatible server.", + ) + parser.add_argument( + "--model", + default=env_optional_str("ERGON_LLAMA_CPP_MODEL"), + help="Optional llama.cpp model name; encoded as # on the target.", + ) + parser.add_argument( + "--model-target", + default=None, + help="Optional full Ergon model target. Overrides --base-url and --model.", + ) + parser.add_argument( + "--max-iterations", + type=_positive_int, + default=env_positive_int( + "ERGON_MINIF2F_MAX_ITERATIONS", + DEFAULT_MINIF2F_MAX_ITERATIONS, + ), + help="Maximum ReAct tool iterations per MiniF2F sample.", + ) + parser.add_argument( + "--base-model", + default=None, + help=( + "Local GGUF path or Hugging Face ':' ref. " + "Starts a managed llama.cpp server for this sample." + ), + ) + parser.add_argument( + "--model-cache-dir", + default=None, + help="Directory for downloaded Hugging Face GGUF files.", + ) + parser.add_argument( + "--llama-server-bin", + default="llama-server", + help="llama.cpp server command to run with --base-model.", + ) + parser.add_argument( + "--host", + default="127.0.0.1", + help="Host for the managed llama.cpp server.", + ) + parser.add_argument( + "--port", + type=_positive_int, + default=8080, + help="Port for the managed llama.cpp server.", + ) + parser.add_argument( + "--startup-timeout", + type=_positive_int, + default=60, + help="Seconds to wait for a managed llama.cpp server to become ready.", + ) + parser.add_argument( + "--keep-llama-server", + action="store_true", + help="Leave the managed llama.cpp server running after the example exits.", + ) + parser.add_argument( + "--json", + action="store_true", + help="Print machine-readable submission output.", + ) + args = parser.parse_args(argv) + _validate_model_routing(args, parser) + return args + + +def model_target_from_args(args: argparse.Namespace) -> str: + """Build the worker model target from parsed CLI args.""" + return build_llamacpp_model_target( + base_url=args.base_url, + model_name=args.model, + model_target=args.model_target, + ) + + +def preflight_base_url_from_args(args: argparse.Namespace) -> str: + """Choose the llama.cpp-compatible endpoint to check before launch.""" + if args.model_target is None: + return args.base_url + base_url = base_url_from_model_target(args.model_target) + if base_url is None: + raise ExampleSetupError( + "--model-target must include an http(s) endpoint for this local llama.cpp example." + ) + return base_url + + +def experiment_submission_service() -> ExperimentSubmissionService: + """Create the concrete service behind this Python composition example.""" + ensure_db() + return ExperimentSubmissionService.for_session(get_session()) + + +async def async_main(argv: Sequence[str] | None = None) -> int: + """Run the example and return a process exit code.""" + server: ManagedLlamaServer | None = None + keep_llama_server = False + try: + args = parse_args(argv) + keep_llama_server = args.keep_llama_server + if args.base_model is not None: + print(f"Resolving base model: {args.base_model}") + base_model_path = resolve_base_model( + args.base_model, + cache_dir=args.model_cache_dir, + ) + print(f"Using base model: {base_model_path}") + print(f"Starting llama.cpp on {args.host}:{args.port}") + server = start_llama_server( + base_model=str(base_model_path), + llama_server_bin=args.llama_server_bin, + host=args.host, + port=args.port, + startup_timeout=args.startup_timeout, + ) + args.base_url = server.base_url + args.model = server.discovered_model + + model_target = model_target_from_args(args) + preflight_llamacpp_and_e2b(base_url=preflight_base_url_from_args(args)) + except ExampleSetupError as exc: + if server is not None: + server.close(keep_running=keep_llama_server) + print(f"Setup error: {exc}", file=sys.stderr) + return 2 + + def make_worker() -> Worker: + return make_minif2f_worker( + model=model_target, + max_iterations=args.max_iterations, + ) + + try: + env = MiniF2FEnvironment( + name="mini-validation", + split="validation", + limit=args.limit, + worker=make_worker(), + evaluators=[make_minif2f_rubric()], + sandbox=LeanSandbox(), + ) + experiment = Experiment(name="minif2f-local-llamacpp", environments=[env]) + result = await experiment.submit( + service=experiment_submission_service(), + k=args.limit, + sampler=RandomSampler(seed=0), + ) + _print_submission_summary( + experiment_id=result.experiment_id, + sampler_invocation_id=result.sampler_invocation_id, + batch_id=result.batch_id, + sample_ids=result.sample_ids, + model_target=model_target, + limit=args.limit, + max_iterations=args.max_iterations, + as_json=args.json, + ) + return 0 + finally: + if server is not None: + server.close(keep_running=keep_llama_server) + + +def main(argv: Sequence[str] | None = None) -> int: + """Synchronous entrypoint for direct script execution.""" + return asyncio.run(async_main(argv)) + + +def _positive_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"{value!r} is not a positive integer") from exc + if parsed < 1: + raise argparse.ArgumentTypeError(f"{value!r} is not a positive integer") + return parsed + + +def _validate_model_routing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: + if args.base_model is None: + return + conflicts = { + "--model": args.model is not None, + "--model-target": args.model_target is not None, + } + for flag, is_set in conflicts.items(): + if is_set: + parser.error(f"--base-model cannot be combined with {flag}") + + +def _print_submission_summary( + *, + experiment_id: UUID, + sampler_invocation_id: UUID | None, + batch_id: UUID | None, + sample_ids: Sequence[UUID], + model_target: str, + limit: int, + max_iterations: int, + as_json: bool, +) -> None: + if as_json: + print_submission_summary( + experiment_id=experiment_id, + sampler_invocation_id=sampler_invocation_id, + batch_id=batch_id, + sample_ids=sample_ids, + as_json=True, + ) + return + + print("MiniF2F llama.cpp samples submitted") + print_submission_summary( + experiment_id=experiment_id, + sampler_invocation_id=sampler_invocation_id, + batch_id=batch_id, + sample_ids=sample_ids, + as_json=False, + ) + print(f"Model target: {model_target}") + print(f"Limit: {limit}") + print(f"Max iterations: {max_iterations}") + if sample_ids: + first_sample_id = sample_ids[0] + print(f"CLI status: {cli_status_command(first_sample_id)}") + url = dashboard_sample_url(first_sample_id) + if url: + print(f"Dashboard: {url}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/getting_started/_shared/launch.py b/examples/getting_started/_shared/launch.py index 17ddbd42c..5039d131a 100644 --- a/examples/getting_started/_shared/launch.py +++ b/examples/getting_started/_shared/launch.py @@ -1,5 +1,7 @@ """Launch helpers for getting-started examples.""" +import json +from collections.abc import Sequence from uuid import UUID @@ -7,3 +9,32 @@ def first_sample_id(sample_result: object) -> UUID: """Return the first launched sample id from an Ergon launch result.""" sample_ids = getattr(sample_result, "sample_ids") return sample_ids[0] + + +def print_submission_summary( + *, + experiment_id: UUID | str, + sampler_invocation_id: UUID | str | None, + batch_id: UUID | str | None, + sample_ids: Sequence[UUID | str], + as_json: bool = True, +) -> None: + """Print the stable machine-readable submission contract for examples.""" + payload = { + "experiment_id": str(experiment_id), + "sampler_invocation_id": ( + str(sampler_invocation_id) if sampler_invocation_id is not None else None + ), + "batch_id": str(batch_id) if batch_id is not None else None, + "sample_ids": [str(sample_id) for sample_id in sample_ids], + } + if as_json: + print(json.dumps(payload, indent=2)) + return + + print(f"Experiment id: {payload['experiment_id']}") + print(f"Sampler invocation id: {payload['sampler_invocation_id']}") + print(f"Batch id: {payload['batch_id']}") + print("Sample ids:") + for sample_id in payload["sample_ids"]: + print(f" - {sample_id}") diff --git a/examples/pyproject.toml b/examples/pyproject.toml index e1cc2d9c6..562ce52f4 100644 --- a/examples/pyproject.toml +++ b/examples/pyproject.toml @@ -20,4 +20,4 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["."] -include = ["getting_started*"] +include = ["getting_started*", "experiment_api*"] diff --git a/tests/examples/test_experiment_api_examples.py b/tests/examples/test_experiment_api_examples.py new file mode 100644 index 000000000..95662c2e2 --- /dev/null +++ b/tests/examples/test_experiment_api_examples.py @@ -0,0 +1,204 @@ +"""Smoke tests for public experiment API examples.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +from collections.abc import Iterator, Sequence +from pathlib import Path +from uuid import UUID, uuid4 + +from pydantic import ConfigDict + +from ergon_core.api import Environment, Experiment, ExperimentSubmitResult, Sample +from ergon_core.api.experiment.sampling import SamplingContext +from ergon_core.test_support.task_factory import task_with_id + +_EXAMPLES_ROOT = Path(__file__).parents[2] / "examples" / "experiment_api" + + +def _load_submit_module(example_dir: str): + path = _EXAMPLES_ROOT / example_dir / "submit.py" + spec = importlib.util.spec_from_file_location(f"example_{example_dir}", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sample(key: str = "sample-1", environment_name: str = "fake") -> Sample: + return Sample.from_tasks( + name=f"{environment_name}:{key}", + sample_key=key, + environment_name=environment_name, + tasks=[ + task_with_id( + uuid4(), + task_slug="solve", + instance_key=key, + description=f"Solve {key}", + ) + ], + ) + + +class FakeEnvironment(Environment): + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") + + def iter_samples(self) -> Iterator[Sample]: + yield _sample(environment_name=self.name) + + +class FakeSubmissionService: + def __init__(self) -> None: + self.submitted_experiments: list[Experiment] = [] + self.submit_calls: list[dict[str, object]] = [] + + async def submit( + self, + *, + experiment: Experiment, + k: int, + sampler, + candidate_pool_size: int | None, + policy_version: int | None, + ) -> ExperimentSubmitResult: + del policy_version + samples = [_sample(str(index), "fake") for index in range(candidate_pool_size or k)] + selected = await sampler.select( + samples=samples, + k=k, + context=SamplingContext(experiment_ref_id=uuid4(), candidate_pool_size=len(samples)), + ) + self.submitted_experiments.append(experiment) + self.submit_calls.append( + { + "k": k, + "candidate_pool_size": candidate_pool_size, + "sampler": sampler, + "candidate_count_seen_by_sampler": len(selected), + } + ) + return ExperimentSubmitResult( + experiment_ref_id=UUID("00000000-0000-0000-0000-000000000001"), + sampler_invocation_id=UUID("00000000-0000-0000-0000-000000000002"), + requested_k=k, + candidate_pool_size=candidate_pool_size or k, + selected_count=min(k, len(selected)), + sample_ids=[UUID("00000000-0000-0000-0000-000000000003")], + ) + + +def _stub_runtime_components(monkeypatch, module) -> None: + for name in ( + "make_minif2f_worker", + "make_minif2f_rubric", + "make_gdpeval_worker", + "make_gdpeval_rubric", + "make_research_rubric", + "make_swebench_worker", + "make_swebench_rubric", + ): + if hasattr(module, name): + monkeypatch.setattr(module, name, lambda *args, **kwargs: object()) + for name in ( + "LeanSandbox", + "_default_gdpeval_sandbox", + "_default_research_sandbox", + "_default_swebench_sandbox", + "SWEBenchSandbox", + ): + if hasattr(module, name): + monkeypatch.setattr(module, name, lambda *args, **kwargs: object()) + + +def test_minif2f_example_prints_sample_ids_not_run_ids(monkeypatch, capsys) -> None: + module = _load_submit_module("01_minif2f_single_environment") + service = FakeSubmissionService() + _stub_runtime_components(monkeypatch, module) + monkeypatch.setattr(module, "experiment_submission_service", lambda: service) + monkeypatch.setattr(module, "MiniF2FEnvironment", FakeEnvironment) + + asyncio.run(module.main()) + + output = json.loads(capsys.readouterr().out) + assert output["experiment_ref_id"] + assert output["sampler_invocation_id"] + assert output["sample_ids"] + assert "run_ids" not in output + assert "definition_id" not in output + + +def test_mixed_environment_example_composes_all_builtin_environments(monkeypatch) -> None: + module = _load_submit_module("02_mixed_environment_training") + service = FakeSubmissionService() + _stub_runtime_components(monkeypatch, module) + for name in ( + "MiniF2FEnvironment", + "ResearchRubricsEnvironment", + "GDPEvalEnvironment", + "SweBenchVerifiedEnvironment", + ): + monkeypatch.setattr(module, name, FakeEnvironment) + monkeypatch.setattr(module, "experiment_submission_service", lambda: service) + + asyncio.run(module.main()) + + experiment = service.submitted_experiments[0] + assert [env.name for env in experiment.environments] == [ + "mini-validation", + "research-validation", + "gdp-validation", + "swebench-train", + ] + + +def test_streaming_example_uses_candidate_pool_larger_than_k(monkeypatch) -> None: + module = _load_submit_module("03_streaming_hf_dataset") + service = FakeSubmissionService() + _stub_runtime_components(monkeypatch, module) + monkeypatch.setattr(module, "SweBenchVerifiedEnvironment", FakeEnvironment) + monkeypatch.setattr(module, "experiment_submission_service", lambda: service) + + asyncio.run(module.main()) + + call = service.submit_calls[0] + assert call["k"] == 8 + assert call["candidate_pool_size"] == 32 + + +def test_curriculum_example_uses_custom_sampler_and_larger_pool(monkeypatch) -> None: + module = _load_submit_module("04_curriculum_sampler") + service = FakeSubmissionService() + monkeypatch.setattr(module, "MiniF2FEnvironment", FakeEnvironment) + monkeypatch.setattr(module, "experiment_submission_service", lambda: service) + + asyncio.run(module.main()) + + call = service.submit_calls[0] + assert call["k"] == 16 + assert call["candidate_pool_size"] == 64 + assert call["sampler"].name == "easy-first" + assert call["candidate_count_seen_by_sampler"] == 64 + + +def test_row_dependent_example_passes_runtime_config_callables(monkeypatch) -> None: + module = _load_submit_module("05_row_dependent_runtime_configs") + captured: dict[str, object] = {} + + class CapturingEnvironment(FakeEnvironment): + def __init__(self, **kwargs) -> None: + captured.update(kwargs) + super().__init__(**kwargs) + + _stub_runtime_components(monkeypatch, module) + monkeypatch.setattr(module, "SweBenchVerifiedEnvironment", CapturingEnvironment) + monkeypatch.setattr(module, "experiment_submission_service", lambda: FakeSubmissionService()) + + asyncio.run(module.main()) + + assert callable(captured["worker"]) + assert callable(captured["evaluators"]) + assert callable(captured["sandbox"]) diff --git a/tests/examples/test_getting_started_examples.py b/tests/examples/test_getting_started_examples.py new file mode 100644 index 000000000..9a87c377f --- /dev/null +++ b/tests/examples/test_getting_started_examples.py @@ -0,0 +1,132 @@ +"""Sample-centered getting-started example tests.""" + +from __future__ import annotations + +import importlib.util +import json +from collections.abc import Iterator +from pathlib import Path +from uuid import UUID + +import pytest +from pydantic import ConfigDict + +from ergon_core.api import Environment, ExperimentSubmitResult, Sample +from ergon_core.test_support.task_factory import task_with_id + +_SUBMIT_PATH = ( + Path(__file__).parents[2] + / "examples" + / "getting_started" + / "01_minif2f_local_llamacpp" + / "submit.py" +) + + +def _load_submit_module(): + spec = importlib.util.spec_from_file_location("minif2f_local_llamacpp_submit", _SUBMIT_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeEnvironment(Environment): + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") + + def iter_samples(self) -> Iterator[Sample]: + yield Sample.from_tasks( + name="mini-validation:sample-1", + sample_key="sample-1", + environment_name=self.name, + tasks=[ + task_with_id( + UUID("00000000-0000-0000-0000-000000000011"), + description="Solve sample 1", + ) + ], + ) + + +class FakeSubmissionService: + async def submit( + self, + *, + experiment, + k: int, + sampler, + candidate_pool_size: int | None, + policy_version: int | None, + ) -> ExperimentSubmitResult: + del experiment, sampler, candidate_pool_size, policy_version + return ExperimentSubmitResult( + experiment_ref_id=UUID("11111111-1111-1111-1111-111111111111"), + sampler_invocation_id=UUID("22222222-2222-2222-2222-222222222222"), + requested_k=k, + candidate_pool_size=k, + selected_count=k, + sample_ids=[UUID("33333333-3333-3333-3333-333333333333")], + ) + + +@pytest.mark.asyncio +async def test_getting_started_submit_uses_experiment_api(monkeypatch, capsys) -> None: + module = _load_submit_module() + observed: dict[str, object] = {} + + def fake_preflight(*, base_url: str) -> object: + observed["preflight_base_url"] = base_url + return object() + + def fake_make_worker(*, model: str, max_iterations: int) -> object: + observed["worker_model"] = model + observed["worker_max_iterations"] = max_iterations + return object() + + class CapturingEnvironment(FakeEnvironment): + def __init__(self, **kwargs) -> None: + observed["environment_kwargs"] = kwargs + super().__init__(**kwargs) + + monkeypatch.setattr(module, "preflight_llamacpp_and_e2b", fake_preflight) + monkeypatch.setattr(module, "MiniF2FEnvironment", CapturingEnvironment) + monkeypatch.setattr(module, "make_minif2f_worker", fake_make_worker) + monkeypatch.setattr(module, "make_minif2f_rubric", lambda: object()) + monkeypatch.setattr(module, "LeanSandbox", lambda: object()) + monkeypatch.setattr(module, "experiment_submission_service", lambda: FakeSubmissionService()) + + exit_code = await module.async_main( + [ + "--limit", + "2", + "--base-url", + "http://localhost:8080", + "--model", + "local-proof-model", + "--max-iterations", + "4", + "--json", + ] + ) + + assert exit_code == 0 + assert observed["preflight_base_url"] == "http://localhost:8080" + assert observed["worker_model"] == "llamacpp:http://localhost:8080#local-proof-model" + assert observed["worker_max_iterations"] == 4 + assert observed["environment_kwargs"]["limit"] == 2 + output = json.loads(capsys.readouterr().out) + assert output == { + "experiment_id": "11111111-1111-1111-1111-111111111111", + "sample_ids": ["33333333-3333-3333-3333-333333333333"], + } + + +def test_getting_started_source_has_no_definition_launch_path() -> None: + source = _SUBMIT_PATH.read_text() + + assert "MiniF2FBenchmark" not in source + assert "persist_benchmark" not in source + assert "launch_run" not in source + assert "definition_id" not in source + assert "run_ids" not in source diff --git a/tests/examples/test_minif2f_local_llamacpp_example.py b/tests/examples/test_minif2f_local_llamacpp_example.py index 37a6414c3..5195ee12c 100644 --- a/tests/examples/test_minif2f_local_llamacpp_example.py +++ b/tests/examples/test_minif2f_local_llamacpp_example.py @@ -9,7 +9,9 @@ from uuid import UUID import pytest +from pydantic import ConfigDict +from ergon_core.api import Environment, ExperimentSubmitResult from getting_started._shared import llamacpp from getting_started._shared import model_cache from getting_started._shared import env @@ -19,12 +21,12 @@ / "examples" / "getting_started" / "01_minif2f_local_llamacpp" - / "run.py" + / "submit.py" ) def _load_example_module(): - spec = importlib.util.spec_from_file_location("minif2f_local_llamacpp_run", _EXAMPLE_PATH) + spec = importlib.util.spec_from_file_location("minif2f_local_llamacpp_submit", _EXAMPLE_PATH) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) @@ -37,7 +39,10 @@ def test_example_uses_packaged_imports_without_repo_root_sys_path_hack() -> None assert "sys.path.insert" not in source assert "Path(__file__).resolve().parents" not in source - assert "from ergon_builtins.benchmarks.minif2f.benchmark import MiniF2FBenchmark" in source + assert "MiniF2FBenchmark" not in source + assert "persist_benchmark" not in source + assert "launch_run" not in source + assert "from ergon_builtins.environments import MiniF2FEnvironment" in source assert "from getting_started._shared.env import" in source @@ -321,11 +326,11 @@ def fake_discover(*, base_url: str, timeout_seconds: float) -> str: @pytest.mark.asyncio -async def test_main_persists_and_launches_minif2f_with_local_worker(monkeypatch, capsys) -> None: +async def test_main_submits_minif2f_with_local_worker(monkeypatch, capsys) -> None: module = _load_example_module() observed: dict[str, object] = {} - definition_id = UUID("11111111-1111-1111-1111-111111111111") sample_id = UUID("22222222-2222-2222-2222-222222222222") + experiment_id = UUID("11111111-1111-1111-1111-111111111111") def fake_preflight(*, base_url: str) -> object: observed["preflight_base_url"] = base_url @@ -341,32 +346,42 @@ def fake_make_worker(*, model: str, max_iterations: int) -> FakeWorker: observed["worker_max_iterations"] = max_iterations return FakeWorker(model=model, max_iterations=max_iterations) - class FakeBenchmark: - def __init__(self, *, limit: int, worker_factory) -> None: - observed["benchmark_limit"] = limit - self.worker = worker_factory() - - class FakeHandle: - def __init__(self, persisted_definition_id: UUID) -> None: - self.definition_id = persisted_definition_id - - class FakeRunResult: - def __init__(self, launched_run_id: UUID) -> None: - self.run_ids = [launched_run_id] - - def fake_persist_benchmark(benchmark: FakeBenchmark) -> FakeHandle: - observed["persisted_worker_model"] = benchmark.worker.model - return FakeHandle(definition_id) - - async def fake_launch_run(persisted_definition_id: UUID) -> FakeRunResult: - observed["launched_definition_id"] = persisted_definition_id - return FakeRunResult(sample_id) + class FakeEnvironment(Environment): + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") + + def __init__(self, **kwargs) -> None: + observed["environment_kwargs"] = kwargs + super().__init__(**kwargs) + + class FakeService: + async def submit( + self, + *, + experiment, + k: int, + sampler, + candidate_pool_size, + policy_version, + ): + observed["experiment_name"] = experiment.name + observed["k"] = k + observed["sampler_name"] = sampler.name + observed["candidate_pool_size"] = candidate_pool_size + observed["policy_version"] = policy_version + return ExperimentSubmitResult( + experiment_ref_id=experiment_id, + requested_k=k, + candidate_pool_size=k, + selected_count=1, + sample_ids=[sample_id], + ) monkeypatch.setattr(module, "preflight_llamacpp_and_e2b", fake_preflight) - monkeypatch.setattr(module, "MiniF2FBenchmark", FakeBenchmark) + monkeypatch.setattr(module, "MiniF2FEnvironment", FakeEnvironment) monkeypatch.setattr(module, "make_minif2f_worker", fake_make_worker) - monkeypatch.setattr(module, "persist_benchmark", fake_persist_benchmark) - monkeypatch.setattr(module, "launch_run", fake_launch_run) + monkeypatch.setattr(module, "make_minif2f_rubric", lambda: object()) + monkeypatch.setattr(module, "LeanSandbox", lambda: object()) + monkeypatch.setattr(module, "experiment_submission_service", lambda: FakeService()) monkeypatch.setenv("ERGON_DASHBOARD_URL", "http://localhost:3000") exit_code = await module.async_main( @@ -383,19 +398,26 @@ async def fake_launch_run(persisted_definition_id: UUID) -> FakeRunResult: ) assert exit_code == 0 - assert observed == { - "preflight_base_url": "http://localhost:8080", - "benchmark_limit": 2, - "worker_model": "llamacpp:http://localhost:8080#local-proof-model", - "worker_max_iterations": 4, - "persisted_worker_model": "llamacpp:http://localhost:8080#local-proof-model", - "launched_definition_id": definition_id, - } + assert observed["preflight_base_url"] == "http://localhost:8080" + assert observed["worker_model"] == "llamacpp:http://localhost:8080#local-proof-model" + assert observed["worker_max_iterations"] == 4 + environment_kwargs = observed["environment_kwargs"] + assert environment_kwargs["name"] == "mini-validation" + assert environment_kwargs["split"] == "validation" + assert environment_kwargs["limit"] == 2 + assert isinstance(environment_kwargs["worker"], FakeWorker) + assert len(environment_kwargs["evaluators"]) == 1 + assert observed["experiment_name"] == "minif2f-local-llamacpp" + assert observed["k"] == 2 + assert observed["sampler_name"] == "random" + assert observed["candidate_pool_size"] is None + assert observed["policy_version"] is None output = capsys.readouterr().out - assert str(definition_id) in output + assert str(experiment_id) in output assert str(sample_id) in output - assert "uv run ergon run status 22222222-2222-2222-2222-222222222222" in output - assert "http://localhost:3000/run/22222222-2222-2222-2222-222222222222" in output + assert "Definition id" not in output + assert "uv run ergon sample status 22222222-2222-2222-2222-222222222222" in output + assert "http://localhost:3000/samples/22222222-2222-2222-2222-222222222222" in output @pytest.mark.asyncio @@ -405,7 +427,7 @@ async def test_main_resolves_base_model_starts_llamacpp_and_cleans_up( ) -> None: module = _load_example_module() observed: dict[str, object] = {} - definition_id = UUID("33333333-3333-3333-3333-333333333333") + experiment_id = UUID("33333333-3333-3333-3333-333333333333") sample_id = UUID("44444444-4444-4444-4444-444444444444") resolved_model = tmp_path / "model.gguf" resolved_model.write_text("fake model") @@ -428,30 +450,44 @@ def fake_make_worker(*, model: str, max_iterations: int) -> _ObservedWorker: observed["worker_max_iterations"] = max_iterations return _ObservedWorker(model=model, max_iterations=max_iterations) - class FakeHandle: - def __init__(self, persisted_definition_id: UUID) -> None: - self.definition_id = persisted_definition_id - - class FakeRunResult: - def __init__(self, launched_run_id: UUID) -> None: - self.run_ids = [launched_run_id] - - def fake_persist_benchmark(benchmark: _ObservedBenchmark) -> FakeHandle: - observed["persisted_worker_model"] = benchmark.worker.model - return FakeHandle(definition_id) - - async def fake_launch_run(persisted_definition_id: UUID) -> FakeRunResult: - observed["launched_definition_id"] = persisted_definition_id - return FakeRunResult(sample_id) + class FakeEnvironment(Environment): + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") + + def __init__(self, **kwargs) -> None: + observed["environment_kwargs"] = kwargs + super().__init__(**kwargs) + + class FakeService: + async def submit( + self, + *, + experiment, + k: int, + sampler, + candidate_pool_size, + policy_version, + ): + observed["experiment_name"] = experiment.name + observed["k"] = k + observed["sampler_name"] = sampler.name + observed["candidate_pool_size"] = candidate_pool_size + observed["policy_version"] = policy_version + return ExperimentSubmitResult( + experiment_ref_id=experiment_id, + requested_k=k, + candidate_pool_size=k, + selected_count=1, + sample_ids=[sample_id], + ) monkeypatch.setattr(module, "resolve_base_model", fake_resolve) monkeypatch.setattr(module, "start_llama_server", fake_start) monkeypatch.setattr(module, "preflight_llamacpp_and_e2b", fake_preflight) - _ObservedBenchmark.observed = observed - monkeypatch.setattr(module, "MiniF2FBenchmark", _ObservedBenchmark) + monkeypatch.setattr(module, "MiniF2FEnvironment", FakeEnvironment) monkeypatch.setattr(module, "make_minif2f_worker", fake_make_worker) - monkeypatch.setattr(module, "persist_benchmark", fake_persist_benchmark) - monkeypatch.setattr(module, "launch_run", fake_launch_run) + monkeypatch.setattr(module, "make_minif2f_rubric", lambda: object()) + monkeypatch.setattr(module, "LeanSandbox", lambda: object()) + monkeypatch.setattr(module, "experiment_submission_service", lambda: FakeService()) exit_code = await module.async_main( [ @@ -485,10 +521,20 @@ async def fake_launch_run(persisted_definition_id: UUID) -> FakeRunResult: "startup_timeout": 15, }, "preflight_base_url": "http://127.0.0.1:8123", - "benchmark_limit": 3, "worker_model": "llamacpp:http://127.0.0.1:8123#mini-proof-local", "worker_max_iterations": 12, - "persisted_worker_model": "llamacpp:http://127.0.0.1:8123#mini-proof-local", - "launched_definition_id": definition_id, + "environment_kwargs": { + "name": "mini-validation", + "split": "validation", + "limit": 3, + "worker": observed["environment_kwargs"]["worker"], + "evaluators": observed["environment_kwargs"]["evaluators"], + "sandbox": observed["environment_kwargs"]["sandbox"], + }, + "experiment_name": "minif2f-local-llamacpp", + "k": 3, + "sampler_name": "random", + "candidate_pool_size": None, + "policy_version": None, "server_closed_keep_running": True, }